[thirdparty/git.git] / ppc / sha1ppc.S

/*
 * SHA-1 implementation for PowerPC.
 *
 * Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
 */

/*
 * PowerPC calling convention:
 * %r0 - volatile temp
 * %r1 - stack pointer.
 * %r2 - reserved
 * %r3-%r12 - Incoming arguments & return values; volatile.
 * %r13-%r31 - Callee-save registers
 * %lr - Return address, volatile
 * %ctr - volatile
 *
 * Register usage in this routine:
 * %r0 - temp
 * %r3 - argument (pointer to 5 words of SHA state)
 * %r4 - argument (pointer to data to hash)
 * %r5 - Constant K in SHA round (initially number of blocks to hash)
 * %r6-%r10 - Working copies of SHA variables A..E (actually E..A order)
 * %r11-%r26 - Data being hashed W[].
 * %r27-%r31 - Previous copies of A..E, for final add back.
 * %ctr - loop count
 */


/*
 * We roll the registers for A, B, C, D, E around on each
 * iteration; E on iteration t is D on iteration t+1, and so on.
 * We use registers 6 - 10 for this.  (Registers 27 - 31 hold
 * the previous values.)
 */
#define RA(t)	(((t)+4)%5+6)
#define RB(t)	(((t)+3)%5+6)
#define RC(t)	(((t)+2)%5+6)
#define RD(t)	(((t)+1)%5+6)
#define RE(t)	(((t)+0)%5+6)

/* We use registers 11 - 26 for the W values */
#define W(t)	((t)%16+11)

/* Register 5 is used for the constant k */

/*
 * The basic SHA-1 round function is:
 * E += ROTL(A,5) + F(B,C,D) + W[i] + K;  B = ROTL(B,30)
 * Then the variables are renamed: (A,B,C,D,E) = (E,A,B,C,D).
 *
 * Every 20 rounds, the function F() and the constant K changes:
 * - 20 rounds of f0(b,c,d) = "bit wise b ? c : d" =  (^b & d) + (b & c)
 * - 20 rounds of f1(b,c,d) = b^c^d = (b^d)^c
 * - 20 rounds of f2(b,c,d) = majority(b,c,d) = (b&d) + ((b^d)&c)
 * - 20 more rounds of f1(b,c,d)
 *
 * These are all scheduled for near-optimal performance on a G4.
 * The G4 is a 3-issue out-of-order machine with 3 ALUs, but it can only
 * *consider* starting the oldest 3 instructions per cycle.  So to get
 * maximum performance out of it, you have to treat it as an in-order
 * machine.  Which means interleaving the computation round t with the
 * computation of W[t+4].
 *
 * The first 16 rounds use W values loaded directly from memory, while the
 * remaining 64 use values computed from those first 16.  We preload
 * 4 values before starting, so there are three kinds of rounds:
 * - The first 12 (all f0) also load the W values from memory.
 * - The next 64 compute W(i+4) in parallel. 8*f0, 20*f1, 20*f2, 16*f1.
 * - The last 4 (all f1) do not do anything with W.
 *
 * Therefore, we have 6 different round functions:
 * STEPD0_LOAD(t,s) - Perform round t and load W(s).  s < 16
 * STEPD0_UPDATE(t,s) - Perform round t and compute W(s).  s >= 16.
 * STEPD1_UPDATE(t,s)
 * STEPD2_UPDATE(t,s)
 * STEPD1(t) - Perform round t with no load or update.
 *
 * The G5 is more fully out-of-order, and can find the parallelism
 * by itself.  The big limit is that it has a 2-cycle ALU latency, so
 * even though it's 2-way, the code has to be scheduled as if it's
 * 4-way, which can be a limit.  To help it, we try to schedule the
 * read of RA(t) as late as possible so it doesn't stall waiting for
 * the previous round's RE(t-1), and we try to rotate RB(t) as early
 * as possible while reading RC(t) (= RB(t-1)) as late as possible.
 */

/* the initial loads. */
#define LOADW(s) \
	lwz	W(s),(s)*4(%r4)

/*
 * Perform a step with F0, and load W(s).  Uses W(s) as a temporary
 * before loading it.
 * This is actually 10 instructions, which is an awkward fit.
 * It can execute grouped as listed, or delayed one instruction.
 * (If delayed two instructions, there is a stall before the start of the
 * second line.)  Thus, two iterations take 7 cycles, 3.5 cycles per round.
 */
#define STEPD0_LOAD(t,s) \
add RE(t),RE(t),W(t); andc   %r0,RD(t),RB(t);  and    W(s),RC(t),RB(t); \
add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;      rotlwi RB(t),RB(t),30;   \
add RE(t),RE(t),W(s); add    %r0,%r0,%r5;      lwz    W(s),(s)*4(%r4);  \
add RE(t),RE(t),%r0

/*
 * This is likewise awkward, 13 instructions.  However, it can also
 * execute starting with 2 out of 3 possible moduli, so it does 2 rounds
 * in 9 cycles, 4.5 cycles/round.
 */
#define STEPD0_UPDATE(t,s,loadk...) \
add RE(t),RE(t),W(t); andc   %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
add RE(t),RE(t),%r0;  and    %r0,RC(t),RB(t); xor    W(s),W(s),W((s)-8);      \
add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     xor    W(s),W(s),W((s)-14);     \
add RE(t),RE(t),%r5;  loadk; rotlwi RB(t),RB(t),30;  rotlwi W(s),W(s),1;     \
add RE(t),RE(t),%r0

/* Nicely optimal.  Conveniently, also the most common. */
#define STEPD1_UPDATE(t,s,loadk...) \
add RE(t),RE(t),W(t); xor    %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
add RE(t),RE(t),%r5;  loadk; xor %r0,%r0,RC(t);  xor W(s),W(s),W((s)-8);      \
add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     xor    W(s),W(s),W((s)-14);     \
add RE(t),RE(t),%r0;  rotlwi RB(t),RB(t),30;  rotlwi W(s),W(s),1

/*
 * The naked version, no UPDATE, for the last 4 rounds.  3 cycles per.
 * We could use W(s) as a temp register, but we don't need it.
 */
#define STEPD1(t) \
                        add   RE(t),RE(t),W(t); xor    %r0,RD(t),RB(t); \
rotlwi RB(t),RB(t),30;  add   RE(t),RE(t),%r5;  xor    %r0,%r0,RC(t);   \
add    RE(t),RE(t),%r0; rotlwi %r0,RA(t),5;     /* spare slot */        \
add    RE(t),RE(t),%r0

/*
 * 14 instructions, 5 cycles per.  The majority function is a bit
 * awkward to compute.  This can execute with a 1-instruction delay,
 * but it causes a 2-instruction delay, which triggers a stall.
 */
#define STEPD2_UPDATE(t,s,loadk...) \
add RE(t),RE(t),W(t); and    %r0,RD(t),RB(t); xor    W(s),W((s)-16),W((s)-3); \
add RE(t),RE(t),%r0;  xor    %r0,RD(t),RB(t); xor    W(s),W(s),W((s)-8);      \
add RE(t),RE(t),%r5;  loadk; and %r0,%r0,RC(t);  xor W(s),W(s),W((s)-14);     \
add RE(t),RE(t),%r0;  rotlwi %r0,RA(t),5;     rotlwi W(s),W(s),1;             \
add RE(t),RE(t),%r0;  rotlwi RB(t),RB(t),30

#define STEP0_LOAD4(t,s)		\
	STEPD0_LOAD(t,s);		\
	STEPD0_LOAD((t+1),(s)+1);	\
	STEPD0_LOAD((t)+2,(s)+2);	\
	STEPD0_LOAD((t)+3,(s)+3)

#define STEPUP4(fn, t, s, loadk...)		\
	STEP##fn##_UPDATE(t,s,);		\
	STEP##fn##_UPDATE((t)+1,(s)+1,);	\
	STEP##fn##_UPDATE((t)+2,(s)+2,);	\
	STEP##fn##_UPDATE((t)+3,(s)+3,loadk)

#define STEPUP20(fn, t, s, loadk...)	\
	STEPUP4(fn, t, s,);		\
	STEPUP4(fn, (t)+4, (s)+4,);	\
	STEPUP4(fn, (t)+8, (s)+8,);	\
	STEPUP4(fn, (t)+12, (s)+12,);	\
	STEPUP4(fn, (t)+16, (s)+16, loadk)

	.globl	ppc_sha1_core
ppc_sha1_core:
	stwu	%r1,-80(%r1)
	stmw	%r13,4(%r1)

	/* Load up A - E */
	lmw	%r27,0(%r3)

	mtctr	%r5

1:
	LOADW(0)
	lis	%r5,0x5a82
	mr	RE(0),%r31
	LOADW(1)
	mr	RD(0),%r30
	mr	RC(0),%r29
	LOADW(2)
	ori	%r5,%r5,0x7999	/* K0-19 */
	mr	RB(0),%r28
	LOADW(3)
	mr	RA(0),%r27

	STEP0_LOAD4(0, 4)
	STEP0_LOAD4(4, 8)
	STEP0_LOAD4(8, 12)
	STEPUP4(D0, 12, 16,)
	STEPUP4(D0, 16, 20, lis %r5,0x6ed9)

	ori	%r5,%r5,0xeba1	/* K20-39 */
	STEPUP20(D1, 20, 24, lis %r5,0x8f1b)

	ori	%r5,%r5,0xbcdc	/* K40-59 */
	STEPUP20(D2, 40, 44, lis %r5,0xca62)

	ori	%r5,%r5,0xc1d6	/* K60-79 */
	STEPUP4(D1, 60, 64,)
	STEPUP4(D1, 64, 68,)
	STEPUP4(D1, 68, 72,)
	STEPUP4(D1, 72, 76,)
	addi	%r4,%r4,64
	STEPD1(76)
	STEPD1(77)
	STEPD1(78)
	STEPD1(79)

	/* Add results to original values */
	add	%r31,%r31,RE(0)
	add	%r30,%r30,RD(0)
	add	%r29,%r29,RC(0)
	add	%r28,%r28,RB(0)
	add	%r27,%r27,RA(0)

	bdnz	1b

	/* Save final hash, restore registers, and return */
	stmw	%r27,0(%r3)
	lmw	%r13,4(%r1)
	addi	%r1,%r1,80
	blr
Commit	Line	Data
a6ef3518 PM	1	/*
	2	* SHA-1 implementation for PowerPC.
	3	*
	4	* Copyright (C) 2005 Paul Mackerras <paulus@samba.org>
	5	*/
a6ef3518 PM	6
a6ef3518 PM	7	/*
84702995	8	* PowerPC calling convention:
	9	* %r0 - volatile temp
	10	* %r1 - stack pointer.
	11	* %r2 - reserved
	12	* %r3-%r12 - Incoming arguments & return values; volatile.
	13	* %r13-%r31 - Callee-save registers
	14	* %lr - Return address, volatile
	15	* %ctr - volatile
	16	*
	17	* Register usage in this routine:
	18	* %r0 - temp
	19	* %r3 - argument (pointer to 5 words of SHA state)
	20	* %r4 - argument (pointer to data to hash)
3dff5379	21	* %r5 - Constant K in SHA round (initially number of blocks to hash)
84702995	22	* %r6-%r10 - Working copies of SHA variables A..E (actually E..A order)
	23	* %r11-%r26 - Data being hashed W[].
	24	* %r27-%r31 - Previous copies of A..E, for final add back.
	25	* %ctr - loop count
	26	*/
	27
	28
	29	/*
	30	* We roll the registers for A, B, C, D, E around on each
	31	* iteration; E on iteration t is D on iteration t+1, and so on.
	32	* We use registers 6 - 10 for this. (Registers 27 - 31 hold
	33	* the previous values.)
a6ef3518	34	*/
84702995	35	#define RA(t) (((t)+4)%5+6)
	36	#define RB(t) (((t)+3)%5+6)
	37	#define RC(t) (((t)+2)%5+6)
	38	#define RD(t) (((t)+1)%5+6)
	39	#define RE(t) (((t)+0)%5+6)
	40
	41	/* We use registers 11 - 26 for the W values */
	42	#define W(t) ((t)%16+11)
	43
	44	/* Register 5 is used for the constant k */
	45
	46	/*
	47	* The basic SHA-1 round function is:
	48	* E += ROTL(A,5) + F(B,C,D) + W[i] + K; B = ROTL(B,30)
	49	* Then the variables are renamed: (A,B,C,D,E) = (E,A,B,C,D).
	50	*
3dff5379	51	* Every 20 rounds, the function F() and the constant K changes:
84702995	52	* - 20 rounds of f0(b,c,d) = "bit wise b ? c : d" = (^b & d) + (b & c)
	53	* - 20 rounds of f1(b,c,d) = b^c^d = (b^d)^c
	54	* - 20 rounds of f2(b,c,d) = majority(b,c,d) = (b&d) + ((b^d)&c)
	55	* - 20 more rounds of f1(b,c,d)
	56	*
	57	* These are all scheduled for near-optimal performance on a G4.
	58	* The G4 is a 3-issue out-of-order machine with 3 ALUs, but it can only
	59	* consider starting the oldest 3 instructions per cycle. So to get
3dff5379	60	* maximum performance out of it, you have to treat it as an in-order
84702995	61	* machine. Which means interleaving the computation round t with the
	62	* computation of W[t+4].
	63	*
	64	* The first 16 rounds use W values loaded directly from memory, while the
82e5a82f	65	* remaining 64 use values computed from those first 16. We preload
84702995	66	* 4 values before starting, so there are three kinds of rounds:
	67	* - The first 12 (all f0) also load the W values from memory.
	68	* - The next 64 compute W(i+4) in parallel. 8f0, 20f1, 20f2, 16f1.
	69	* - The last 4 (all f1) do not do anything with W.
	70	*
	71	* Therefore, we have 6 different round functions:
	72	* STEPD0_LOAD(t,s) - Perform round t and load W(s). s < 16
	73	* STEPD0_UPDATE(t,s) - Perform round t and compute W(s). s >= 16.
	74	* STEPD1_UPDATE(t,s)
	75	* STEPD2_UPDATE(t,s)
	76	* STEPD1(t) - Perform round t with no load or update.
	77	*
	78	* The G5 is more fully out-of-order, and can find the parallelism
	79	* by itself. The big limit is that it has a 2-cycle ALU latency, so
	80	* even though it's 2-way, the code has to be scheduled as if it's
	81	* 4-way, which can be a limit. To help it, we try to schedule the
	82	* read of RA(t) as late as possible so it doesn't stall waiting for
	83	* the previous round's RE(t-1), and we try to rotate RB(t) as early
	84	* as possible while reading RC(t) (= RB(t-1)) as late as possible.
	85	*/
	86
	87	/* the initial loads. */
	88	#define LOADW(s) \
	89	lwz W(s),(s)*4(%r4)
	90
	91	/*
	92	* Perform a step with F0, and load W(s). Uses W(s) as a temporary
	93	* before loading it.
	94	* This is actually 10 instructions, which is an awkward fit.
	95	* It can execute grouped as listed, or delayed one instruction.
	96	* (If delayed two instructions, there is a stall before the start of the
	97	* second line.) Thus, two iterations take 7 cycles, 3.5 cycles per round.
	98	*/
	99	#define STEPD0_LOAD(t,s) \
	100	add RE(t),RE(t),W(t); andc %r0,RD(t),RB(t); and W(s),RC(t),RB(t); \
	101	add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; rotlwi RB(t),RB(t),30; \
	102	add RE(t),RE(t),W(s); add %r0,%r0,%r5; lwz W(s),(s)*4(%r4); \
	103	add RE(t),RE(t),%r0
	104
	105	/*
	106	* This is likewise awkward, 13 instructions. However, it can also
	107	* execute starting with 2 out of 3 possible moduli, so it does 2 rounds
	108	* in 9 cycles, 4.5 cycles/round.
	109	*/
	110	#define STEPD0_UPDATE(t,s,loadk...) \
	111	add RE(t),RE(t),W(t); andc %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
	112	add RE(t),RE(t),%r0; and %r0,RC(t),RB(t); xor W(s),W(s),W((s)-8); \
	113	add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; xor W(s),W(s),W((s)-14); \
	114	add RE(t),RE(t),%r5; loadk; rotlwi RB(t),RB(t),30; rotlwi W(s),W(s),1; \
	115	add RE(t),RE(t),%r0
	116
	117	/* Nicely optimal. Conveniently, also the most common. */
	118	#define STEPD1_UPDATE(t,s,loadk...) \
	119	add RE(t),RE(t),W(t); xor %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
	120	add RE(t),RE(t),%r5; loadk; xor %r0,%r0,RC(t); xor W(s),W(s),W((s)-8); \
	121	add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; xor W(s),W(s),W((s)-14); \
	122	add RE(t),RE(t),%r0; rotlwi RB(t),RB(t),30; rotlwi W(s),W(s),1
	123
	124	/*
	125	* The naked version, no UPDATE, for the last 4 rounds. 3 cycles per.
	126	* We could use W(s) as a temp register, but we don't need it.
	127	*/
	128	#define STEPD1(t) \
	129	add RE(t),RE(t),W(t); xor %r0,RD(t),RB(t); \
130	rotlwi RB(t),RB(t),30; add RE(t),RE(t),%r5; xor %r0,%r0,RC(t); \
131	add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; /* spare slot */ \
132	add RE(t),RE(t),%r0
133
134	/*
135	* 14 instructions, 5 cycles per. The majority function is a bit
136	* awkward to compute. This can execute with a 1-instruction delay,
137	* but it causes a 2-instruction delay, which triggers a stall.
138	*/
139	#define STEPD2_UPDATE(t,s,loadk...) \
140	add RE(t),RE(t),W(t); and %r0,RD(t),RB(t); xor W(s),W((s)-16),W((s)-3); \
141	add RE(t),RE(t),%r0; xor %r0,RD(t),RB(t); xor W(s),W(s),W((s)-8); \
142	add RE(t),RE(t),%r5; loadk; and %r0,%r0,RC(t); xor W(s),W(s),W((s)-14); \
143	add RE(t),RE(t),%r0; rotlwi %r0,RA(t),5; rotlwi W(s),W(s),1; \
144	add RE(t),RE(t),%r0; rotlwi RB(t),RB(t),30
145
146	#define STEP0_LOAD4(t,s) \
147	STEPD0_LOAD(t,s); \
148	STEPD0_LOAD((t+1),(s)+1); \
149	STEPD0_LOAD((t)+2,(s)+2); \
150	STEPD0_LOAD((t)+3,(s)+3)
151
152	#define STEPUP4(fn, t, s, loadk...) \
153	STEP##fn##_UPDATE(t,s,); \
154	STEP##fn##_UPDATE((t)+1,(s)+1,); \
155	STEP##fn##_UPDATE((t)+2,(s)+2,); \
156	STEP##fn##_UPDATE((t)+3,(s)+3,loadk)
157
158	#define STEPUP20(fn, t, s, loadk...) \
159	STEPUP4(fn, t, s,); \
160	STEPUP4(fn, (t)+4, (s)+4,); \
161	STEPUP4(fn, (t)+8, (s)+8,); \
162	STEPUP4(fn, (t)+12, (s)+12,); \
163	STEPUP4(fn, (t)+16, (s)+16, loadk)
a6ef3518	164
9126f009 NP	165	.globl ppc_sha1_core
9126f009 NP	166	ppc_sha1_core:
84702995	167	stwu %r1,-80(%r1)
84702995	168	stmw %r13,4(%r1)
a6ef3518 PM	169
a6ef3518 PM	170	/* Load up A - E */
84702995	171	lmw %r27,0(%r3)
a6ef3518 PM	172
	173	mtctr %r5
	174
84702995	175	1:
	176	LOADW(0)
	177	lis %r5,0x5a82
	178	mr RE(0),%r31
a6ef3518	179	LOADW(1)
84702995	180	mr RD(0),%r30
84702995	181	mr RC(0),%r29
a6ef3518	182	LOADW(2)
84702995	183	ori %r5,%r5,0x7999 /* K0-19 */
84702995	184	mr RB(0),%r28
a6ef3518	185	LOADW(3)
84702995	186	mr RA(0),%r27
	187
	188	STEP0_LOAD4(0, 4)
	189	STEP0_LOAD4(4, 8)
	190	STEP0_LOAD4(8, 12)
	191	STEPUP4(D0, 12, 16,)
	192	STEPUP4(D0, 16, 20, lis %r5,0x6ed9)
a6ef3518	193
84702995	194	ori %r5,%r5,0xeba1 /* K20-39 */
	195	STEPUP20(D1, 20, 24, lis %r5,0x8f1b)
	196
	197	ori %r5,%r5,0xbcdc /* K40-59 */
	198	STEPUP20(D2, 40, 44, lis %r5,0xca62)
	199
	200	ori %r5,%r5,0xc1d6 /* K60-79 */
	201	STEPUP4(D1, 60, 64,)
	202	STEPUP4(D1, 64, 68,)
	203	STEPUP4(D1, 68, 72,)
	204	STEPUP4(D1, 72, 76,)
	205	addi %r4,%r4,64
a6ef3518 PM	206	STEPD1(76)
	207	STEPD1(77)
	208	STEPD1(78)
	209	STEPD1(79)
	210
84702995	211	/* Add results to original values */
	212	add %r31,%r31,RE(0)
	213	add %r30,%r30,RD(0)
	214	add %r29,%r29,RC(0)
	215	add %r28,%r28,RB(0)
	216	add %r27,%r27,RA(0)
a6ef3518	217
a6ef3518 PM	218	bdnz 1b
a6ef3518 PM	219
84702995	220	/* Save final hash, restore registers, and return */
	221	stmw %r27,0(%r3)
	222	lmw %r13,4(%r1)
	223	addi %r1,%r1,80
a6ef3518	224	blr