[thirdparty/linux.git] / lib / crypto / poly1305-donna64.c

// SPDX-License-Identifier: GPL-2.0 OR MIT
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 *
 * This is based in part on Andrew Moon's poly1305-donna, which is in the
 * public domain.
 */

#include <linux/kernel.h>
#include <asm/unaligned.h>
#include <crypto/internal/poly1305.h>

typedef __uint128_t u128;

void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16])
{
	u64 t0, t1;

	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
	t0 = get_unaligned_le64(&raw_key[0]);
	t1 = get_unaligned_le64(&raw_key[8]);

	key->key.r64[0] = t0 & 0xffc0fffffffULL;
	key->key.r64[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffffULL;
	key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;

	/* s = 20*r */
	key->precomputed_s.r64[0] = key->key.r64[1] * 20;
	key->precomputed_s.r64[1] = key->key.r64[2] * 20;
}
EXPORT_SYMBOL(poly1305_core_setkey);

void poly1305_core_blocks(struct poly1305_state *state,
			  const struct poly1305_core_key *key, const void *src,
			  unsigned int nblocks, u32 hibit)
{
	const u8 *input = src;
	u64 hibit64;
	u64 r0, r1, r2;
	u64 s1, s2;
	u64 h0, h1, h2;
	u64 c;
	u128 d0, d1, d2, d;

	if (!nblocks)
		return;

	hibit64 = ((u64)hibit) << 40;

	r0 = key->key.r64[0];
	r1 = key->key.r64[1];
	r2 = key->key.r64[2];

	h0 = state->h64[0];
	h1 = state->h64[1];
	h2 = state->h64[2];

	s1 = key->precomputed_s.r64[0];
	s2 = key->precomputed_s.r64[1];

	do {
		u64 t0, t1;

		/* h += m[i] */
		t0 = get_unaligned_le64(&input[0]);
		t1 = get_unaligned_le64(&input[8]);

		h0 += t0 & 0xfffffffffffULL;
		h1 += ((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL;
		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) | hibit64;

		/* h *= r */
		d0 = (u128)h0 * r0;
		d = (u128)h1 * s2;
		d0 += d;
		d = (u128)h2 * s1;
		d0 += d;
		d1 = (u128)h0 * r1;
		d = (u128)h1 * r0;
		d1 += d;
		d = (u128)h2 * s2;
		d1 += d;
		d2 = (u128)h0 * r2;
		d = (u128)h1 * r1;
		d2 += d;
		d = (u128)h2 * r0;
		d2 += d;

		/* (partial) h %= p */
		c = (u64)(d0 >> 44);
		h0 = (u64)d0 & 0xfffffffffffULL;
		d1 += c;
		c = (u64)(d1 >> 44);
		h1 = (u64)d1 & 0xfffffffffffULL;
		d2 += c;
		c = (u64)(d2 >> 42);
		h2 = (u64)d2 & 0x3ffffffffffULL;
		h0 += c * 5;
		c = h0 >> 44;
		h0 = h0 & 0xfffffffffffULL;
		h1 += c;

		input += POLY1305_BLOCK_SIZE;
	} while (--nblocks);

	state->h64[0] = h0;
	state->h64[1] = h1;
	state->h64[2] = h2;
}
EXPORT_SYMBOL(poly1305_core_blocks);

void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
			void *dst)
{
	u8 *mac = dst;
	u64 h0, h1, h2, c;
	u64 g0, g1, g2;
	u64 t0, t1;

	/* fully carry h */
	h0 = state->h64[0];
	h1 = state->h64[1];
	h2 = state->h64[2];

	c = h1 >> 44;
	h1 &= 0xfffffffffffULL;
	h2 += c;
	c = h2 >> 42;
	h2 &= 0x3ffffffffffULL;
	h0 += c * 5;
	c = h0 >> 44;
	h0 &= 0xfffffffffffULL;
	h1 += c;
	c = h1 >> 44;
	h1 &= 0xfffffffffffULL;
	h2 += c;
	c = h2 >> 42;
	h2 &= 0x3ffffffffffULL;
	h0 += c * 5;
	c = h0 >> 44;
	h0 &= 0xfffffffffffULL;
	h1 += c;

	/* compute h + -p */
	g0 = h0 + 5;
	c  = g0 >> 44;
	g0 &= 0xfffffffffffULL;
	g1 = h1 + c;
	c  = g1 >> 44;
	g1 &= 0xfffffffffffULL;
	g2 = h2 + c - (1ULL << 42);

	/* select h if h < p, or h + -p if h >= p */
	c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
	g0 &= c;
	g1 &= c;
	g2 &= c;
	c  = ~c;
	h0 = (h0 & c) | g0;
	h1 = (h1 & c) | g1;
	h2 = (h2 & c) | g2;

	if (likely(nonce)) {
		/* h = (h + nonce) */
		t0 = ((u64)nonce[1] << 32) | nonce[0];
		t1 = ((u64)nonce[3] << 32) | nonce[2];

		h0 += t0 & 0xfffffffffffULL;
		c = h0 >> 44;
		h0 &= 0xfffffffffffULL;
		h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffffULL) + c;
		c = h1 >> 44;
		h1 &= 0xfffffffffffULL;
		h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
		h2 &= 0x3ffffffffffULL;
	}

	/* mac = h % (2^128) */
	h0 = h0 | (h1 << 44);
	h1 = (h1 >> 20) | (h2 << 24);

	put_unaligned_le64(h0, &mac[0]);
	put_unaligned_le64(h1, &mac[8]);
}
EXPORT_SYMBOL(poly1305_core_emit);
Commit	Line	Data
1c08a104 JD	1	// SPDX-License-Identifier: GPL-2.0 OR MIT
	2	/*
	3	* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
	4	*
	5	* This is based in part on Andrew Moon's poly1305-donna, which is in the
	6	* public domain.
	7	*/
	8
	9	#include <linux/kernel.h>
	10	#include <asm/unaligned.h>
	11	#include <crypto/internal/poly1305.h>
	12
	13	typedef __uint128_t u128;
	14
	15	void poly1305_core_setkey(struct poly1305_core_key *key, const u8 raw_key[16])
	16	{
	17	u64 t0, t1;
	18
	19	/* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
	20	t0 = get_unaligned_le64(&raw_key[0]);
	21	t1 = get_unaligned_le64(&raw_key[8]);
	22
	23	key->key.r64[0] = t0 & 0xffc0fffffffULL;
	24	key->key.r64[1] = ((t0 >> 44) \| (t1 << 20)) & 0xfffffc0ffffULL;
	25	key->key.r64[2] = ((t1 >> 24)) & 0x00ffffffc0fULL;
	26
	27	/* s = 20r /
	28	key->precomputed_s.r64[0] = key->key.r64[1] * 20;
	29	key->precomputed_s.r64[1] = key->key.r64[2] * 20;
	30	}
	31	EXPORT_SYMBOL(poly1305_core_setkey);
	32
	33	void poly1305_core_blocks(struct poly1305_state *state,
	34	const struct poly1305_core_key key, const void src,
	35	unsigned int nblocks, u32 hibit)
	36	{
	37	const u8 *input = src;
	38	u64 hibit64;
	39	u64 r0, r1, r2;
	40	u64 s1, s2;
	41	u64 h0, h1, h2;
	42	u64 c;
	43	u128 d0, d1, d2, d;
	44
	45	if (!nblocks)
	46	return;
	47
	48	hibit64 = ((u64)hibit) << 40;
	49
	50	r0 = key->key.r64[0];
	51	r1 = key->key.r64[1];
	52	r2 = key->key.r64[2];
	53
	54	h0 = state->h64[0];
	55	h1 = state->h64[1];
	56	h2 = state->h64[2];
	57
	58	s1 = key->precomputed_s.r64[0];
	59	s2 = key->precomputed_s.r64[1];
	60
	61	do {
	62	u64 t0, t1;
	63
	64	/* h += m[i] */
65	t0 = get_unaligned_le64(&input[0]);
66	t1 = get_unaligned_le64(&input[8]);
67
68	h0 += t0 & 0xfffffffffffULL;
69	h1 += ((t0 >> 44) \| (t1 << 20)) & 0xfffffffffffULL;
70	h2 += (((t1 >> 24)) & 0x3ffffffffffULL) \| hibit64;
71
72	/* h = r /
73	d0 = (u128)h0 * r0;
74	d = (u128)h1 * s2;
75	d0 += d;
76	d = (u128)h2 * s1;
77	d0 += d;
78	d1 = (u128)h0 * r1;
79	d = (u128)h1 * r0;
80	d1 += d;
81	d = (u128)h2 * s2;
82	d1 += d;
83	d2 = (u128)h0 * r2;
84	d = (u128)h1 * r1;
85	d2 += d;
86	d = (u128)h2 * r0;
87	d2 += d;
88
89	/* (partial) h %= p */
90	c = (u64)(d0 >> 44);
91	h0 = (u64)d0 & 0xfffffffffffULL;
92	d1 += c;
93	c = (u64)(d1 >> 44);
94	h1 = (u64)d1 & 0xfffffffffffULL;
95	d2 += c;
96	c = (u64)(d2 >> 42);
97	h2 = (u64)d2 & 0x3ffffffffffULL;
98	h0 += c * 5;
99	c = h0 >> 44;
100	h0 = h0 & 0xfffffffffffULL;
101	h1 += c;
102
103	input += POLY1305_BLOCK_SIZE;
104	} while (--nblocks);
105
106	state->h64[0] = h0;
107	state->h64[1] = h1;
108	state->h64[2] = h2;
109	}
110	EXPORT_SYMBOL(poly1305_core_blocks);
111
112	void poly1305_core_emit(const struct poly1305_state *state, const u32 nonce[4],
113	void *dst)
114	{
115	u8 *mac = dst;
116	u64 h0, h1, h2, c;
117	u64 g0, g1, g2;
118	u64 t0, t1;
119
120	/* fully carry h */
121	h0 = state->h64[0];
122	h1 = state->h64[1];
123	h2 = state->h64[2];
124
125	c = h1 >> 44;
126	h1 &= 0xfffffffffffULL;
127	h2 += c;
128	c = h2 >> 42;
129	h2 &= 0x3ffffffffffULL;
130	h0 += c * 5;
131	c = h0 >> 44;
132	h0 &= 0xfffffffffffULL;
133	h1 += c;
134	c = h1 >> 44;
135	h1 &= 0xfffffffffffULL;
136	h2 += c;
137	c = h2 >> 42;
138	h2 &= 0x3ffffffffffULL;
139	h0 += c * 5;
140	c = h0 >> 44;
141	h0 &= 0xfffffffffffULL;
142	h1 += c;
143
144	/* compute h + -p */
145	g0 = h0 + 5;
146	c = g0 >> 44;
147	g0 &= 0xfffffffffffULL;
148	g1 = h1 + c;
149	c = g1 >> 44;
150	g1 &= 0xfffffffffffULL;
151	g2 = h2 + c - (1ULL << 42);
152
153	/* select h if h < p, or h + -p if h >= p */
154	c = (g2 >> ((sizeof(u64) * 8) - 1)) - 1;
155	g0 &= c;
156	g1 &= c;
157	g2 &= c;
158	c = ~c;
159	h0 = (h0 & c) \| g0;
160	h1 = (h1 & c) \| g1;
161	h2 = (h2 & c) \| g2;
162
163	if (likely(nonce)) {
164	/* h = (h + nonce) */
165	t0 = ((u64)nonce[1] << 32) \| nonce[0];
166	t1 = ((u64)nonce[3] << 32) \| nonce[2];
167
168	h0 += t0 & 0xfffffffffffULL;
169	c = h0 >> 44;
170	h0 &= 0xfffffffffffULL;
171	h1 += (((t0 >> 44) \| (t1 << 20)) & 0xfffffffffffULL) + c;
172	c = h1 >> 44;
173	h1 &= 0xfffffffffffULL;
174	h2 += (((t1 >> 24)) & 0x3ffffffffffULL) + c;
175	h2 &= 0x3ffffffffffULL;
176	}
177
178	/* mac = h % (2^128) */
179	h0 = h0 \| (h1 << 44);
180	h1 = (h1 >> 20) \| (h2 << 24);
181
182	put_unaligned_le64(h0, &mac[0]);
183	put_unaligned_le64(h1, &mac[8]);
184	}
185	EXPORT_SYMBOL(poly1305_core_emit);