2 * Copyright 2015-2018 The OpenSSL Project Authors. All Rights Reserved.
4 * Licensed under the Apache License 2.0 (the "License"). You may not use
5 * this file except in compliance with the License. You can obtain a copy
6 * in the file LICENSE in the source distribution or at
7 * https://www.openssl.org/source/license.html
12 #include <openssl/crypto.h>
14 #include "crypto/poly1305.h"
15 #include "poly1305_local.h"
17 size_t Poly1305_ctx_size(void)
19 return sizeof(struct poly1305_context
);
22 /* pick 32-bit unsigned integer in little endian order */
23 static unsigned int U8TOU32(const unsigned char *p
)
25 return (((unsigned int)(p
[0] & 0xff)) |
26 ((unsigned int)(p
[1] & 0xff) << 8) |
27 ((unsigned int)(p
[2] & 0xff) << 16) |
28 ((unsigned int)(p
[3] & 0xff) << 24));
32 * Implementations can be classified by amount of significant bits in
33 * words making up the multi-precision value, or in other words radix
34 * or base of numerical representation, e.g. base 2^64, base 2^32,
35 * base 2^26. Complementary characteristic is how wide is the result of
36 * multiplication of pair of digits, e.g. it would take 128 bits to
37 * accommodate multiplication result in base 2^64 case. These are used
38 * interchangeably. To describe implementation that is. But interface
39 * is designed to isolate this so that low-level primitives implemented
40 * in assembly can be self-contained/self-coherent.
44 * Even though there is __int128 reference implementation targeting
45 * 64-bit platforms provided below, it's not obvious that it's optimal
46 * choice for every one of them. Depending on instruction set overall
47 * amount of instructions can be comparable to one in __int64
48 * implementation. Amount of multiplication instructions would be lower,
49 * but not necessarily overall. And in out-of-order execution context,
50 * it is the latter that can be crucial...
52 * On related note. Poly1305 author, D. J. Bernstein, discusses and
53 * provides floating-point implementations of the algorithm in question.
54 * It made a lot of sense by the time of introduction, because most
55 * then-modern processors didn't have pipelined integer multiplier.
56 * [Not to mention that some had non-constant timing for integer
57 * multiplications.] Floating-point instructions on the other hand could
58 * be issued every cycle, which allowed to achieve better performance.
59 * Nowadays, with SIMD and/or out-or-order execution, shared or
60 * even emulated FPU, it's more complicated, and floating-point
61 * implementation is not necessarily optimal choice in every situation,
67 typedef unsigned int u32
;
70 * poly1305_blocks processes a multiple of POLY1305_BLOCK_SIZE blocks
71 * of |inp| no longer than |len|. Behaviour for |len| not divisible by
72 * block size is unspecified in general case, even though in reference
73 * implementation the trailing chunk is simply ignored. Per algorithm
74 * specification, every input block, complete or last partial, is to be
75 * padded with a bit past most significant byte. The latter kind is then
76 * padded with zeros till block size. This last partial block padding
77 * is caller(*)'s responsibility, and because of this the last partial
78 * block is always processed with separate call with |len| set to
79 * POLY1305_BLOCK_SIZE and |padbit| to 0. In all other cases |padbit|
80 * should be set to 1 to perform implicit padding with 128th bit.
81 * poly1305_blocks does not actually check for this constraint though,
82 * it's caller(*)'s responsibility to comply.
84 * (*) In the context "caller" is not application code, but higher
85 * level Poly1305_* from this very module, so that quirks are
89 poly1305_blocks(void *ctx
, const unsigned char *inp
, size_t len
, u32 padbit
);
92 * Type-agnostic "rip-off" from constant_time.h
94 # define CONSTANT_TIME_CARRY(a,b) ( \
95 (a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1) \
98 # if (defined(__SIZEOF_INT128__) && __SIZEOF_INT128__==16) && \
99 (defined(__SIZEOF_LONG__) && __SIZEOF_LONG__==8)
101 typedef unsigned long u64
;
102 typedef __uint128_t u128
;
109 /* pick 32-bit unsigned integer in little endian order */
110 static u64
U8TOU64(const unsigned char *p
)
112 return (((u64
)(p
[0] & 0xff)) |
113 ((u64
)(p
[1] & 0xff) << 8) |
114 ((u64
)(p
[2] & 0xff) << 16) |
115 ((u64
)(p
[3] & 0xff) << 24) |
116 ((u64
)(p
[4] & 0xff) << 32) |
117 ((u64
)(p
[5] & 0xff) << 40) |
118 ((u64
)(p
[6] & 0xff) << 48) |
119 ((u64
)(p
[7] & 0xff) << 56));
122 /* store a 32-bit unsigned integer in little endian */
123 static void U64TO8(unsigned char *p
, u64 v
)
125 p
[0] = (unsigned char)((v
) & 0xff);
126 p
[1] = (unsigned char)((v
>> 8) & 0xff);
127 p
[2] = (unsigned char)((v
>> 16) & 0xff);
128 p
[3] = (unsigned char)((v
>> 24) & 0xff);
129 p
[4] = (unsigned char)((v
>> 32) & 0xff);
130 p
[5] = (unsigned char)((v
>> 40) & 0xff);
131 p
[6] = (unsigned char)((v
>> 48) & 0xff);
132 p
[7] = (unsigned char)((v
>> 56) & 0xff);
135 static void poly1305_init(void *ctx
, const unsigned char key
[16])
137 poly1305_internal
*st
= (poly1305_internal
*) ctx
;
144 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
145 st
->r
[0] = U8TOU64(&key
[0]) & 0x0ffffffc0fffffff;
146 st
->r
[1] = U8TOU64(&key
[8]) & 0x0ffffffc0ffffffc;
150 poly1305_blocks(void *ctx
, const unsigned char *inp
, size_t len
, u32 padbit
)
152 poly1305_internal
*st
= (poly1305_internal
*)ctx
;
167 while (len
>= POLY1305_BLOCK_SIZE
) {
169 h0
= (u64
)(d0
= (u128
)h0
+ U8TOU64(inp
+ 0));
170 h1
= (u64
)(d1
= (u128
)h1
+ (d0
>> 64) + U8TOU64(inp
+ 8));
172 * padbit can be zero only when original len was
173 * POLY1306_BLOCK_SIZE, but we don't check
175 h2
+= (u64
)(d1
>> 64) + padbit
;
177 /* h *= r "%" p, where "%" stands for "partial remainder" */
178 d0
= ((u128
)h0
* r0
) +
180 d1
= ((u128
)h0
* r1
) +
185 /* last reduction step: */
186 /* a) h2:h0 = h2<<128 + d1<<64 + d0 */
188 h1
= (u64
)(d1
+= d0
>> 64);
189 h2
+= (u64
)(d1
>> 64);
190 /* b) (h2:h0 += (h2:h0>>130) * 5) %= 2^130 */
191 c
= (h2
>> 2) + (h2
& ~3UL);
194 h1
+= (c
= CONSTANT_TIME_CARRY(h0
,c
));
195 h2
+= CONSTANT_TIME_CARRY(h1
,c
);
197 * Occasional overflows to 3rd bit of h2 are taken care of
198 * "naturally". If after this point we end up at the top of
199 * this loop, then the overflow bit will be accounted for
200 * in next iteration. If we end up in poly1305_emit, then
201 * comparison to modulus below will still count as "carry
202 * into 131st bit", so that properly reduced value will be
203 * picked in conditional move.
206 inp
+= POLY1305_BLOCK_SIZE
;
207 len
-= POLY1305_BLOCK_SIZE
;
215 static void poly1305_emit(void *ctx
, unsigned char mac
[16],
218 poly1305_internal
*st
= (poly1305_internal
*) ctx
;
228 /* compare to modulus by computing h + -p */
229 g0
= (u64
)(t
= (u128
)h0
+ 5);
230 g1
= (u64
)(t
= (u128
)h1
+ (t
>> 64));
231 g2
= h2
+ (u64
)(t
>> 64);
233 /* if there was carry into 131st bit, h1:h0 = g1:g0 */
234 mask
= 0 - (g2
>> 2);
238 h0
= (h0
& mask
) | g0
;
239 h1
= (h1
& mask
) | g1
;
241 /* mac = (h + nonce) % (2^128) */
242 h0
= (u64
)(t
= (u128
)h0
+ nonce
[0] + ((u64
)nonce
[1]<<32));
243 h1
= (u64
)(t
= (u128
)h1
+ nonce
[2] + ((u64
)nonce
[3]<<32) + (t
>> 64));
251 # if defined(_WIN32) && !defined(__MINGW32__)
252 typedef unsigned __int64 u64
;
253 # elif defined(__arch64__)
254 typedef unsigned long u64
;
256 typedef unsigned long long u64
;
264 /* store a 32-bit unsigned integer in little endian */
265 static void U32TO8(unsigned char *p
, unsigned int v
)
267 p
[0] = (unsigned char)((v
) & 0xff);
268 p
[1] = (unsigned char)((v
>> 8) & 0xff);
269 p
[2] = (unsigned char)((v
>> 16) & 0xff);
270 p
[3] = (unsigned char)((v
>> 24) & 0xff);
273 static void poly1305_init(void *ctx
, const unsigned char key
[16])
275 poly1305_internal
*st
= (poly1305_internal
*) ctx
;
284 /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
285 st
->r
[0] = U8TOU32(&key
[0]) & 0x0fffffff;
286 st
->r
[1] = U8TOU32(&key
[4]) & 0x0ffffffc;
287 st
->r
[2] = U8TOU32(&key
[8]) & 0x0ffffffc;
288 st
->r
[3] = U8TOU32(&key
[12]) & 0x0ffffffc;
292 poly1305_blocks(void *ctx
, const unsigned char *inp
, size_t len
, u32 padbit
)
294 poly1305_internal
*st
= (poly1305_internal
*)ctx
;
297 u32 h0
, h1
, h2
, h3
, h4
, c
;
315 while (len
>= POLY1305_BLOCK_SIZE
) {
317 h0
= (u32
)(d0
= (u64
)h0
+ U8TOU32(inp
+ 0));
318 h1
= (u32
)(d1
= (u64
)h1
+ (d0
>> 32) + U8TOU32(inp
+ 4));
319 h2
= (u32
)(d2
= (u64
)h2
+ (d1
>> 32) + U8TOU32(inp
+ 8));
320 h3
= (u32
)(d3
= (u64
)h3
+ (d2
>> 32) + U8TOU32(inp
+ 12));
321 h4
+= (u32
)(d3
>> 32) + padbit
;
323 /* h *= r "%" p, where "%" stands for "partial remainder" */
324 d0
= ((u64
)h0
* r0
) +
328 d1
= ((u64
)h0
* r1
) +
333 d2
= ((u64
)h0
* r2
) +
338 d3
= ((u64
)h0
* r3
) +
345 /* last reduction step: */
346 /* a) h4:h0 = h4<<128 + d3<<96 + d2<<64 + d1<<32 + d0 */
348 h1
= (u32
)(d1
+= d0
>> 32);
349 h2
= (u32
)(d2
+= d1
>> 32);
350 h3
= (u32
)(d3
+= d2
>> 32);
351 h4
+= (u32
)(d3
>> 32);
352 /* b) (h4:h0 += (h4:h0>>130) * 5) %= 2^130 */
353 c
= (h4
>> 2) + (h4
& ~3U);
356 h1
+= (c
= CONSTANT_TIME_CARRY(h0
,c
));
357 h2
+= (c
= CONSTANT_TIME_CARRY(h1
,c
));
358 h3
+= (c
= CONSTANT_TIME_CARRY(h2
,c
));
359 h4
+= CONSTANT_TIME_CARRY(h3
,c
);
361 * Occasional overflows to 3rd bit of h4 are taken care of
362 * "naturally". If after this point we end up at the top of
363 * this loop, then the overflow bit will be accounted for
364 * in next iteration. If we end up in poly1305_emit, then
365 * comparison to modulus below will still count as "carry
366 * into 131st bit", so that properly reduced value will be
367 * picked in conditional move.
370 inp
+= POLY1305_BLOCK_SIZE
;
371 len
-= POLY1305_BLOCK_SIZE
;
381 static void poly1305_emit(void *ctx
, unsigned char mac
[16],
384 poly1305_internal
*st
= (poly1305_internal
*) ctx
;
385 u32 h0
, h1
, h2
, h3
, h4
;
386 u32 g0
, g1
, g2
, g3
, g4
;
396 /* compare to modulus by computing h + -p */
397 g0
= (u32
)(t
= (u64
)h0
+ 5);
398 g1
= (u32
)(t
= (u64
)h1
+ (t
>> 32));
399 g2
= (u32
)(t
= (u64
)h2
+ (t
>> 32));
400 g3
= (u32
)(t
= (u64
)h3
+ (t
>> 32));
401 g4
= h4
+ (u32
)(t
>> 32);
403 /* if there was carry into 131st bit, h3:h0 = g3:g0 */
404 mask
= 0 - (g4
>> 2);
410 h0
= (h0
& mask
) | g0
;
411 h1
= (h1
& mask
) | g1
;
412 h2
= (h2
& mask
) | g2
;
413 h3
= (h3
& mask
) | g3
;
415 /* mac = (h + nonce) % (2^128) */
416 h0
= (u32
)(t
= (u64
)h0
+ nonce
[0]);
417 h1
= (u32
)(t
= (u64
)h1
+ (t
>> 32) + nonce
[1]);
418 h2
= (u32
)(t
= (u64
)h2
+ (t
>> 32) + nonce
[2]);
419 h3
= (u32
)(t
= (u64
)h3
+ (t
>> 32) + nonce
[3]);
424 U32TO8(mac
+ 12, h3
);
428 int poly1305_init(void *ctx
, const unsigned char key
[16], void *func
);
429 void poly1305_blocks(void *ctx
, const unsigned char *inp
, size_t len
,
430 unsigned int padbit
);
431 void poly1305_emit(void *ctx
, unsigned char mac
[16],
432 const unsigned int nonce
[4]);
435 void Poly1305_Init(POLY1305
*ctx
, const unsigned char key
[32])
437 ctx
->nonce
[0] = U8TOU32(&key
[16]);
438 ctx
->nonce
[1] = U8TOU32(&key
[20]);
439 ctx
->nonce
[2] = U8TOU32(&key
[24]);
440 ctx
->nonce
[3] = U8TOU32(&key
[28]);
443 poly1305_init(ctx
->opaque
, key
);
446 * Unlike reference poly1305_init assembly counterpart is expected
447 * to return a value: non-zero if it initializes ctx->func, and zero
448 * otherwise. Latter is to simplify assembly in cases when there no
449 * multiple code paths to switch between.
451 if (!poly1305_init(ctx
->opaque
, key
, &ctx
->func
)) {
452 ctx
->func
.blocks
= poly1305_blocks
;
453 ctx
->func
.emit
= poly1305_emit
;
463 * This "eclipses" poly1305_blocks and poly1305_emit, but it's
464 * conscious choice imposed by -Wshadow compiler warnings.
466 # define poly1305_blocks (*poly1305_blocks_p)
467 # define poly1305_emit (*poly1305_emit_p)
470 void Poly1305_Update(POLY1305
*ctx
, const unsigned char *inp
, size_t len
)
474 * As documented, poly1305_blocks is never called with input
475 * longer than single block and padbit argument set to 0. This
476 * property is fluently used in assembly modules to optimize
477 * padbit handling on loop boundary.
479 poly1305_blocks_f poly1305_blocks_p
= ctx
->func
.blocks
;
483 if ((num
= ctx
->num
)) {
484 rem
= POLY1305_BLOCK_SIZE
- num
;
486 memcpy(ctx
->data
+ num
, inp
, rem
);
487 poly1305_blocks(ctx
->opaque
, ctx
->data
, POLY1305_BLOCK_SIZE
, 1);
491 /* Still not enough data to process a block. */
492 memcpy(ctx
->data
+ num
, inp
, len
);
493 ctx
->num
= num
+ len
;
498 rem
= len
% POLY1305_BLOCK_SIZE
;
501 if (len
>= POLY1305_BLOCK_SIZE
) {
502 poly1305_blocks(ctx
->opaque
, inp
, len
, 1);
507 memcpy(ctx
->data
, inp
, rem
);
512 void Poly1305_Final(POLY1305
*ctx
, unsigned char mac
[16])
515 poly1305_blocks_f poly1305_blocks_p
= ctx
->func
.blocks
;
516 poly1305_emit_f poly1305_emit_p
= ctx
->func
.emit
;
520 if ((num
= ctx
->num
)) {
521 ctx
->data
[num
++] = 1; /* pad bit */
522 while (num
< POLY1305_BLOCK_SIZE
)
523 ctx
->data
[num
++] = 0;
524 poly1305_blocks(ctx
->opaque
, ctx
->data
, POLY1305_BLOCK_SIZE
, 0);
527 poly1305_emit(ctx
->opaque
, mac
, ctx
->nonce
);
529 /* zero out the state */
530 OPENSSL_cleanse(ctx
, sizeof(*ctx
));