*
* @v multiplicand Multiplicand element
* @v multiplier Multiplier element
- * @v result Result element pair
+ * @v result Result element
* @v carry Carry element
*/
static inline __attribute__ (( always_inline )) void
uint32_t discard_high;
__asm__ __volatile__ ( /* Perform multiplication */
- "umull %0, %1, %5, %6\n\t"
+ "umull %0, %1, %4, %5\n\t"
/* Accumulate result */
"adds %2, %0\n\t"
- "adcs %3, %1\n\t"
+ "adc %1, #0\n\t"
/* Accumulate carry (cannot overflow) */
- "adc %4, #0\n\t"
+ "adds %2, %3\n\t"
+ "adc %3, %1, #0\n\t"
: "=r" ( discard_low ),
"=r" ( discard_high ),
- "+r" ( result[0] ),
- "+r" ( result[1] ),
+ "+r" ( *result ),
"+r" ( *carry )
: "r" ( multiplicand ),
- "r" ( multiplier ) );
+ "r" ( multiplier )
+ : "cc" );
}
#endif /* _BITS_BIGINT_H */
*
* @v multiplicand Multiplicand element
* @v multiplier Multiplier element
- * @v result Result element pair
+ * @v result Result element
* @v carry Carry element
*/
static inline __attribute__ (( always_inline )) void
uint64_t discard_high;
__asm__ __volatile__ ( /* Perform multiplication */
- "mul %0, %5, %6\n\t"
- "umulh %1, %5, %6\n\t"
+ "mul %0, %4, %5\n\t"
+ "umulh %1, %4, %5\n\t"
/* Accumulate result */
"adds %2, %2, %0\n\t"
- "adcs %3, %3, %1\n\t"
+ "adc %1, %1, xzr\n\t"
/* Accumulate carry (cannot overflow) */
- "adc %4, %4, xzr\n\t"
+ "adds %2, %2, %3\n\t"
+ "adc %3, %1, xzr\n\t"
: "=&r" ( discard_low ),
"=r" ( discard_high ),
- "+r" ( result[0] ),
- "+r" ( result[1] ),
+ "+r" ( *result ),
"+r" ( *carry )
: "r" ( multiplicand ),
- "r" ( multiplier ) );
+ "r" ( multiplier )
+ : "cc" );
}
#endif /* _BITS_BIGINT_H */
*
* @v multiplicand Multiplicand element
* @v multiplier Multiplier element
- * @v result Result element pair
+ * @v result Result element
* @v carry Carry element
*/
static inline __attribute__ (( always_inline )) void
uint64_t discard_carry;
__asm__ __volatile__ ( /* Perform multiplication */
- "mul.d %0, %6, %7\n\t"
- "mulh.du %1, %6, %7\n\t"
+ "mul.d %0, %5, %6\n\t"
+ "mulh.du %1, %5, %6\n\t"
/* Accumulate low half */
"add.d %3, %3, %0\n\t"
"sltu %2, %3, %0\n\t"
- /* Add carry to high half (cannot overflow) */
"add.d %1, %1, %2\n\t"
- /* Accumulate high half */
- "add.d %4, %4, %1\n\t"
- "sltu %2, %4, %1\n\t"
/* Accumulate carry (cannot overflow) */
- "add.d %5, %5, %2\n\t"
+ "add.d %3, %3, %4\n\t"
+ "sltu %2, %3, %4\n\t"
+ "add.d %4, %1, %2\n\t"
: "=&r" ( discard_low ),
"=r" ( discard_high ),
"=r" ( discard_carry ),
- "+r" ( result[0] ),
- "+r" ( result[1] ),
+ "+r" ( *result ),
"+r" ( *carry )
: "r" ( multiplicand ),
"r" ( multiplier ) );
*
* @v multiplicand Multiplicand element
* @v multiplier Multiplier element
- * @v result Result element pair
+ * @v result Result element
* @v carry Carry element
*/
static inline __attribute__ (( always_inline )) void
unsigned long discard_carry;
__asm__ __volatile__ ( /* Perform multiplication */
- "mulhu %1, %6, %7\n\t"
- "mul %0, %6, %7\n\t"
+ "mulhu %1, %5, %6\n\t"
+ "mul %0, %5, %6\n\t"
/* Accumulate low half */
"add %3, %3, %0\n\t"
"sltu %2, %3, %0\n\t"
- /* Add carry to high half (cannot overflow) */
"add %1, %1, %2\n\t"
- /* Accumulate high half */
- "add %4, %4, %1\n\t"
- "sltu %2, %4, %1\n\t"
/* Accumulate carry (cannot overflow) */
- "add %5, %5, %2\n\t"
+ "add %3, %3, %4\n\t"
+ "sltu %2, %3, %4\n\t"
+ "add %4, %1, %2\n\t"
: "=r" ( discard_low ),
"=&r" ( discard_high ),
"=r" ( discard_carry ),
- "+r" ( result[0] ),
- "+r" ( result[1] ),
+ "+r" ( *result ),
"+r" ( *carry )
: "r" ( multiplicand ),
"r" ( multiplier ) );
*
* @v multiplicand Multiplicand element
* @v multiplier Multiplier element
- * @v result Result element pair
+ * @v result Result element
* @v carry Carry element
*/
static inline __attribute__ (( always_inline )) void
bigint_multiply_one ( const uint32_t multiplicand, const uint32_t multiplier,
uint32_t *result, uint32_t *carry ) {
uint32_t discard_a;
- uint32_t discard_d;
__asm__ __volatile__ ( /* Perform multiplication */
- "mull %6\n\t"
+ "mull %3\n\t"
+ /* Accumulate carry */
+ "addl %5, %0\n\t"
+ "adcl $0, %1\n\t"
/* Accumulate result */
"addl %0, %2\n\t"
- "adcl %1, %3\n\t"
- /* Accumulate carry (cannot overflow) */
- "adcl $0, %4\n\t"
- : "=a" ( discard_a ),
- "=d" ( discard_d ),
- "+m" ( result[0] ),
- "+m" ( result[1] ),
- "+m" ( *carry )
- : "0" ( multiplicand ),
- "g" ( multiplier ) );
+ "adcl $0, %1\n\t"
+ : "=&a" ( discard_a ),
+ "=&d" ( *carry ),
+ "+m" ( *result )
+ : "g" ( multiplicand ),
+ "0" ( multiplier ),
+ "r" ( *carry ) );
}
#endif /* _BITS_BIGINT_H */
* @v multiplier0 Element 0 of big integer to be multiplied
* @v multiplier_size Number of elements in multiplier
* @v result0 Element 0 of big integer to hold result
- * @v carry0 Element 0 of big integer to hold temporary carry
*/
void bigint_multiply_raw ( const bigint_element_t *multiplicand0,
unsigned int multiplicand_size,
const bigint_element_t *multiplier0,
unsigned int multiplier_size,
- bigint_element_t *result0,
- bigint_element_t *carry0 ) {
+ bigint_element_t *result0 ) {
unsigned int result_size = ( multiplicand_size + multiplier_size );
const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
*multiplicand = ( ( const void * ) multiplicand0 );
*multiplier = ( ( const void * ) multiplier0 );
bigint_t ( result_size ) __attribute__ (( may_alias ))
*result = ( ( void * ) result0 );
- bigint_t ( result_size ) __attribute__ (( may_alias ))
- *carry = ( ( void * ) carry0 );
bigint_element_t multiplicand_element;
const bigint_element_t *multiplier_element;
- bigint_element_t *result_elements;
- bigint_element_t *carry_element;
+ bigint_element_t *result_element;
+ bigint_element_t carry_element;
unsigned int i;
unsigned int j;
- /* Zero result and temporary carry space */
- memset ( result, 0, sizeof ( *result ) );
- memset ( carry, 0, sizeof ( *carry ) );
+ /* Zero required portion of result
+ *
+ * All elements beyond the length of the multiplier will be
+ * written before they are read, and so do not need to be
+ * zeroed in advance.
+ */
+ memset ( result, 0, sizeof ( *multiplier ) );
- /* Multiply integers one element at a time, adding the double
- * element directly into the result and accumulating any
- * overall carry out from this double-element addition into
- * the temporary carry space.
+ /* Multiply integers one element at a time, adding the low
+ * half of the double-element product directly into the
+ * result, and maintaining a running single-element carry.
+ *
+ * The running carry can never overflow beyond a single
+ * element. At each step, the calculation we perform is:
*
- * We could propagate the carry immediately instead of using a
- * temporary carry space. However, this would cause the
- * multiplication to run in non-constant time, which is
- * undesirable.
+ * carry:result[i+j] := ( ( multiplicand[i] * multiplier[j] )
+ * + result[i+j] + carry )
*
- * The carry elements can never overflow, provided that the
- * element size is large enough to accommodate any plausible
- * big integer. The total number of potential carries (across
- * all elements) is the sum of the number of elements in the
- * multiplicand and multiplier. With a 16-bit element size,
- * this therefore allows for up to a 1Mbit multiplication
- * result (e.g. a 512kbit integer multiplied by another
- * 512kbit integer), which is around 100x higher than could be
- * needed in practice. With a more realistic 32-bit element
- * size, the limit becomes a totally implausible 128Gbit
- * multiplication result.
+ * The maximum value (for n-bit elements) is therefore:
+ *
+ * (2^n - 1)*(2^n - 1) + (2^n - 1) + (2^n - 1) = 2^(2n) - 1
+ *
+ * This is precisely the maximum value for a 2n-bit integer,
+ * and so the carry out remains within the range of an n-bit
+ * integer, i.e. a single element.
*/
for ( i = 0 ; i < multiplicand_size ; i++ ) {
multiplicand_element = multiplicand->element[i];
multiplier_element = &multiplier->element[0];
- result_elements = &result->element[i];
- carry_element = &carry->element[i];
+ result_element = &result->element[i];
+ carry_element = 0;
for ( j = 0 ; j < multiplier_size ; j++ ) {
bigint_multiply_one ( multiplicand_element,
*(multiplier_element++),
- result_elements++,
- carry_element++ );
+ result_element++,
+ &carry_element );
}
- }
-
- /* Add the temporary carry into the result. The least
- * significant element of the carry represents the carry out
- * from multiplying the least significant elements of the
- * multiplicand and multiplier, and therefore must be added to
- * the third-least significant element of the result (i.e. the
- * carry needs to be shifted left by two elements before being
- * adding to the result).
- *
- * The most significant two elements of the carry are
- * guaranteed to be zero, since:
- *
- * a < 2^{n}, b < 2^{m} => ab < 2^{n+m}
- *
- * and the overall result of the multiplication (including
- * adding in the shifted carries) is therefore guaranteed not
- * to overflow beyond the end of the result.
- *
- * We could avoid this shifting by writing the carry directly
- * into the "correct" element during the element-by-element
- * multiplication stage above. However, this would add
- * complexity to the loop since we would have to arrange for
- * the (provably zero) most significant two carry out results
- * to be discarded, in order to avoid writing beyond the end
- * of the temporary carry space.
- *
- * Performing the logical shift is essentially free, since we
- * simply adjust the element pointers.
- *
- * To avoid requiring additional checks in each architecture's
- * implementation of bigint_add_raw(), we explicitly avoid
- * calling bigint_add_raw() with a size of zero.
- */
- if ( result_size > 2 ) {
- bigint_add_raw ( &carry->element[0], &result->element[2],
- ( result_size - 2 ) );
+ *result_element = carry_element;
}
}
( ( void * ) result0 );
struct {
bigint_t ( size * 2 ) result;
- union {
- bigint_t ( size * 2 ) modulus;
- bigint_t ( size * 2 ) carry;
- };
+ bigint_t ( size * 2 ) modulus;
} *temp = tmp;
int rotation;
int i;
/* Perform multiplication */
profile_start ( &bigint_mod_multiply_multiply_profiler );
- bigint_multiply ( multiplicand, multiplier, &temp->result,
- &temp->carry );
+ bigint_multiply ( multiplicand, multiplier, &temp->result );
profile_stop ( &bigint_mod_multiply_multiply_profiler );
/* Rescale modulus to match result */
* Storage size of each big integer 128 40
* (in bytes)
*
- * Stack usage for key exchange 1144 424
+ * Stack usage for key exchange 1144 360
* (in bytes, large objects only)
*
* Cost of big integer addition 16 5
* We overlap the buffers used by each step of the multiplication
* calculation to reduce the total stack space required:
*
- * |--------------------------------------------------------------------------|
- * | <------- step 1 carry ------> | <----------- step 1 result ------------> |
- * | | <- low 256 bits -> | <- high 260 bits -> |
- * | <- step 2 carry -> | <-- step 2 result --> | <pad> | |
- * | <- s3 carry -> | <--------- pad ---------> | <- step 3 result -> | |
- * |--------------------------------------------------------------------------|
+ * |--------------------------------------------------------|
+ * | <- pad -> | <------------ step 1 result -------------> |
+ * | | <- low 256 bits -> | <-- high 260 bits --> |
+ * | <------- step 2 result ------> | <-- step 3 result --> |
+ * |--------------------------------------------------------|
*/
union x25519_multiply_workspace {
- /** Step 1 */
+ /** Step 1 result */
struct {
- /** Step 1 temporary carry workspace */
- union x25519_multiply_step1 carry;
+ /** Padding to avoid collision between steps 1 and 2
+ *
+ * The step 2 multiplication consumes the high 260
+ * bits of step 1, and so the step 2 multiplication
+ * result must not overlap this portion of the step 1
+ * result.
+ */
+ uint8_t pad[ sizeof ( union x25519_multiply_step2 ) -
+ offsetof ( union x25519_multiply_step1,
+ parts.high_260bit ) ];
/** Step 1 result */
- union x25519_multiply_step1 result;
- } __attribute__ (( packed )) step1;
- /** Step 2
- *
- * The step 2 multiplication consumes the high 260 bits of
- * step 1, and so the step 2 multiplication result (and
- * temporary carry workspace) must not overlap this portion of
- * the step 1 result.
- */
+ union x25519_multiply_step1 step1;
+ } __attribute__ (( packed ));
+ /** Steps 2 and 3 results */
struct {
- /** Step 2 temporary carry workspace */
- union x25519_multiply_step2 carry;
/** Step 2 result */
- union x25519_multiply_step2 result;
- /** Avoid collision between step 1 result and step 2 result */
- uint8_t pad[ ( int )
- ( sizeof ( union x25519_multiply_step1 ) +
- offsetof ( union x25519_multiply_step1,
- parts.high_260bit ) -
- sizeof ( union x25519_multiply_step2 ) -
- sizeof ( union x25519_multiply_step2 ) ) ];
- } __attribute__ (( packed )) step2;
- /** Step 3
- *
- * The step 3 multiplication consumes the high 11 bits of step
- * 2, and so the step 3 multiplication result (and temporary
- * carry workspace) must not overlap this portion of the step
- * 2 result.
- */
- struct {
- /** Step 3 temporary carry workspace */
- union x25519_multiply_step3 carry;
- /** Avoid collision between step 2 result and step 3 carry */
- uint8_t pad1[ ( int )
- ( sizeof ( union x25519_multiply_step2 ) -
- sizeof ( union x25519_multiply_step3 ) ) ];
- /** Avoid collision between step 2 result and step 3 result */
- uint8_t pad2[ sizeof ( union x25519_multiply_step2 ) ];
+ union x25519_multiply_step2 step2;
/** Step 3 result */
- union x25519_multiply_step3 result;
- } __attribute__ (( packed )) step3;
+ union x25519_multiply_step3 step3;
+ } __attribute__ (( packed ));
};
/** An X25519 elliptic curve point in projective coordinates
const union x25519_oct258 *multiplier,
union x25519_quad257 *result ) {
union x25519_multiply_workspace tmp;
- union x25519_multiply_step1 *step1 = &tmp.step1.result;
- union x25519_multiply_step2 *step2 = &tmp.step2.result;
- union x25519_multiply_step3 *step3 = &tmp.step3.result;
+ union x25519_multiply_step1 *step1 = &tmp.step1;
+ union x25519_multiply_step2 *step2 = &tmp.step2;
+ union x25519_multiply_step3 *step3 = &tmp.step3;
/* Step 1: perform raw multiplication
*
*/
static_assert ( sizeof ( step1->product ) >= sizeof ( step1->parts ) );
bigint_multiply ( &multiplicand->value, &multiplier->value,
- &step1->product, &tmp.step1.carry.product );
+ &step1->product );
/* Step 2: reduce high-order 516-256=260 bits of step 1 result
*
static_assert ( sizeof ( step2->product ) >= sizeof ( step2->parts ) );
bigint_grow ( &step1->parts.low_256bit, &result->value );
bigint_multiply ( &step1->parts.high_260bit, &x25519_reduce_256,
- &step2->product, &tmp.step2.carry.product );
+ &step2->product );
bigint_add ( &result->value, &step2->value );
/* Step 3: reduce high-order 267-256=11 bits of step 2 result
memset ( &step3->value, 0, sizeof ( step3->value ) );
bigint_grow ( &step2->parts.low_256bit, &result->value );
bigint_multiply ( &step2->parts.high_11bit, &x25519_reduce_256,
- &step3->product, &tmp.step3.carry.product );
+ &step3->product );
bigint_add ( &step3->value, &result->value );
/* Step 1 calculates the product of the input operands, and
* @v multiplicand Big integer to be multiplied
* @v multiplier Big integer to be multiplied
* @v result Big integer to hold result
- * @v carry Big integer to hold temporary carry space
*/
-#define bigint_multiply( multiplicand, multiplier, result, carry ) do { \
+#define bigint_multiply( multiplicand, multiplier, result ) do { \
unsigned int multiplicand_size = bigint_size (multiplicand); \
unsigned int multiplier_size = bigint_size (multiplier); \
bigint_multiply_raw ( (multiplicand)->element, \
multiplicand_size, (multiplier)->element, \
- multiplier_size, (result)->element, \
- (carry)->element ); \
+ multiplier_size, (result)->element ); \
} while ( 0 )
/**
unsigned int size = bigint_size (modulus); \
sizeof ( struct { \
bigint_t ( size * 2 ) temp_result; \
- union { \
- bigint_t ( size * 2 ) temp_modulus; \
- bigint_t ( size * 2 ) temp_carry; \
- }; \
+ bigint_t ( size * 2 ) temp_modulus; \
} ); } )
/**
unsigned int multiplicand_size,
const bigint_element_t *multiplier0,
unsigned int multiplier_size,
- bigint_element_t *result0,
- bigint_element_t *carry0 );
+ bigint_element_t *result0 );
void bigint_mod_multiply_raw ( const bigint_element_t *multiplicand0,
const bigint_element_t *multiplier0,
const bigint_element_t *modulus0,
unsigned int multiplicand_size,
const bigint_element_t *multiplier0,
unsigned int multiplier_size,
- bigint_element_t *result0,
- bigint_element_t *carry0 ) {
+ bigint_element_t *result0 ) {
unsigned int result_size = ( multiplicand_size + multiplier_size );
const bigint_t ( multiplicand_size ) __attribute__ (( may_alias ))
*multiplicand = ( ( const void * ) multiplicand0 );
*multiplier = ( ( const void * ) multiplier0 );
bigint_t ( result_size ) __attribute__ (( may_alias ))
*result = ( ( void * ) result0 );
- bigint_t ( result_size ) __attribute__ (( may_alias ))
- *carry = ( ( void * ) carry0 );
- bigint_multiply ( multiplicand, multiplier, result, carry );
+ bigint_multiply ( multiplicand, multiplier, result );
}
void bigint_mod_multiply_sample ( const bigint_element_t *multiplicand0,
bigint_t ( multiplicand_size ) multiplicand_temp; \
bigint_t ( multiplier_size ) multiplier_temp; \
bigint_t ( multiplicand_size + multiplier_size ) result_temp; \
- bigint_t ( multiplicand_size + multiplier_size ) carry_temp; \
{} /* Fix emacs alignment */ \
\
assert ( bigint_size ( &result_temp ) == \
( bigint_size ( &multiplicand_temp ) + \
bigint_size ( &multiplier_temp ) ) ); \
- assert ( bigint_size ( &carry_temp ) == \
- bigint_size ( &result_temp ) ); \
bigint_init ( &multiplicand_temp, multiplicand_raw, \
sizeof ( multiplicand_raw ) ); \
bigint_init ( &multiplier_temp, multiplier_raw, \
DBG_HDA ( 0, &multiplicand_temp, sizeof ( multiplicand_temp ) );\
DBG_HDA ( 0, &multiplier_temp, sizeof ( multiplier_temp ) ); \
bigint_multiply ( &multiplicand_temp, &multiplier_temp, \
- &result_temp, &carry_temp ); \
+ &result_temp ); \
DBG_HDA ( 0, &result_temp, sizeof ( result_temp ) ); \
bigint_done ( &result_temp, result_raw, sizeof ( result_raw ) );\
\
projects / thirdparty / ipxe.git / commitdiff
