hashtable, mdfour: avoid signed left-shift overflow

UBSan flags two spots that shift a value into the top bits of a word via a
signed operand:

  * lib/mdfour.c copy64(): `in[i] << 24` promotes the uchar to int, so a
    byte >= 128 overflows int (UB).  Cast each byte to uint32.
  * hashtable.c NON_ZERO_64(): `(int64)(x) << 32` overflows int64 whenever
    x's high bit is set.  Shift as uint64_t (covers all four call sites).

Behavior-preserving -- only the intermediate type changes; the resulting
bit pattern is identical.

diff --git a/hashtable.c b/hashtable.c
index 2cc4e55047004d0541a7889531daffae8f7c90ca..f4aa85f1cbbd113109854b88b351b346d1b9f4fe 100644 (file)
--- a/hashtable.c
+++ b/hashtable.c
@@ -351,7 +351,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new)
 */
 
 #define NON_ZERO_32(x) ((x) ? (x) : (uint32_t)1)
-#define NON_ZERO_64(x, y) ((x) || (y) ? (y) | (int64)(x) << 32 | (y) : (int64)1)
+#define NON_ZERO_64(x, y) ((x) || (y) ? (y) | (uint64_t)(x) << 32 | (y) : (int64)1)
 
 uint32_t hashlittle(const void *key, size_t length)
 {

diff --git a/lib/mdfour.c b/lib/mdfour.c
index 6203658d53761b01a16789f443ef1f6c2c1a6bae..7df180618607e2a1f405731eec77c2d1ddaa264a 100644 (file)
--- a/lib/mdfour.c
+++ b/lib/mdfour.c
@@ -89,8 +89,8 @@ static void copy64(uint32 *M, const uchar *in)
        int i;
 
        for (i = 0; i < MD4_DIGEST_LEN; i++) {
-               M[i] = (in[i*4+3] << 24) | (in[i*4+2] << 16)
-                    | (in[i*4+1] << 8)  | (in[i*4+0] << 0);
+               M[i] = ((uint32)in[i*4+3] << 24) | ((uint32)in[i*4+2] << 16)
+                    | ((uint32)in[i*4+1] << 8)  | ((uint32)in[i*4+0] << 0);
        }
 }