From: Andrew Tridgell <andrew@tridgell.net>
Date: Sun, 7 Jun 2026 21:18:02 +0000 (+1000)
Subject: hashtable, mdfour: avoid signed left-shift overflow
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=4148419736dd3ab459b44168dedb52ef877b6e90;p=thirdparty%2Frsync.git

hashtable, mdfour: avoid signed left-shift overflow

UBSan flags two spots that shift a value into the top bits of a word via a
signed operand:

  * lib/mdfour.c copy64(): `in[i] << 24` promotes the uchar to int, so a
    byte >= 128 overflows int (UB).  Cast each byte to uint32.
  * hashtable.c NON_ZERO_64(): `(int64)(x) << 32` overflows int64 whenever
    x's high bit is set.  Shift as uint64_t (covers all four call sites).

Behavior-preserving -- only the intermediate type changes; the resulting
bit pattern is identical.
---

diff --git a/hashtable.c b/hashtable.c
index 2cc4e550..f4aa85f1 100644
--- a/hashtable.c
+++ b/hashtable.c
@@ -351,7 +351,7 @@ void *hashtable_find(struct hashtable *tbl, int64 key, void *data_when_new)
 */
 
 #define NON_ZERO_32(x) ((x) ? (x) : (uint32_t)1)
-#define NON_ZERO_64(x, y) ((x) || (y) ? (y) | (int64)(x) << 32 | (y) : (int64)1)
+#define NON_ZERO_64(x, y) ((x) || (y) ? (y) | (uint64_t)(x) << 32 | (y) : (int64)1)
 
 uint32_t hashlittle(const void *key, size_t length)
 {
diff --git a/lib/mdfour.c b/lib/mdfour.c
index 6203658d..7df18061 100644
--- a/lib/mdfour.c
+++ b/lib/mdfour.c
@@ -89,8 +89,8 @@ static void copy64(uint32 *M, const uchar *in)
 	int i;
 
 	for (i = 0; i < MD4_DIGEST_LEN; i++) {
-		M[i] = (in[i*4+3] << 24) | (in[i*4+2] << 16)
-		     | (in[i*4+1] << 8)  | (in[i*4+0] << 0);
+		M[i] = ((uint32)in[i*4+3] << 24) | ((uint32)in[i*4+2] << 16)
+		     | ((uint32)in[i*4+1] << 8)  | ((uint32)in[i*4+0] << 0);
 	}
 }