detect: bytemath do not left shift more than 64

As it is undefined behavior by C standard.
In this case, zeroes the value.

Ticket: #5900
(cherry picked from commit 473ca6dcf4789259ec4543d7d648bd1bd19986fc)

diff --git a/src/detect-bytemath.c b/src/detect-bytemath.c
index 9a7005d81b8d262c45df8622c416df423ba09cbe..83593110b7f27b900313c13252496a6cb0e5029a 100644 (file)
--- a/src/detect-bytemath.c
+++ b/src/detect-bytemath.c
@@ -226,7 +226,11 @@ int DetectByteMathDoMatch(DetectEngineThreadCtx *det_ctx, const SigMatchData *sm
             val *= rvalue;
             break;
         case DETECT_BYTEMATH_OPERATOR_LSHIFT:
-            val <<= rvalue;
+            if (rvalue < 64) {
+                val <<= rvalue;
+            } else {
+                val = 0;
+            }
             break;
         case DETECT_BYTEMATH_OPERATOR_RSHIFT:
             val >>= rvalue;