detect: bytemath do not left shift more than 64

As it is undefined behavior by C standard.
In this case, zeroes the value.

Ticket: #5900

diff --git a/src/detect-bytemath.c b/src/detect-bytemath.c
index 1aa55b5e630e16f66d66c894e5e878444fe52001..a93144e26544235d623ce962160352fa69e8a7c3 100644 (file)
--- a/src/detect-bytemath.c
+++ b/src/detect-bytemath.c
@@ -168,7 +168,11 @@ int DetectByteMathDoMatch(DetectEngineThreadCtx *det_ctx, const SigMatchData *sm
             val *= rvalue;
             break;
         case LeftShift:
-            val <<= rvalue;
+            if (rvalue < 64) {
+                val <<= rvalue;
+            } else {
+                val = 0;
+            }
             break;
         case RightShift:
             val >>= rvalue;