detect/engine: fix -Wshorten-64-to-32 warnings

Ticket: #6186

Especially take care of the case where byte_extract extracts a u64
value that does not fit in a u32

diff --git a/src/detect-engine-content-inspection.c b/src/detect-engine-content-inspection.c
index 3e9841314ef07aa9a919e0982ce6a930f080f59a..d885bef5be76c1b80e785f6f1af0186206c47438 100644 (file)
--- a/src/detect-engine-content-inspection.c
+++ b/src/detect-engine-content-inspection.c
@@ -157,7 +157,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                 int distance = cd->distance;
                 if (cd->flags & DETECT_CONTENT_DISTANCE) {
                     if (cd->flags & DETECT_CONTENT_DISTANCE_VAR) {
-                        distance = det_ctx->byte_values[cd->distance];
+                        // This cast is wrong if a 64-bit value was extracted
+                        distance = (uint32_t)det_ctx->byte_values[cd->distance];
                     }
                     if (distance < 0 && (uint32_t)(abs(distance)) > offset)
                         offset = 0;
@@ -170,8 +171,10 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
 
                 if (cd->flags & DETECT_CONTENT_WITHIN) {
                     if (cd->flags & DETECT_CONTENT_WITHIN_VAR) {
+                        // This cast is wrong if a 64-bit value was extracted for within
                         if ((int32_t)depth > (int32_t)(prev_buffer_offset + det_ctx->byte_values[cd->within] + distance)) {
-                            depth = prev_buffer_offset + det_ctx->byte_values[cd->within] + distance;
+                            depth = prev_buffer_offset +
+                                    (uint32_t)det_ctx->byte_values[cd->within] + distance;
                         }
                     } else {
                         if ((int32_t)depth > (int32_t)(prev_buffer_offset + cd->within + distance)) {
@@ -188,14 +191,15 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                         } else if (depth >= (stream_start_offset + buffer_len)) {
                             ;
                         } else {
-                            depth = depth - stream_start_offset;
+                            depth = depth - (uint32_t)stream_start_offset;
                         }
                     }
                 }
 
                 if (cd->flags & DETECT_CONTENT_DEPTH_VAR) {
                     if ((det_ctx->byte_values[cd->depth] + prev_buffer_offset) < depth) {
-                        depth = prev_buffer_offset + det_ctx->byte_values[cd->depth];
+                        // ok to cast as we checked the byte value fits in a u32
+                        depth = prev_buffer_offset + (uint32_t)det_ctx->byte_values[cd->depth];
                     }
                 } else {
                     if (cd->depth != 0) {
@@ -208,8 +212,10 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                 }
 
                 if (cd->flags & DETECT_CONTENT_OFFSET_VAR) {
-                    if (det_ctx->byte_values[cd->offset] > offset)
-                        offset = det_ctx->byte_values[cd->offset];
+                    if (det_ctx->byte_values[cd->offset] > offset) {
+                        // This cast is wrong if a 64-bit value was extracted
+                        offset = (uint32_t)det_ctx->byte_values[cd->offset];
+                    }
                 } else {
                     if (cd->offset > offset) {
                         offset = cd->offset;
@@ -219,7 +225,8 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
             } else { /* implied no relative matches */
                 /* set depth */
                 if (cd->flags & DETECT_CONTENT_DEPTH_VAR) {
-                    depth = det_ctx->byte_values[cd->depth];
+                    // This cast is wrong if a 64-bit value was extracted
+                    depth = (uint32_t)det_ctx->byte_values[cd->depth];
                 } else {
                     if (cd->depth != 0) {
                         depth = cd->depth;
@@ -232,15 +239,17 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
                     } else if (depth >= (stream_start_offset + buffer_len)) {
                         ;
                     } else {
-                        depth = depth - stream_start_offset;
+                        depth = (uint32_t)(depth - stream_start_offset);
                     }
                 }
 
                 /* set offset */
-                if (cd->flags & DETECT_CONTENT_OFFSET_VAR)
-                    offset = det_ctx->byte_values[cd->offset];
-                else
+                if (cd->flags & DETECT_CONTENT_OFFSET_VAR) {
+                    // This cast is wrong if a 64-bit value was extracted
+                    offset = (uint32_t)det_ctx->byte_values[cd->offset];
+                } else {
                     offset = cd->offset;
+                }
                 prev_buffer_offset = 0;
             }
 
@@ -499,13 +508,15 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
         uint64_t value = btd->value;
         int32_t nbytes = btd->nbytes;
         if (btflags & DETECT_BYTETEST_OFFSET_VAR) {
-            offset = det_ctx->byte_values[offset];
+            // This cast is wrong if a 64-bit value was extracted
+            offset = (int32_t)det_ctx->byte_values[offset];
         }
         if (btflags & DETECT_BYTETEST_VALUE_VAR) {
             value = det_ctx->byte_values[value];
         }
         if (btflags & DETECT_BYTETEST_NBYTES_VAR) {
-            nbytes = det_ctx->byte_values[nbytes];
+            // This cast is wrong if a 64-bit value was extracted
+            nbytes = (int32_t)det_ctx->byte_values[nbytes];
         }
 
         /* if we have dce enabled we will have to use the endianness
@@ -531,12 +542,14 @@ static int DetectEngineContentInspectionInternal(DetectEngineThreadCtx *det_ctx,
         int32_t nbytes;
 
         if (bjflags & DETECT_BYTEJUMP_OFFSET_VAR) {
-            offset = det_ctx->byte_values[offset];
+            // This cast is wrong if a 64-bit value was extracted
+            offset = (int32_t)det_ctx->byte_values[offset];
             SCLogDebug("[BJ] using offset value %d", offset);
         }
 
         if (bjflags & DETECT_BYTEJUMP_NBYTES_VAR) {
-            nbytes = det_ctx->byte_values[bjd->nbytes];
+            // This cast is wrong if a 64-bit value was extracted
+            nbytes = (int32_t)det_ctx->byte_values[bjd->nbytes];
             SCLogDebug("[BJ] using nbytes value %d [index %d]", nbytes, bjd->nbytes);
         } else {
             nbytes = bjd->nbytes;