community-id: Fix IPv6 address sorting not respecting byte order

When comparing IPv6 addresses based on uint32_t chunks, one needs to
apply ntohl() conversion to the individual parts, otherwise on little
endian systems individual bytes are compared in the wrong order.
Avoid this all and leverage memcmp(), it'll short circuit on the first
differing byte and its return values tells us which address sorts lower.

Bug: #6276
(cherry picked from commit 0ddc44f4c116945e0a8fa846d4f2c5ba5a7d8b63)

diff --git a/src/output-json.c b/src/output-json.c
index f26297257fa1f6fbea0bb37172e5669adcea9599..347697b4497efe6a98d0b20e1975dc9706d7c7b8 100644 (file)
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -679,18 +679,6 @@ static bool CalculateCommunityFlowIdv4(const Flow *f,
     return false;
 }
 
-static inline bool FlowHashRawAddressIPv6LtU32(const uint32_t *a, const uint32_t *b)
-{
-    for (int i = 0; i < 4; i++) {
-        if (a[i] < b[i])
-            return true;
-        if (a[i] > b[i])
-            break;
-    }
-
-    return false;
-}
-
 static bool CalculateCommunityFlowIdv6(const Flow *f,
         const uint16_t seed, unsigned char *base64buf)
 {
@@ -714,9 +702,8 @@ static bool CalculateCommunityFlowIdv6(const Flow *f,
     dp = htons(dp);
 
     ipv6.seed = htons(seed);
-    if (FlowHashRawAddressIPv6LtU32(f->src.addr_data32, f->dst.addr_data32) ||
-            ((memcmp(&f->src, &f->dst, sizeof(f->src)) == 0) && sp < dp))
-    {
+    int cmp_r = memcmp(&f->src, &f->dst, sizeof(f->src));
+    if ((cmp_r < 0) || (cmp_r == 0 && sp < dp)) {
         memcpy(&ipv6.src, &f->src.addr_data32, 16);
         memcpy(&ipv6.dst, &f->dst.addr_data32, 16);
         ipv6.sp = sp;