community-id: Fix IPv6 address sorting not respecting byte order

When comparing IPv6 addresses based on uint32_t chunks, one needs to
apply ntohl() conversion to the individual parts, otherwise on little
endian systems individual bytes are compared in the wrong order.
Avoid this all and leverage memcmp(), it'll short circuit on the first
differing byte and its return values tells us which address sorts lower.

Bug: #6276

diff --git a/src/output-json.c b/src/output-json.c
index d19bc3da1545f2a98e9dd0bd8ad341c8fe88ae5a..5d4255cd2897550c3189fdc1bd8bfd06e9f67f8b 100644 (file)
--- a/src/output-json.c
+++ b/src/output-json.c
@@ -638,18 +638,6 @@ static bool CalculateCommunityFlowIdv4(const Flow *f,
     return false;
 }
 
-static inline bool FlowHashRawAddressIPv6LtU32(const uint32_t *a, const uint32_t *b)
-{
-    for (int i = 0; i < 4; i++) {
-        if (a[i] < b[i])
-            return true;
-        if (a[i] > b[i])
-            break;
-    }
-
-    return false;
-}
-
 static bool CalculateCommunityFlowIdv6(const Flow *f,
         const uint16_t seed, unsigned char *base64buf)
 {
@@ -673,9 +661,8 @@ static bool CalculateCommunityFlowIdv6(const Flow *f,
     dp = htons(dp);
 
     ipv6.seed = htons(seed);
-    if (FlowHashRawAddressIPv6LtU32(f->src.addr_data32, f->dst.addr_data32) ||
-            ((memcmp(&f->src, &f->dst, sizeof(f->src)) == 0) && sp < dp))
-    {
+    int cmp_r = memcmp(&f->src, &f->dst, sizeof(f->src));
+    if ((cmp_r < 0) || (cmp_r == 0 && sp < dp)) {
         memcpy(&ipv6.src, &f->src.addr_data32, 16);
         memcpy(&ipv6.dst, &f->dst.addr_data32, 16);
         ipv6.sp = sp;