Fix BRIN minmax-multi distance for inet type

The distance calculation ignored the mask, unlike the inet comparator,
which resulted in negative distance in some cases. Fixed by applying the
mask in brin_minmax_multi_distance_inet. I've considered simply calling
inetmi() to calculate the delta, but that does not consider mask either.

Reviewed-by: Zhihong Yu
Discussion: https://postgr.es/m/1a0a7b9d-9bda-e3a2-7fa4-88f15042a051%40enterprisedb.com

diff --git a/src/backend/access/brin/brin_minmax_multi.c b/src/backend/access/brin/brin_minmax_multi.c
index e182cd95ecd007457213613e827b51845ef30a02..19ded7d933c9e52c91a395449fa63d499d8f3661 100644 (file)
--- a/src/backend/access/brin/brin_minmax_multi.c
+++ b/src/backend/access/brin/brin_minmax_multi.c
@@ -2297,6 +2297,9 @@ brin_minmax_multi_distance_inet(PG_FUNCTION_ARGS)
        inet       *ipa = PG_GETARG_INET_PP(0);
        inet       *ipb = PG_GETARG_INET_PP(1);
 
+       int                     lena,
+                               lenb;
+
        /*
         * If the addresses are from different families, consider them to be in
         * maximal possible distance (which is 1.0).
@@ -2304,24 +2307,64 @@ brin_minmax_multi_distance_inet(PG_FUNCTION_ARGS)
        if (ip_family(ipa) != ip_family(ipb))
                PG_RETURN_FLOAT8(1.0);
 
-       /* ipv4 or ipv6 */
-       if (ip_family(ipa) == PGSQL_AF_INET)
-               len = 4;
-       else
-               len = 16;                               /* NS_IN6ADDRSZ */
+       addra = (unsigned char *) palloc(ip_addrsize(ipa));
+       memcpy(addra, ip_addr(ipa), ip_addrsize(ipa));
+
+       addrb = (unsigned char *) palloc(ip_addrsize(ipb));
+       memcpy(addrb, ip_addr(ipb), ip_addrsize(ipb));
+
+       /*
+        * The length is calculated from the mask length, because we sort the
+        * addresses by first address in the range, so A.B.C.D/24 < A.B.C.1
+        * (the first range starts at A.B.C.0, which is before A.B.C.1). We
+        * don't want to produce negative delta in this case, so we just cut
+        * the extra bytes.
+        *
+        * XXX Maybe this should be a bit more careful and cut the bits, not
+        * just whole bytes.
+        */
+       lena = ip_bits(ipa);
+       lenb = ip_bits(ipb);
+
+       len = ip_addrsize(ipa);
+
+       /* apply the network mask to both addresses */
+       for (i = 0; i < len; i++)
+       {
+               unsigned char   mask;
+               int                             nbits;
 
-       addra = ip_addr(ipa);
-       addrb = ip_addr(ipb);
+               nbits = lena - (i * 8);
+               if (nbits < 8)
+               {
+                       mask = (0xFF << (8 - nbits));
+                       addra[i] = (addra[i] & mask);
+               }
 
+               nbits = lenb - (i * 8);
+               if (nbits < 8)
+               {
+                       mask = (0xFF << (8 - nbits));
+                       addrb[i] = (addrb[i] & mask);
+               }
+       }
+
+       /* Calculate the difference between the addresses. */
        delta = 0;
        for (i = len - 1; i >= 0; i--)
        {
-               delta += (float8) addrb[i] - (float8) addra[i];
+               unsigned char a = addra[i];
+               unsigned char b = addrb[i];
+
+               delta += (float8) b - (float8) a;
                delta /= 256;
        }
 
        Assert((delta >= 0) && (delta <= 1));
 
+       pfree(addra);
+       pfree(addrb);
+
        PG_RETURN_FLOAT8(delta);
 }