Catch invalid typlens in a couple of places

Rearrange the logic in record_image_cmp() and record_image_eq() to
error out on unexpected typlens (either not supported there or
completely invalid due to corruption).  Barring corruption, this is
not possible today but it seems more future-proof and robust to fix
this.

Reported-by: Peter Geoghegan <pg@bowt.ie>

diff --git a/src/backend/utils/adt/rowtypes.c b/src/backend/utils/adt/rowtypes.c
index 29e711a4b4bf47f332873c3169dbbd25f453e69a..99e9c941f4dc56ed091366b86df44519c03f33ef 100644 (file)
--- a/src/backend/utils/adt/rowtypes.c
+++ b/src/backend/utils/adt/rowtypes.c
@@ -1426,30 +1426,7 @@ record_image_cmp(FunctionCallInfo fcinfo)
                        }
 
                        /* Compare the pair of elements */
-                       if (tupdesc1->attrs[i1]->attlen == -1)
-                       {
-                               Size            len1,
-                                                       len2;
-                               struct varlena *arg1val;
-                               struct varlena *arg2val;
-
-                               len1 = toast_raw_datum_size(values1[i1]);
-                               len2 = toast_raw_datum_size(values2[i2]);
-                               arg1val = PG_DETOAST_DATUM_PACKED(values1[i1]);
-                               arg2val = PG_DETOAST_DATUM_PACKED(values2[i2]);
-
-                               cmpresult = memcmp(VARDATA_ANY(arg1val),
-                                                                  VARDATA_ANY(arg2val),
-                                                                  Min(len1, len2) - VARHDRSZ);
-                               if ((cmpresult == 0) && (len1 != len2))
-                                       cmpresult = (len1 < len2) ? -1 : 1;
-
-                               if ((Pointer) arg1val != (Pointer) values1[i1])
-                                       pfree(arg1val);
-                               if ((Pointer) arg2val != (Pointer) values2[i2])
-                                       pfree(arg2val);
-                       }
-                       else if (tupdesc1->attrs[i1]->attbyval)
+                       if (tupdesc1->attrs[i1]->attbyval)
                        {
                                switch (tupdesc1->attrs[i1]->attlen)
                                {
@@ -1491,12 +1468,37 @@ record_image_cmp(FunctionCallInfo fcinfo)
                                                Assert(false);  /* cannot happen */
                                }
                        }
-                       else
+                       else if (tupdesc1->attrs[i1]->attlen > 0)
                        {
                                cmpresult = memcmp(DatumGetPointer(values1[i1]),
                                                                   DatumGetPointer(values2[i2]),
                                                                   tupdesc1->attrs[i1]->attlen);
                        }
+                       else if (tupdesc1->attrs[i1]->attlen == -1)
+                       {
+                               Size            len1,
+                                                       len2;
+                               struct varlena *arg1val;
+                               struct varlena *arg2val;
+
+                               len1 = toast_raw_datum_size(values1[i1]);
+                               len2 = toast_raw_datum_size(values2[i2]);
+                               arg1val = PG_DETOAST_DATUM_PACKED(values1[i1]);
+                               arg2val = PG_DETOAST_DATUM_PACKED(values2[i2]);
+
+                               cmpresult = memcmp(VARDATA_ANY(arg1val),
+                                                                  VARDATA_ANY(arg2val),
+                                                                  Min(len1, len2) - VARHDRSZ);
+                               if ((cmpresult == 0) && (len1 != len2))
+                                       cmpresult = (len1 < len2) ? -1 : 1;
+
+                               if ((Pointer) arg1val != (Pointer) values1[i1])
+                                       pfree(arg1val);
+                               if ((Pointer) arg2val != (Pointer) values2[i2])
+                                       pfree(arg2val);
+                       }
+                       else
+                               elog(ERROR, "unexpected attlen: %d", tupdesc1->attrs[i1]->attlen);
 
                        if (cmpresult < 0)
                        {
@@ -1687,36 +1689,7 @@ record_image_eq(PG_FUNCTION_ARGS)
                        }
 
                        /* Compare the pair of elements */
-                       if (tupdesc1->attrs[i1]->attlen == -1)
-                       {
-                               Size            len1,
-                                                       len2;
-
-                               len1 = toast_raw_datum_size(values1[i1]);
-                               len2 = toast_raw_datum_size(values2[i2]);
-                               /* No need to de-toast if lengths don't match. */
-                               if (len1 != len2)
-                                       result = false;
-                               else
-                               {
-                                       struct varlena *arg1val;
-                                       struct varlena *arg2val;
-
-                                       arg1val = PG_DETOAST_DATUM_PACKED(values1[i1]);
-                                       arg2val = PG_DETOAST_DATUM_PACKED(values2[i2]);
-
-                                       result = (memcmp(VARDATA_ANY(arg1val),
-                                                                        VARDATA_ANY(arg2val),
-                                                                        len1 - VARHDRSZ) == 0);
-
-                                       /* Only free memory if it's a copy made here. */
-                                       if ((Pointer) arg1val != (Pointer) values1[i1])
-                                               pfree(arg1val);
-                                       if ((Pointer) arg2val != (Pointer) values2[i2])
-                                               pfree(arg2val);
-                               }
-                       }
-                       else if (tupdesc1->attrs[i1]->attbyval)
+                       if (tupdesc1->attrs[i1]->attbyval)
                        {
                                switch (tupdesc1->attrs[i1]->attlen)
                                {
@@ -1742,12 +1715,44 @@ record_image_eq(PG_FUNCTION_ARGS)
                                                Assert(false);  /* cannot happen */
                                }
                        }
-                       else
+                       else if (tupdesc1->attrs[i1]->attlen > 0)
                        {
                                result = (memcmp(DatumGetPointer(values1[i1]),
                                                                 DatumGetPointer(values2[i2]),
                                                                 tupdesc1->attrs[i1]->attlen) == 0);
                        }
+                       else if (tupdesc1->attrs[i1]->attlen == -1)
+                       {
+                               Size            len1,
+                                                       len2;
+
+                               len1 = toast_raw_datum_size(values1[i1]);
+                               len2 = toast_raw_datum_size(values2[i2]);
+                               /* No need to de-toast if lengths don't match. */
+                               if (len1 != len2)
+                                       result = false;
+                               else
+                               {
+                                       struct varlena *arg1val;
+                                       struct varlena *arg2val;
+
+                                       arg1val = PG_DETOAST_DATUM_PACKED(values1[i1]);
+                                       arg2val = PG_DETOAST_DATUM_PACKED(values2[i2]);
+
+                                       result = (memcmp(VARDATA_ANY(arg1val),
+                                                                        VARDATA_ANY(arg2val),
+                                                                        len1 - VARHDRSZ) == 0);
+
+                                       /* Only free memory if it's a copy made here. */
+                                       if ((Pointer) arg1val != (Pointer) values1[i1])
+                                               pfree(arg1val);
+                                       if ((Pointer) arg2val != (Pointer) values2[i2])
+                                               pfree(arg2val);
+                               }
+                       }
+                       else
+                               elog(ERROR, "unexpected attlen: %d", tupdesc1->attrs[i1]->attlen);
+
                        if (!result)
                                break;
                }