Future-proof sort template against undefined behavior

Commit 176dffdf7 added a NULL array pointer check before performing
a qsort in order to prevent undefined behavior when passing NULL
pointer and zero length. To head off future degenerate cases, check
that there are at least two elements to sort before proceeding with
insertion sort. This has the added advantage of allowing us to remove
four equivalent checks that guarded against recursion/iteration.

There might be a tiny performance penalty from unproductive
recursions, but we can buy that back by increasing the insertion sort
threshold. That is left for future work.

Discussion: https://postgr.es/m/CANWCAZZWvds_35nXc4vXD-eBQa_=mxVtqZf-PM_ps=SD7ghhJg@mail.gmail.com

diff --git a/src/include/lib/sort_template.h b/src/include/lib/sort_template.h
index e02aa73cd4d8efd12931eddb75281bc9b8b01670..22b2092d03b7b0f2f31a01da0ecf21be823b627d 100644 (file)
--- a/src/include/lib/sort_template.h
+++ b/src/include/lib/sort_template.h
@@ -311,6 +311,14 @@ loop:
        DO_CHECK_FOR_INTERRUPTS();
        if (n < 7)
        {
+               /*
+                * Not strictly necessary, but a caller may pass a NULL pointer input
+                * and zero length, and this silences warnings about applying offsets
+                * to NULL pointers.
+                */
+               if (n < 2)
+                       return;
+
                for (pm = a + ST_POINTER_STEP; pm < a + n * ST_POINTER_STEP;
                         pm += ST_POINTER_STEP)
                        for (pl = pm; pl > a && DO_COMPARE(pl - ST_POINTER_STEP, pl) > 0;
@@ -387,29 +395,23 @@ loop:
        if (d1 <= d2)
        {
                /* Recurse on left partition, then iterate on right partition */
-               if (d1 > ST_POINTER_STEP)
-                       DO_SORT(a, d1 / ST_POINTER_STEP);
-               if (d2 > ST_POINTER_STEP)
-               {
-                       /* Iterate rather than recurse to save stack space */
-                       /* DO_SORT(pn - d2, d2 / ST_POINTER_STEP) */
-                       a = pn - d2;
-                       n = d2 / ST_POINTER_STEP;
-                       goto loop;
-               }
+               DO_SORT(a, d1 / ST_POINTER_STEP);
+
+               /* Iterate rather than recurse to save stack space */
+               /* DO_SORT(pn - d2, d2 / ST_POINTER_STEP) */
+               a = pn - d2;
+               n = d2 / ST_POINTER_STEP;
+               goto loop;
        }
        else
        {
                /* Recurse on right partition, then iterate on left partition */
-               if (d2 > ST_POINTER_STEP)
-                       DO_SORT(pn - d2, d2 / ST_POINTER_STEP);
-               if (d1 > ST_POINTER_STEP)
-               {
-                       /* Iterate rather than recurse to save stack space */
-                       /* DO_SORT(a, d1 / ST_POINTER_STEP) */
-                       n = d1 / ST_POINTER_STEP;
-                       goto loop;
-               }
+               DO_SORT(pn - d2, d2 / ST_POINTER_STEP);
+
+               /* Iterate rather than recurse to save stack space */
+               /* DO_SORT(a, d1 / ST_POINTER_STEP) */
+               n = d1 / ST_POINTER_STEP;
+               goto loop;
        }
 }
 #endif