sort: fix unlikely int overflow with -r

* src/sort.c (keycompare, compare): Don’t overflow if -r is
specified and a comparison function returns INT_MIN, as this
causes the comparison to have undefined behavior (typically the
reverse of correct).  glibc memcmp on s390x reportedly returns
INT_MIN in some cases, so this is not a purely academic issue.

diff --git a/src/sort.c b/src/sort.c
index 29c9f39f3c2b7ff095e9d7bd89ed0f743d09dd24..81045b1abc4e63a3cda824c25591d2914df6f302 100644 (file)
--- a/src/sort.c
+++ b/src/sort.c
@@ -2794,7 +2794,9 @@ keycompare (struct line const *a, struct line const *b)
         }
     }
 
-  return key->reverse ? -diff : diff;
+  if (key->reverse)
+    diff = diff < 0 ? -1 : -diff;
+  return diff;
 }
 
 /* Compare two lines A and B, returning negative, zero, or positive
@@ -2839,7 +2841,9 @@ compare (struct line const *a, struct line const *b)
         diff = (alen > blen) - (alen < blen);
     }
 
-  return reverse ? -diff : diff;
+  if (reverse)
+    diff = diff < 0 ? 1 : -diff;
+  return diff;
 }
 
 /* Write LINE to output stream FP; the output file's name is