From a351fdd6fcbc48ccd337a111d24fe9fa3c0d58c3 Mon Sep 17 00:00:00 2001 From: Paul Eggert Date: Tue, 17 May 2022 19:30:09 -0700 Subject: [PATCH] sort: fix unlikely int overflow with -r MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit * src/sort.c (keycompare, compare): Don’t overflow if -r is specified and a comparison function returns INT_MIN, as this causes the comparison to have undefined behavior (typically the reverse of correct). glibc memcmp on s390x reportedly returns INT_MIN in some cases, so this is not a purely academic issue. --- src/sort.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/sort.c b/src/sort.c index 29c9f39f3c..81045b1abc 100644 --- a/src/sort.c +++ b/src/sort.c @@ -2794,7 +2794,9 @@ keycompare (struct line const *a, struct line const *b) } } - return key->reverse ? -diff : diff; + if (key->reverse) + diff = diff < 0 ? -1 : -diff; + return diff; } /* Compare two lines A and B, returning negative, zero, or positive @@ -2839,7 +2841,9 @@ compare (struct line const *a, struct line const *b) diff = (alen > blen) - (alen < blen); } - return reverse ? -diff : diff; + if (reverse) + diff = diff < 0 ? 1 : -diff; + return diff; } /* Write LINE to output stream FP; the output file's name is -- 2.47.2