libsmartcols: fix reduction stages use

There is no proper check for the number of reduction stages, so in
some cases, the code can loop indefinitely.

The patch also fixes 'rc' variable shadowing.

Fixes: https://github.com/util-linux/util-linux/issues/3046
Signed-off-by: Karel Zak <kzak@redhat.com>
(cherry picked from commit 8a01276a3c5bc09dce342e087019c704d8c56a32)

diff --git a/libsmartcols/src/calculate.c b/libsmartcols/src/calculate.c
index ad0b15d92d5266d70a2bffce69186488a15c6b6d..1426d235a167358c3a3af6b5ffc15f91cae1a3a3 100644 (file)
--- a/libsmartcols/src/calculate.c
+++ b/libsmartcols/src/calculate.c
@@ -302,6 +302,8 @@ static int reduce_column(struct libscols_table *tb,
        size_t wanted, org_width, reduce = 1;
        int is_trunc = 0;
 
+       if (stage > 6)
+               return -1;
        if (tb->termwidth >= *width)
                return 1;
        /* ignore hidden columns */
@@ -481,7 +483,7 @@ int __scols_calculate(struct libscols_table *tb, struct ul_buffer *buf)
        /* reduce columns width */
        while (width > tb->termwidth) {
                size_t org_width = width;
-               int rc = 0, n = 0;
+               int xrc = 0, n = 0;
 
                if (!sorted) {
                        DBG(TAB, ul_debugobj(tb, "sorting by deviation"));
@@ -496,12 +498,12 @@ int __scols_calculate(struct libscols_table *tb, struct ul_buffer *buf)
                scols_reset_iter(&itr, SCOLS_ITER_BACKWARD);
 
                while (width > tb->termwidth
-                      && rc == 0
+                      && xrc == 0
                       && scols_table_next_column(tb, &itr, &cl) == 0) {
-                       rc = reduce_column(tb, cl, &width, stage, n++);
+                       xrc = reduce_column(tb, cl, &width, stage, n++);
                }
 
-               if (rc != 0)
+               if (xrc != 0)
                        break;
                if (org_width == width)
                        stage++;