libsmartcols: fix reduction stages use

There is no proper check for the number of reduction stages, so in
some cases, the code can loop indefinitely.

The patch also fixes 'rc' variable shadowing.

Fixes: https://github.com/util-linux/util-linux/issues/3046
Signed-off-by: Karel Zak <kzak@redhat.com>

diff --git a/libsmartcols/src/calculate.c b/libsmartcols/src/calculate.c
index 86e1b27992f6185cf6650b70e527b212d761ae10..41daf74b3cc85b89b9900bf4382cc5ef301ec367 100644 (file)
--- a/libsmartcols/src/calculate.c
+++ b/libsmartcols/src/calculate.c
@@ -306,6 +306,8 @@ static int reduce_column(struct libscols_table *tb,
        size_t wanted, org_width, reduce = 1;
        int is_trunc = 0;
 
+       if (stage > 6)
+               return -1;
        if (tb->termwidth >= *width)
                return 1;
        /* ignore hidden columns */
@@ -493,7 +495,7 @@ int __scols_calculate(struct libscols_table *tb, struct ul_buffer *buf)
        /* reduce columns width */
        while (width > tb->termwidth) {
                size_t org_width = width;
-               int rc = 0, n = 0;
+               int xrc = 0, n = 0;
 
                if (!sorted) {
                        DBG(TAB, ul_debugobj(tb, "sorting by deviation"));
@@ -508,12 +510,12 @@ int __scols_calculate(struct libscols_table *tb, struct ul_buffer *buf)
                scols_reset_iter(&itr, SCOLS_ITER_BACKWARD);
 
                while (width > tb->termwidth
-                      && rc == 0
+                      && xrc == 0
                       && scols_table_next_column(tb, &itr, &cl) == 0) {
-                       rc = reduce_column(tb, cl, &width, stage, n++);
+                       xrc = reduce_column(tb, cl, &width, stage, n++);
                }
 
-               if (rc != 0)
+               if (xrc != 0)
                        break;
                if (org_width == width)
                        stage++;