e2fsck, resize2fs: fix a fp precision error that can lead to a seg fault

Commit 641b66b fixed a floating point precision error which can result
in a search algorithm looping forever.  It can also result in an array
index being out of bounds and causing a segfault.  Here are two more
cases in e2fsck and resize2fs that need to be fixed.  I've just used
the same fix from the that commit.

Signed-off-by: Lachlan McIlroy <lmcilroy@redhat.com>
Signed-off-by: "Theodore Ts'o" <tytso@mit.edu>

diff --git a/e2fsck/ea_refcount.c b/e2fsck/ea_refcount.c
index 39f2db7fc6c5b5f623978ce7ef09ce71cb9431be..b10cfffaf250d3bff8d7f57e73da8e594b7efbeb 100644 (file)
--- a/e2fsck/ea_refcount.c
+++ b/e2fsck/ea_refcount.c
@@ -196,9 +196,14 @@ retry:
                                range = 0;
                        else if (blk > highval)
                                range = 1;
-                       else
+                       else {
                                range = ((float) (blk - lowval)) /
                                        (highval - lowval);
+                               if (range > 0.9)
+                                       range = 0.9;
+                               if (range < 0.1)
+                                       range = 0.1;
+                       }
                        mid = low + ((int) (range * (high-low)));
                }
 #endif

diff --git a/resize/extent.c b/resize/extent.c
index 2ed7591d1675bd3bce50d31725f368b5b2452297..f0fb1e0067e6f9a541cc3d803ebf48f363a510d9 100644 (file)
--- a/resize/extent.c
+++ b/resize/extent.c
@@ -167,9 +167,14 @@ __u32 ext2fs_extent_translate(ext2_extent extent, __u32 old_loc)
                                range = 0;
                        else if (old_loc > highval)
                                range = 1;
-                       else
+                       else {
                                range = ((float) (old_loc - lowval)) /
                                        (highval - lowval);
+                               if (range > 0.9)
+                                       range = 0.9;
+                               if (range < 0.1)
+                                       range = 0.1;
+                       }
                        mid = low + ((int) (range * (high-low)));
                }
 #endif