selftests/mm: fix incorrect mmap() error handling with NULL instead of MAP_FAILED

mmap() returns MAP_FAILED, which is defined as (void *)-1, on error, not
NULL.  Several selftests incorrectly check the return value of mmap()
using !ptr or ptr == NULL, which would erroneously treat MAP_FAILED as a
valid pointer since MAP_FAILED is non-zero and non-NULL.  This can lead to
segfaults when mmap() actually fails under memory pressure.

Link: https://lore.kernel.org/20260513025223.592766-1-lihongfu@kylinos.cn
Signed-off-by: Hongfu Li <lihongfu@kylinos.cn>
Reviewed-by: Dev Jain <dev.jain@arm.com>
Reviewed-by: Lorenzo Stoakes <ljs@kernel.org>
Cc: David Hildenbrand <david@kernel.org>
Cc: Liam R. Howlett <liam@infradead.org>
Cc: Michal Hocko <mhocko@suse.com>
Cc: Mike Rapoport <rppt@kernel.org>
Cc: Shuah Khan <shuah@kernel.org>
Cc: Suren Baghdasaryan <surenb@google.com>
Cc: Vlastimil Babka <vbabka@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/tools/testing/selftests/mm/ksm_tests.c b/tools/testing/selftests/mm/ksm_tests.c
index 64d1c63ae8c0783ede41e53dc387c9f25f98e28a..a050f4840cfa3a726dbde8afdf6fa2d99ddc0250 100644 (file)
--- a/tools/testing/selftests/mm/ksm_tests.c
+++ b/tools/testing/selftests/mm/ksm_tests.c
@@ -174,7 +174,7 @@ static void  *allocate_memory(void *ptr, int prot, int mapping, char data, size_
 {
        void *map_ptr = mmap(ptr, map_size, PROT_WRITE, mapping, -1, 0);
 
-       if (!map_ptr) {
+       if (map_ptr == MAP_FAILED) {
                ksft_perror("mmap");
                return NULL;
        }

diff --git a/tools/testing/selftests/mm/madv_populate.c b/tools/testing/selftests/mm/madv_populate.c
index 88050e0f829a0e61d188de34fe71fdc5b030bd7d..7fce5d0b622be1ffedccbb696fea18d285a86c85 100644 (file)
--- a/tools/testing/selftests/mm/madv_populate.c
+++ b/tools/testing/selftests/mm/madv_populate.c
@@ -34,7 +34,7 @@ static void sense_support(void)
 
        addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
                    MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
-       if (!addr)
+       if (addr == MAP_FAILED)
                ksft_exit_fail_msg("mmap failed\n");
 
        ret = madvise(addr, pagesize, MADV_POPULATE_READ);

diff --git a/tools/testing/selftests/mm/soft-dirty.c b/tools/testing/selftests/mm/soft-dirty.c
index c426c4636ef573f537896339bbf0ff6d311a27b9..fb1864a68e1c6b10cbe61116bc90fb9d41dcc294 100644 (file)
--- a/tools/testing/selftests/mm/soft-dirty.c
+++ b/tools/testing/selftests/mm/soft-dirty.c
@@ -143,7 +143,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
        if (anon) {
                map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
                           MAP_ANONYMOUS|MAP_PRIVATE, -1, 0);
-               if (!map)
+               if (map == MAP_FAILED)
                        ksft_exit_fail_msg("anon mmap failed\n");
        } else {
                test_fd = open(fname, O_RDWR | O_CREAT, 0664);
@@ -155,7 +155,7 @@ static void test_mprotect(int pagemap_fd, int pagesize, bool anon)
                ftruncate(test_fd, pagesize);
                map = mmap(NULL, pagesize, PROT_READ|PROT_WRITE,
                           MAP_SHARED, test_fd, 0);
-               if (!map)
+               if (map == MAP_FAILED)
                        ksft_exit_fail_msg("file mmap failed\n");
        }
 

diff --git a/tools/testing/selftests/mm/vm_util.c b/tools/testing/selftests/mm/vm_util.c
index 336a26751d3faec7a586021e15b48bcd5b62805b..311fc5b4513eb43ebf0bee6bf1177f146f688020 100644 (file)
--- a/tools/testing/selftests/mm/vm_util.c
+++ b/tools/testing/selftests/mm/vm_util.c
@@ -396,7 +396,7 @@ bool softdirty_supported(void)
        /* New mappings are expected to be marked with VM_SOFTDIRTY (sd). */
        addr = mmap(0, pagesize, PROT_READ | PROT_WRITE,
                    MAP_ANONYMOUS | MAP_PRIVATE, 0, 0);
-       if (!addr)
+       if (addr == MAP_FAILED)
                ksft_exit_fail_msg("mmap failed\n");
 
        supported = check_vmflag(addr, "sd");