kasan: Move generic KASAN page tables out of BSS too

Make sure that all KASAN page tables are emitted into the .pgtbl section
(provided that the arch has one - otherwise, fall back to page aligned
BSS)

This is needed because BSS itself is no longer accessible via the linear
map on arm64.

Cc: Andrey Ryabinin <ryabinin.a.a@gmail.com>
Cc: Alexander Potapenko <glider@google.com>
Cc: Andrey Konovalov <andreyknvl@gmail.com>
Cc: Dmitry Vyukov <dvyukov@google.com>
Cc: Vincenzo Frascino <vincenzo.frascino@arm.com>
Cc: kasan-dev@googlegroups.com
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/include/linux/linkage.h b/include/linux/linkage.h
index b11660b706c584b65541611c962410a8620a0bef..53fe1f48fd28dad27ac1d8be808573cf1cbd3fcd 100644 (file)
--- a/include/linux/linkage.h
+++ b/include/linux/linkage.h
@@ -39,6 +39,10 @@
 #define __page_aligned_data    __section(".data..page_aligned") __aligned(PAGE_SIZE)
 #define __page_aligned_bss     __section(".bss..page_aligned") __aligned(PAGE_SIZE)
 
+#ifndef __bss_pgtbl
+#define __bss_pgtbl            __page_aligned_bss
+#endif
+
 /*
  * For assembly routines.
  *

diff --git a/mm/kasan/init.c b/mm/kasan/init.c
index 9c880f607c6a2def461dfd7b32a5f948cfe956fb..66a88388798768d96af0a5c9e6a7d0a415cc9711 100644 (file)
--- a/mm/kasan/init.c
+++ b/mm/kasan/init.c
@@ -26,10 +26,10 @@
  *   - Latter it reused it as zero shadow to cover large ranges of memory
  *     that allowed to access, but not handled by kasan (vmalloc/vmemmap ...).
  */
-unsigned char kasan_early_shadow_page[PAGE_SIZE] __page_aligned_bss;
+unsigned char kasan_early_shadow_page[PAGE_SIZE] __bss_pgtbl;
 
 #if CONFIG_PGTABLE_LEVELS > 4
-p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __page_aligned_bss;
+p4d_t kasan_early_shadow_p4d[MAX_PTRS_PER_P4D] __bss_pgtbl;
 static inline bool kasan_p4d_table(pgd_t pgd)
 {
        return pgd_page(pgd) == virt_to_page(lm_alias(kasan_early_shadow_p4d));
@@ -41,7 +41,7 @@ static inline bool kasan_p4d_table(pgd_t pgd)
 }
 #endif
 #if CONFIG_PGTABLE_LEVELS > 3
-pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __page_aligned_bss;
+pud_t kasan_early_shadow_pud[MAX_PTRS_PER_PUD] __bss_pgtbl;
 static inline bool kasan_pud_table(p4d_t p4d)
 {
        return p4d_page(p4d) == virt_to_page(lm_alias(kasan_early_shadow_pud));
@@ -53,7 +53,7 @@ static inline bool kasan_pud_table(p4d_t p4d)
 }
 #endif
 #if CONFIG_PGTABLE_LEVELS > 2
-pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __page_aligned_bss;
+pmd_t kasan_early_shadow_pmd[MAX_PTRS_PER_PMD] __bss_pgtbl;
 static inline bool kasan_pmd_table(pud_t pud)
 {
        return pud_page(pud) == virt_to_page(lm_alias(kasan_early_shadow_pmd));
@@ -65,7 +65,7 @@ static inline bool kasan_pmd_table(pud_t pud)
 }
 #endif
 pte_t kasan_early_shadow_pte[MAX_PTRS_PER_PTE + PTE_HWTABLE_PTRS]
-       __page_aligned_bss;
+       __bss_pgtbl;
 
 static inline bool kasan_pte_table(pmd_t pmd)
 {