arm64: mm: avoid fixmap for early swapper_pg_dir updates

Early in the boot, when .rodata is still writable, we can poke
swapper_pg_dir entries directly, and there is no need to go through the
fixmap. After a future patch, we will enter the kernel with
swapper_pg_dir already active, and early swapper_pg_dir updates for
creating the fixmap page table hierarchy itself cannot go through the
fixmap for obvious reaons. So let's keep track of whether rodata is
writable, and update the descriptor directly in that case.

As the same reasoning applies to early KASAN init, make the function
noinstr as well.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20240214122845.2033971-67-ardb+git@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 14a62c773201530bff9cbb576d4d229893adefe8..9758f7e3f4b6228c2a0baad7a85d5ae96719d587 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -55,6 +55,8 @@ EXPORT_SYMBOL(kimage_voffset);
 
 u32 __boot_cpu_mode[] = { BOOT_CPU_MODE_EL2, BOOT_CPU_MODE_EL1 };
 
+static bool rodata_is_rw __ro_after_init = true;
+
 /*
  * The booting CPU updates the failed status @__early_cpu_boot_status,
  * with MMU turned off.
@@ -71,10 +73,21 @@ EXPORT_SYMBOL(empty_zero_page);
 static DEFINE_SPINLOCK(swapper_pgdir_lock);
 static DEFINE_MUTEX(fixmap_lock);
 
-void set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
+void noinstr set_swapper_pgd(pgd_t *pgdp, pgd_t pgd)
 {
        pgd_t *fixmap_pgdp;
 
+       /*
+        * Don't bother with the fixmap if swapper_pg_dir is still mapped
+        * writable in the kernel mapping.
+        */
+       if (rodata_is_rw) {
+               WRITE_ONCE(*pgdp, pgd);
+               dsb(ishst);
+               isb();
+               return;
+       }
+
        spin_lock(&swapper_pgdir_lock);
        fixmap_pgdp = pgd_set_fixmap(__pa_symbol(pgdp));
        WRITE_ONCE(*fixmap_pgdp, pgd);
@@ -628,6 +641,7 @@ void mark_rodata_ro(void)
         * to cover NOTES and EXCEPTION_TABLE.
         */
        section_size = (unsigned long)__init_begin - (unsigned long)__start_rodata;
+       WRITE_ONCE(rodata_is_rw, false);
        update_mapping_prot(__pa_symbol(__start_rodata), (unsigned long)__start_rodata,
                            section_size, PAGE_KERNEL_RO);