KVM: arm64: Add .hyp.data section

The hypervisor has not needed its own .data section because all globals
were either .rodata or .bss. To avoid having to initialize future
data-structures at run-time, let's introduce add a .data section to the
hypervisor.

Signed-off-by: David Brazdil <dbrazdil@google.com>
Signed-off-by: Quentin Perret <qperret@google.com>
Link: https://lore.kernel.org/r/20250416160900.3078417-2-qperret@google.com
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/sections.h b/arch/arm64/include/asm/sections.h
index 40971ac1303f9aa0067d9e860b81e01a444a4b54..51b0d594239eb9a444c0b89015d6d3f8916e3c79 100644 (file)
--- a/arch/arm64/include/asm/sections.h
+++ b/arch/arm64/include/asm/sections.h
@@ -11,6 +11,7 @@ extern char __alt_instructions[], __alt_instructions_end[];
 extern char __hibernate_exit_text_start[], __hibernate_exit_text_end[];
 extern char __hyp_idmap_text_start[], __hyp_idmap_text_end[];
 extern char __hyp_text_start[], __hyp_text_end[];
+extern char __hyp_data_start[], __hyp_data_end[];
 extern char __hyp_rodata_start[], __hyp_rodata_end[];
 extern char __hyp_reloc_begin[], __hyp_reloc_end[];
 extern char __hyp_bss_start[], __hyp_bss_end[];

diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index 5e3c4b58f279049253814cc4c524404fcfcc9228..9c5345f1ab66ca0637b6967f25cff9576c422a6a 100644 (file)
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -131,6 +131,8 @@ KVM_NVHE_ALIAS(__hyp_text_start);
 KVM_NVHE_ALIAS(__hyp_text_end);
 KVM_NVHE_ALIAS(__hyp_bss_start);
 KVM_NVHE_ALIAS(__hyp_bss_end);
+KVM_NVHE_ALIAS(__hyp_data_start);
+KVM_NVHE_ALIAS(__hyp_data_end);
 KVM_NVHE_ALIAS(__hyp_rodata_start);
 KVM_NVHE_ALIAS(__hyp_rodata_end);
 

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index e73326bd3ff7e9282f3b80727e89fb8ec1fe88ea..7c770053f072e31fcb37b3187ce607799940fcb4 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -13,7 +13,7 @@
        *(__kvm_ex_table)                                       \
        __stop___kvm_ex_table = .;
 
-#define HYPERVISOR_DATA_SECTIONS                               \
+#define HYPERVISOR_RODATA_SECTIONS                             \
        HYP_SECTION_NAME(.rodata) : {                           \
                . = ALIGN(PAGE_SIZE);                           \
                __hyp_rodata_start = .;                         \
@@ -23,6 +23,15 @@
                __hyp_rodata_end = .;                           \
        }
 
+#define HYPERVISOR_DATA_SECTION                                        \
+       HYP_SECTION_NAME(.data) : {                             \
+               . = ALIGN(PAGE_SIZE);                           \
+               __hyp_data_start = .;                           \
+               *(HYP_SECTION_NAME(.data))                      \
+               . = ALIGN(PAGE_SIZE);                           \
+               __hyp_data_end = .;                             \
+       }
+
 #define HYPERVISOR_PERCPU_SECTION                              \
        . = ALIGN(PAGE_SIZE);                                   \
        HYP_SECTION_NAME(.data..percpu) : {                     \
@@ -51,7 +60,8 @@
 #define SBSS_ALIGN                     PAGE_SIZE
 #else /* CONFIG_KVM */
 #define HYPERVISOR_EXTABLE
-#define HYPERVISOR_DATA_SECTIONS
+#define HYPERVISOR_RODATA_SECTIONS
+#define HYPERVISOR_DATA_SECTION
 #define HYPERVISOR_PERCPU_SECTION
 #define HYPERVISOR_RELOC_SECTION
 #define SBSS_ALIGN                     0
@@ -190,7 +200,7 @@ SECTIONS
        /* everything from this point to __init_begin will be marked RO NX */
        RO_DATA(PAGE_SIZE)
 
-       HYPERVISOR_DATA_SECTIONS
+       HYPERVISOR_RODATA_SECTIONS
 
        .got : { *(.got) }
        /*
@@ -295,6 +305,8 @@ SECTIONS
        _sdata = .;
        RW_DATA(L1_CACHE_BYTES, PAGE_SIZE, THREAD_ALIGN)
 
+       HYPERVISOR_DATA_SECTION
+
        /*
         * Data written with the MMU off but read with the MMU on requires
         * cache lines to be invalidated, discarding up to a Cache Writeback

diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c
index 68fec8c95feef96d6c73c4fdbc69729518c69b70..58e8119f66bd96667ae355c4728b0c3b8e4338a6 100644 (file)
--- a/arch/arm64/kvm/arm.c
+++ b/arch/arm64/kvm/arm.c
@@ -2604,6 +2604,13 @@ static int __init init_hyp_mode(void)
                goto out_err;
        }
 
+       err = create_hyp_mappings(kvm_ksym_ref(__hyp_data_start),
+                                 kvm_ksym_ref(__hyp_data_end), PAGE_HYP);
+       if (err) {
+               kvm_err("Cannot map .hyp.data section\n");
+               goto out_err;
+       }
+
        err = create_hyp_mappings(kvm_ksym_ref(__hyp_rodata_start),
                                  kvm_ksym_ref(__hyp_rodata_end), PAGE_HYP_RO);
        if (err) {

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index f4562f417d3fc1d54351c7897d22c7df6a35637a..d724f6d69302a0500f7c3d5cde6390fc4bc30b00 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -25,5 +25,7 @@ SECTIONS {
        BEGIN_HYP_SECTION(.data..percpu)
                PERCPU_INPUT(L1_CACHE_BYTES)
        END_HYP_SECTION
+
        HYP_SECTION(.bss)
+       HYP_SECTION(.data)
 }

diff --git a/arch/arm64/kvm/hyp/nvhe/setup.c b/arch/arm64/kvm/hyp/nvhe/setup.c
index d62bcb5634a21d66179488c7c0e7710a52a2f4f6..46d9bd04348f48e1a3883b6108635046f4336889 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/setup.c
+++ b/arch/arm64/kvm/hyp/nvhe/setup.c
@@ -119,6 +119,10 @@ static int recreate_hyp_mappings(phys_addr_t phys, unsigned long size,
        if (ret)
                return ret;
 
+       ret = pkvm_create_mappings(__hyp_data_start, __hyp_data_end, PAGE_HYP);
+       if (ret)
+               return ret;
+
        ret = pkvm_create_mappings(__hyp_rodata_start, __hyp_rodata_end, PAGE_HYP_RO);
        if (ret)
                return ret;

diff --git a/arch/arm64/kvm/pkvm.c b/arch/arm64/kvm/pkvm.c
index 0f89157d31fd6fbaacc3dc80f823524fe242686f..c462140e640ab9999c5ec7d7b8685a0099c2aaef 100644 (file)
--- a/arch/arm64/kvm/pkvm.c
+++ b/arch/arm64/kvm/pkvm.c
@@ -262,6 +262,7 @@ static int __init finalize_pkvm(void)
         * at, which would end badly once inaccessible.
         */
        kmemleak_free_part(__hyp_bss_start, __hyp_bss_end - __hyp_bss_start);
+       kmemleak_free_part(__hyp_data_start, __hyp_data_end - __hyp_data_start);
        kmemleak_free_part(__hyp_rodata_start, __hyp_rodata_end - __hyp_rodata_start);
        kmemleak_free_part_phys(hyp_mem_base, hyp_mem_size);