KVM: arm64: Add event support to the nVHE/pKVM hyp and trace remote

Allow the creation of hypervisor and trace remote events with a single
macro HYP_EVENT(). That macro expands in the kernel side to add all
the required declarations (based on REMOTE_EVENT()) as well as in the
hypervisor side to create the trace_<event>() function.

Signed-off-by: Vincent Donnefort <vdonnefort@google.com>
Link: https://patch.msgid.link/20260309162516.2623589-28-vdonnefort@google.com
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index 66abf77cf3711c2ff91161570052b25de14e9a92..47d250436f8cab9b782106eb515dc21a3f012b55 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -95,6 +95,7 @@ enum __kvm_host_smccc_func {
        __KVM_HOST_SMCCC_FUNC___tracing_swap_reader,
        __KVM_HOST_SMCCC_FUNC___tracing_update_clock,
        __KVM_HOST_SMCCC_FUNC___tracing_reset,
+       __KVM_HOST_SMCCC_FUNC___tracing_enable_event,
 };
 
 #define DECLARE_KVM_VHE_SYM(sym)       extern char sym[]

diff --git a/arch/arm64/include/asm/kvm_define_hypevents.h b/arch/arm64/include/asm/kvm_define_hypevents.h
new file mode 100644 (file)
index 0000000..77d6790
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_define_hypevents.h
@@ -0,0 +1,16 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#define REMOTE_EVENT_INCLUDE_FILE arch/arm64/include/asm/kvm_hypevents.h
+
+#define REMOTE_EVENT_SECTION "_hyp_events"
+
+#define HE_STRUCT(__args)              __args
+#define HE_PRINTK(__args...)           __args
+#define he_field                       re_field
+
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk) \
+       REMOTE_EVENT(__name, 0, RE_STRUCT(__struct), RE_PRINTK(__printk))
+
+#define HYP_EVENT_MULTI_READ
+#include <trace/define_remote_events.h>
+#undef HYP_EVENT_MULTI_READ

diff --git a/arch/arm64/include/asm/kvm_hypevents.h b/arch/arm64/include/asm/kvm_hypevents.h
new file mode 100644 (file)
index 0000000..d6e033c
--- /dev/null
+++ b/arch/arm64/include/asm/kvm_hypevents.h
@@ -0,0 +1,10 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#if !defined(__ARM64_KVM_HYPEVENTS_H_) || defined(HYP_EVENT_MULTI_READ)
+#define __ARM64_KVM_HYPEVENTS_H_
+
+#ifdef __KVM_NVHE_HYPERVISOR__
+#include <nvhe/trace.h>
+#endif
+
+#endif

diff --git a/arch/arm64/include/asm/kvm_hyptrace.h b/arch/arm64/include/asm/kvm_hyptrace.h
index 9c30a479bc36a84310e17296f82fddb9cec25cf2..de133b735f72b9bf611da69374c2fbc0455b578e 100644 (file)
--- a/arch/arm64/include/asm/kvm_hyptrace.h
+++ b/arch/arm64/include/asm/kvm_hyptrace.h
@@ -10,4 +10,17 @@ struct hyp_trace_desc {
        struct trace_buffer_desc        trace_buffer_desc;
 
 };
+
+struct hyp_event_id {
+       unsigned short  id;
+       atomic_t        enabled;
+};
+
+extern struct remote_event __hyp_events_start[];
+extern struct remote_event __hyp_events_end[];
+
+/* hyp_event section used by the hypervisor */
+extern struct hyp_event_id __hyp_event_ids_start[];
+extern struct hyp_event_id __hyp_event_ids_end[];
+
 #endif

diff --git a/arch/arm64/kernel/image-vars.h b/arch/arm64/kernel/image-vars.h
index d7b0d12b1015569af0e7990575c8e3ed1b8660f7..d4c7d45ae6bc8321401c75af269f3f99085893ed 100644 (file)
--- a/arch/arm64/kernel/image-vars.h
+++ b/arch/arm64/kernel/image-vars.h
@@ -138,6 +138,10 @@ KVM_NVHE_ALIAS(__hyp_data_start);
 KVM_NVHE_ALIAS(__hyp_data_end);
 KVM_NVHE_ALIAS(__hyp_rodata_start);
 KVM_NVHE_ALIAS(__hyp_rodata_end);
+#ifdef CONFIG_NVHE_EL2_TRACING
+KVM_NVHE_ALIAS(__hyp_event_ids_start);
+KVM_NVHE_ALIAS(__hyp_event_ids_end);
+#endif
 
 /* pKVM static key */
 KVM_NVHE_ALIAS(kvm_protected_mode_initialized);

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 2964aad0362e4e14a4798509f00872bddc429997..4bf2a83c3448ba5ed56a4b1b0e951d9a0b8c9593 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -13,12 +13,23 @@
        *(__kvm_ex_table)                                       \
        __stop___kvm_ex_table = .;
 
+#ifdef CONFIG_NVHE_EL2_TRACING
+#define HYPERVISOR_EVENT_IDS                                   \
+       . = ALIGN(PAGE_SIZE);                                   \
+       __hyp_event_ids_start = .;                              \
+       *(HYP_SECTION_NAME(.event_ids))                         \
+       __hyp_event_ids_end = .;
+#else
+#define HYPERVISOR_EVENT_IDS
+#endif
+
 #define HYPERVISOR_RODATA_SECTIONS                             \
        HYP_SECTION_NAME(.rodata) : {                           \
                . = ALIGN(PAGE_SIZE);                           \
                __hyp_rodata_start = .;                         \
                *(HYP_SECTION_NAME(.data..ro_after_init))       \
                *(HYP_SECTION_NAME(.rodata))                    \
+               HYPERVISOR_EVENT_IDS                            \
                . = ALIGN(PAGE_SIZE);                           \
                __hyp_rodata_end = .;                           \
        }
@@ -307,6 +318,13 @@ SECTIONS
 
        HYPERVISOR_DATA_SECTION
 
+#ifdef CONFIG_NVHE_EL2_TRACING
+       .data.hyp_events : {
+               __hyp_events_start = .;
+               *(SORT(_hyp_events.*))
+               __hyp_events_end = .;
+       }
+#endif
        /*
         * Data written with the MMU off but read with the MMU on requires
         * cache lines to be invalidated, discarding up to a Cache Writeback

diff --git a/arch/arm64/kvm/hyp/include/nvhe/define_events.h b/arch/arm64/kvm/hyp/include/nvhe/define_events.h
new file mode 100644 (file)
index 0000000..776d4c6
--- /dev/null
+++ b/arch/arm64/kvm/hyp/include/nvhe/define_events.h
@@ -0,0 +1,14 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+
+#undef HYP_EVENT
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)       \
+       struct hyp_event_id hyp_event_id_##__name                       \
+       __section(".hyp.event_ids."#__name) = {                         \
+               .enabled = ATOMIC_INIT(0),                              \
+       }
+
+#define HYP_EVENT_MULTI_READ
+#include <asm/kvm_hypevents.h>
+#undef HYP_EVENT_MULTI_READ
+
+#undef HYP_EVENT

diff --git a/arch/arm64/kvm/hyp/include/nvhe/trace.h b/arch/arm64/kvm/hyp/include/nvhe/trace.h
index 44912869d18491f47f5e4afee1a99556f0571308..802a18b77c56f6104d39a920631f206b12db8272 100644 (file)
--- a/arch/arm64/kvm/hyp/include/nvhe/trace.h
+++ b/arch/arm64/kvm/hyp/include/nvhe/trace.h
@@ -1,9 +1,36 @@
 /* SPDX-License-Identifier: GPL-2.0-only */
 #ifndef __ARM64_KVM_HYP_NVHE_TRACE_H
 #define __ARM64_KVM_HYP_NVHE_TRACE_H
+
+#include <linux/trace_remote_event.h>
+
 #include <asm/kvm_hyptrace.h>
 
+#define HE_PROTO(__args...)    __args
+#define HE_ASSIGN(__args...)   __args
+#define HE_STRUCT              RE_STRUCT
+#define he_field               re_field
+
 #ifdef CONFIG_NVHE_EL2_TRACING
+
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)               \
+       REMOTE_EVENT_FORMAT(__name, __struct);                                  \
+       extern struct hyp_event_id hyp_event_id_##__name;                       \
+       static __always_inline void trace_##__name(__proto)                     \
+       {                                                                       \
+               struct remote_event_format_##__name *__entry;                   \
+               size_t length = sizeof(*__entry);                               \
+                                                                               \
+               if (!atomic_read(&hyp_event_id_##__name.enabled))               \
+                       return;                                                 \
+               __entry = tracing_reserve_entry(length);                        \
+               if (!__entry)                                                   \
+                       return;                                                 \
+               __entry->hdr.id = hyp_event_id_##__name.id;                     \
+               __assign                                                        \
+               tracing_commit_entry();                                         \
+       }
+
 void *tracing_reserve_entry(unsigned long length);
 void tracing_commit_entry(void);
 
@@ -13,9 +40,12 @@ int __tracing_enable(bool enable);
 int __tracing_swap_reader(unsigned int cpu);
 void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc);
 int __tracing_reset(unsigned int cpu);
+int __tracing_enable_event(unsigned short id, bool enable);
 #else
 static inline void *tracing_reserve_entry(unsigned long length) { return NULL; }
 static inline void tracing_commit_entry(void) { }
+#define HYP_EVENT(__name, __proto, __struct, __assign, __printk)      \
+       static inline void trace_##__name(__proto) {}
 
 static inline int __tracing_load(unsigned long desc_va, size_t desc_size) { return -ENODEV; }
 static inline void __tracing_unload(void) { }
@@ -23,5 +53,6 @@ static inline int __tracing_enable(bool enable) { return -ENODEV; }
 static inline int __tracing_swap_reader(unsigned int cpu) { return -ENODEV; }
 static inline void __tracing_update_clock(u32 mult, u32 shift, u64 epoch_ns, u64 epoch_cyc) { }
 static inline int __tracing_reset(unsigned int cpu) { return -ENODEV; }
+static inline int __tracing_enable_event(unsigned short id, bool enable)  { return -ENODEV; }
 #endif
 #endif

diff --git a/arch/arm64/kvm/hyp/nvhe/Makefile b/arch/arm64/kvm/hyp/nvhe/Makefile
index f1840628d2d6e94b2801050ca95bb3391adb75d7..143d55ec7298753ac7564f402f9c2985a22c4438 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/Makefile
+++ b/arch/arm64/kvm/hyp/nvhe/Makefile
@@ -29,7 +29,7 @@ hyp-obj-y += ../vgic-v3-sr.o ../aarch32.o ../vgic-v2-cpuif-proxy.o ../entry.o \
         ../fpsimd.o ../hyp-entry.o ../exception.o ../pgtable.o
 hyp-obj-y += ../../../kernel/smccc-call.o
 hyp-obj-$(CONFIG_LIST_HARDENED) += list_debug.o
-hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o
+hyp-obj-$(CONFIG_NVHE_EL2_TRACING) += clock.o trace.o events.o
 hyp-obj-y += $(lib-objs)
 
 # Path to simple_ring_buffer.c

diff --git a/arch/arm64/kvm/hyp/nvhe/events.c b/arch/arm64/kvm/hyp/nvhe/events.c
new file mode 100644 (file)
index 0000000..add9383
--- /dev/null
+++ b/arch/arm64/kvm/hyp/nvhe/events.c
@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * Copyright (C) 2025 Google LLC
+ * Author: Vincent Donnefort <vdonnefort@google.com>
+ */
+
+#include <nvhe/mm.h>
+#include <nvhe/trace.h>
+
+#include <nvhe/define_events.h>
+
+int __tracing_enable_event(unsigned short id, bool enable)
+{
+       struct hyp_event_id *event_id = &__hyp_event_ids_start[id];
+       atomic_t *enabled;
+
+       if (event_id >= __hyp_event_ids_end)
+               return -EINVAL;
+
+       enabled = hyp_fixmap_map(__hyp_pa(&event_id->enabled));
+       atomic_set(enabled, enable);
+       hyp_fixmap_unmap();
+
+       return 0;
+}

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-main.c b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
index 9b05f0c875861ee42817b5520a63b7fad3991070..fc5953f31b4bf10fa2bbf176ed8ab8ee146efefb 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp-main.c
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-main.c
@@ -632,6 +632,14 @@ static void handle___tracing_reset(struct kvm_cpu_context *host_ctxt)
        cpu_reg(host_ctxt, 1) = __tracing_reset(cpu);
 }
 
+static void handle___tracing_enable_event(struct kvm_cpu_context *host_ctxt)
+{
+       DECLARE_REG(unsigned short, id, host_ctxt, 1);
+       DECLARE_REG(bool, enable, host_ctxt, 2);
+
+       cpu_reg(host_ctxt, 1) = __tracing_enable_event(id, enable);
+}
+
 typedef void (*hcall_t)(struct kvm_cpu_context *);
 
 #define HANDLE_FUNC(x) [__KVM_HOST_SMCCC_FUNC_##x] = (hcall_t)handle_##x
@@ -679,6 +687,7 @@ static const hcall_t host_hcall[] = {
        HANDLE_FUNC(__tracing_swap_reader),
        HANDLE_FUNC(__tracing_update_clock),
        HANDLE_FUNC(__tracing_reset),
+       HANDLE_FUNC(__tracing_enable_event),
 };
 
 static void handle_host_hcall(struct kvm_cpu_context *host_ctxt)

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
index d724f6d69302a0500f7c3d5cde6390fc4bc30b00..7a02837203d15d7a800bc8ff67d011ffadc5824c 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp.lds.S
@@ -16,6 +16,12 @@ SECTIONS {
        HYP_SECTION(.text)
        HYP_SECTION(.data..ro_after_init)
        HYP_SECTION(.rodata)
+#ifdef CONFIG_NVHE_EL2_TRACING
+       . = ALIGN(PAGE_SIZE);
+       BEGIN_HYP_SECTION(.event_ids)
+               *(SORT(.hyp.event_ids.*))
+       END_HYP_SECTION
+#endif
 
        /*
         * .hyp..data..percpu needs to be page aligned to maintain the same

diff --git a/arch/arm64/kvm/hyp_trace.c b/arch/arm64/kvm/hyp_trace.c
index 09bc192e3514a2d499ef294f6d716b6ee7b64026..0144cd26703e98c324097bc8f132b7d2611b319e 100644 (file)
--- a/arch/arm64/kvm/hyp_trace.c
+++ b/arch/arm64/kvm/hyp_trace.c
@@ -318,6 +318,25 @@ static int hyp_trace_reset(unsigned int cpu, void *priv)
 
 static int hyp_trace_enable_event(unsigned short id, bool enable, void *priv)
 {
+       struct hyp_event_id *event_id = lm_alias(&__hyp_event_ids_start[id]);
+       struct page *page;
+       atomic_t *enabled;
+       void *map;
+
+       if (is_protected_kvm_enabled())
+               return kvm_call_hyp_nvhe(__tracing_enable_event, id, enable);
+
+       enabled = &event_id->enabled;
+       page = virt_to_page(enabled);
+       map = vmap(&page, 1, VM_MAP, PAGE_KERNEL);
+       if (!map)
+               return -ENOMEM;
+
+       enabled = map + offset_in_page(enabled);
+       atomic_set(enabled, enable);
+
+       vunmap(map);
+
        return 0;
 }
 
@@ -345,6 +364,19 @@ static struct trace_remote_callbacks trace_remote_callbacks = {
        .enable_event           = hyp_trace_enable_event,
 };
 
+#include <asm/kvm_define_hypevents.h>
+
+static void __init hyp_trace_init_events(void)
+{
+       struct hyp_event_id *hyp_event_id = __hyp_event_ids_start;
+       struct remote_event *event = __hyp_events_start;
+       int id = 0;
+
+       /* Events on both sides hypervisor are sorted */
+       for (; event < __hyp_events_end; event++, hyp_event_id++, id++)
+               event->id = hyp_event_id->id = id;
+}
+
 int __init kvm_hyp_trace_init(void)
 {
        int cpu;
@@ -364,5 +396,8 @@ int __init kvm_hyp_trace_init(void)
        }
 #endif
 
-       return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer, NULL, 0);
+       hyp_trace_init_events();
+
+       return trace_remote_register("hypervisor", &trace_remote_callbacks, &trace_buffer,
+                                    __hyp_events_start, __hyp_events_end - __hyp_events_start);
 }