KVM: arm64: Generalise kvm_pgtable_stage2_set_owner()

kvm_pgtable_stage2_set_owner() can be generalised into a way to store
up to 59 bits in the page tables alongside a 4-bit 'type' identifier
specific to the format of the 59-bit payload.

Introduce kvm_pgtable_stage2_annotate() and move the existing invalid
ptes (for locked ptes and donated pages) over to the new scheme.

Tested-by: Fuad Tabba <tabba@google.com>
Tested-by: Mostafa Saleh <smostafa@google.com>
Signed-off-by: Will Deacon <will@kernel.org>
Link: https://patch.msgid.link/20260330144841.26181-22-will@kernel.org
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/include/asm/kvm_pgtable.h b/arch/arm64/include/asm/kvm_pgtable.h
index 50caca311ef5fa039d3fd97e1f9f05d1ee88e970..e36c2908bdb280c25bfa9bfd09aa6dfc92c2485a 100644 (file)
--- a/arch/arm64/include/asm/kvm_pgtable.h
+++ b/arch/arm64/include/asm/kvm_pgtable.h
@@ -100,13 +100,25 @@ typedef u64 kvm_pte_t;
                                         KVM_PTE_LEAF_ATTR_HI_S2_XN)
 
 #define KVM_INVALID_PTE_OWNER_MASK     GENMASK(9, 2)
-#define KVM_MAX_OWNER_ID               2
 
-/*
- * Used to indicate a pte for which a 'break-before-make' sequence is in
- * progress.
- */
-#define KVM_INVALID_PTE_LOCKED         BIT(10)
+/* pKVM invalid pte encodings */
+#define KVM_INVALID_PTE_TYPE_MASK      GENMASK(63, 60)
+#define KVM_INVALID_PTE_ANNOT_MASK     ~(KVM_PTE_VALID | \
+                                         KVM_INVALID_PTE_TYPE_MASK)
+
+enum kvm_invalid_pte_type {
+       /*
+        * Used to indicate a pte for which a 'break-before-make'
+        * sequence is in progress.
+        */
+       KVM_INVALID_PTE_TYPE_LOCKED     = 1,
+
+       /*
+        * pKVM has unmapped the page from the host due to a change of
+        * ownership.
+        */
+       KVM_HOST_INVALID_PTE_TYPE_DONATION,
+};
 
 static inline bool kvm_pte_valid(kvm_pte_t pte)
 {
@@ -658,14 +670,18 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
                           void *mc, enum kvm_pgtable_walk_flags flags);
 
 /**
- * kvm_pgtable_stage2_set_owner() - Unmap and annotate pages in the IPA space to
- *                                 track ownership.
+ * kvm_pgtable_stage2_annotate() - Unmap and annotate pages in the IPA space
+ *                                to track ownership (and more).
  * @pgt:       Page-table structure initialised by kvm_pgtable_stage2_init*().
  * @addr:      Base intermediate physical address to annotate.
  * @size:      Size of the annotated range.
  * @mc:                Cache of pre-allocated and zeroed memory from which to allocate
  *             page-table pages.
- * @owner_id:  Unique identifier for the owner of the page.
+ * @type:      The type of the annotation, determining its meaning and format.
+ * @annotation:        A 59-bit value that will be stored in the page tables.
+ *             @annotation[0] and @annotation[63:60] must be 0.
+ *             @annotation[59:1] is stored in the page tables, along
+ *             with @type.
  *
  * By default, all page-tables are owned by identifier 0. This function can be
  * used to mark portions of the IPA space as owned by other entities. When a
@@ -674,8 +690,9 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
  *
  * Return: 0 on success, negative error code on failure.
  */
-int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
-                                void *mc, u8 owner_id);
+int kvm_pgtable_stage2_annotate(struct kvm_pgtable *pgt, u64 addr, u64 size,
+                               void *mc, enum kvm_invalid_pte_type type,
+                               kvm_pte_t annotation);
 
 /**
  * kvm_pgtable_stage2_unmap() - Remove a mapping from a guest stage-2 page-table.

diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index bf5102594fc82a0ea23e93aba8b7eed1b749a6a9..aea6ec981801b21b741c1d5f3b5ce2b456b9c847 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -549,10 +549,19 @@ static void __host_update_page_state(phys_addr_t addr, u64 size, enum pkvm_page_
                set_host_state(page, state);
 }
 
+static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)
+{
+       return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
+}
+
 int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
 {
+       kvm_pte_t annotation;
        int ret = -EINVAL;
 
+       if (!FIELD_FIT(KVM_INVALID_PTE_OWNER_MASK, owner_id))
+               return -EINVAL;
+
        if (!range_is_memory(addr, addr + size))
                return -EPERM;
 
@@ -564,8 +573,11 @@ int host_stage2_set_owner_locked(phys_addr_t addr, u64 size, u8 owner_id)
                break;
        case PKVM_ID_GUEST:
        case PKVM_ID_HYP:
-               ret = host_stage2_try(kvm_pgtable_stage2_set_owner, &host_mmu.pgt,
-                                     addr, size, &host_s2_pool, owner_id);
+               annotation = kvm_init_invalid_leaf_owner(owner_id);
+               ret = host_stage2_try(kvm_pgtable_stage2_annotate, &host_mmu.pgt,
+                                     addr, size, &host_s2_pool,
+                                     KVM_HOST_INVALID_PTE_TYPE_DONATION,
+                                     annotation);
                if (!ret)
                        __host_update_page_state(addr, size, PKVM_NOPAGE);
                break;

diff --git a/arch/arm64/kvm/hyp/pgtable.c b/arch/arm64/kvm/hyp/pgtable.c
index 9b480f947da26d0be8b3843b7279250470e87a94..84c7a1df845d43e56a1389bd2df6f918915d489e 100644 (file)
--- a/arch/arm64/kvm/hyp/pgtable.c
+++ b/arch/arm64/kvm/hyp/pgtable.c
@@ -114,11 +114,6 @@ static kvm_pte_t kvm_init_valid_leaf_pte(u64 pa, kvm_pte_t attr, s8 level)
        return pte;
 }
 
-static kvm_pte_t kvm_init_invalid_leaf_owner(u8 owner_id)
-{
-       return FIELD_PREP(KVM_INVALID_PTE_OWNER_MASK, owner_id);
-}
-
 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data,
                                  const struct kvm_pgtable_visit_ctx *ctx,
                                  enum kvm_pgtable_walk_flags visit)
@@ -581,7 +576,7 @@ void kvm_pgtable_hyp_destroy(struct kvm_pgtable *pgt)
 struct stage2_map_data {
        const u64                       phys;
        kvm_pte_t                       attr;
-       u8                              owner_id;
+       kvm_pte_t                       pte_annot;
 
        kvm_pte_t                       *anchor;
        kvm_pte_t                       *childp;
@@ -798,7 +793,11 @@ static bool stage2_pte_is_counted(kvm_pte_t pte)
 
 static bool stage2_pte_is_locked(kvm_pte_t pte)
 {
-       return !kvm_pte_valid(pte) && (pte & KVM_INVALID_PTE_LOCKED);
+       if (kvm_pte_valid(pte))
+               return false;
+
+       return FIELD_GET(KVM_INVALID_PTE_TYPE_MASK, pte) ==
+              KVM_INVALID_PTE_TYPE_LOCKED;
 }
 
 static bool stage2_try_set_pte(const struct kvm_pgtable_visit_ctx *ctx, kvm_pte_t new)
@@ -829,6 +828,7 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
                                 struct kvm_s2_mmu *mmu)
 {
        struct kvm_pgtable_mm_ops *mm_ops = ctx->mm_ops;
+       kvm_pte_t locked_pte;
 
        if (stage2_pte_is_locked(ctx->old)) {
                /*
@@ -839,7 +839,9 @@ static bool stage2_try_break_pte(const struct kvm_pgtable_visit_ctx *ctx,
                return false;
        }
 
-       if (!stage2_try_set_pte(ctx, KVM_INVALID_PTE_LOCKED))
+       locked_pte = FIELD_PREP(KVM_INVALID_PTE_TYPE_MASK,
+                               KVM_INVALID_PTE_TYPE_LOCKED);
+       if (!stage2_try_set_pte(ctx, locked_pte))
                return false;
 
        if (!kvm_pgtable_walk_skip_bbm_tlbi(ctx)) {
@@ -964,7 +966,7 @@ static int stage2_map_walker_try_leaf(const struct kvm_pgtable_visit_ctx *ctx,
        if (!data->annotation)
                new = kvm_init_valid_leaf_pte(phys, data->attr, ctx->level);
        else
-               new = kvm_init_invalid_leaf_owner(data->owner_id);
+               new = data->pte_annot;
 
        /*
         * Skip updating the PTE if we are trying to recreate the exact
@@ -1118,16 +1120,18 @@ int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size,
        return ret;
 }
 
-int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
-                                void *mc, u8 owner_id)
+int kvm_pgtable_stage2_annotate(struct kvm_pgtable *pgt, u64 addr, u64 size,
+                               void *mc, enum kvm_invalid_pte_type type,
+                               kvm_pte_t pte_annot)
 {
        int ret;
        struct stage2_map_data map_data = {
                .mmu            = pgt->mmu,
                .memcache       = mc,
-               .owner_id       = owner_id,
                .force_pte      = true,
                .annotation     = true,
+               .pte_annot      = pte_annot |
+                                 FIELD_PREP(KVM_INVALID_PTE_TYPE_MASK, type),
        };
        struct kvm_pgtable_walker walker = {
                .cb             = stage2_map_walker,
@@ -1136,7 +1140,10 @@ int kvm_pgtable_stage2_set_owner(struct kvm_pgtable *pgt, u64 addr, u64 size,
                .arg            = &map_data,
        };
 
-       if (owner_id > KVM_MAX_OWNER_ID)
+       if (pte_annot & ~KVM_INVALID_PTE_ANNOT_MASK)
+               return -EINVAL;
+
+       if (!type || type == KVM_INVALID_PTE_TYPE_LOCKED)
                return -EINVAL;
 
        ret = kvm_pgtable_walk(pgt, addr, size, &walker);