x86/mm: Enable AMD translation cache extensions

With AMD TCE (translation cache extensions) only the intermediate mappings
that cover the address range zapped by INVLPG / INVLPGB get invalidated,
rather than all intermediate mappings getting zapped at every TLB invalidation.

This can help reduce the TLB miss rate, by keeping more intermediate mappings
in the cache.

From the AMD manual:

Translation Cache Extension (TCE) Bit. Bit 15, read/write. Setting this bit to
1 changes how the INVLPG, INVLPGB, and INVPCID instructions operate on TLB
entries. When this bit is 0, these instructions remove the target PTE from the
TLB as well as all upper-level table entries that are cached in the TLB,
whether or not they are associated with the target PTE.  When this bit is set,
these instructions will remove the target PTE and only those upper-level
entries that lead to the target PTE in the page table hierarchy, leaving
unrelated upper-level entries intact.

  [ bp: use cpu_has()... I know, it is a mess. ]

Signed-off-by: Rik van Riel <riel@surriel.com>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Link: https://lore.kernel.org/r/20250226030129.530345-13-riel@surriel.com

diff --git a/arch/x86/include/asm/msr-index.h b/arch/x86/include/asm/msr-index.h
index 72765b2fe0d874b498fb77b90e2f121890ca49f0..1aacd6b68fab7b280f1d3bebe7aaf41d0fd5727b 100644 (file)
--- a/arch/x86/include/asm/msr-index.h
+++ b/arch/x86/include/asm/msr-index.h
@@ -25,6 +25,7 @@
 #define _EFER_SVME             12 /* Enable virtualization */
 #define _EFER_LMSLE            13 /* Long Mode Segment Limit Enable */
 #define _EFER_FFXSR            14 /* Enable Fast FXSAVE/FXRSTOR */
+#define _EFER_TCE              15 /* Enable Translation Cache Extensions */
 #define _EFER_AUTOIBRS         21 /* Enable Automatic IBRS */
 
 #define EFER_SCE               (1<<_EFER_SCE)
@@ -34,6 +35,7 @@
 #define EFER_SVME              (1<<_EFER_SVME)
 #define EFER_LMSLE             (1<<_EFER_LMSLE)
 #define EFER_FFXSR             (1<<_EFER_FFXSR)
+#define EFER_TCE               (1<<_EFER_TCE)
 #define EFER_AUTOIBRS          (1<<_EFER_AUTOIBRS)
 
 /*

diff --git a/arch/x86/kernel/cpu/amd.c b/arch/x86/kernel/cpu/amd.c
index 351c03025a8e1927b73f2429aa35876e30307f7b..79569f72b8ee5006297e1912209c426b472ec771 100644 (file)
--- a/arch/x86/kernel/cpu/amd.c
+++ b/arch/x86/kernel/cpu/amd.c
@@ -1075,6 +1075,10 @@ static void init_amd(struct cpuinfo_x86 *c)
 
        /* AMD CPUs don't need fencing after x2APIC/TSC_DEADLINE MSR writes. */
        clear_cpu_cap(c, X86_FEATURE_APIC_MSRS_FENCE);
+
+       /* Enable Translation Cache Extension */
+       if (cpu_has(c, X86_FEATURE_TCE))
+               msr_set_bit(MSR_EFER, _EFER_TCE);
 }
 
 #ifdef CONFIG_X86_32

diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h
index 3ae84c3b8e6dba73f0f44d3e25a1948a68daea91..dc1c1057f26e45a847796c4718200deb8561e6ba 100644 (file)
--- a/tools/arch/x86/include/asm/msr-index.h
+++ b/tools/arch/x86/include/asm/msr-index.h
@@ -25,6 +25,7 @@
 #define _EFER_SVME             12 /* Enable virtualization */
 #define _EFER_LMSLE            13 /* Long Mode Segment Limit Enable */
 #define _EFER_FFXSR            14 /* Enable Fast FXSAVE/FXRSTOR */
+#define _EFER_TCE              15 /* Enable Translation Cache Extensions */
 #define _EFER_AUTOIBRS         21 /* Enable Automatic IBRS */
 
 #define EFER_SCE               (1<<_EFER_SCE)
@@ -34,6 +35,7 @@
 #define EFER_SVME              (1<<_EFER_SVME)
 #define EFER_LMSLE             (1<<_EFER_LMSLE)
 #define EFER_FFXSR             (1<<_EFER_FFXSR)
+#define EFER_TCE               (1<<_EFER_TCE)
 #define EFER_AUTOIBRS          (1<<_EFER_AUTOIBRS)
 
 /*