[thirdparty/linux.git] / arch / x86 / mm / dump_pagetables.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Debug helper to dump the current kernel pagetables of the system
 * so that we can see what the various memory ranges are set to.
 *
 * (C) Copyright 2008 Intel Corporation
 *
 * Author: Arjan van de Ven <arjan@linux.intel.com>
 */

#include <linux/debugfs.h>
#include <linux/kasan.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/highmem.h>
#include <linux/pci.h>
#include <linux/ptdump.h>

#include <asm/e820/types.h>
#include <asm/pgtable.h>

/*
 * The dumper groups pagetable entries of the same type into one, and for
 * that it needs to keep some state when walking, and flush this state
 * when a "break" in the continuity is found.
 */
struct pg_state {
	struct ptdump_state ptdump;
	int level;
	pgprotval_t current_prot;
	pgprotval_t effective_prot;
	pgprotval_t prot_levels[5];
	unsigned long start_address;
	const struct addr_marker *marker;
	unsigned long lines;
	bool to_dmesg;
	bool check_wx;
	unsigned long wx_pages;
	struct seq_file *seq;
};

struct addr_marker {
	unsigned long start_address;
	const char *name;
	unsigned long max_lines;
};

/* Address space markers hints */

#ifdef CONFIG_X86_64

enum address_markers_idx {
	USER_SPACE_NR = 0,
	KERNEL_SPACE_NR,
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	LDT_NR,
#endif
	LOW_KERNEL_NR,
	VMALLOC_START_NR,
	VMEMMAP_START_NR,
#ifdef CONFIG_KASAN
	KASAN_SHADOW_START_NR,
	KASAN_SHADOW_END_NR,
#endif
	CPU_ENTRY_AREA_NR,
#ifdef CONFIG_X86_ESPFIX64
	ESPFIX_START_NR,
#endif
#ifdef CONFIG_EFI
	EFI_END_NR,
#endif
	HIGH_KERNEL_NR,
	MODULES_VADDR_NR,
	MODULES_END_NR,
	FIXADDR_START_NR,
	END_OF_SPACE_NR,
};

static struct addr_marker address_markers[] = {
	[USER_SPACE_NR]		= { 0,			"User Space" },
	[KERNEL_SPACE_NR]	= { (1UL << 63),	"Kernel Space" },
	[LOW_KERNEL_NR]		= { 0UL,		"Low Kernel Mapping" },
	[VMALLOC_START_NR]	= { 0UL,		"vmalloc() Area" },
	[VMEMMAP_START_NR]	= { 0UL,		"Vmemmap" },
#ifdef CONFIG_KASAN
	/*
	 * These fields get initialized with the (dynamic)
	 * KASAN_SHADOW_{START,END} values in pt_dump_init().
	 */
	[KASAN_SHADOW_START_NR]	= { 0UL,		"KASAN shadow" },
	[KASAN_SHADOW_END_NR]	= { 0UL,		"KASAN shadow end" },
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	[LDT_NR]		= { 0UL,		"LDT remap" },
#endif
	[CPU_ENTRY_AREA_NR]	= { CPU_ENTRY_AREA_BASE,"CPU entry Area" },
#ifdef CONFIG_X86_ESPFIX64
	[ESPFIX_START_NR]	= { ESPFIX_BASE_ADDR,	"ESPfix Area", 16 },
#endif
#ifdef CONFIG_EFI
	[EFI_END_NR]		= { EFI_VA_END,		"EFI Runtime Services" },
#endif
	[HIGH_KERNEL_NR]	= { __START_KERNEL_map,	"High Kernel Mapping" },
	[MODULES_VADDR_NR]	= { MODULES_VADDR,	"Modules" },
	[MODULES_END_NR]	= { MODULES_END,	"End Modules" },
	[FIXADDR_START_NR]	= { FIXADDR_START,	"Fixmap Area" },
	[END_OF_SPACE_NR]	= { -1,			NULL }
};

#define INIT_PGD	((pgd_t *) &init_top_pgt)

#else /* CONFIG_X86_64 */

enum address_markers_idx {
	USER_SPACE_NR = 0,
	KERNEL_SPACE_NR,
	VMALLOC_START_NR,
	VMALLOC_END_NR,
#ifdef CONFIG_HIGHMEM
	PKMAP_BASE_NR,
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	LDT_NR,
#endif
	CPU_ENTRY_AREA_NR,
	FIXADDR_START_NR,
	END_OF_SPACE_NR,
};

static struct addr_marker address_markers[] = {
	[USER_SPACE_NR]		= { 0,			"User Space" },
	[KERNEL_SPACE_NR]	= { PAGE_OFFSET,	"Kernel Mapping" },
	[VMALLOC_START_NR]	= { 0UL,		"vmalloc() Area" },
	[VMALLOC_END_NR]	= { 0UL,		"vmalloc() End" },
#ifdef CONFIG_HIGHMEM
	[PKMAP_BASE_NR]		= { 0UL,		"Persistent kmap() Area" },
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	[LDT_NR]		= { 0UL,		"LDT remap" },
#endif
	[CPU_ENTRY_AREA_NR]	= { 0UL,		"CPU entry area" },
	[FIXADDR_START_NR]	= { 0UL,		"Fixmap area" },
	[END_OF_SPACE_NR]	= { -1,			NULL }
};

#define INIT_PGD	(swapper_pg_dir)

#endif /* !CONFIG_X86_64 */

/* Multipliers for offsets within the PTEs */
#define PTE_LEVEL_MULT (PAGE_SIZE)
#define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT)
#define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT)
#define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT)
#define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT)

#define pt_dump_seq_printf(m, to_dmesg, fmt, args...)		\
({								\
	if (to_dmesg)					\
		printk(KERN_INFO fmt, ##args);			\
	else							\
		if (m)						\
			seq_printf(m, fmt, ##args);		\
})

#define pt_dump_cont_printf(m, to_dmesg, fmt, args...)		\
({								\
	if (to_dmesg)					\
		printk(KERN_CONT fmt, ##args);			\
	else							\
		if (m)						\
			seq_printf(m, fmt, ##args);		\
})

/*
 * Print a readable form of a pgprot_t to the seq_file
 */
static void printk_prot(struct seq_file *m, pgprotval_t pr, int level, bool dmsg)
{
	static const char * const level_name[] =
		{ "pgd", "p4d", "pud", "pmd", "pte" };

	if (!(pr & _PAGE_PRESENT)) {
		/* Not present */
		pt_dump_cont_printf(m, dmsg, "                              ");
	} else {
		if (pr & _PAGE_USER)
			pt_dump_cont_printf(m, dmsg, "USR ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_RW)
			pt_dump_cont_printf(m, dmsg, "RW ");
		else
			pt_dump_cont_printf(m, dmsg, "ro ");
		if (pr & _PAGE_PWT)
			pt_dump_cont_printf(m, dmsg, "PWT ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_PCD)
			pt_dump_cont_printf(m, dmsg, "PCD ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");

		/* Bit 7 has a different meaning on level 3 vs 4 */
		if (level <= 3 && pr & _PAGE_PSE)
			pt_dump_cont_printf(m, dmsg, "PSE ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if ((level == 4 && pr & _PAGE_PAT) ||
		    ((level == 3 || level == 2) && pr & _PAGE_PAT_LARGE))
			pt_dump_cont_printf(m, dmsg, "PAT ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_GLOBAL)
			pt_dump_cont_printf(m, dmsg, "GLB ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_NX)
			pt_dump_cont_printf(m, dmsg, "NX ");
		else
			pt_dump_cont_printf(m, dmsg, "x  ");
	}
	pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]);
}

static void note_wx(struct pg_state *st, unsigned long addr)
{
	unsigned long npages;

	npages = (addr - st->start_address) / PAGE_SIZE;

#ifdef CONFIG_PCI_BIOS
	/*
	 * If PCI BIOS is enabled, the PCI BIOS area is forced to WX.
	 * Inform about it, but avoid the warning.
	 */
	if (pcibios_enabled && st->start_address >= PAGE_OFFSET + BIOS_BEGIN &&
	    addr <= PAGE_OFFSET + BIOS_END) {
		pr_warn_once("x86/mm: PCI BIOS W+X mapping %lu pages\n", npages);
		return;
	}
#endif
	/* Account the WX pages */
	st->wx_pages += npages;
	WARN_ONCE(__supported_pte_mask & _PAGE_NX,
		  "x86/mm: Found insecure W+X mapping at address %pS\n",
		  (void *)st->start_address);
}

static void effective_prot(struct ptdump_state *pt_st, int level, u64 val)
{
	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	pgprotval_t prot = val & PTE_FLAGS_MASK;
	pgprotval_t effective;

	if (level > 0) {
		pgprotval_t higher_prot = st->prot_levels[level - 1];

		effective = (higher_prot & prot & (_PAGE_USER | _PAGE_RW)) |
			    ((higher_prot | prot) & _PAGE_NX);
	} else {
		effective = prot;
	}

	st->prot_levels[level] = effective;
}

/*
 * This function gets called on a break in a continuous series
 * of PTE entries; the next one is different so we need to
 * print what we collected so far.
 */
static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
		      unsigned long val)
{
	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	pgprotval_t new_prot, new_eff;
	pgprotval_t cur, eff;
	static const char units[] = "BKMGTPE";
	struct seq_file *m = st->seq;

	new_prot = val & PTE_FLAGS_MASK;
	if (!val)
		new_eff = 0;
	else
		new_eff = st->prot_levels[level];

	/*
	 * If we have a "break" in the series, we need to flush the state that
	 * we have now. "break" is either changing perms, levels or
	 * address space marker.
	 */
	cur = st->current_prot;
	eff = st->effective_prot;

	if (st->level == -1) {
		/* First entry */
		st->current_prot = new_prot;
		st->effective_prot = new_eff;
		st->level = level;
		st->marker = address_markers;
		st->lines = 0;
		pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
				   st->marker->name);
	} else if (new_prot != cur || new_eff != eff || level != st->level ||
		   addr >= st->marker[1].start_address) {
		const char *unit = units;
		unsigned long delta;
		int width = sizeof(unsigned long) * 2;

		if (st->check_wx && (eff & _PAGE_RW) && !(eff & _PAGE_NX))
			note_wx(st, addr);

		/*
		 * Now print the actual finished series
		 */
		if (!st->marker->max_lines ||
		    st->lines < st->marker->max_lines) {
			pt_dump_seq_printf(m, st->to_dmesg,
					   "0x%0*lx-0x%0*lx   ",
					   width, st->start_address,
					   width, addr);

			delta = addr - st->start_address;
			while (!(delta & 1023) && unit[1]) {
				delta >>= 10;
				unit++;
			}
			pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ",
					    delta, *unit);
			printk_prot(m, st->current_prot, st->level,
				    st->to_dmesg);
		}
		st->lines++;

		/*
		 * We print markers for special areas of address space,
		 * such as the start of vmalloc space etc.
		 * This helps in the interpretation.
		 */
		if (addr >= st->marker[1].start_address) {
			if (st->marker->max_lines &&
			    st->lines > st->marker->max_lines) {
				unsigned long nskip =
					st->lines - st->marker->max_lines;
				pt_dump_seq_printf(m, st->to_dmesg,
						   "... %lu entr%s skipped ... \n",
						   nskip,
						   nskip == 1 ? "y" : "ies");
			}
			st->marker++;
			st->lines = 0;
			pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
					   st->marker->name);
		}

		st->start_address = addr;
		st->current_prot = new_prot;
		st->effective_prot = new_eff;
		st->level = level;
	}
}

static void ptdump_walk_pgd_level_core(struct seq_file *m,
				       struct mm_struct *mm, pgd_t *pgd,
				       bool checkwx, bool dmesg)
{
	const struct ptdump_range ptdump_ranges[] = {
#ifdef CONFIG_X86_64
	{0, PTRS_PER_PGD * PGD_LEVEL_MULT / 2},
	{GUARD_HOLE_END_ADDR, ~0UL},
#else
	{0, ~0UL},
#endif
	{0, 0}
};

	struct pg_state st = {
		.ptdump = {
			.note_page	= note_page,
			.effective_prot = effective_prot,
			.range		= ptdump_ranges
		},
		.level = -1,
		.to_dmesg	= dmesg,
		.check_wx	= checkwx,
		.seq		= m
	};

	ptdump_walk_pgd(&st.ptdump, mm, pgd);

	if (!checkwx)
		return;
	if (st.wx_pages)
		pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n",
			st.wx_pages);
	else
		pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n");
}

void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm)
{
	ptdump_walk_pgd_level_core(m, mm, mm->pgd, false, true);
}

void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
				   bool user)
{
	pgd_t *pgd = mm->pgd;
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	if (user && boot_cpu_has(X86_FEATURE_PTI))
		pgd = kernel_to_user_pgdp(pgd);
#endif
	ptdump_walk_pgd_level_core(m, mm, pgd, false, false);
}
EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs);

void ptdump_walk_user_pgd_level_checkwx(void)
{
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	pgd_t *pgd = INIT_PGD;

	if (!(__supported_pte_mask & _PAGE_NX) ||
	    !boot_cpu_has(X86_FEATURE_PTI))
		return;

	pr_info("x86/mm: Checking user space page tables\n");
	pgd = kernel_to_user_pgdp(pgd);
	ptdump_walk_pgd_level_core(NULL, &init_mm, pgd, true, false);
#endif
}

void ptdump_walk_pgd_level_checkwx(void)
{
	ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false);
}

static int __init pt_dump_init(void)
{
	/*
	 * Various markers are not compile-time constants, so assign them
	 * here.
	 */
#ifdef CONFIG_X86_64
	address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET;
	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START;
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
#endif
#ifdef CONFIG_KASAN
	address_markers[KASAN_SHADOW_START_NR].start_address = KASAN_SHADOW_START;
	address_markers[KASAN_SHADOW_END_NR].start_address = KASAN_SHADOW_END;
#endif
#endif
#ifdef CONFIG_X86_32
	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
# ifdef CONFIG_HIGHMEM
	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
# endif
	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
	address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE;
# ifdef CONFIG_MODIFY_LDT_SYSCALL
	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
# endif
#endif
	return 0;
}
__initcall(pt_dump_init);
Commit	Line	Data
b886d83c	1	// SPDX-License-Identifier: GPL-2.0-only
926e5392 AV	2	/*
	3	* Debug helper to dump the current kernel pagetables of the system
	4	* so that we can see what the various memory ranges are set to.
	5	*
	6	* (C) Copyright 2008 Intel Corporation
	7	*
	8	* Author: Arjan van de Ven <arjan@linux.intel.com>
926e5392 AV	9	*/
926e5392 AV	10
fe770bf0	11	#include <linux/debugfs.h>
04b67022	12	#include <linux/kasan.h>
fe770bf0	13	#include <linux/mm.h>
84e629b6	14	#include <linux/init.h>
146fbb76	15	#include <linux/sched.h>
926e5392	16	#include <linux/seq_file.h>
d6ef1f19	17	#include <linux/highmem.h>
c200dac7	18	#include <linux/pci.h>
2ae27137	19	#include <linux/ptdump.h>
926e5392	20
c200dac7	21	#include <asm/e820/types.h>
926e5392 AV	22	#include <asm/pgtable.h>
	23
	24	/*
	25	* The dumper groups pagetable entries of the same type into one, and for
	26	* that it needs to keep some state when walking, and flush this state
	27	* when a "break" in the continuity is found.
	28	*/
	29	struct pg_state {
2ae27137	30	struct ptdump_state ptdump;
926e5392	31	int level;
2ae27137	32	pgprotval_t current_prot;
672c0ae0	33	pgprotval_t effective_prot;
2ae27137	34	pgprotval_t prot_levels[5];
926e5392	35	unsigned long start_address;
fe770bf0	36	const struct addr_marker *marker;
3891a04a	37	unsigned long lines;
ef6bea6d	38	bool to_dmesg;
e1a58320 SS	39	bool check_wx;
e1a58320 SS	40	unsigned long wx_pages;
74d2aaa1	41	struct seq_file *seq;
926e5392 AV	42	};
926e5392 AV	43
fe770bf0 PA	44	struct addr_marker {
	45	unsigned long start_address;
	46	const char *name;
3891a04a	47	unsigned long max_lines;
fe770bf0 PA	48	};
fe770bf0 PA	49
146122e2 TG	50	/* Address space markers hints */
	51
	52	#ifdef CONFIG_X86_64
	53
92851e2f AS	54	enum address_markers_idx {
92851e2f AS	55	USER_SPACE_NR = 0,
92851e2f	56	KERNEL_SPACE_NR,
254eb550	57	#ifdef CONFIG_MODIFY_LDT_SYSCALL
f55f0501 AL	58	LDT_NR,
f55f0501 AL	59	#endif
254eb550	60	LOW_KERNEL_NR,
92851e2f AS	61	VMALLOC_START_NR,
92851e2f AS	62	VMEMMAP_START_NR,
025205f8 AR	63	#ifdef CONFIG_KASAN
	64	KASAN_SHADOW_START_NR,
	65	KASAN_SHADOW_END_NR,
f55f0501	66	#endif
f2078904	67	CPU_ENTRY_AREA_NR,
146122e2	68	#ifdef CONFIG_X86_ESPFIX64
3891a04a	69	ESPFIX_START_NR,
146122e2 TG	70	#endif
	71	#ifdef CONFIG_EFI
	72	EFI_END_NR,
	73	#endif
92851e2f AS	74	HIGH_KERNEL_NR,
	75	MODULES_VADDR_NR,
	76	MODULES_END_NR,
146122e2 TG	77	FIXADDR_START_NR,
	78	END_OF_SPACE_NR,
	79	};
	80
	81	static struct addr_marker address_markers[] = {
	82	[USER_SPACE_NR] = { 0, "User Space" },
	83	[KERNEL_SPACE_NR] = { (1UL << 63), "Kernel Space" },
	84	[LOW_KERNEL_NR] = { 0UL, "Low Kernel Mapping" },
	85	[VMALLOC_START_NR] = { 0UL, "vmalloc() Area" },
	86	[VMEMMAP_START_NR] = { 0UL, "Vmemmap" },
	87	#ifdef CONFIG_KASAN
09e61a77 KS	88	/*
	89	* These fields get initialized with the (dynamic)
	90	* KASAN_SHADOW_{START,END} values in pt_dump_init().
	91	*/
	92	[KASAN_SHADOW_START_NR] = { 0UL, "KASAN shadow" },
	93	[KASAN_SHADOW_END_NR] = { 0UL, "KASAN shadow end" },
f55f0501 AL	94	#endif
f55f0501 AL	95	#ifdef CONFIG_MODIFY_LDT_SYSCALL
5c7919bb	96	[LDT_NR] = { 0UL, "LDT remap" },
146122e2	97	#endif
92a0f81d	98	[CPU_ENTRY_AREA_NR] = { CPU_ENTRY_AREA_BASE,"CPU entry Area" },
146122e2 TG	99	#ifdef CONFIG_X86_ESPFIX64
	100	[ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 },
	101	#endif
	102	#ifdef CONFIG_EFI
	103	[EFI_END_NR] = { EFI_VA_END, "EFI Runtime Services" },
	104	#endif
	105	[HIGH_KERNEL_NR] = { __START_KERNEL_map, "High Kernel Mapping" },
	106	[MODULES_VADDR_NR] = { MODULES_VADDR, "Modules" },
	107	[MODULES_END_NR] = { MODULES_END, "End Modules" },
	108	[FIXADDR_START_NR] = { FIXADDR_START, "Fixmap Area" },
	109	[END_OF_SPACE_NR] = { -1, NULL }
	110	};
	111
4e8537e4 JR	112	#define INIT_PGD ((pgd_t *) &init_top_pgt)
4e8537e4 JR	113
146122e2 TG	114	#else /* CONFIG_X86_64 */
	115
	116	enum address_markers_idx {
	117	USER_SPACE_NR = 0,
92851e2f AS	118	KERNEL_SPACE_NR,
	119	VMALLOC_START_NR,
	120	VMALLOC_END_NR,
146122e2	121	#ifdef CONFIG_HIGHMEM
92851e2f	122	PKMAP_BASE_NR,
f3e48e54 JR	123	#endif
	124	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	125	LDT_NR,
92851e2f	126	#endif
92a0f81d	127	CPU_ENTRY_AREA_NR,
146122e2 TG	128	FIXADDR_START_NR,
146122e2 TG	129	END_OF_SPACE_NR,
92851e2f AS	130	};
92851e2f AS	131
fe770bf0	132	static struct addr_marker address_markers[] = {
146122e2 TG	133	[USER_SPACE_NR] = { 0, "User Space" },
	134	[KERNEL_SPACE_NR] = { PAGE_OFFSET, "Kernel Mapping" },
	135	[VMALLOC_START_NR] = { 0UL, "vmalloc() Area" },
	136	[VMALLOC_END_NR] = { 0UL, "vmalloc() End" },
	137	#ifdef CONFIG_HIGHMEM
	138	[PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" },
f3e48e54 JR	139	#endif
	140	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	141	[LDT_NR] = { 0UL, "LDT remap" },
fe770bf0	142	#endif
92a0f81d	143	[CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" },
146122e2 TG	144	[FIXADDR_START_NR] = { 0UL, "Fixmap area" },
146122e2 TG	145	[END_OF_SPACE_NR] = { -1, NULL }
fe770bf0	146	};
926e5392	147
4e8537e4 JR	148	#define INIT_PGD (swapper_pg_dir)
4e8537e4 JR	149
146122e2 TG	150	#endif /* !CONFIG_X86_64 */
146122e2 TG	151
fe770bf0 PA	152	/* Multipliers for offsets within the PTEs */
	153	#define PTE_LEVEL_MULT (PAGE_SIZE)
	154	#define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT)
	155	#define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT)
fdd3d8ce	156	#define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT)
84bbabc3	157	#define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT)
926e5392	158
ef6bea6d BP	159	#define pt_dump_seq_printf(m, to_dmesg, fmt, args...) \
	160	({ \
	161	if (to_dmesg) \
	162	printk(KERN_INFO fmt, ##args); \
	163	else \
	164	if (m) \
	165	seq_printf(m, fmt, ##args); \
	166	})
	167
	168	#define pt_dump_cont_printf(m, to_dmesg, fmt, args...) \
	169	({ \
	170	if (to_dmesg) \
	171	printk(KERN_CONT fmt, ##args); \
	172	else \
	173	if (m) \
	174	seq_printf(m, fmt, ##args); \
	175	})
	176
926e5392 AV	177	/*
	178	* Print a readable form of a pgprot_t to the seq_file
	179	*/
2ae27137	180	static void printk_prot(struct seq_file *m, pgprotval_t pr, int level, bool dmsg)
926e5392	181	{
fe770bf0	182	static const char * const level_name[] =
f8f0d0b6	183	{ "pgd", "p4d", "pud", "pmd", "pte" };
fe770bf0	184
c0534494	185	if (!(pr & _PAGE_PRESENT)) {
fe770bf0	186	/* Not present */
f439c429	187	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0 PA	188	} else {
fe770bf0 PA	189	if (pr & _PAGE_USER)
ef6bea6d	190	pt_dump_cont_printf(m, dmsg, "USR ");
926e5392	191	else
ef6bea6d	192	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	193	if (pr & _PAGE_RW)
ef6bea6d	194	pt_dump_cont_printf(m, dmsg, "RW ");
fe770bf0	195	else
ef6bea6d	196	pt_dump_cont_printf(m, dmsg, "ro ");
fe770bf0	197	if (pr & _PAGE_PWT)
ef6bea6d	198	pt_dump_cont_printf(m, dmsg, "PWT ");
fe770bf0	199	else
ef6bea6d	200	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	201	if (pr & _PAGE_PCD)
ef6bea6d	202	pt_dump_cont_printf(m, dmsg, "PCD ");
926e5392	203	else
ef6bea6d	204	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	205
f439c429	206	/* Bit 7 has a different meaning on level 3 vs 4 */
f8f0d0b6	207	if (level <= 3 && pr & _PAGE_PSE)
f439c429 JG	208	pt_dump_cont_printf(m, dmsg, "PSE ");
	209	else
	210	pt_dump_cont_printf(m, dmsg, " ");
f8f0d0b6 SP	211	if ((level == 4 && pr & _PAGE_PAT) \|\|
f8f0d0b6 SP	212	((level == 3 \|\| level == 2) && pr & _PAGE_PAT_LARGE))
da25e628	213	pt_dump_cont_printf(m, dmsg, "PAT ");
f439c429 JG	214	else
f439c429 JG	215	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	216	if (pr & _PAGE_GLOBAL)
ef6bea6d	217	pt_dump_cont_printf(m, dmsg, "GLB ");
fe770bf0	218	else
ef6bea6d	219	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	220	if (pr & _PAGE_NX)
ef6bea6d	221	pt_dump_cont_printf(m, dmsg, "NX ");
fe770bf0	222	else
ef6bea6d	223	pt_dump_cont_printf(m, dmsg, "x ");
926e5392	224	}
ef6bea6d	225	pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]);
926e5392 AV	226	}
926e5392 AV	227
2ae27137	228	static void note_wx(struct pg_state *st, unsigned long addr)
c200dac7 TG	229	{
	230	unsigned long npages;
	231
2ae27137	232	npages = (addr - st->start_address) / PAGE_SIZE;
c200dac7 TG	233
	234	#ifdef CONFIG_PCI_BIOS
	235	/*
	236	* If PCI BIOS is enabled, the PCI BIOS area is forced to WX.
	237	* Inform about it, but avoid the warning.
	238	*/
	239	if (pcibios_enabled && st->start_address >= PAGE_OFFSET + BIOS_BEGIN &&
2ae27137	240	addr <= PAGE_OFFSET + BIOS_END) {
c200dac7 TG	241	pr_warn_once("x86/mm: PCI BIOS W+X mapping %lu pages\n", npages);
	242	return;
	243	}
	244	#endif
	245	/* Account the WX pages */
	246	st->wx_pages += npages;
510bb96f TG	247	WARN_ONCE(__supported_pte_mask & _PAGE_NX,
510bb96f TG	248	"x86/mm: Found insecure W+X mapping at address %pS\n",
c200dac7 TG	249	(void *)st->start_address);
	250	}
	251
1494e0c3	252	static void effective_prot(struct ptdump_state *pt_st, int level, u64 val)
2ae27137	253	{
1494e0c3 SP	254	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	255	pgprotval_t prot = val & PTE_FLAGS_MASK;
	256	pgprotval_t effective;
	257
	258	if (level > 0) {
	259	pgprotval_t higher_prot = st->prot_levels[level - 1];
	260
	261	effective = (higher_prot & prot & (_PAGE_USER \| _PAGE_RW)) \|
	262	((higher_prot \| prot) & _PAGE_NX);
	263	} else {
	264	effective = prot;
	265	}
	266
	267	st->prot_levels[level] = effective;
2ae27137 SP	268	}
2ae27137 SP	269
926e5392 AV	270	/*
	271	* This function gets called on a break in a continuous series
	272	* of PTE entries; the next one is different so we need to
	273	* print what we collected so far.
	274	*/
2ae27137 SP	275	static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
2ae27137 SP	276	unsigned long val)
926e5392	277	{
2ae27137 SP	278	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	279	pgprotval_t new_prot, new_eff;
	280	pgprotval_t cur, eff;
3891a04a	281	static const char units[] = "BKMGTPE";
74d2aaa1	282	struct seq_file *m = st->seq;
926e5392	283
2ae27137	284	new_prot = val & PTE_FLAGS_MASK;
1494e0c3 SP	285	if (!val)
	286	new_eff = 0;
	287	else
	288	new_eff = st->prot_levels[level];
2ae27137	289
926e5392 AV	290	/*
926e5392 AV	291	* If we have a "break" in the series, we need to flush the state that
fe770bf0 PA	292	* we have now. "break" is either changing perms, levels or
fe770bf0 PA	293	* address space marker.
926e5392	294	*/
2ae27137	295	cur = st->current_prot;
672c0ae0	296	eff = st->effective_prot;
926e5392	297
f8f0d0b6	298	if (st->level == -1) {
fe770bf0 PA	299	/* First entry */
fe770bf0 PA	300	st->current_prot = new_prot;
672c0ae0	301	st->effective_prot = new_eff;
fe770bf0 PA	302	st->level = level;
fe770bf0 PA	303	st->marker = address_markers;
3891a04a	304	st->lines = 0;
ef6bea6d BP	305	pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
ef6bea6d BP	306	st->marker->name);
2ae27137 SP	307	} else if (new_prot != cur \|\| new_eff != eff \|\| level != st->level \|\|
2ae27137 SP	308	addr >= st->marker[1].start_address) {
fe770bf0	309	const char *unit = units;
926e5392	310	unsigned long delta;
6424fb38	311	int width = sizeof(unsigned long) * 2;
e1a58320	312
c200dac7	313	if (st->check_wx && (eff & _PAGE_RW) && !(eff & _PAGE_NX))
2ae27137	314	note_wx(st, addr);
926e5392	315
926e5392 AV	316	/*
	317	* Now print the actual finished series
	318	*/
3891a04a PA	319	if (!st->marker->max_lines \|\|
	320	st->lines < st->marker->max_lines) {
	321	pt_dump_seq_printf(m, st->to_dmesg,
	322	"0x%0lx-0x%0lx ",
	323	width, st->start_address,
2ae27137	324	width, addr);
926e5392	325
2ae27137	326	delta = addr - st->start_address;
3891a04a PA	327	while (!(delta & 1023) && unit[1]) {
	328	delta >>= 10;
	329	unit++;
	330	}
	331	pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ",
	332	delta, *unit);
	333	printk_prot(m, st->current_prot, st->level,
	334	st->to_dmesg);
926e5392	335	}
3891a04a	336	st->lines++;
fe770bf0 PA	337
	338	/*
	339	* We print markers for special areas of address space,
	340	* such as the start of vmalloc space etc.
	341	* This helps in the interpretation.
	342	*/
2ae27137	343	if (addr >= st->marker[1].start_address) {
3891a04a PA	344	if (st->marker->max_lines &&
	345	st->lines > st->marker->max_lines) {
	346	unsigned long nskip =
	347	st->lines - st->marker->max_lines;
	348	pt_dump_seq_printf(m, st->to_dmesg,
	349	"... %lu entr%s skipped ... \n",
	350	nskip,
	351	nskip == 1 ? "y" : "ies");
	352	}
fe770bf0	353	st->marker++;
3891a04a	354	st->lines = 0;
ef6bea6d BP	355	pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
ef6bea6d BP	356	st->marker->name);
926e5392	357	}
fe770bf0	358
2ae27137	359	st->start_address = addr;
926e5392	360	st->current_prot = new_prot;
672c0ae0	361	st->effective_prot = new_eff;
926e5392	362	st->level = level;
fe770bf0	363	}
926e5392 AV	364	}
926e5392 AV	365
e47690d7 SP	366	static void ptdump_walk_pgd_level_core(struct seq_file *m,
e47690d7 SP	367	struct mm_struct mm, pgd_t pgd,
2ae27137	368	bool checkwx, bool dmesg)
926e5392	369	{
2ae27137 SP	370	const struct ptdump_range ptdump_ranges[] = {
2ae27137 SP	371	#ifdef CONFIG_X86_64
2ae27137	372	{0, PTRS_PER_PGD * PGD_LEVEL_MULT / 2},
bba42aff	373	{GUARD_HOLE_END_ADDR, ~0UL},
fe770bf0	374	#else
2ae27137	375	{0, ~0UL},
fdd3d8ce	376	#endif
2ae27137 SP	377	{0, 0}
2ae27137 SP	378	};
fdd3d8ce	379
2ae27137 SP	380	struct pg_state st = {
	381	.ptdump = {
	382	.note_page = note_page,
1494e0c3	383	.effective_prot = effective_prot,
2ae27137 SP	384	.range = ptdump_ranges
2ae27137 SP	385	},
f8f0d0b6	386	.level = -1,
2ae27137 SP	387	.to_dmesg = dmesg,
	388	.check_wx = checkwx,
	389	.seq = m
	390	};
fdd3d8ce	391
e47690d7	392	ptdump_walk_pgd(&st.ptdump, mm, pgd);
fe770bf0	393
e1a58320 SS	394	if (!checkwx)
	395	return;
	396	if (st.wx_pages)
	397	pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n",
	398	st.wx_pages);
	399	else
	400	pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n");
	401	}
	402
e455248d	403	void ptdump_walk_pgd_level(struct seq_file m, struct mm_struct mm)
e1a58320	404	{
e47690d7	405	ptdump_walk_pgd_level_core(m, mm, mm->pgd, false, true);
b4bf4f92 TG	406	}
b4bf4f92 TG	407
c5cfae12 SP	408	void ptdump_walk_pgd_level_debugfs(struct seq_file m, struct mm_struct mm,
c5cfae12 SP	409	bool user)
b4bf4f92	410	{
c5cfae12	411	pgd_t *pgd = mm->pgd;
a4b51ef6	412	#ifdef CONFIG_PAGE_TABLE_ISOLATION
28e3ace7	413	if (user && boot_cpu_has(X86_FEATURE_PTI))
a4b51ef6 TG	414	pgd = kernel_to_user_pgdp(pgd);
a4b51ef6 TG	415	#endif
e47690d7	416	ptdump_walk_pgd_level_core(m, mm, pgd, false, false);
b4bf4f92 TG	417	}
	418	EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs);
	419
d878efce	420	void ptdump_walk_user_pgd_level_checkwx(void)
b4bf4f92 TG	421	{
b4bf4f92 TG	422	#ifdef CONFIG_PAGE_TABLE_ISOLATION
4e8537e4	423	pgd_t *pgd = INIT_PGD;
b4bf4f92	424
d878efce	425	if (!(__supported_pte_mask & _PAGE_NX) \|\|
28e3ace7	426	!boot_cpu_has(X86_FEATURE_PTI))
b4bf4f92 TG	427	return;
	428
	429	pr_info("x86/mm: Checking user space page tables\n");
	430	pgd = kernel_to_user_pgdp(pgd);
e47690d7	431	ptdump_walk_pgd_level_core(NULL, &init_mm, pgd, true, false);
b4bf4f92	432	#endif
926e5392 AV	433	}
926e5392 AV	434
e1a58320 SS	435	void ptdump_walk_pgd_level_checkwx(void)
e1a58320 SS	436	{
e47690d7	437	ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false);
e1a58320 SS	438	}
e1a58320 SS	439
8609d1b5	440	static int __init pt_dump_init(void)
926e5392	441	{
0483e1fa TG	442	/*
	443	* Various markers are not compile-time constants, so assign them
	444	* here.
	445	*/
	446	#ifdef CONFIG_X86_64
	447	address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET;
	448	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	449	address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START;
5c7919bb KS	450	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	451	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
	452	#endif
09e61a77 KS	453	#ifdef CONFIG_KASAN
	454	address_markers[KASAN_SHADOW_START_NR].start_address = KASAN_SHADOW_START;
	455	address_markers[KASAN_SHADOW_END_NR].start_address = KASAN_SHADOW_END;
	456	#endif
0483e1fa	457	#endif
fe770bf0	458	#ifdef CONFIG_X86_32
92851e2f AS	459	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
92851e2f AS	460	address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
fe770bf0	461	# ifdef CONFIG_HIGHMEM
92851e2f	462	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
fe770bf0	463	# endif
92851e2f	464	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
92a0f81d	465	address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE;
f3e48e54 JR	466	# ifdef CONFIG_MODIFY_LDT_SYSCALL
	467	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
	468	# endif
fe770bf0	469	#endif
926e5392 AV	470	return 0;
926e5392 AV	471	}
926e5392	472	__initcall(pt_dump_init);