[thirdparty/linux.git] / arch / x86 / mm / dump_pagetables.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Debug helper to dump the current kernel pagetables of the system
 * so that we can see what the various memory ranges are set to.
 *
 * (C) Copyright 2008 Intel Corporation
 *
 * Author: Arjan van de Ven <arjan@linux.intel.com>
 */

#include <linux/debugfs.h>
#include <linux/kasan.h>
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/sched.h>
#include <linux/seq_file.h>
#include <linux/highmem.h>
#include <linux/pci.h>
#include <linux/ptdump.h>

#include <asm/e820/types.h>

/*
 * The dumper groups pagetable entries of the same type into one, and for
 * that it needs to keep some state when walking, and flush this state
 * when a "break" in the continuity is found.
 */
struct pg_state {
	struct ptdump_state ptdump;
	int level;
	pgprotval_t current_prot;
	pgprotval_t effective_prot;
	pgprotval_t prot_levels[5];
	unsigned long start_address;
	const struct addr_marker *marker;
	unsigned long lines;
	bool to_dmesg;
	bool check_wx;
	unsigned long wx_pages;
	struct seq_file *seq;
};

struct addr_marker {
	unsigned long start_address;
	const char *name;
	unsigned long max_lines;
};

/* Address space markers hints */

#ifdef CONFIG_X86_64

enum address_markers_idx {
	USER_SPACE_NR = 0,
	KERNEL_SPACE_NR,
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	LDT_NR,
#endif
	LOW_KERNEL_NR,
	VMALLOC_START_NR,
	VMEMMAP_START_NR,
#ifdef CONFIG_KASAN
	KASAN_SHADOW_START_NR,
	KASAN_SHADOW_END_NR,
#endif
	CPU_ENTRY_AREA_NR,
#ifdef CONFIG_X86_ESPFIX64
	ESPFIX_START_NR,
#endif
#ifdef CONFIG_EFI
	EFI_END_NR,
#endif
	HIGH_KERNEL_NR,
	MODULES_VADDR_NR,
	MODULES_END_NR,
	FIXADDR_START_NR,
	END_OF_SPACE_NR,
};

static struct addr_marker address_markers[] = {
	[USER_SPACE_NR]		= { 0,			"User Space" },
	[KERNEL_SPACE_NR]	= { (1UL << 63),	"Kernel Space" },
	[LOW_KERNEL_NR]		= { 0UL,		"Low Kernel Mapping" },
	[VMALLOC_START_NR]	= { 0UL,		"vmalloc() Area" },
	[VMEMMAP_START_NR]	= { 0UL,		"Vmemmap" },
#ifdef CONFIG_KASAN
	/*
	 * These fields get initialized with the (dynamic)
	 * KASAN_SHADOW_{START,END} values in pt_dump_init().
	 */
	[KASAN_SHADOW_START_NR]	= { 0UL,		"KASAN shadow" },
	[KASAN_SHADOW_END_NR]	= { 0UL,		"KASAN shadow end" },
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	[LDT_NR]		= { 0UL,		"LDT remap" },
#endif
	[CPU_ENTRY_AREA_NR]	= { CPU_ENTRY_AREA_BASE,"CPU entry Area" },
#ifdef CONFIG_X86_ESPFIX64
	[ESPFIX_START_NR]	= { ESPFIX_BASE_ADDR,	"ESPfix Area", 16 },
#endif
#ifdef CONFIG_EFI
	[EFI_END_NR]		= { EFI_VA_END,		"EFI Runtime Services" },
#endif
	[HIGH_KERNEL_NR]	= { __START_KERNEL_map,	"High Kernel Mapping" },
	[MODULES_VADDR_NR]	= { MODULES_VADDR,	"Modules" },
	[MODULES_END_NR]	= { MODULES_END,	"End Modules" },
	[FIXADDR_START_NR]	= { FIXADDR_START,	"Fixmap Area" },
	[END_OF_SPACE_NR]	= { -1,			NULL }
};

#define INIT_PGD	((pgd_t *) &init_top_pgt)

#else /* CONFIG_X86_64 */

enum address_markers_idx {
	USER_SPACE_NR = 0,
	KERNEL_SPACE_NR,
	VMALLOC_START_NR,
	VMALLOC_END_NR,
#ifdef CONFIG_HIGHMEM
	PKMAP_BASE_NR,
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	LDT_NR,
#endif
	CPU_ENTRY_AREA_NR,
	FIXADDR_START_NR,
	END_OF_SPACE_NR,
};

static struct addr_marker address_markers[] = {
	[USER_SPACE_NR]		= { 0,			"User Space" },
	[KERNEL_SPACE_NR]	= { PAGE_OFFSET,	"Kernel Mapping" },
	[VMALLOC_START_NR]	= { 0UL,		"vmalloc() Area" },
	[VMALLOC_END_NR]	= { 0UL,		"vmalloc() End" },
#ifdef CONFIG_HIGHMEM
	[PKMAP_BASE_NR]		= { 0UL,		"Persistent kmap() Area" },
#endif
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	[LDT_NR]		= { 0UL,		"LDT remap" },
#endif
	[CPU_ENTRY_AREA_NR]	= { 0UL,		"CPU entry area" },
	[FIXADDR_START_NR]	= { 0UL,		"Fixmap area" },
	[END_OF_SPACE_NR]	= { -1,			NULL }
};

#define INIT_PGD	(swapper_pg_dir)

#endif /* !CONFIG_X86_64 */

/* Multipliers for offsets within the PTEs */
#define PTE_LEVEL_MULT (PAGE_SIZE)
#define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT)
#define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT)
#define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT)
#define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT)

#define pt_dump_seq_printf(m, to_dmesg, fmt, args...)		\
({								\
	if (to_dmesg)					\
		printk(KERN_INFO fmt, ##args);			\
	else							\
		if (m)						\
			seq_printf(m, fmt, ##args);		\
})

#define pt_dump_cont_printf(m, to_dmesg, fmt, args...)		\
({								\
	if (to_dmesg)					\
		printk(KERN_CONT fmt, ##args);			\
	else							\
		if (m)						\
			seq_printf(m, fmt, ##args);		\
})

/*
 * Print a readable form of a pgprot_t to the seq_file
 */
static void printk_prot(struct seq_file *m, pgprotval_t pr, int level, bool dmsg)
{
	static const char * const level_name[] =
		{ "pgd", "p4d", "pud", "pmd", "pte" };

	if (!(pr & _PAGE_PRESENT)) {
		/* Not present */
		pt_dump_cont_printf(m, dmsg, "                              ");
	} else {
		if (pr & _PAGE_USER)
			pt_dump_cont_printf(m, dmsg, "USR ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_RW)
			pt_dump_cont_printf(m, dmsg, "RW ");
		else
			pt_dump_cont_printf(m, dmsg, "ro ");
		if (pr & _PAGE_PWT)
			pt_dump_cont_printf(m, dmsg, "PWT ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_PCD)
			pt_dump_cont_printf(m, dmsg, "PCD ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");

		/* Bit 7 has a different meaning on level 3 vs 4 */
		if (level <= 3 && pr & _PAGE_PSE)
			pt_dump_cont_printf(m, dmsg, "PSE ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if ((level == 4 && pr & _PAGE_PAT) ||
		    ((level == 3 || level == 2) && pr & _PAGE_PAT_LARGE))
			pt_dump_cont_printf(m, dmsg, "PAT ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_GLOBAL)
			pt_dump_cont_printf(m, dmsg, "GLB ");
		else
			pt_dump_cont_printf(m, dmsg, "    ");
		if (pr & _PAGE_NX)
			pt_dump_cont_printf(m, dmsg, "NX ");
		else
			pt_dump_cont_printf(m, dmsg, "x  ");
	}
	pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]);
}

static void note_wx(struct pg_state *st, unsigned long addr)
{
	unsigned long npages;

	npages = (addr - st->start_address) / PAGE_SIZE;

#ifdef CONFIG_PCI_BIOS
	/*
	 * If PCI BIOS is enabled, the PCI BIOS area is forced to WX.
	 * Inform about it, but avoid the warning.
	 */
	if (pcibios_enabled && st->start_address >= PAGE_OFFSET + BIOS_BEGIN &&
	    addr <= PAGE_OFFSET + BIOS_END) {
		pr_warn_once("x86/mm: PCI BIOS W+X mapping %lu pages\n", npages);
		return;
	}
#endif
	/* Account the WX pages */
	st->wx_pages += npages;
	WARN_ONCE(__supported_pte_mask & _PAGE_NX,
		  "x86/mm: Found insecure W+X mapping at address %pS\n",
		  (void *)st->start_address);
}

static void effective_prot(struct ptdump_state *pt_st, int level, u64 val)
{
	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	pgprotval_t prot = val & PTE_FLAGS_MASK;
	pgprotval_t effective;

	if (level > 0) {
		pgprotval_t higher_prot = st->prot_levels[level - 1];

		effective = (higher_prot & prot & (_PAGE_USER | _PAGE_RW)) |
			    ((higher_prot | prot) & _PAGE_NX);
	} else {
		effective = prot;
	}

	st->prot_levels[level] = effective;
}

/*
 * This function gets called on a break in a continuous series
 * of PTE entries; the next one is different so we need to
 * print what we collected so far.
 */
static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
		      u64 val)
{
	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	pgprotval_t new_prot, new_eff;
	pgprotval_t cur, eff;
	static const char units[] = "BKMGTPE";
	struct seq_file *m = st->seq;

	new_prot = val & PTE_FLAGS_MASK;
	if (!val)
		new_eff = 0;
	else
		new_eff = st->prot_levels[level];

	/*
	 * If we have a "break" in the series, we need to flush the state that
	 * we have now. "break" is either changing perms, levels or
	 * address space marker.
	 */
	cur = st->current_prot;
	eff = st->effective_prot;

	if (st->level == -1) {
		/* First entry */
		st->current_prot = new_prot;
		st->effective_prot = new_eff;
		st->level = level;
		st->marker = address_markers;
		st->lines = 0;
		pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
				   st->marker->name);
	} else if (new_prot != cur || new_eff != eff || level != st->level ||
		   addr >= st->marker[1].start_address) {
		const char *unit = units;
		unsigned long delta;
		int width = sizeof(unsigned long) * 2;

		if (st->check_wx && (eff & _PAGE_RW) && !(eff & _PAGE_NX))
			note_wx(st, addr);

		/*
		 * Now print the actual finished series
		 */
		if (!st->marker->max_lines ||
		    st->lines < st->marker->max_lines) {
			pt_dump_seq_printf(m, st->to_dmesg,
					   "0x%0*lx-0x%0*lx   ",
					   width, st->start_address,
					   width, addr);

			delta = addr - st->start_address;
			while (!(delta & 1023) && unit[1]) {
				delta >>= 10;
				unit++;
			}
			pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ",
					    delta, *unit);
			printk_prot(m, st->current_prot, st->level,
				    st->to_dmesg);
		}
		st->lines++;

		/*
		 * We print markers for special areas of address space,
		 * such as the start of vmalloc space etc.
		 * This helps in the interpretation.
		 */
		if (addr >= st->marker[1].start_address) {
			if (st->marker->max_lines &&
			    st->lines > st->marker->max_lines) {
				unsigned long nskip =
					st->lines - st->marker->max_lines;
				pt_dump_seq_printf(m, st->to_dmesg,
						   "... %lu entr%s skipped ... \n",
						   nskip,
						   nskip == 1 ? "y" : "ies");
			}
			st->marker++;
			st->lines = 0;
			pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
					   st->marker->name);
		}

		st->start_address = addr;
		st->current_prot = new_prot;
		st->effective_prot = new_eff;
		st->level = level;
	}
}

bool ptdump_walk_pgd_level_core(struct seq_file *m,
				struct mm_struct *mm, pgd_t *pgd,
				bool checkwx, bool dmesg)
{
	const struct ptdump_range ptdump_ranges[] = {
#ifdef CONFIG_X86_64
	{0, PTRS_PER_PGD * PGD_LEVEL_MULT / 2},
	{GUARD_HOLE_END_ADDR, ~0UL},
#else
	{0, ~0UL},
#endif
	{0, 0}
};

	struct pg_state st = {
		.ptdump = {
			.note_page	= note_page,
			.effective_prot = effective_prot,
			.range		= ptdump_ranges
		},
		.level = -1,
		.to_dmesg	= dmesg,
		.check_wx	= checkwx,
		.seq		= m
	};

	ptdump_walk_pgd(&st.ptdump, mm, pgd);

	if (!checkwx)
		return true;
	if (st.wx_pages) {
		pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n",
			st.wx_pages);

		return false;
	} else {
		pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n");

		return true;
	}
}

void ptdump_walk_pgd_level(struct seq_file *m, struct mm_struct *mm)
{
	ptdump_walk_pgd_level_core(m, mm, mm->pgd, false, true);
}

void ptdump_walk_pgd_level_debugfs(struct seq_file *m, struct mm_struct *mm,
				   bool user)
{
	pgd_t *pgd = mm->pgd;
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	if (user && boot_cpu_has(X86_FEATURE_PTI))
		pgd = kernel_to_user_pgdp(pgd);
#endif
	ptdump_walk_pgd_level_core(m, mm, pgd, false, false);
}
EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs);

void ptdump_walk_user_pgd_level_checkwx(void)
{
#ifdef CONFIG_PAGE_TABLE_ISOLATION
	pgd_t *pgd = INIT_PGD;

	if (!(__supported_pte_mask & _PAGE_NX) ||
	    !boot_cpu_has(X86_FEATURE_PTI))
		return;

	pr_info("x86/mm: Checking user space page tables\n");
	pgd = kernel_to_user_pgdp(pgd);
	ptdump_walk_pgd_level_core(NULL, &init_mm, pgd, true, false);
#endif
}

bool ptdump_walk_pgd_level_checkwx(void)
{
	if (!(__supported_pte_mask & _PAGE_NX))
		return true;

	return ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false);
}

static int __init pt_dump_init(void)
{
	/*
	 * Various markers are not compile-time constants, so assign them
	 * here.
	 */
#ifdef CONFIG_X86_64
	address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET;
	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START;
#ifdef CONFIG_MODIFY_LDT_SYSCALL
	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
#endif
#ifdef CONFIG_KASAN
	address_markers[KASAN_SHADOW_START_NR].start_address = KASAN_SHADOW_START;
	address_markers[KASAN_SHADOW_END_NR].start_address = KASAN_SHADOW_END;
#endif
#endif
#ifdef CONFIG_X86_32
	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
# ifdef CONFIG_HIGHMEM
	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
# endif
	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
	address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE;
# ifdef CONFIG_MODIFY_LDT_SYSCALL
	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
# endif
#endif
	return 0;
}
__initcall(pt_dump_init);
Commit	Line	Data
b886d83c	1	// SPDX-License-Identifier: GPL-2.0-only
926e5392 AV	2	/*
	3	* Debug helper to dump the current kernel pagetables of the system
	4	* so that we can see what the various memory ranges are set to.
	5	*
	6	* (C) Copyright 2008 Intel Corporation
	7	*
	8	* Author: Arjan van de Ven <arjan@linux.intel.com>
926e5392 AV	9	*/
926e5392 AV	10
fe770bf0	11	#include <linux/debugfs.h>
04b67022	12	#include <linux/kasan.h>
fe770bf0	13	#include <linux/mm.h>
84e629b6	14	#include <linux/init.h>
146fbb76	15	#include <linux/sched.h>
926e5392	16	#include <linux/seq_file.h>
d6ef1f19	17	#include <linux/highmem.h>
c200dac7	18	#include <linux/pci.h>
2ae27137	19	#include <linux/ptdump.h>
926e5392	20
c200dac7	21	#include <asm/e820/types.h>
926e5392 AV	22
	23	/*
	24	* The dumper groups pagetable entries of the same type into one, and for
	25	* that it needs to keep some state when walking, and flush this state
	26	* when a "break" in the continuity is found.
	27	*/
	28	struct pg_state {
2ae27137	29	struct ptdump_state ptdump;
926e5392	30	int level;
2ae27137	31	pgprotval_t current_prot;
672c0ae0	32	pgprotval_t effective_prot;
2ae27137	33	pgprotval_t prot_levels[5];
926e5392	34	unsigned long start_address;
fe770bf0	35	const struct addr_marker *marker;
3891a04a	36	unsigned long lines;
ef6bea6d	37	bool to_dmesg;
e1a58320 SS	38	bool check_wx;
e1a58320 SS	39	unsigned long wx_pages;
74d2aaa1	40	struct seq_file *seq;
926e5392 AV	41	};
926e5392 AV	42
fe770bf0 PA	43	struct addr_marker {
	44	unsigned long start_address;
	45	const char *name;
3891a04a	46	unsigned long max_lines;
fe770bf0 PA	47	};
fe770bf0 PA	48
146122e2 TG	49	/* Address space markers hints */
	50
	51	#ifdef CONFIG_X86_64
	52
92851e2f AS	53	enum address_markers_idx {
92851e2f AS	54	USER_SPACE_NR = 0,
92851e2f	55	KERNEL_SPACE_NR,
254eb550	56	#ifdef CONFIG_MODIFY_LDT_SYSCALL
f55f0501 AL	57	LDT_NR,
f55f0501 AL	58	#endif
254eb550	59	LOW_KERNEL_NR,
92851e2f AS	60	VMALLOC_START_NR,
92851e2f AS	61	VMEMMAP_START_NR,
025205f8 AR	62	#ifdef CONFIG_KASAN
	63	KASAN_SHADOW_START_NR,
	64	KASAN_SHADOW_END_NR,
f55f0501	65	#endif
f2078904	66	CPU_ENTRY_AREA_NR,
146122e2	67	#ifdef CONFIG_X86_ESPFIX64
3891a04a	68	ESPFIX_START_NR,
146122e2 TG	69	#endif
	70	#ifdef CONFIG_EFI
	71	EFI_END_NR,
	72	#endif
92851e2f AS	73	HIGH_KERNEL_NR,
	74	MODULES_VADDR_NR,
	75	MODULES_END_NR,
146122e2 TG	76	FIXADDR_START_NR,
	77	END_OF_SPACE_NR,
	78	};
	79
	80	static struct addr_marker address_markers[] = {
	81	[USER_SPACE_NR] = { 0, "User Space" },
	82	[KERNEL_SPACE_NR] = { (1UL << 63), "Kernel Space" },
	83	[LOW_KERNEL_NR] = { 0UL, "Low Kernel Mapping" },
	84	[VMALLOC_START_NR] = { 0UL, "vmalloc() Area" },
	85	[VMEMMAP_START_NR] = { 0UL, "Vmemmap" },
	86	#ifdef CONFIG_KASAN
09e61a77 KS	87	/*
	88	* These fields get initialized with the (dynamic)
	89	* KASAN_SHADOW_{START,END} values in pt_dump_init().
	90	*/
	91	[KASAN_SHADOW_START_NR] = { 0UL, "KASAN shadow" },
	92	[KASAN_SHADOW_END_NR] = { 0UL, "KASAN shadow end" },
f55f0501 AL	93	#endif
f55f0501 AL	94	#ifdef CONFIG_MODIFY_LDT_SYSCALL
5c7919bb	95	[LDT_NR] = { 0UL, "LDT remap" },
146122e2	96	#endif
92a0f81d	97	[CPU_ENTRY_AREA_NR] = { CPU_ENTRY_AREA_BASE,"CPU entry Area" },
146122e2 TG	98	#ifdef CONFIG_X86_ESPFIX64
	99	[ESPFIX_START_NR] = { ESPFIX_BASE_ADDR, "ESPfix Area", 16 },
	100	#endif
	101	#ifdef CONFIG_EFI
	102	[EFI_END_NR] = { EFI_VA_END, "EFI Runtime Services" },
	103	#endif
	104	[HIGH_KERNEL_NR] = { __START_KERNEL_map, "High Kernel Mapping" },
	105	[MODULES_VADDR_NR] = { MODULES_VADDR, "Modules" },
	106	[MODULES_END_NR] = { MODULES_END, "End Modules" },
	107	[FIXADDR_START_NR] = { FIXADDR_START, "Fixmap Area" },
	108	[END_OF_SPACE_NR] = { -1, NULL }
	109	};
	110
4e8537e4 JR	111	#define INIT_PGD ((pgd_t *) &init_top_pgt)
4e8537e4 JR	112
146122e2 TG	113	#else /* CONFIG_X86_64 */
	114
	115	enum address_markers_idx {
	116	USER_SPACE_NR = 0,
92851e2f AS	117	KERNEL_SPACE_NR,
	118	VMALLOC_START_NR,
	119	VMALLOC_END_NR,
146122e2	120	#ifdef CONFIG_HIGHMEM
92851e2f	121	PKMAP_BASE_NR,
f3e48e54 JR	122	#endif
	123	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	124	LDT_NR,
92851e2f	125	#endif
92a0f81d	126	CPU_ENTRY_AREA_NR,
146122e2 TG	127	FIXADDR_START_NR,
146122e2 TG	128	END_OF_SPACE_NR,
92851e2f AS	129	};
92851e2f AS	130
fe770bf0	131	static struct addr_marker address_markers[] = {
146122e2 TG	132	[USER_SPACE_NR] = { 0, "User Space" },
	133	[KERNEL_SPACE_NR] = { PAGE_OFFSET, "Kernel Mapping" },
	134	[VMALLOC_START_NR] = { 0UL, "vmalloc() Area" },
	135	[VMALLOC_END_NR] = { 0UL, "vmalloc() End" },
	136	#ifdef CONFIG_HIGHMEM
	137	[PKMAP_BASE_NR] = { 0UL, "Persistent kmap() Area" },
f3e48e54 JR	138	#endif
	139	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	140	[LDT_NR] = { 0UL, "LDT remap" },
fe770bf0	141	#endif
92a0f81d	142	[CPU_ENTRY_AREA_NR] = { 0UL, "CPU entry area" },
146122e2 TG	143	[FIXADDR_START_NR] = { 0UL, "Fixmap area" },
146122e2 TG	144	[END_OF_SPACE_NR] = { -1, NULL }
fe770bf0	145	};
926e5392	146
4e8537e4 JR	147	#define INIT_PGD (swapper_pg_dir)
4e8537e4 JR	148
146122e2 TG	149	#endif /* !CONFIG_X86_64 */
146122e2 TG	150
fe770bf0 PA	151	/* Multipliers for offsets within the PTEs */
	152	#define PTE_LEVEL_MULT (PAGE_SIZE)
	153	#define PMD_LEVEL_MULT (PTRS_PER_PTE * PTE_LEVEL_MULT)
	154	#define PUD_LEVEL_MULT (PTRS_PER_PMD * PMD_LEVEL_MULT)
fdd3d8ce	155	#define P4D_LEVEL_MULT (PTRS_PER_PUD * PUD_LEVEL_MULT)
84bbabc3	156	#define PGD_LEVEL_MULT (PTRS_PER_P4D * P4D_LEVEL_MULT)
926e5392	157
ef6bea6d BP	158	#define pt_dump_seq_printf(m, to_dmesg, fmt, args...) \
	159	({ \
	160	if (to_dmesg) \
	161	printk(KERN_INFO fmt, ##args); \
	162	else \
	163	if (m) \
	164	seq_printf(m, fmt, ##args); \
	165	})
	166
	167	#define pt_dump_cont_printf(m, to_dmesg, fmt, args...) \
	168	({ \
	169	if (to_dmesg) \
	170	printk(KERN_CONT fmt, ##args); \
	171	else \
	172	if (m) \
	173	seq_printf(m, fmt, ##args); \
	174	})
	175
926e5392 AV	176	/*
	177	* Print a readable form of a pgprot_t to the seq_file
	178	*/
2ae27137	179	static void printk_prot(struct seq_file *m, pgprotval_t pr, int level, bool dmsg)
926e5392	180	{
fe770bf0	181	static const char * const level_name[] =
f8f0d0b6	182	{ "pgd", "p4d", "pud", "pmd", "pte" };
fe770bf0	183
c0534494	184	if (!(pr & _PAGE_PRESENT)) {
fe770bf0	185	/* Not present */
f439c429	186	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0 PA	187	} else {
fe770bf0 PA	188	if (pr & _PAGE_USER)
ef6bea6d	189	pt_dump_cont_printf(m, dmsg, "USR ");
926e5392	190	else
ef6bea6d	191	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	192	if (pr & _PAGE_RW)
ef6bea6d	193	pt_dump_cont_printf(m, dmsg, "RW ");
fe770bf0	194	else
ef6bea6d	195	pt_dump_cont_printf(m, dmsg, "ro ");
fe770bf0	196	if (pr & _PAGE_PWT)
ef6bea6d	197	pt_dump_cont_printf(m, dmsg, "PWT ");
fe770bf0	198	else
ef6bea6d	199	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	200	if (pr & _PAGE_PCD)
ef6bea6d	201	pt_dump_cont_printf(m, dmsg, "PCD ");
926e5392	202	else
ef6bea6d	203	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	204
f439c429	205	/* Bit 7 has a different meaning on level 3 vs 4 */
f8f0d0b6	206	if (level <= 3 && pr & _PAGE_PSE)
f439c429 JG	207	pt_dump_cont_printf(m, dmsg, "PSE ");
	208	else
	209	pt_dump_cont_printf(m, dmsg, " ");
f8f0d0b6 SP	210	if ((level == 4 && pr & _PAGE_PAT) \|\|
f8f0d0b6 SP	211	((level == 3 \|\| level == 2) && pr & _PAGE_PAT_LARGE))
da25e628	212	pt_dump_cont_printf(m, dmsg, "PAT ");
f439c429 JG	213	else
f439c429 JG	214	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	215	if (pr & _PAGE_GLOBAL)
ef6bea6d	216	pt_dump_cont_printf(m, dmsg, "GLB ");
fe770bf0	217	else
ef6bea6d	218	pt_dump_cont_printf(m, dmsg, " ");
fe770bf0	219	if (pr & _PAGE_NX)
ef6bea6d	220	pt_dump_cont_printf(m, dmsg, "NX ");
fe770bf0	221	else
ef6bea6d	222	pt_dump_cont_printf(m, dmsg, "x ");
926e5392	223	}
ef6bea6d	224	pt_dump_cont_printf(m, dmsg, "%s\n", level_name[level]);
926e5392 AV	225	}
926e5392 AV	226
2ae27137	227	static void note_wx(struct pg_state *st, unsigned long addr)
c200dac7 TG	228	{
	229	unsigned long npages;
	230
2ae27137	231	npages = (addr - st->start_address) / PAGE_SIZE;
c200dac7 TG	232
	233	#ifdef CONFIG_PCI_BIOS
	234	/*
	235	* If PCI BIOS is enabled, the PCI BIOS area is forced to WX.
	236	* Inform about it, but avoid the warning.
	237	*/
	238	if (pcibios_enabled && st->start_address >= PAGE_OFFSET + BIOS_BEGIN &&
2ae27137	239	addr <= PAGE_OFFSET + BIOS_END) {
c200dac7 TG	240	pr_warn_once("x86/mm: PCI BIOS W+X mapping %lu pages\n", npages);
	241	return;
	242	}
	243	#endif
	244	/* Account the WX pages */
	245	st->wx_pages += npages;
510bb96f TG	246	WARN_ONCE(__supported_pte_mask & _PAGE_NX,
510bb96f TG	247	"x86/mm: Found insecure W+X mapping at address %pS\n",
c200dac7 TG	248	(void *)st->start_address);
	249	}
	250
1494e0c3	251	static void effective_prot(struct ptdump_state *pt_st, int level, u64 val)
2ae27137	252	{
1494e0c3 SP	253	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	254	pgprotval_t prot = val & PTE_FLAGS_MASK;
	255	pgprotval_t effective;
	256
	257	if (level > 0) {
	258	pgprotval_t higher_prot = st->prot_levels[level - 1];
	259
	260	effective = (higher_prot & prot & (_PAGE_USER \| _PAGE_RW)) \|
	261	((higher_prot \| prot) & _PAGE_NX);
	262	} else {
	263	effective = prot;
	264	}
	265
	266	st->prot_levels[level] = effective;
2ae27137 SP	267	}
2ae27137 SP	268
926e5392 AV	269	/*
	270	* This function gets called on a break in a continuous series
	271	* of PTE entries; the next one is different so we need to
	272	* print what we collected so far.
	273	*/
2ae27137	274	static void note_page(struct ptdump_state *pt_st, unsigned long addr, int level,
99395ee3	275	u64 val)
926e5392	276	{
2ae27137 SP	277	struct pg_state *st = container_of(pt_st, struct pg_state, ptdump);
	278	pgprotval_t new_prot, new_eff;
	279	pgprotval_t cur, eff;
3891a04a	280	static const char units[] = "BKMGTPE";
74d2aaa1	281	struct seq_file *m = st->seq;
926e5392	282
2ae27137	283	new_prot = val & PTE_FLAGS_MASK;
1494e0c3 SP	284	if (!val)
	285	new_eff = 0;
	286	else
	287	new_eff = st->prot_levels[level];
2ae27137	288
926e5392 AV	289	/*
926e5392 AV	290	* If we have a "break" in the series, we need to flush the state that
fe770bf0 PA	291	* we have now. "break" is either changing perms, levels or
fe770bf0 PA	292	* address space marker.
926e5392	293	*/
2ae27137	294	cur = st->current_prot;
672c0ae0	295	eff = st->effective_prot;
926e5392	296
f8f0d0b6	297	if (st->level == -1) {
fe770bf0 PA	298	/* First entry */
fe770bf0 PA	299	st->current_prot = new_prot;
672c0ae0	300	st->effective_prot = new_eff;
fe770bf0 PA	301	st->level = level;
fe770bf0 PA	302	st->marker = address_markers;
3891a04a	303	st->lines = 0;
ef6bea6d BP	304	pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
ef6bea6d BP	305	st->marker->name);
2ae27137 SP	306	} else if (new_prot != cur \|\| new_eff != eff \|\| level != st->level \|\|
2ae27137 SP	307	addr >= st->marker[1].start_address) {
fe770bf0	308	const char *unit = units;
926e5392	309	unsigned long delta;
6424fb38	310	int width = sizeof(unsigned long) * 2;
e1a58320	311
c200dac7	312	if (st->check_wx && (eff & _PAGE_RW) && !(eff & _PAGE_NX))
2ae27137	313	note_wx(st, addr);
926e5392	314
926e5392 AV	315	/*
	316	* Now print the actual finished series
	317	*/
3891a04a PA	318	if (!st->marker->max_lines \|\|
	319	st->lines < st->marker->max_lines) {
	320	pt_dump_seq_printf(m, st->to_dmesg,
	321	"0x%0lx-0x%0lx ",
	322	width, st->start_address,
2ae27137	323	width, addr);
926e5392	324
2ae27137	325	delta = addr - st->start_address;
3891a04a PA	326	while (!(delta & 1023) && unit[1]) {
	327	delta >>= 10;
	328	unit++;
	329	}
	330	pt_dump_cont_printf(m, st->to_dmesg, "%9lu%c ",
	331	delta, *unit);
	332	printk_prot(m, st->current_prot, st->level,
	333	st->to_dmesg);
926e5392	334	}
3891a04a	335	st->lines++;
fe770bf0 PA	336
	337	/*
	338	* We print markers for special areas of address space,
	339	* such as the start of vmalloc space etc.
	340	* This helps in the interpretation.
	341	*/
2ae27137	342	if (addr >= st->marker[1].start_address) {
3891a04a PA	343	if (st->marker->max_lines &&
	344	st->lines > st->marker->max_lines) {
	345	unsigned long nskip =
	346	st->lines - st->marker->max_lines;
	347	pt_dump_seq_printf(m, st->to_dmesg,
	348	"... %lu entr%s skipped ... \n",
	349	nskip,
	350	nskip == 1 ? "y" : "ies");
	351	}
fe770bf0	352	st->marker++;
3891a04a	353	st->lines = 0;
ef6bea6d BP	354	pt_dump_seq_printf(m, st->to_dmesg, "---[ %s ]---\n",
ef6bea6d BP	355	st->marker->name);
926e5392	356	}
fe770bf0	357
2ae27137	358	st->start_address = addr;
926e5392	359	st->current_prot = new_prot;
672c0ae0	360	st->effective_prot = new_eff;
926e5392	361	st->level = level;
fe770bf0	362	}
926e5392 AV	363	}
926e5392 AV	364
6cdc82db CL	365	bool ptdump_walk_pgd_level_core(struct seq_file *m,
	366	struct mm_struct mm, pgd_t pgd,
	367	bool checkwx, bool dmesg)
926e5392	368	{
2ae27137 SP	369	const struct ptdump_range ptdump_ranges[] = {
2ae27137 SP	370	#ifdef CONFIG_X86_64
2ae27137	371	{0, PTRS_PER_PGD * PGD_LEVEL_MULT / 2},
bba42aff	372	{GUARD_HOLE_END_ADDR, ~0UL},
fe770bf0	373	#else
2ae27137	374	{0, ~0UL},
fdd3d8ce	375	#endif
2ae27137 SP	376	{0, 0}
2ae27137 SP	377	};
fdd3d8ce	378
2ae27137 SP	379	struct pg_state st = {
	380	.ptdump = {
	381	.note_page = note_page,
1494e0c3	382	.effective_prot = effective_prot,
2ae27137 SP	383	.range = ptdump_ranges
2ae27137 SP	384	},
f8f0d0b6	385	.level = -1,
2ae27137 SP	386	.to_dmesg = dmesg,
	387	.check_wx = checkwx,
	388	.seq = m
	389	};
fdd3d8ce	390
e47690d7	391	ptdump_walk_pgd(&st.ptdump, mm, pgd);
fe770bf0	392
e1a58320	393	if (!checkwx)
6cdc82db CL	394	return true;
6cdc82db CL	395	if (st.wx_pages) {
e1a58320 SS	396	pr_info("x86/mm: Checked W+X mappings: FAILED, %lu W+X pages found.\n",
e1a58320 SS	397	st.wx_pages);
6cdc82db CL	398
	399	return false;
	400	} else {
e1a58320	401	pr_info("x86/mm: Checked W+X mappings: passed, no W+X pages found.\n");
6cdc82db CL	402
	403	return true;
	404	}
e1a58320 SS	405	}
e1a58320 SS	406
e455248d	407	void ptdump_walk_pgd_level(struct seq_file m, struct mm_struct mm)
e1a58320	408	{
e47690d7	409	ptdump_walk_pgd_level_core(m, mm, mm->pgd, false, true);
b4bf4f92 TG	410	}
b4bf4f92 TG	411
c5cfae12 SP	412	void ptdump_walk_pgd_level_debugfs(struct seq_file m, struct mm_struct mm,
c5cfae12 SP	413	bool user)
b4bf4f92	414	{
c5cfae12	415	pgd_t *pgd = mm->pgd;
a4b51ef6	416	#ifdef CONFIG_PAGE_TABLE_ISOLATION
28e3ace7	417	if (user && boot_cpu_has(X86_FEATURE_PTI))
a4b51ef6 TG	418	pgd = kernel_to_user_pgdp(pgd);
a4b51ef6 TG	419	#endif
e47690d7	420	ptdump_walk_pgd_level_core(m, mm, pgd, false, false);
b4bf4f92 TG	421	}
	422	EXPORT_SYMBOL_GPL(ptdump_walk_pgd_level_debugfs);
	423
d878efce	424	void ptdump_walk_user_pgd_level_checkwx(void)
b4bf4f92 TG	425	{
b4bf4f92 TG	426	#ifdef CONFIG_PAGE_TABLE_ISOLATION
4e8537e4	427	pgd_t *pgd = INIT_PGD;
b4bf4f92	428
d878efce	429	if (!(__supported_pte_mask & _PAGE_NX) \|\|
28e3ace7	430	!boot_cpu_has(X86_FEATURE_PTI))
b4bf4f92 TG	431	return;
	432
	433	pr_info("x86/mm: Checking user space page tables\n");
	434	pgd = kernel_to_user_pgdp(pgd);
e47690d7	435	ptdump_walk_pgd_level_core(NULL, &init_mm, pgd, true, false);
b4bf4f92	436	#endif
926e5392 AV	437	}
926e5392 AV	438
6cdc82db	439	bool ptdump_walk_pgd_level_checkwx(void)
e1a58320	440	{
a5e8131a	441	if (!(__supported_pte_mask & _PAGE_NX))
6cdc82db	442	return true;
a5e8131a	443
6cdc82db	444	return ptdump_walk_pgd_level_core(NULL, &init_mm, INIT_PGD, true, false);
e1a58320 SS	445	}
e1a58320 SS	446
8609d1b5	447	static int __init pt_dump_init(void)
926e5392	448	{
0483e1fa TG	449	/*
	450	* Various markers are not compile-time constants, so assign them
	451	* here.
	452	*/
	453	#ifdef CONFIG_X86_64
	454	address_markers[LOW_KERNEL_NR].start_address = PAGE_OFFSET;
	455	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
	456	address_markers[VMEMMAP_START_NR].start_address = VMEMMAP_START;
5c7919bb KS	457	#ifdef CONFIG_MODIFY_LDT_SYSCALL
	458	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
	459	#endif
09e61a77 KS	460	#ifdef CONFIG_KASAN
	461	address_markers[KASAN_SHADOW_START_NR].start_address = KASAN_SHADOW_START;
	462	address_markers[KASAN_SHADOW_END_NR].start_address = KASAN_SHADOW_END;
	463	#endif
0483e1fa	464	#endif
fe770bf0	465	#ifdef CONFIG_X86_32
92851e2f AS	466	address_markers[VMALLOC_START_NR].start_address = VMALLOC_START;
92851e2f AS	467	address_markers[VMALLOC_END_NR].start_address = VMALLOC_END;
fe770bf0	468	# ifdef CONFIG_HIGHMEM
92851e2f	469	address_markers[PKMAP_BASE_NR].start_address = PKMAP_BASE;
fe770bf0	470	# endif
92851e2f	471	address_markers[FIXADDR_START_NR].start_address = FIXADDR_START;
92a0f81d	472	address_markers[CPU_ENTRY_AREA_NR].start_address = CPU_ENTRY_AREA_BASE;
f3e48e54 JR	473	# ifdef CONFIG_MODIFY_LDT_SYSCALL
	474	address_markers[LDT_NR].start_address = LDT_BASE_ADDR;
	475	# endif
fe770bf0	476	#endif
926e5392 AV	477	return 0;
926e5392 AV	478	}
926e5392	479	__initcall(pt_dump_init);