[thirdparty/linux.git] / mm / early_ioremap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * Provide common bits of early_ioremap() support for architectures needing
 * temporary mappings during boot before ioremap() is available.
 *
 * This is mostly a direct copy of the x86 early_ioremap implementation.
 *
 * (C) Copyright 1995 1996, 2014 Linus Torvalds
 *
 */
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/io.h>
#include <linux/module.h>
#include <linux/slab.h>
#include <linux/mm.h>
#include <linux/vmalloc.h>
#include <asm/fixmap.h>
#include <asm/early_ioremap.h>
#include "internal.h"

#ifdef CONFIG_MMU
static int early_ioremap_debug __initdata;

static int __init early_ioremap_debug_setup(char *str)
{
	early_ioremap_debug = 1;

	return 0;
}
early_param("early_ioremap_debug", early_ioremap_debug_setup);

static int after_paging_init __initdata;

pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr,
						    unsigned long size,
						    pgprot_t prot)
{
	return prot;
}

void __init early_ioremap_reset(void)
{
	after_paging_init = 1;
}

/*
 * Generally, ioremap() is available after paging_init() has been called.
 * Architectures wanting to allow early_ioremap after paging_init() can
 * define __late_set_fixmap and __late_clear_fixmap to do the right thing.
 */
#ifndef __late_set_fixmap
static inline void __init __late_set_fixmap(enum fixed_addresses idx,
					    phys_addr_t phys, pgprot_t prot)
{
	BUG();
}
#endif

#ifndef __late_clear_fixmap
static inline void __init __late_clear_fixmap(enum fixed_addresses idx)
{
	BUG();
}
#endif

static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;

void __init early_ioremap_setup(void)
{
	int i;

	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
		if (WARN_ON(prev_map[i]))
			break;

	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
		slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
}

static int __init check_early_ioremap_leak(void)
{
	int count = 0;
	int i;

	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
		if (prev_map[i])
			count++;

	if (WARN(count, KERN_WARNING
		 "Debug warning: early ioremap leak of %d areas detected.\n"
		 "please boot with early_ioremap_debug and report the dmesg.\n",
		 count))
		return 1;
	return 0;
}
late_initcall(check_early_ioremap_leak);

static void __init __iomem *
__early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
{
	unsigned long offset;
	resource_size_t last_addr;
	unsigned int nrpages;
	enum fixed_addresses idx;
	int i, slot;

	WARN_ON(system_state >= SYSTEM_RUNNING);

	slot = -1;
	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
		if (!prev_map[i]) {
			slot = i;
			break;
		}
	}

	if (WARN(slot < 0, "%s(%pa, %08lx) not found slot\n",
		 __func__, &phys_addr, size))
		return NULL;

	/* Don't allow wraparound or zero size */
	last_addr = phys_addr + size - 1;
	if (WARN_ON(!size || last_addr < phys_addr))
		return NULL;

	prev_size[slot] = size;
	/*
	 * Mappings have to be page-aligned
	 */
	offset = offset_in_page(phys_addr);
	phys_addr &= PAGE_MASK;
	size = PAGE_ALIGN(last_addr + 1) - phys_addr;

	/*
	 * Mappings have to fit in the FIX_BTMAP area.
	 */
	nrpages = size >> PAGE_SHIFT;
	if (WARN_ON(nrpages > NR_FIX_BTMAPS))
		return NULL;

	/*
	 * Ok, go for it..
	 */
	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
	while (nrpages > 0) {
		if (after_paging_init)
			__late_set_fixmap(idx, phys_addr, prot);
		else
			__early_set_fixmap(idx, phys_addr, prot);
		phys_addr += PAGE_SIZE;
		--idx;
		--nrpages;
	}
	WARN(early_ioremap_debug, "%s(%pa, %08lx) [%d] => %08lx + %08lx\n",
	     __func__, &phys_addr, size, slot, offset, slot_virt[slot]);

	prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
	return prev_map[slot];
}

void __init early_iounmap(void __iomem *addr, unsigned long size)
{
	unsigned long virt_addr;
	unsigned long offset;
	unsigned int nrpages;
	enum fixed_addresses idx;
	int i, slot;

	slot = -1;
	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
		if (prev_map[i] == addr) {
			slot = i;
			break;
		}
	}

	if (WARN(slot < 0, "%s(%p, %08lx) not found slot\n",
		  __func__, addr, size))
		return;

	if (WARN(prev_size[slot] != size,
		 "%s(%p, %08lx) [%d] size not consistent %08lx\n",
		  __func__, addr, size, slot, prev_size[slot]))
		return;

	WARN(early_ioremap_debug, "%s(%p, %08lx) [%d]\n",
	      __func__, addr, size, slot);

	virt_addr = (unsigned long)addr;
	if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)))
		return;

	offset = offset_in_page(virt_addr);
	nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;

	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
	while (nrpages > 0) {
		if (after_paging_init)
			__late_clear_fixmap(idx);
		else
			__early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR);
		--idx;
		--nrpages;
	}
	prev_map[slot] = NULL;
}

/* Remap an IO device */
void __init __iomem *
early_ioremap(resource_size_t phys_addr, unsigned long size)
{
	return __early_ioremap(phys_addr, size, FIXMAP_PAGE_IO);
}

/* Remap memory */
void __init *
early_memremap(resource_size_t phys_addr, unsigned long size)
{
	pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
						     FIXMAP_PAGE_NORMAL);

	return (__force void *)__early_ioremap(phys_addr, size, prot);
}
#ifdef FIXMAP_PAGE_RO
void __init *
early_memremap_ro(resource_size_t phys_addr, unsigned long size)
{
	pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
						     FIXMAP_PAGE_RO);

	return (__force void *)__early_ioremap(phys_addr, size, prot);
}
#endif

#ifdef CONFIG_ARCH_USE_MEMREMAP_PROT
void __init *
early_memremap_prot(resource_size_t phys_addr, unsigned long size,
		    unsigned long prot_val)
{
	return (__force void *)__early_ioremap(phys_addr, size,
					       __pgprot(prot_val));
}
#endif

#define MAX_MAP_CHUNK	(NR_FIX_BTMAPS << PAGE_SHIFT)

void __init copy_from_early_mem(void *dest, phys_addr_t src, unsigned long size)
{
	unsigned long slop, clen;
	char *p;

	while (size) {
		slop = offset_in_page(src);
		clen = size;
		if (clen > MAX_MAP_CHUNK - slop)
			clen = MAX_MAP_CHUNK - slop;
		p = early_memremap(src & PAGE_MASK, clen + slop);
		memcpy(dest, p + slop, clen);
		early_memunmap(p, clen + slop);
		dest += clen;
		src += clen;
		size -= clen;
	}
}

#else /* CONFIG_MMU */

void __init __iomem *
early_ioremap(resource_size_t phys_addr, unsigned long size)
{
	return (__force void __iomem *)phys_addr;
}

/* Remap memory */
void __init *
early_memremap(resource_size_t phys_addr, unsigned long size)
{
	return (void *)phys_addr;
}
void __init *
early_memremap_ro(resource_size_t phys_addr, unsigned long size)
{
	return (void *)phys_addr;
}

void __init early_iounmap(void __iomem *addr, unsigned long size)
{
}

#endif /* CONFIG_MMU */


void __init early_memunmap(void *addr, unsigned long size)
{
	early_iounmap((__force void __iomem *)addr, size);
}
Commit	Line	Data
b2441318	1	// SPDX-License-Identifier: GPL-2.0
9e5c33d7 MS	2	/*
	3	* Provide common bits of early_ioremap() support for architectures needing
	4	* temporary mappings during boot before ioremap() is available.
	5	*
	6	* This is mostly a direct copy of the x86 early_ioremap implementation.
	7	*
	8	* (C) Copyright 1995 1996, 2014 Linus Torvalds
	9	*
	10	*/
	11	#include <linux/kernel.h>
	12	#include <linux/init.h>
	13	#include <linux/io.h>
	14	#include <linux/module.h>
	15	#include <linux/slab.h>
	16	#include <linux/mm.h>
	17	#include <linux/vmalloc.h>
	18	#include <asm/fixmap.h>
4f1af60b	19	#include <asm/early_ioremap.h>
be4893d9	20	#include "internal.h"
9e5c33d7 MS	21
	22	#ifdef CONFIG_MMU
	23	static int early_ioremap_debug __initdata;
	24
	25	static int __init early_ioremap_debug_setup(char *str)
	26	{
	27	early_ioremap_debug = 1;
	28
	29	return 0;
	30	}
	31	early_param("early_ioremap_debug", early_ioremap_debug_setup);
	32
	33	static int after_paging_init __initdata;
	34
8f716c9b TL	35	pgprot_t __init __weak early_memremap_pgprot_adjust(resource_size_t phys_addr,
	36	unsigned long size,
	37	pgprot_t prot)
	38	{
	39	return prot;
	40	}
	41
9e5c33d7 MS	42	void __init early_ioremap_reset(void)
9e5c33d7 MS	43	{
9e5c33d7 MS	44	after_paging_init = 1;
	45	}
	46
	47	/*
	48	* Generally, ioremap() is available after paging_init() has been called.
	49	* Architectures wanting to allow early_ioremap after paging_init() can
	50	* define __late_set_fixmap and __late_clear_fixmap to do the right thing.
	51	*/
	52	#ifndef __late_set_fixmap
	53	static inline void __init __late_set_fixmap(enum fixed_addresses idx,
	54	phys_addr_t phys, pgprot_t prot)
	55	{
	56	BUG();
	57	}
	58	#endif
	59
	60	#ifndef __late_clear_fixmap
	61	static inline void __init __late_clear_fixmap(enum fixed_addresses idx)
	62	{
	63	BUG();
	64	}
	65	#endif
	66
	67	static void __iomem *prev_map[FIX_BTMAPS_SLOTS] __initdata;
	68	static unsigned long prev_size[FIX_BTMAPS_SLOTS] __initdata;
	69	static unsigned long slot_virt[FIX_BTMAPS_SLOTS] __initdata;
	70
	71	void __init early_ioremap_setup(void)
	72	{
	73	int i;
	74
	75	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
	76	if (WARN_ON(prev_map[i]))
	77	break;
	78
	79	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
	80	slot_virt[i] = __fix_to_virt(FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*i);
	81	}
	82
	83	static int __init check_early_ioremap_leak(void)
	84	{
	85	int count = 0;
	86	int i;
	87
	88	for (i = 0; i < FIX_BTMAPS_SLOTS; i++)
	89	if (prev_map[i])
	90	count++;
	91
	92	if (WARN(count, KERN_WARNING
	93	"Debug warning: early ioremap leak of %d areas detected.\n"
	94	"please boot with early_ioremap_debug and report the dmesg.\n",
	95	count))
	96	return 1;
	97	return 0;
	98	}
	99	late_initcall(check_early_ioremap_leak);
	100
	101	static void __init __iomem *
	102	__early_ioremap(resource_size_t phys_addr, unsigned long size, pgprot_t prot)
	103	{
	104	unsigned long offset;
	105	resource_size_t last_addr;
	106	unsigned int nrpages;
	107	enum fixed_addresses idx;
108	int i, slot;
109
7f6f60a1	110	WARN_ON(system_state >= SYSTEM_RUNNING);
9e5c33d7 MS	111
	112	slot = -1;
	113	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
	114	if (!prev_map[i]) {
	115	slot = i;
	116	break;
	117	}
	118	}
	119
7b69d79f AS	120	if (WARN(slot < 0, "%s(%pa, %08lx) not found slot\n",
7b69d79f AS	121	__func__, &phys_addr, size))
9e5c33d7 MS	122	return NULL;
	123
	124	/* Don't allow wraparound or zero size */
	125	last_addr = phys_addr + size - 1;
	126	if (WARN_ON(!size \|\| last_addr < phys_addr))
	127	return NULL;
	128
	129	prev_size[slot] = size;
	130	/*
	131	* Mappings have to be page-aligned
	132	*/
5d57b014	133	offset = offset_in_page(phys_addr);
9e5c33d7 MS	134	phys_addr &= PAGE_MASK;
	135	size = PAGE_ALIGN(last_addr + 1) - phys_addr;
	136
	137	/*
	138	* Mappings have to fit in the FIX_BTMAP area.
	139	*/
	140	nrpages = size >> PAGE_SHIFT;
	141	if (WARN_ON(nrpages > NR_FIX_BTMAPS))
	142	return NULL;
	143
	144	/*
	145	* Ok, go for it..
	146	*/
	147	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
	148	while (nrpages > 0) {
	149	if (after_paging_init)
	150	__late_set_fixmap(idx, phys_addr, prot);
	151	else
	152	__early_set_fixmap(idx, phys_addr, prot);
	153	phys_addr += PAGE_SIZE;
	154	--idx;
	155	--nrpages;
	156	}
7b69d79f AS	157	WARN(early_ioremap_debug, "%s(%pa, %08lx) [%d] => %08lx + %08lx\n",
7b69d79f AS	158	__func__, &phys_addr, size, slot, offset, slot_virt[slot]);
9e5c33d7 MS	159
	160	prev_map[slot] = (void __iomem *)(offset + slot_virt[slot]);
	161	return prev_map[slot];
	162	}
	163
	164	void __init early_iounmap(void __iomem *addr, unsigned long size)
	165	{
	166	unsigned long virt_addr;
	167	unsigned long offset;
	168	unsigned int nrpages;
	169	enum fixed_addresses idx;
	170	int i, slot;
	171
	172	slot = -1;
	173	for (i = 0; i < FIX_BTMAPS_SLOTS; i++) {
	174	if (prev_map[i] == addr) {
	175	slot = i;
	176	break;
	177	}
	178	}
	179
87005394 SZ	180	if (WARN(slot < 0, "%s(%p, %08lx) not found slot\n",
87005394 SZ	181	__func__, addr, size))
9e5c33d7 MS	182	return;
	183
	184	if (WARN(prev_size[slot] != size,
87005394 SZ	185	"%s(%p, %08lx) [%d] size not consistent %08lx\n",
87005394 SZ	186	__func__, addr, size, slot, prev_size[slot]))
9e5c33d7 MS	187	return;
9e5c33d7 MS	188
87005394 SZ	189	WARN(early_ioremap_debug, "%s(%p, %08lx) [%d]\n",
87005394 SZ	190	__func__, addr, size, slot);
9e5c33d7 MS	191
	192	virt_addr = (unsigned long)addr;
	193	if (WARN_ON(virt_addr < fix_to_virt(FIX_BTMAP_BEGIN)))
	194	return;
	195
5d57b014	196	offset = offset_in_page(virt_addr);
9e5c33d7 MS	197	nrpages = PAGE_ALIGN(offset + size) >> PAGE_SHIFT;
	198
	199	idx = FIX_BTMAP_BEGIN - NR_FIX_BTMAPS*slot;
	200	while (nrpages > 0) {
	201	if (after_paging_init)
	202	__late_clear_fixmap(idx);
	203	else
	204	__early_set_fixmap(idx, 0, FIXMAP_PAGE_CLEAR);
	205	--idx;
	206	--nrpages;
	207	}
	208	prev_map[slot] = NULL;
	209	}
	210
	211	/* Remap an IO device */
	212	void __init __iomem *
	213	early_ioremap(resource_size_t phys_addr, unsigned long size)
	214	{
	215	return __early_ioremap(phys_addr, size, FIXMAP_PAGE_IO);
	216	}
	217
	218	/* Remap memory */
	219	void __init *
	220	early_memremap(resource_size_t phys_addr, unsigned long size)
	221	{
8f716c9b TL	222	pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
	223	FIXMAP_PAGE_NORMAL);
	224
	225	return (__force void *)__early_ioremap(phys_addr, size, prot);
9e5c33d7	226	}
2592dbbb JG	227	#ifdef FIXMAP_PAGE_RO
	228	void __init *
	229	early_memremap_ro(resource_size_t phys_addr, unsigned long size)
	230	{
8f716c9b TL	231	pgprot_t prot = early_memremap_pgprot_adjust(phys_addr, size,
	232	FIXMAP_PAGE_RO);
	233
	234	return (__force void *)__early_ioremap(phys_addr, size, prot);
2592dbbb JG	235	}
2592dbbb JG	236	#endif
6b0f68e3	237
f88a68fa TL	238	#ifdef CONFIG_ARCH_USE_MEMREMAP_PROT
	239	void __init *
	240	early_memremap_prot(resource_size_t phys_addr, unsigned long size,
	241	unsigned long prot_val)
	242	{
	243	return (__force void *)__early_ioremap(phys_addr, size,
	244	__pgprot(prot_val));
	245	}
	246	#endif
	247
6b0f68e3 MS	248	#define MAX_MAP_CHUNK (NR_FIX_BTMAPS << PAGE_SHIFT)
	249
	250	void __init copy_from_early_mem(void *dest, phys_addr_t src, unsigned long size)
	251	{
	252	unsigned long slop, clen;
	253	char *p;
	254
	255	while (size) {
5d57b014	256	slop = offset_in_page(src);
6b0f68e3 MS	257	clen = size;
	258	if (clen > MAX_MAP_CHUNK - slop)
	259	clen = MAX_MAP_CHUNK - slop;
	260	p = early_memremap(src & PAGE_MASK, clen + slop);
	261	memcpy(dest, p + slop, clen);
	262	early_memunmap(p, clen + slop);
	263	dest += clen;
	264	src += clen;
	265	size -= clen;
	266	}
	267	}
	268
9e5c33d7 MS	269	#else /* CONFIG_MMU */
	270
	271	void __init __iomem *
	272	early_ioremap(resource_size_t phys_addr, unsigned long size)
	273	{
	274	return (__force void __iomem *)phys_addr;
	275	}
	276
	277	/* Remap memory */
	278	void __init *
	279	early_memremap(resource_size_t phys_addr, unsigned long size)
	280	{
	281	return (void *)phys_addr;
	282	}
2592dbbb JG	283	void __init *
	284	early_memremap_ro(resource_size_t phys_addr, unsigned long size)
	285	{
	286	return (void *)phys_addr;
	287	}
9e5c33d7 MS	288
	289	void __init early_iounmap(void __iomem *addr, unsigned long size)
	290	{
	291	}
	292
	293	#endif /* CONFIG_MMU */
	294
	295
	296	void __init early_memunmap(void *addr, unsigned long size)
	297	{
	298	early_iounmap((__force void __iomem *)addr, size);
	299	}