[thirdparty/kernel/stable.git] / mm / kmsan / hooks.c

// SPDX-License-Identifier: GPL-2.0
/*
 * KMSAN hooks for kernel subsystems.
 *
 * These functions handle creation of KMSAN metadata for memory allocations.
 *
 * Copyright (C) 2018-2022 Google LLC
 * Author: Alexander Potapenko <glider@google.com>
 *
 */

#include <linux/cacheflush.h>
#include <linux/dma-direction.h>
#include <linux/gfp.h>
#include <linux/kmsan.h>
#include <linux/mm.h>
#include <linux/mm_types.h>
#include <linux/scatterlist.h>
#include <linux/slab.h>
#include <linux/uaccess.h>
#include <linux/usb.h>

#include "../internal.h"
#include "../slab.h"
#include "kmsan.h"

/*
 * Instrumented functions shouldn't be called under
 * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
 * skipping effects of functions like memset() inside instrumented code.
 */

void kmsan_task_create(struct task_struct *task)
{
	kmsan_enter_runtime();
	kmsan_internal_task_create(task);
	kmsan_leave_runtime();
}

void kmsan_task_exit(struct task_struct *task)
{
	struct kmsan_ctx *ctx = &task->kmsan_ctx;

	if (!kmsan_enabled || kmsan_in_runtime())
		return;

	ctx->allow_reporting = false;
}

void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags)
{
	if (unlikely(object == NULL))
		return;
	if (!kmsan_enabled || kmsan_in_runtime())
		return;
	/*
	 * There's a ctor or this is an RCU cache - do nothing. The memory
	 * status hasn't changed since last use.
	 */
	if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU))
		return;

	kmsan_enter_runtime();
	if (flags & __GFP_ZERO)
		kmsan_internal_unpoison_memory(object, s->object_size,
					       KMSAN_POISON_CHECK);
	else
		kmsan_internal_poison_memory(object, s->object_size, flags,
					     KMSAN_POISON_CHECK);
	kmsan_leave_runtime();
}

void kmsan_slab_free(struct kmem_cache *s, void *object)
{
	if (!kmsan_enabled || kmsan_in_runtime())
		return;

	/* RCU slabs could be legally used after free within the RCU period */
	if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)))
		return;
	/*
	 * If there's a constructor, freed memory must remain in the same state
	 * until the next allocation. We cannot save its state to detect
	 * use-after-free bugs, instead we just keep it unpoisoned.
	 */
	if (s->ctor)
		return;
	kmsan_enter_runtime();
	kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
				     KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
	kmsan_leave_runtime();
}

void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
{
	if (unlikely(ptr == NULL))
		return;
	if (!kmsan_enabled || kmsan_in_runtime())
		return;
	kmsan_enter_runtime();
	if (flags & __GFP_ZERO)
		kmsan_internal_unpoison_memory((void *)ptr, size,
					       /*checked*/ true);
	else
		kmsan_internal_poison_memory((void *)ptr, size, flags,
					     KMSAN_POISON_CHECK);
	kmsan_leave_runtime();
}

void kmsan_kfree_large(const void *ptr)
{
	struct page *page;

	if (!kmsan_enabled || kmsan_in_runtime())
		return;
	kmsan_enter_runtime();
	page = virt_to_head_page((void *)ptr);
	KMSAN_WARN_ON(ptr != page_address(page));
	kmsan_internal_poison_memory((void *)ptr,
				     page_size(page),
				     GFP_KERNEL,
				     KMSAN_POISON_CHECK | KMSAN_POISON_FREE);
	kmsan_leave_runtime();
}

static unsigned long vmalloc_shadow(unsigned long addr)
{
	return (unsigned long)kmsan_get_metadata((void *)addr,
						 KMSAN_META_SHADOW);
}

static unsigned long vmalloc_origin(unsigned long addr)
{
	return (unsigned long)kmsan_get_metadata((void *)addr,
						 KMSAN_META_ORIGIN);
}

void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
{
	__vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
	__vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
}

/*
 * This function creates new shadow/origin pages for the physical pages mapped
 * into the virtual memory. If those physical pages already had shadow/origin,
 * those are ignored.
 */
int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
			     phys_addr_t phys_addr, pgprot_t prot,
			     unsigned int page_shift)
{
	gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO;
	struct page *shadow, *origin;
	unsigned long off = 0;
	int nr, err = 0, clean = 0, mapped;

	if (!kmsan_enabled || kmsan_in_runtime())
		return 0;

	nr = (end - start) / PAGE_SIZE;
	kmsan_enter_runtime();
	for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
		shadow = alloc_pages(gfp_mask, 1);
		origin = alloc_pages(gfp_mask, 1);
		if (!shadow || !origin) {
			err = -ENOMEM;
			goto ret;
		}
		mapped = __vmap_pages_range_noflush(
			vmalloc_shadow(start + off),
			vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
			PAGE_SHIFT);
		if (mapped) {
			err = mapped;
			goto ret;
		}
		shadow = NULL;
		mapped = __vmap_pages_range_noflush(
			vmalloc_origin(start + off),
			vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
			PAGE_SHIFT);
		if (mapped) {
			__vunmap_range_noflush(
				vmalloc_shadow(start + off),
				vmalloc_shadow(start + off + PAGE_SIZE));
			err = mapped;
			goto ret;
		}
		origin = NULL;
	}
	/* Page mapping loop finished normally, nothing to clean up. */
	clean = 0;

ret:
	if (clean > 0) {
		/*
		 * Something went wrong. Clean up shadow/origin pages allocated
		 * on the last loop iteration, then delete mappings created
		 * during the previous iterations.
		 */
		if (shadow)
			__free_pages(shadow, 1);
		if (origin)
			__free_pages(origin, 1);
		__vunmap_range_noflush(
			vmalloc_shadow(start),
			vmalloc_shadow(start + clean * PAGE_SIZE));
		__vunmap_range_noflush(
			vmalloc_origin(start),
			vmalloc_origin(start + clean * PAGE_SIZE));
	}
	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
	kmsan_leave_runtime();
	return err;
}

void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
{
	unsigned long v_shadow, v_origin;
	struct page *shadow, *origin;
	int nr;

	if (!kmsan_enabled || kmsan_in_runtime())
		return;

	nr = (end - start) / PAGE_SIZE;
	kmsan_enter_runtime();
	v_shadow = (unsigned long)vmalloc_shadow(start);
	v_origin = (unsigned long)vmalloc_origin(start);
	for (int i = 0; i < nr;
	     i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
		shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
		origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
		__vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
		__vunmap_range_noflush(v_origin, vmalloc_origin(end));
		if (shadow)
			__free_pages(shadow, 1);
		if (origin)
			__free_pages(origin, 1);
	}
	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
	kmsan_leave_runtime();
}

void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy,
			size_t left)
{
	unsigned long ua_flags;

	if (!kmsan_enabled || kmsan_in_runtime())
		return;
	/*
	 * At this point we've copied the memory already. It's hard to check it
	 * before copying, as the size of actually copied buffer is unknown.
	 */

	/* copy_to_user() may copy zero bytes. No need to check. */
	if (!to_copy)
		return;
	/* Or maybe copy_to_user() failed to copy anything. */
	if (to_copy <= left)
		return;

	ua_flags = user_access_save();
	if ((u64)to < TASK_SIZE) {
		/* This is a user memory access, check it. */
		kmsan_internal_check_memory((void *)from, to_copy - left, to,
					    REASON_COPY_TO_USER);
	} else {
		/* Otherwise this is a kernel memory access. This happens when a
		 * compat syscall passes an argument allocated on the kernel
		 * stack to a real syscall.
		 * Don't check anything, just copy the shadow of the copied
		 * bytes.
		 */
		kmsan_internal_memmove_metadata((void *)to, (void *)from,
						to_copy - left);
	}
	user_access_restore(ua_flags);
}
EXPORT_SYMBOL(kmsan_copy_to_user);

void kmsan_memmove(void *to, const void *from, size_t size)
{
	if (!kmsan_enabled || kmsan_in_runtime())
		return;

	kmsan_enter_runtime();
	kmsan_internal_memmove_metadata(to, (void *)from, size);
	kmsan_leave_runtime();
}
EXPORT_SYMBOL(kmsan_memmove);

/* Helper function to check an URB. */
void kmsan_handle_urb(const struct urb *urb, bool is_out)
{
	if (!urb)
		return;
	if (is_out)
		kmsan_internal_check_memory(urb->transfer_buffer,
					    urb->transfer_buffer_length,
					    /*user_addr*/ 0, REASON_SUBMIT_URB);
	else
		kmsan_internal_unpoison_memory(urb->transfer_buffer,
					       urb->transfer_buffer_length,
					       /*checked*/ false);
}
EXPORT_SYMBOL_GPL(kmsan_handle_urb);

static void kmsan_handle_dma_page(const void *addr, size_t size,
				  enum dma_data_direction dir)
{
	switch (dir) {
	case DMA_BIDIRECTIONAL:
		kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
					    REASON_ANY);
		kmsan_internal_unpoison_memory((void *)addr, size,
					       /*checked*/ false);
		break;
	case DMA_TO_DEVICE:
		kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
					    REASON_ANY);
		break;
	case DMA_FROM_DEVICE:
		kmsan_internal_unpoison_memory((void *)addr, size,
					       /*checked*/ false);
		break;
	case DMA_NONE:
		break;
	}
}

/* Helper function to handle DMA data transfers. */
void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
		      enum dma_data_direction dir)
{
	u64 page_offset, to_go, addr;

	if (PageHighMem(page))
		return;
	addr = (u64)page_address(page) + offset;
	/*
	 * The kernel may occasionally give us adjacent DMA pages not belonging
	 * to the same allocation. Process them separately to avoid triggering
	 * internal KMSAN checks.
	 */
	while (size > 0) {
		page_offset = offset_in_page(addr);
		to_go = min(PAGE_SIZE - page_offset, (u64)size);
		kmsan_handle_dma_page((void *)addr, to_go, dir);
		addr += to_go;
		size -= to_go;
	}
}

void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
			 enum dma_data_direction dir)
{
	struct scatterlist *item;
	int i;

	for_each_sg(sg, item, nents, i)
		kmsan_handle_dma(sg_page(item), item->offset, item->length,
				 dir);
}

/* Functions from kmsan-checks.h follow. */

/*
 * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it
 * into the stack depot. This may cause deadlocks if done from within KMSAN
 * runtime, therefore we bail out if kmsan_in_runtime().
 */
void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
{
	if (!kmsan_enabled || kmsan_in_runtime())
		return;
	kmsan_enter_runtime();
	/* The users may want to poison/unpoison random memory. */
	kmsan_internal_poison_memory((void *)address, size, flags,
				     KMSAN_POISON_NOCHECK);
	kmsan_leave_runtime();
}
EXPORT_SYMBOL(kmsan_poison_memory);

/*
 * Unlike kmsan_poison_memory(), this function can be used from within KMSAN
 * runtime, because it does not trigger allocations or call instrumented code.
 */
void kmsan_unpoison_memory(const void *address, size_t size)
{
	unsigned long ua_flags;

	if (!kmsan_enabled)
		return;

	ua_flags = user_access_save();
	/* The users may want to poison/unpoison random memory. */
	kmsan_internal_unpoison_memory((void *)address, size,
				       KMSAN_POISON_NOCHECK);
	user_access_restore(ua_flags);
}
EXPORT_SYMBOL(kmsan_unpoison_memory);

/*
 * Version of kmsan_unpoison_memory() called from IRQ entry functions.
 */
void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
{
	kmsan_unpoison_memory((void *)regs, sizeof(*regs));
}

void kmsan_check_memory(const void *addr, size_t size)
{
	if (!kmsan_enabled)
		return;
	return kmsan_internal_check_memory((void *)addr, size, /*user_addr*/ 0,
					   REASON_ANY);
}
EXPORT_SYMBOL(kmsan_check_memory);
Commit	Line	Data
f80be457 AP	1	// SPDX-License-Identifier: GPL-2.0
	2	/*
	3	* KMSAN hooks for kernel subsystems.
	4	*
	5	* These functions handle creation of KMSAN metadata for memory allocations.
	6	*
	7	* Copyright (C) 2018-2022 Google LLC
	8	* Author: Alexander Potapenko <glider@google.com>
	9	*
	10	*/
	11
	12	#include <linux/cacheflush.h>
7ade4f10	13	#include <linux/dma-direction.h>
f80be457	14	#include <linux/gfp.h>
b073d7f8	15	#include <linux/kmsan.h>
f80be457 AP	16	#include <linux/mm.h>
f80be457 AP	17	#include <linux/mm_types.h>
7ade4f10	18	#include <linux/scatterlist.h>
f80be457 AP	19	#include <linux/slab.h>
f80be457 AP	20	#include <linux/uaccess.h>
553a8018	21	#include <linux/usb.h>
f80be457 AP	22
	23	#include "../internal.h"
	24	#include "../slab.h"
	25	#include "kmsan.h"
	26
	27	/*
	28	* Instrumented functions shouldn't be called under
	29	* kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to
	30	* skipping effects of functions like memset() inside instrumented code.
	31	*/
	32
50b5e49c AP	33	void kmsan_task_create(struct task_struct *task)
	34	{
	35	kmsan_enter_runtime();
	36	kmsan_internal_task_create(task);
	37	kmsan_leave_runtime();
	38	}
	39
	40	void kmsan_task_exit(struct task_struct *task)
	41	{
	42	struct kmsan_ctx *ctx = &task->kmsan_ctx;
	43
	44	if (!kmsan_enabled \|\| kmsan_in_runtime())
	45	return;
	46
	47	ctx->allow_reporting = false;
	48	}
	49
68ef169a AP	50	void kmsan_slab_alloc(struct kmem_cache s, void object, gfp_t flags)
	51	{
	52	if (unlikely(object == NULL))
	53	return;
	54	if (!kmsan_enabled \|\| kmsan_in_runtime())
	55	return;
	56	/*
	57	* There's a ctor or this is an RCU cache - do nothing. The memory
	58	* status hasn't changed since last use.
	59	*/
	60	if (s->ctor \|\| (s->flags & SLAB_TYPESAFE_BY_RCU))
	61	return;
	62
	63	kmsan_enter_runtime();
	64	if (flags & __GFP_ZERO)
	65	kmsan_internal_unpoison_memory(object, s->object_size,
	66	KMSAN_POISON_CHECK);
	67	else
	68	kmsan_internal_poison_memory(object, s->object_size, flags,
	69	KMSAN_POISON_CHECK);
	70	kmsan_leave_runtime();
	71	}
	72
	73	void kmsan_slab_free(struct kmem_cache s, void object)
	74	{
	75	if (!kmsan_enabled \|\| kmsan_in_runtime())
	76	return;
	77
	78	/* RCU slabs could be legally used after free within the RCU period */
	79	if (unlikely(s->flags & (SLAB_TYPESAFE_BY_RCU \| SLAB_POISON)))
	80	return;
	81	/*
	82	* If there's a constructor, freed memory must remain in the same state
	83	* until the next allocation. We cannot save its state to detect
	84	* use-after-free bugs, instead we just keep it unpoisoned.
	85	*/
	86	if (s->ctor)
	87	return;
	88	kmsan_enter_runtime();
	89	kmsan_internal_poison_memory(object, s->object_size, GFP_KERNEL,
	90	KMSAN_POISON_CHECK \| KMSAN_POISON_FREE);
	91	kmsan_leave_runtime();
	92	}
	93
	94	void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags)
	95	{
	96	if (unlikely(ptr == NULL))
	97	return;
	98	if (!kmsan_enabled \|\| kmsan_in_runtime())
	99	return;
	100	kmsan_enter_runtime();
	101	if (flags & __GFP_ZERO)
	102	kmsan_internal_unpoison_memory((void *)ptr, size,
	103	/checked/ true);
	104	else
	105	kmsan_internal_poison_memory((void *)ptr, size, flags,
	106	KMSAN_POISON_CHECK);
	107	kmsan_leave_runtime();
	108	}
	109
	110	void kmsan_kfree_large(const void *ptr)
	111	{
	112	struct page *page;
	113
114	if (!kmsan_enabled \|\| kmsan_in_runtime())
115	return;
116	kmsan_enter_runtime();
117	page = virt_to_head_page((void *)ptr);
118	KMSAN_WARN_ON(ptr != page_address(page));
119	kmsan_internal_poison_memory((void *)ptr,
5d7800d9	120	page_size(page),
68ef169a AP	121	GFP_KERNEL,
	122	KMSAN_POISON_CHECK \| KMSAN_POISON_FREE);
	123	kmsan_leave_runtime();
	124	}
	125
b073d7f8 AP	126	static unsigned long vmalloc_shadow(unsigned long addr)
	127	{
	128	return (unsigned long)kmsan_get_metadata((void *)addr,
	129	KMSAN_META_SHADOW);
	130	}
	131
	132	static unsigned long vmalloc_origin(unsigned long addr)
	133	{
	134	return (unsigned long)kmsan_get_metadata((void *)addr,
	135	KMSAN_META_ORIGIN);
	136	}
	137
	138	void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end)
	139	{
	140	__vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end));
	141	__vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end));
	142	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	143	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
	144	}
	145
	146	/*
	147	* This function creates new shadow/origin pages for the physical pages mapped
	148	* into the virtual memory. If those physical pages already had shadow/origin,
	149	* those are ignored.
	150	*/
fdea03e1 AP	151	int kmsan_ioremap_page_range(unsigned long start, unsigned long end,
	152	phys_addr_t phys_addr, pgprot_t prot,
	153	unsigned int page_shift)
b073d7f8 AP	154	{
	155	gfp_t gfp_mask = GFP_KERNEL \| __GFP_ZERO;
	156	struct page shadow, origin;
	157	unsigned long off = 0;
fdea03e1	158	int nr, err = 0, clean = 0, mapped;
b073d7f8 AP	159
b073d7f8 AP	160	if (!kmsan_enabled \|\| kmsan_in_runtime())
fdea03e1	161	return 0;
b073d7f8 AP	162
	163	nr = (end - start) / PAGE_SIZE;
	164	kmsan_enter_runtime();
fdea03e1	165	for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) {
b073d7f8 AP	166	shadow = alloc_pages(gfp_mask, 1);
b073d7f8 AP	167	origin = alloc_pages(gfp_mask, 1);
fdea03e1 AP	168	if (!shadow \|\| !origin) {
	169	err = -ENOMEM;
	170	goto ret;
	171	}
	172	mapped = __vmap_pages_range_noflush(
b073d7f8 AP	173	vmalloc_shadow(start + off),
	174	vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow,
	175	PAGE_SHIFT);
fdea03e1 AP	176	if (mapped) {
	177	err = mapped;
	178	goto ret;
	179	}
	180	shadow = NULL;
	181	mapped = __vmap_pages_range_noflush(
b073d7f8 AP	182	vmalloc_origin(start + off),
	183	vmalloc_origin(start + off + PAGE_SIZE), prot, &origin,
	184	PAGE_SHIFT);
fdea03e1 AP	185	if (mapped) {
	186	__vunmap_range_noflush(
	187	vmalloc_shadow(start + off),
	188	vmalloc_shadow(start + off + PAGE_SIZE));
	189	err = mapped;
	190	goto ret;
	191	}
	192	origin = NULL;
	193	}
	194	/* Page mapping loop finished normally, nothing to clean up. */
	195	clean = 0;
	196
	197	ret:
	198	if (clean > 0) {
	199	/*
	200	* Something went wrong. Clean up shadow/origin pages allocated
	201	* on the last loop iteration, then delete mappings created
	202	* during the previous iterations.
	203	*/
	204	if (shadow)
	205	__free_pages(shadow, 1);
	206	if (origin)
	207	__free_pages(origin, 1);
	208	__vunmap_range_noflush(
	209	vmalloc_shadow(start),
	210	vmalloc_shadow(start + clean * PAGE_SIZE));
	211	__vunmap_range_noflush(
	212	vmalloc_origin(start),
	213	vmalloc_origin(start + clean * PAGE_SIZE));
b073d7f8 AP	214	}
	215	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	216	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
	217	kmsan_leave_runtime();
fdea03e1	218	return err;
b073d7f8 AP	219	}
	220
	221	void kmsan_iounmap_page_range(unsigned long start, unsigned long end)
	222	{
	223	unsigned long v_shadow, v_origin;
	224	struct page shadow, origin;
	225	int nr;
	226
	227	if (!kmsan_enabled \|\| kmsan_in_runtime())
	228	return;
	229
	230	nr = (end - start) / PAGE_SIZE;
	231	kmsan_enter_runtime();
	232	v_shadow = (unsigned long)vmalloc_shadow(start);
	233	v_origin = (unsigned long)vmalloc_origin(start);
	234	for (int i = 0; i < nr;
	235	i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) {
	236	shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow);
	237	origin = kmsan_vmalloc_to_page_or_null((void *)v_origin);
	238	__vunmap_range_noflush(v_shadow, vmalloc_shadow(end));
	239	__vunmap_range_noflush(v_origin, vmalloc_origin(end));
	240	if (shadow)
	241	__free_pages(shadow, 1);
	242	if (origin)
	243	__free_pages(origin, 1);
	244	}
	245	flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end));
	246	flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end));
	247	kmsan_leave_runtime();
	248	}
	249
75cf0290 AP	250	void kmsan_copy_to_user(void __user to, const void from, size_t to_copy,
	251	size_t left)
	252	{
	253	unsigned long ua_flags;
	254
	255	if (!kmsan_enabled \|\| kmsan_in_runtime())
	256	return;
	257	/*
	258	* At this point we've copied the memory already. It's hard to check it
	259	* before copying, as the size of actually copied buffer is unknown.
	260	*/
	261
	262	/* copy_to_user() may copy zero bytes. No need to check. */
	263	if (!to_copy)
	264	return;
	265	/* Or maybe copy_to_user() failed to copy anything. */
	266	if (to_copy <= left)
	267	return;
	268
	269	ua_flags = user_access_save();
	270	if ((u64)to < TASK_SIZE) {
	271	/* This is a user memory access, check it. */
	272	kmsan_internal_check_memory((void *)from, to_copy - left, to,
	273	REASON_COPY_TO_USER);
	274	} else {
	275	/* Otherwise this is a kernel memory access. This happens when a
	276	* compat syscall passes an argument allocated on the kernel
	277	* stack to a real syscall.
	278	* Don't check anything, just copy the shadow of the copied
	279	* bytes.
	280	*/
	281	kmsan_internal_memmove_metadata((void )to, (void )from,
	282	to_copy - left);
	283	}
	284	user_access_restore(ua_flags);
	285	}
	286	EXPORT_SYMBOL(kmsan_copy_to_user);
	287
3429055f AP	288	void kmsan_memmove(void to, const void from, size_t size)
	289	{
	290	if (!kmsan_enabled \|\| kmsan_in_runtime())
	291	return;
	292
	293	kmsan_enter_runtime();
	294	kmsan_internal_memmove_metadata(to, (void *)from, size);
	295	kmsan_leave_runtime();
	296	}
	297	EXPORT_SYMBOL(kmsan_memmove);
	298
553a8018 AP	299	/* Helper function to check an URB. */
	300	void kmsan_handle_urb(const struct urb *urb, bool is_out)
	301	{
	302	if (!urb)
	303	return;
	304	if (is_out)
	305	kmsan_internal_check_memory(urb->transfer_buffer,
	306	urb->transfer_buffer_length,
	307	/user_addr/ 0, REASON_SUBMIT_URB);
	308	else
	309	kmsan_internal_unpoison_memory(urb->transfer_buffer,
	310	urb->transfer_buffer_length,
	311	/checked/ false);
	312	}
7ba594d7	313	EXPORT_SYMBOL_GPL(kmsan_handle_urb);
553a8018	314
7ade4f10 AP	315	static void kmsan_handle_dma_page(const void *addr, size_t size,
	316	enum dma_data_direction dir)
	317	{
	318	switch (dir) {
	319	case DMA_BIDIRECTIONAL:
	320	kmsan_internal_check_memory((void )addr, size, /user_addr*/ 0,
	321	REASON_ANY);
	322	kmsan_internal_unpoison_memory((void *)addr, size,
	323	/checked/ false);
	324	break;
	325	case DMA_TO_DEVICE:
	326	kmsan_internal_check_memory((void )addr, size, /user_addr*/ 0,
	327	REASON_ANY);
	328	break;
	329	case DMA_FROM_DEVICE:
	330	kmsan_internal_unpoison_memory((void *)addr, size,
	331	/checked/ false);
	332	break;
	333	case DMA_NONE:
	334	break;
	335	}
	336	}
	337
	338	/* Helper function to handle DMA data transfers. */
	339	void kmsan_handle_dma(struct page *page, size_t offset, size_t size,
	340	enum dma_data_direction dir)
	341	{
	342	u64 page_offset, to_go, addr;
	343
	344	if (PageHighMem(page))
	345	return;
	346	addr = (u64)page_address(page) + offset;
	347	/*
	348	* The kernel may occasionally give us adjacent DMA pages not belonging
	349	* to the same allocation. Process them separately to avoid triggering
	350	* internal KMSAN checks.
	351	*/
	352	while (size > 0) {
4852a805	353	page_offset = offset_in_page(addr);
7ade4f10 AP	354	to_go = min(PAGE_SIZE - page_offset, (u64)size);
	355	kmsan_handle_dma_page((void *)addr, to_go, dir);
	356	addr += to_go;
	357	size -= to_go;
	358	}
	359	}
	360
	361	void kmsan_handle_dma_sg(struct scatterlist *sg, int nents,
	362	enum dma_data_direction dir)
	363	{
	364	struct scatterlist *item;
	365	int i;
	366
	367	for_each_sg(sg, item, nents, i)
	368	kmsan_handle_dma(sg_page(item), item->offset, item->length,
	369	dir);
	370	}
	371
f80be457	372	/* Functions from kmsan-checks.h follow. */
d749cc75 AP	373
	374	/*
	375	* To create an origin, kmsan_poison_memory() unwinds the stacks and stores it
	376	* into the stack depot. This may cause deadlocks if done from within KMSAN
	377	* runtime, therefore we bail out if kmsan_in_runtime().
	378	*/
f80be457 AP	379	void kmsan_poison_memory(const void *address, size_t size, gfp_t flags)
	380	{
	381	if (!kmsan_enabled \|\| kmsan_in_runtime())
	382	return;
	383	kmsan_enter_runtime();
	384	/* The users may want to poison/unpoison random memory. */
	385	kmsan_internal_poison_memory((void *)address, size, flags,
	386	KMSAN_POISON_NOCHECK);
	387	kmsan_leave_runtime();
	388	}
	389	EXPORT_SYMBOL(kmsan_poison_memory);
	390
d749cc75 AP	391	/*
	392	* Unlike kmsan_poison_memory(), this function can be used from within KMSAN
	393	* runtime, because it does not trigger allocations or call instrumented code.
	394	*/
f80be457 AP	395	void kmsan_unpoison_memory(const void *address, size_t size)
	396	{
	397	unsigned long ua_flags;
	398
d749cc75	399	if (!kmsan_enabled)
f80be457 AP	400	return;
	401
	402	ua_flags = user_access_save();
f80be457 AP	403	/* The users may want to poison/unpoison random memory. */
	404	kmsan_internal_unpoison_memory((void *)address, size,
	405	KMSAN_POISON_NOCHECK);
f80be457 AP	406	user_access_restore(ua_flags);
	407	}
	408	EXPORT_SYMBOL(kmsan_unpoison_memory);
	409
6cae637f	410	/*
d749cc75	411	* Version of kmsan_unpoison_memory() called from IRQ entry functions.
6cae637f AP	412	*/
	413	void kmsan_unpoison_entry_regs(const struct pt_regs *regs)
	414	{
d749cc75	415	kmsan_unpoison_memory((void )regs, sizeof(regs));
6cae637f AP	416	}
6cae637f AP	417
f80be457 AP	418	void kmsan_check_memory(const void *addr, size_t size)
	419	{
	420	if (!kmsan_enabled)
	421	return;
	422	return kmsan_internal_check_memory((void )addr, size, /user_addr*/ 0,
	423	REASON_ANY);
	424	}
	425	EXPORT_SYMBOL(kmsan_check_memory);