--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -3612,6 +3612,7 @@ static __cold int io_uring_mmap(struct f
+@@ -3613,6 +3613,7 @@ static __cold int io_uring_mmap(struct f
struct io_ring_ctx *ctx = file->private_data;
size_t sz = vma->vm_end - vma->vm_start;
long offset = vma->vm_pgoff << PAGE_SHIFT;
unsigned long pfn;
void *ptr;
-@@ -3622,8 +3623,8 @@ static __cold int io_uring_mmap(struct f
+@@ -3623,8 +3624,8 @@ static __cold int io_uring_mmap(struct f
switch (offset & IORING_OFF_MMAP_MASK) {
case IORING_OFF_SQ_RING:
case IORING_OFF_CQ_RING:
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -2889,6 +2889,8 @@ static void *io_pages_map(struct page **
+@@ -2890,6 +2890,8 @@ static void *io_pages_map(struct page **
ret = io_mem_alloc_compound(pages, nr_pages, size, gfp);
if (!IS_ERR(ret))
goto done;
ret = io_mem_alloc_single(pages, nr_pages, size, gfp);
if (!IS_ERR(ret)) {
-@@ -2897,7 +2899,7 @@ done:
+@@ -2898,7 +2900,7 @@ done:
*npages = nr_pages;
return ret;
}
-From bfaf932689d7e59e3558977854b74bde1b137fae Mon Sep 17 00:00:00 2001
+From 063cda8806cf277c56e4bfc7163e5faa355e177d Mon Sep 17 00:00:00 2001
From: Jens Axboe <axboe@kernel.dk>
Date: Wed, 13 Mar 2024 09:56:14 -0600
Subject: io_uring: get rid of remap_pfn_range() for mapping rings/sqes
Signed-off-by: Jens Axboe <axboe@kernel.dk>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>
---
- io_uring/io_uring.c | 139 +++++++++++++++++++++++++++++++++++++++++++++++++---
+ io_uring/io_uring.c | 140 +++++++++++++++++++++++++++++++++++++++++++++++++---
io_uring/io_uring.h | 2
- 2 files changed, 133 insertions(+), 8 deletions(-)
+ 2 files changed, 134 insertions(+), 8 deletions(-)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -2683,6 +2683,36 @@ static int io_cqring_wait(struct io_ring
+@@ -71,6 +71,7 @@
+ #include <linux/io_uring.h>
+ #include <linux/audit.h>
+ #include <linux/security.h>
++#include <linux/vmalloc.h>
+ #include <asm/shmparam.h>
+
+ #define CREATE_TRACE_POINTS
+@@ -2683,6 +2684,36 @@ static int io_cqring_wait(struct io_ring
return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
}
void io_mem_free(void *ptr)
{
if (!ptr)
-@@ -2787,8 +2817,8 @@ static void *io_sqes_map(struct io_ring_
+@@ -2787,8 +2818,8 @@ static void *io_sqes_map(struct io_ring_
static void io_rings_free(struct io_ring_ctx *ctx)
{
if (!(ctx->flags & IORING_SETUP_NO_MMAP)) {
} else {
io_pages_free(&ctx->ring_pages, ctx->n_ring_pages);
ctx->n_ring_pages = 0;
-@@ -2800,6 +2830,80 @@ static void io_rings_free(struct io_ring
+@@ -2800,6 +2831,80 @@ static void io_rings_free(struct io_ring
ctx->sq_sqes = NULL;
}
void *io_mem_alloc(size_t size)
{
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
-@@ -3463,14 +3567,12 @@ static void *io_uring_validate_mmap_requ
+@@ -3463,14 +3568,12 @@ static void *io_uring_validate_mmap_requ
/* Don't allow mmap if the ring was setup without it */
if (ctx->flags & IORING_SETUP_NO_MMAP)
return ERR_PTR(-EINVAL);
case IORING_OFF_PBUF_RING: {
struct io_buffer_list *bl;
unsigned int bgid;
-@@ -3494,11 +3596,22 @@ static void *io_uring_validate_mmap_requ
+@@ -3494,11 +3597,22 @@ static void *io_uring_validate_mmap_requ
return ptr;
}
unsigned long pfn;
void *ptr;
-@@ -3506,6 +3619,16 @@ static __cold int io_uring_mmap(struct f
+@@ -3506,6 +3620,16 @@ static __cold int io_uring_mmap(struct f
if (IS_ERR(ptr))
return PTR_ERR(ptr);
pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}
-@@ -3795,7 +3918,7 @@ static __cold int io_allocate_scq_urings
+@@ -3795,7 +3919,7 @@ static __cold int io_allocate_scq_urings
return -EOVERFLOW;
if (!(ctx->flags & IORING_SETUP_NO_MMAP))
else
rings = io_rings_map(ctx, p->cq_off.user_addr, size);
-@@ -3820,7 +3943,7 @@ static __cold int io_allocate_scq_urings
+@@ -3820,7 +3944,7 @@ static __cold int io_allocate_scq_urings
}
if (!(ctx->flags & IORING_SETUP_NO_MMAP))
struct io_restriction restrictions;
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -311,7 +311,6 @@ static __cold struct io_ring_ctx *io_rin
+@@ -312,7 +312,6 @@ static __cold struct io_ring_ctx *io_rin
INIT_LIST_HEAD(&ctx->sqd_list);
INIT_LIST_HEAD(&ctx->cq_overflow_list);
INIT_LIST_HEAD(&ctx->io_buffers_cache);
io_alloc_cache_init(&ctx->rsrc_node_cache, IO_NODE_ALLOC_CACHE_MAX,
sizeof(struct io_rsrc_node));
io_alloc_cache_init(&ctx->apoll_cache, IO_ALLOC_CACHE_MAX,
-@@ -2682,15 +2681,15 @@ static int io_cqring_wait(struct io_ring
+@@ -2683,15 +2682,15 @@ static int io_cqring_wait(struct io_ring
return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
}
struct page **to_free = *pages;
int i;
-@@ -2712,14 +2711,6 @@ static void io_pages_unmap(void *ptr, st
+@@ -2713,14 +2712,6 @@ static void io_pages_unmap(void *ptr, st
*npages = 0;
}
static void io_pages_free(struct page ***pages, int npages)
{
struct page **page_array;
-@@ -2818,8 +2809,10 @@ static void *io_sqes_map(struct io_ring_
+@@ -2819,8 +2810,10 @@ static void *io_sqes_map(struct io_ring_
static void io_rings_free(struct io_ring_ctx *ctx)
{
if (!(ctx->flags & IORING_SETUP_NO_MMAP)) {
} else {
io_pages_free(&ctx->ring_pages, ctx->n_ring_pages);
ctx->n_ring_pages = 0;
-@@ -2876,8 +2869,8 @@ err:
+@@ -2877,8 +2870,8 @@ err:
return ERR_PTR(-ENOMEM);
}
{
gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN;
struct page **pages;
-@@ -2909,17 +2902,6 @@ fail:
+@@ -2910,17 +2903,6 @@ fail:
return ret;
}
static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset)
{
-@@ -3073,7 +3055,6 @@ static __cold void io_ring_ctx_free(stru
+@@ -3074,7 +3056,6 @@ static __cold void io_ring_ctx_free(stru
ctx->mm_account = NULL;
}
io_rings_free(ctx);
percpu_ref_exit(&ctx->refs);
free_uid(ctx->user);
-@@ -3563,10 +3544,8 @@ static void *io_uring_validate_mmap_requ
+@@ -3564,10 +3545,8 @@ static void *io_uring_validate_mmap_requ
{
struct io_ring_ctx *ctx = file->private_data;
loff_t offset = pgoff << PAGE_SHIFT;
case IORING_OFF_SQ_RING:
case IORING_OFF_CQ_RING:
/* Don't allow mmap if the ring was setup without it */
-@@ -3581,6 +3560,7 @@ static void *io_uring_validate_mmap_requ
+@@ -3582,6 +3561,7 @@ static void *io_uring_validate_mmap_requ
case IORING_OFF_PBUF_RING: {
struct io_buffer_list *bl;
unsigned int bgid;
bgid = (offset & ~IORING_OFF_MMAP_MASK) >> IORING_OFF_PBUF_SHIFT;
bl = io_pbuf_get_bl(ctx, bgid);
-@@ -3588,17 +3568,11 @@ static void *io_uring_validate_mmap_requ
+@@ -3589,17 +3569,11 @@ static void *io_uring_validate_mmap_requ
return bl;
ptr = bl->buf_ring;
io_put_bl(ctx, bl);
}
int io_uring_mmap_pages(struct io_ring_ctx *ctx, struct vm_area_struct *vma,
-@@ -3618,7 +3592,6 @@ static __cold int io_uring_mmap(struct f
+@@ -3619,7 +3593,6 @@ static __cold int io_uring_mmap(struct f
size_t sz = vma->vm_end - vma->vm_start;
long offset = vma->vm_pgoff << PAGE_SHIFT;
unsigned int npages;
void *ptr;
ptr = io_uring_validate_mmap_request(file, vma->vm_pgoff, sz);
-@@ -3633,10 +3606,11 @@ static __cold int io_uring_mmap(struct f
+@@ -3634,10 +3607,11 @@ static __cold int io_uring_mmap(struct f
case IORING_OFF_SQES:
return io_uring_mmap_pages(ctx, vma, ctx->sqe_pages,
ctx->n_sqe_pages);
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -2738,33 +2738,57 @@ static void io_pages_free(struct page **
+@@ -2739,33 +2739,57 @@ static void io_pages_free(struct page **
*pages = NULL;
}
page_addr = vmap(page_array, nr_pages, VM_MAP, PAGE_KERNEL);
if (page_addr) {
-@@ -2772,10 +2796,9 @@ static void *__io_uaddr_map(struct page
+@@ -2773,10 +2797,9 @@ static void *__io_uaddr_map(struct page
*npages = nr_pages;
return page_addr;
}
#include <linux/fsnotify.h>
#include <linux/fadvise.h>
#include <linux/task_work.h>
-@@ -2745,7 +2744,7 @@ static void *__io_uaddr_map(struct page
+@@ -2746,7 +2745,7 @@ static void *__io_uaddr_map(struct page
struct page **page_array;
unsigned int nr_pages;
void *page_addr;
*npages = 0;
-@@ -2767,34 +2766,13 @@ static void *__io_uaddr_map(struct page
+@@ -2768,34 +2767,13 @@ static void *__io_uaddr_map(struct page
goto free_pages;
}
free_pages:
io_pages_free(&page_array, pinned > 0 ? pinned : 0);
return ERR_PTR(ret);
-@@ -2824,6 +2802,8 @@ static void io_rings_free(struct io_ring
+@@ -2825,6 +2803,8 @@ static void io_rings_free(struct io_ring
ctx->n_ring_pages = 0;
io_pages_free(&ctx->sqe_pages, ctx->n_sqe_pages);
ctx->n_sqe_pages = 0;