-From 6c7c93b9211b6a6c625d4e8102659470742cb27f Mon Sep 17 00:00:00 2001
+From 8c3571a1ebeca6037fa49f6192fd8fe45c61245c Mon Sep 17 00:00:00 2001
From: Jens Axboe <axboe@kernel.dk>
Date: Wed, 13 Mar 2024 09:56:14 -0600
Subject: io_uring: get rid of remap_pfn_range() for mapping rings/sqes
---
include/linux/io_uring_types.h | 5 +
include/uapi/linux/io_uring.h | 1
- io_uring/io_uring.c | 132 ++++++++++++++++++++++++++++++++++++-----
+ io_uring/io_uring.c | 133 ++++++++++++++++++++++++++++++++++++-----
io_uring/io_uring.h | 2
- 4 files changed, 124 insertions(+), 16 deletions(-)
+ 4 files changed, 125 insertions(+), 16 deletions(-)
--- a/include/linux/io_uring_types.h
+++ b/include/linux/io_uring_types.h
* Filled with the offset for mmap(2)
--- a/io_uring/io_uring.c
+++ b/io_uring/io_uring.c
-@@ -2513,37 +2513,118 @@ static int io_cqring_wait(struct io_ring
+@@ -71,6 +71,7 @@
+ #include <linux/io_uring.h>
+ #include <linux/audit.h>
+ #include <linux/security.h>
++#include <linux/vmalloc.h>
+ #include <asm/shmparam.h>
+
+ #define CREATE_TRACE_POINTS
+@@ -2513,37 +2514,118 @@ static int io_cqring_wait(struct io_ring
return READ_ONCE(rings->cq.head) == READ_ONCE(rings->cq.tail) ? ret : 0;
}
- gfp_t gfp = GFP_KERNEL_ACCOUNT | __GFP_ZERO | __GFP_NOWARN | __GFP_COMP;
void *ret;
+ int i;
-+
+
+- ret = (void *) __get_free_pages(gfp, get_order(size));
+ for (i = 0; i < nr_pages; i++) {
+ pages[i] = alloc_page(gfp);
+ if (!pages[i])
+ goto err;
+ }
-
-- ret = (void *) __get_free_pages(gfp, get_order(size));
++
+ ret = vmap(pages, nr_pages, VM_MAP, PAGE_KERNEL);
if (ret)
return ret;
static unsigned long rings_size(struct io_ring_ctx *ctx, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset)
{
-@@ -3125,11 +3206,9 @@ static void *io_uring_validate_mmap_requ
+@@ -3125,11 +3207,9 @@ static void *io_uring_validate_mmap_requ
switch (offset) {
case IORING_OFF_SQ_RING:
case IORING_OFF_CQ_RING:
default:
return ERR_PTR(-EINVAL);
}
-@@ -3141,11 +3220,22 @@ static void *io_uring_validate_mmap_requ
+@@ -3141,11 +3221,22 @@ static void *io_uring_validate_mmap_requ
return ptr;
}
unsigned long pfn;
void *ptr;
-@@ -3153,6 +3243,16 @@ static __cold int io_uring_mmap(struct f
+@@ -3153,6 +3244,16 @@ static __cold int io_uring_mmap(struct f
if (IS_ERR(ptr))
return PTR_ERR(ptr);
pfn = virt_to_phys(ptr) >> PAGE_SHIFT;
return remap_pfn_range(vma, vma->vm_start, pfn, sz, vma->vm_page_prot);
}
-@@ -3443,7 +3543,7 @@ static __cold int io_allocate_scq_urings
+@@ -3443,7 +3544,7 @@ static __cold int io_allocate_scq_urings
if (size == SIZE_MAX)
return -EOVERFLOW;
if (IS_ERR(rings))
return PTR_ERR(rings);
-@@ -3463,7 +3563,7 @@ static __cold int io_allocate_scq_urings
+@@ -3463,7 +3564,7 @@ static __cold int io_allocate_scq_urings
return -EOVERFLOW;
}