]>
Commit | Line | Data |
---|---|---|
eb42cebb PB |
1 | #include <linux/kernel.h> |
2 | #include <linux/errno.h> | |
3 | #include <linux/file.h> | |
4 | #include <linux/slab.h> | |
5 | #include <linux/net.h> | |
6 | #include <linux/io_uring.h> | |
7 | ||
8 | #include "io_uring.h" | |
9 | #include "notif.h" | |
68ef5578 | 10 | #include "rsrc.h" |
eb42cebb PB |
11 | |
12 | static void __io_notif_complete_tw(struct callback_head *cb) | |
13 | { | |
14 | struct io_notif *notif = container_of(cb, struct io_notif, task_work); | |
68ef5578 | 15 | struct io_rsrc_node *rsrc_node = notif->rsrc_node; |
eb42cebb PB |
16 | struct io_ring_ctx *ctx = notif->ctx; |
17 | ||
e58d498e PB |
18 | if (likely(notif->task)) { |
19 | io_put_task(notif->task, 1); | |
20 | notif->task = NULL; | |
21 | } | |
22 | ||
eb42cebb PB |
23 | io_cq_lock(ctx); |
24 | io_fill_cqe_aux(ctx, notif->tag, 0, notif->seq, true); | |
eb4a299b PB |
25 | |
26 | list_add(¬if->cache_node, &ctx->notif_list_locked); | |
27 | ctx->notif_locked_nr++; | |
eb42cebb PB |
28 | io_cq_unlock_post(ctx); |
29 | ||
68ef5578 | 30 | io_rsrc_put_node(rsrc_node, 1); |
eb42cebb | 31 | percpu_ref_put(&ctx->refs); |
eb42cebb PB |
32 | } |
33 | ||
34 | static inline void io_notif_complete(struct io_notif *notif) | |
35 | { | |
36 | __io_notif_complete_tw(¬if->task_work); | |
37 | } | |
38 | ||
39 | static void io_notif_complete_wq(struct work_struct *work) | |
40 | { | |
41 | struct io_notif *notif = container_of(work, struct io_notif, commit_work); | |
42 | ||
43 | io_notif_complete(notif); | |
44 | } | |
45 | ||
46 | static void io_uring_tx_zerocopy_callback(struct sk_buff *skb, | |
47 | struct ubuf_info *uarg, | |
48 | bool success) | |
49 | { | |
50 | struct io_notif *notif = container_of(uarg, struct io_notif, uarg); | |
51 | ||
52 | if (!refcount_dec_and_test(&uarg->refcnt)) | |
53 | return; | |
e58d498e PB |
54 | |
55 | if (likely(notif->task)) { | |
56 | init_task_work(¬if->task_work, __io_notif_complete_tw); | |
57 | if (likely(!task_work_add(notif->task, ¬if->task_work, | |
58 | TWA_SIGNAL))) | |
59 | return; | |
60 | } | |
61 | ||
eb42cebb PB |
62 | INIT_WORK(¬if->commit_work, io_notif_complete_wq); |
63 | queue_work(system_unbound_wq, ¬if->commit_work); | |
64 | } | |
65 | ||
eb4a299b PB |
66 | static void io_notif_splice_cached(struct io_ring_ctx *ctx) |
67 | __must_hold(&ctx->uring_lock) | |
68 | { | |
69 | spin_lock(&ctx->completion_lock); | |
70 | list_splice_init(&ctx->notif_list_locked, &ctx->notif_list); | |
71 | ctx->notif_locked_nr = 0; | |
72 | spin_unlock(&ctx->completion_lock); | |
73 | } | |
74 | ||
75 | void io_notif_cache_purge(struct io_ring_ctx *ctx) | |
76 | __must_hold(&ctx->uring_lock) | |
77 | { | |
78 | io_notif_splice_cached(ctx); | |
79 | ||
80 | while (!list_empty(&ctx->notif_list)) { | |
81 | struct io_notif *notif = list_first_entry(&ctx->notif_list, | |
82 | struct io_notif, cache_node); | |
83 | ||
84 | list_del(¬if->cache_node); | |
85 | kfree(notif); | |
86 | } | |
87 | } | |
88 | ||
89 | static inline bool io_notif_has_cached(struct io_ring_ctx *ctx) | |
90 | __must_hold(&ctx->uring_lock) | |
91 | { | |
92 | if (likely(!list_empty(&ctx->notif_list))) | |
93 | return true; | |
94 | if (data_race(READ_ONCE(ctx->notif_locked_nr) <= IO_NOTIF_SPLICE_BATCH)) | |
95 | return false; | |
96 | io_notif_splice_cached(ctx); | |
97 | return !list_empty(&ctx->notif_list); | |
98 | } | |
99 | ||
eb42cebb PB |
100 | struct io_notif *io_alloc_notif(struct io_ring_ctx *ctx, |
101 | struct io_notif_slot *slot) | |
102 | __must_hold(&ctx->uring_lock) | |
103 | { | |
104 | struct io_notif *notif; | |
105 | ||
eb4a299b PB |
106 | if (likely(io_notif_has_cached(ctx))) { |
107 | notif = list_first_entry(&ctx->notif_list, | |
108 | struct io_notif, cache_node); | |
109 | list_del(¬if->cache_node); | |
110 | } else { | |
111 | notif = kzalloc(sizeof(*notif), GFP_ATOMIC | __GFP_ACCOUNT); | |
112 | if (!notif) | |
113 | return NULL; | |
114 | /* pre-initialise some fields */ | |
115 | notif->ctx = ctx; | |
116 | notif->uarg.flags = SKBFL_ZEROCOPY_FRAG | SKBFL_DONT_ORPHAN; | |
117 | notif->uarg.callback = io_uring_tx_zerocopy_callback; | |
118 | } | |
eb42cebb PB |
119 | |
120 | notif->seq = slot->seq++; | |
121 | notif->tag = slot->tag; | |
eb42cebb PB |
122 | /* master ref owned by io_notif_slot, will be dropped on flush */ |
123 | refcount_set(¬if->uarg.refcnt, 1); | |
124 | percpu_ref_get(&ctx->refs); | |
68ef5578 PB |
125 | notif->rsrc_node = ctx->rsrc_node; |
126 | io_charge_rsrc_node(ctx); | |
eb42cebb PB |
127 | return notif; |
128 | } | |
129 | ||
130 | static void io_notif_slot_flush(struct io_notif_slot *slot) | |
131 | __must_hold(&ctx->uring_lock) | |
132 | { | |
133 | struct io_notif *notif = slot->notif; | |
134 | ||
135 | slot->notif = NULL; | |
136 | ||
137 | if (WARN_ON_ONCE(in_interrupt())) | |
138 | return; | |
139 | /* drop slot's master ref */ | |
140 | if (refcount_dec_and_test(¬if->uarg.refcnt)) | |
141 | io_notif_complete(notif); | |
142 | } | |
143 | ||
144 | __cold int io_notif_unregister(struct io_ring_ctx *ctx) | |
145 | __must_hold(&ctx->uring_lock) | |
146 | { | |
147 | int i; | |
148 | ||
149 | if (!ctx->notif_slots) | |
150 | return -ENXIO; | |
151 | ||
152 | for (i = 0; i < ctx->nr_notif_slots; i++) { | |
153 | struct io_notif_slot *slot = &ctx->notif_slots[i]; | |
154 | ||
e58d498e PB |
155 | if (!slot->notif) |
156 | continue; | |
157 | if (WARN_ON_ONCE(slot->notif->task)) | |
158 | slot->notif->task = NULL; | |
159 | io_notif_slot_flush(slot); | |
eb42cebb PB |
160 | } |
161 | ||
162 | kvfree(ctx->notif_slots); | |
163 | ctx->notif_slots = NULL; | |
164 | ctx->nr_notif_slots = 0; | |
bc24d6bd PB |
165 | io_notif_cache_purge(ctx); |
166 | return 0; | |
167 | } | |
168 | ||
169 | __cold int io_notif_register(struct io_ring_ctx *ctx, | |
170 | void __user *arg, unsigned int size) | |
171 | __must_hold(&ctx->uring_lock) | |
172 | { | |
173 | struct io_uring_notification_slot __user *slots; | |
174 | struct io_uring_notification_slot slot; | |
175 | struct io_uring_notification_register reg; | |
176 | unsigned i; | |
177 | ||
178 | if (ctx->nr_notif_slots) | |
179 | return -EBUSY; | |
180 | if (size != sizeof(reg)) | |
181 | return -EINVAL; | |
182 | if (copy_from_user(®, arg, sizeof(reg))) | |
183 | return -EFAULT; | |
184 | if (!reg.nr_slots || reg.nr_slots > IORING_MAX_NOTIF_SLOTS) | |
185 | return -EINVAL; | |
186 | if (reg.resv || reg.resv2 || reg.resv3) | |
187 | return -EINVAL; | |
188 | ||
189 | slots = u64_to_user_ptr(reg.data); | |
190 | ctx->notif_slots = kvcalloc(reg.nr_slots, sizeof(ctx->notif_slots[0]), | |
191 | GFP_KERNEL_ACCOUNT); | |
192 | if (!ctx->notif_slots) | |
193 | return -ENOMEM; | |
194 | ||
195 | for (i = 0; i < reg.nr_slots; i++, ctx->nr_notif_slots++) { | |
196 | struct io_notif_slot *notif_slot = &ctx->notif_slots[i]; | |
197 | ||
198 | if (copy_from_user(&slot, &slots[i], sizeof(slot))) { | |
199 | io_notif_unregister(ctx); | |
200 | return -EFAULT; | |
201 | } | |
202 | if (slot.resv[0] | slot.resv[1] | slot.resv[2]) { | |
203 | io_notif_unregister(ctx); | |
204 | return -EINVAL; | |
205 | } | |
206 | notif_slot->tag = slot.tag; | |
207 | } | |
eb42cebb | 208 | return 0; |
e58d498e | 209 | } |