1 #include <linux/kernel.h>
2 #include <linux/errno.h>
3 #include <linux/file.h>
4 #include <linux/slab.h>
6 #include <linux/io_uring.h>
12 static void __io_notif_complete_tw(struct io_kiocb
*notif
, bool *locked
)
14 struct io_notif_data
*nd
= io_notif_to_data(notif
);
15 struct io_ring_ctx
*ctx
= notif
->ctx
;
17 if (nd
->account_pages
&& ctx
->user
) {
18 __io_unaccount_mem(ctx
->user
, nd
->account_pages
);
19 nd
->account_pages
= 0;
21 io_req_task_complete(notif
, locked
);
24 static inline void io_notif_complete(struct io_kiocb
*notif
)
25 __must_hold(¬if
->ctx
->uring_lock
)
29 __io_notif_complete_tw(notif
, &locked
);
32 static void io_uring_tx_zerocopy_callback(struct sk_buff
*skb
,
33 struct ubuf_info
*uarg
,
36 struct io_notif_data
*nd
= container_of(uarg
, struct io_notif_data
, uarg
);
37 struct io_kiocb
*notif
= cmd_to_io_kiocb(nd
);
39 if (refcount_dec_and_test(&uarg
->refcnt
)) {
40 notif
->io_task_work
.func
= __io_notif_complete_tw
;
41 io_req_task_work_add(notif
);
45 struct io_kiocb
*io_alloc_notif(struct io_ring_ctx
*ctx
)
46 __must_hold(&ctx
->uring_lock
)
48 struct io_kiocb
*notif
;
49 struct io_notif_data
*nd
;
51 if (unlikely(!io_alloc_req_refill(ctx
)))
53 notif
= io_alloc_req(ctx
);
54 notif
->opcode
= IORING_OP_NOP
;
57 notif
->task
= current
;
59 notif
->rsrc_node
= NULL
;
60 io_req_set_rsrc_node(notif
, ctx
, 0);
62 nd
= io_notif_to_data(notif
);
63 nd
->account_pages
= 0;
64 nd
->uarg
.flags
= SKBFL_ZEROCOPY_FRAG
| SKBFL_DONT_ORPHAN
;
65 nd
->uarg
.callback
= io_uring_tx_zerocopy_callback
;
66 refcount_set(&nd
->uarg
.refcnt
, 1);
70 void io_notif_flush(struct io_kiocb
*notif
)
71 __must_hold(&slot
->notif
->ctx
->uring_lock
)
73 struct io_notif_data
*nd
= io_notif_to_data(notif
);
75 /* drop slot's master ref */
76 if (refcount_dec_and_test(&nd
->uarg
.refcnt
)) {
77 notif
->io_task_work
.func
= __io_notif_complete_tw
;
78 io_req_task_work_add(notif
);