1 // SPDX-License-Identifier: GPL-2.0
3 * io_uring opcode handling table
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
26 #include "openclose.h"
27 #include "uring_cmd.h"
39 static int io_no_issue(struct io_kiocb
*req
, unsigned int issue_flags
)
45 static __maybe_unused
int io_eopnotsupp_prep(struct io_kiocb
*kiocb
,
46 const struct io_uring_sqe
*sqe
)
51 const struct io_issue_def io_issue_defs
[] = {
60 .unbound_nonreg_file
= 1,
72 [IORING_OP_WRITEV
] = {
75 .unbound_nonreg_file
= 1,
89 .prep
= io_fsync_prep
,
92 [IORING_OP_READ_FIXED
] = {
94 .unbound_nonreg_file
= 1,
101 .prep
= io_prep_rw_fixed
,
104 [IORING_OP_WRITE_FIXED
] = {
107 .unbound_nonreg_file
= 1,
114 .prep
= io_prep_rw_fixed
,
117 [IORING_OP_POLL_ADD
] = {
119 .unbound_nonreg_file
= 1,
121 .prep
= io_poll_add_prep
,
122 .issue
= io_poll_add
,
124 [IORING_OP_POLL_REMOVE
] = {
126 .prep
= io_poll_remove_prep
,
127 .issue
= io_poll_remove
,
129 [IORING_OP_SYNC_FILE_RANGE
] = {
133 .issue
= io_sync_file_range
,
135 [IORING_OP_SENDMSG
] = {
137 .unbound_nonreg_file
= 1,
141 #if defined(CONFIG_NET)
142 .prep
= io_sendmsg_prep
,
145 .prep
= io_eopnotsupp_prep
,
148 [IORING_OP_RECVMSG
] = {
150 .unbound_nonreg_file
= 1,
155 #if defined(CONFIG_NET)
156 .prep
= io_recvmsg_prep
,
159 .prep
= io_eopnotsupp_prep
,
162 [IORING_OP_TIMEOUT
] = {
164 .prep
= io_timeout_prep
,
167 [IORING_OP_TIMEOUT_REMOVE
] = {
168 /* used by timeout updates' prep() */
170 .prep
= io_timeout_remove_prep
,
171 .issue
= io_timeout_remove
,
173 [IORING_OP_ACCEPT
] = {
175 .unbound_nonreg_file
= 1,
178 .ioprio
= 1, /* used for flags */
179 #if defined(CONFIG_NET)
180 .prep
= io_accept_prep
,
183 .prep
= io_eopnotsupp_prep
,
186 [IORING_OP_ASYNC_CANCEL
] = {
188 .prep
= io_async_cancel_prep
,
189 .issue
= io_async_cancel
,
191 [IORING_OP_LINK_TIMEOUT
] = {
193 .prep
= io_link_timeout_prep
,
194 .issue
= io_no_issue
,
196 [IORING_OP_CONNECT
] = {
198 .unbound_nonreg_file
= 1,
200 #if defined(CONFIG_NET)
201 .prep
= io_connect_prep
,
204 .prep
= io_eopnotsupp_prep
,
207 [IORING_OP_FALLOCATE
] = {
209 .prep
= io_fallocate_prep
,
210 .issue
= io_fallocate
,
212 [IORING_OP_OPENAT
] = {
213 .prep
= io_openat_prep
,
216 [IORING_OP_CLOSE
] = {
217 .prep
= io_close_prep
,
220 [IORING_OP_FILES_UPDATE
] = {
223 .prep
= io_files_update_prep
,
224 .issue
= io_files_update
,
226 [IORING_OP_STATX
] = {
228 .prep
= io_statx_prep
,
233 .unbound_nonreg_file
= 1,
244 [IORING_OP_WRITE
] = {
247 .unbound_nonreg_file
= 1,
257 [IORING_OP_FADVISE
] = {
260 .prep
= io_fadvise_prep
,
263 [IORING_OP_MADVISE
] = {
265 .prep
= io_madvise_prep
,
270 .unbound_nonreg_file
= 1,
275 #if defined(CONFIG_NET)
276 .prep
= io_sendmsg_prep
,
279 .prep
= io_eopnotsupp_prep
,
284 .unbound_nonreg_file
= 1,
289 #if defined(CONFIG_NET)
290 .prep
= io_recvmsg_prep
,
293 .prep
= io_eopnotsupp_prep
,
296 [IORING_OP_OPENAT2
] = {
297 .prep
= io_openat2_prep
,
300 [IORING_OP_EPOLL_CTL
] = {
301 .unbound_nonreg_file
= 1,
303 #if defined(CONFIG_EPOLL)
304 .prep
= io_epoll_ctl_prep
,
305 .issue
= io_epoll_ctl
,
307 .prep
= io_eopnotsupp_prep
,
310 [IORING_OP_SPLICE
] = {
313 .unbound_nonreg_file
= 1,
315 .prep
= io_splice_prep
,
318 [IORING_OP_PROVIDE_BUFFERS
] = {
321 .prep
= io_provide_buffers_prep
,
322 .issue
= io_provide_buffers
,
324 [IORING_OP_REMOVE_BUFFERS
] = {
327 .prep
= io_remove_buffers_prep
,
328 .issue
= io_remove_buffers
,
333 .unbound_nonreg_file
= 1,
338 [IORING_OP_SHUTDOWN
] = {
340 #if defined(CONFIG_NET)
341 .prep
= io_shutdown_prep
,
342 .issue
= io_shutdown
,
344 .prep
= io_eopnotsupp_prep
,
347 [IORING_OP_RENAMEAT
] = {
348 .prep
= io_renameat_prep
,
349 .issue
= io_renameat
,
351 [IORING_OP_UNLINKAT
] = {
352 .prep
= io_unlinkat_prep
,
353 .issue
= io_unlinkat
,
355 [IORING_OP_MKDIRAT
] = {
356 .prep
= io_mkdirat_prep
,
359 [IORING_OP_SYMLINKAT
] = {
360 .prep
= io_symlinkat_prep
,
361 .issue
= io_symlinkat
,
363 [IORING_OP_LINKAT
] = {
364 .prep
= io_linkat_prep
,
367 [IORING_OP_MSG_RING
] = {
370 .prep
= io_msg_ring_prep
,
371 .issue
= io_msg_ring
,
373 [IORING_OP_FSETXATTR
] = {
375 .prep
= io_fsetxattr_prep
,
376 .issue
= io_fsetxattr
,
378 [IORING_OP_SETXATTR
] = {
379 .prep
= io_setxattr_prep
,
380 .issue
= io_setxattr
,
382 [IORING_OP_FGETXATTR
] = {
384 .prep
= io_fgetxattr_prep
,
385 .issue
= io_fgetxattr
,
387 [IORING_OP_GETXATTR
] = {
388 .prep
= io_getxattr_prep
,
389 .issue
= io_getxattr
,
391 [IORING_OP_SOCKET
] = {
393 #if defined(CONFIG_NET)
394 .prep
= io_socket_prep
,
397 .prep
= io_eopnotsupp_prep
,
400 [IORING_OP_URING_CMD
] = {
405 .prep
= io_uring_cmd_prep
,
406 .issue
= io_uring_cmd
,
408 [IORING_OP_SEND_ZC
] = {
410 .unbound_nonreg_file
= 1,
415 #if defined(CONFIG_NET)
416 .prep
= io_send_zc_prep
,
419 .prep
= io_eopnotsupp_prep
,
422 [IORING_OP_SENDMSG_ZC
] = {
424 .unbound_nonreg_file
= 1,
428 #if defined(CONFIG_NET)
429 .prep
= io_send_zc_prep
,
430 .issue
= io_sendmsg_zc
,
432 .prep
= io_eopnotsupp_prep
,
435 [IORING_OP_READ_MULTISHOT
] = {
437 .unbound_nonreg_file
= 1,
441 .prep
= io_read_mshot_prep
,
442 .issue
= io_read_mshot
,
444 [IORING_OP_WAITID
] = {
445 .prep
= io_waitid_prep
,
448 [IORING_OP_FUTEX_WAIT
] = {
449 #if defined(CONFIG_FUTEX)
450 .prep
= io_futex_prep
,
451 .issue
= io_futex_wait
,
453 .prep
= io_eopnotsupp_prep
,
456 [IORING_OP_FUTEX_WAKE
] = {
457 #if defined(CONFIG_FUTEX)
458 .prep
= io_futex_prep
,
459 .issue
= io_futex_wake
,
461 .prep
= io_eopnotsupp_prep
,
464 [IORING_OP_FUTEX_WAITV
] = {
465 #if defined(CONFIG_FUTEX)
466 .prep
= io_futexv_prep
,
467 .issue
= io_futexv_wait
,
469 .prep
= io_eopnotsupp_prep
,
474 const struct io_cold_def io_cold_defs
[] = {
478 [IORING_OP_READV
] = {
479 .async_size
= sizeof(struct io_async_rw
),
481 .prep_async
= io_readv_prep_async
,
482 .cleanup
= io_readv_writev_cleanup
,
485 [IORING_OP_WRITEV
] = {
486 .async_size
= sizeof(struct io_async_rw
),
488 .prep_async
= io_writev_prep_async
,
489 .cleanup
= io_readv_writev_cleanup
,
492 [IORING_OP_FSYNC
] = {
495 [IORING_OP_READ_FIXED
] = {
496 .async_size
= sizeof(struct io_async_rw
),
497 .name
= "READ_FIXED",
500 [IORING_OP_WRITE_FIXED
] = {
501 .async_size
= sizeof(struct io_async_rw
),
502 .name
= "WRITE_FIXED",
505 [IORING_OP_POLL_ADD
] = {
508 [IORING_OP_POLL_REMOVE
] = {
509 .name
= "POLL_REMOVE",
511 [IORING_OP_SYNC_FILE_RANGE
] = {
512 .name
= "SYNC_FILE_RANGE",
514 [IORING_OP_SENDMSG
] = {
516 #if defined(CONFIG_NET)
517 .async_size
= sizeof(struct io_async_msghdr
),
518 .prep_async
= io_sendmsg_prep_async
,
519 .cleanup
= io_sendmsg_recvmsg_cleanup
,
520 .fail
= io_sendrecv_fail
,
523 [IORING_OP_RECVMSG
] = {
525 #if defined(CONFIG_NET)
526 .async_size
= sizeof(struct io_async_msghdr
),
527 .prep_async
= io_recvmsg_prep_async
,
528 .cleanup
= io_sendmsg_recvmsg_cleanup
,
529 .fail
= io_sendrecv_fail
,
532 [IORING_OP_TIMEOUT
] = {
533 .async_size
= sizeof(struct io_timeout_data
),
536 [IORING_OP_TIMEOUT_REMOVE
] = {
537 .name
= "TIMEOUT_REMOVE",
539 [IORING_OP_ACCEPT
] = {
542 [IORING_OP_ASYNC_CANCEL
] = {
543 .name
= "ASYNC_CANCEL",
545 [IORING_OP_LINK_TIMEOUT
] = {
546 .async_size
= sizeof(struct io_timeout_data
),
547 .name
= "LINK_TIMEOUT",
549 [IORING_OP_CONNECT
] = {
551 #if defined(CONFIG_NET)
552 .async_size
= sizeof(struct io_async_connect
),
553 .prep_async
= io_connect_prep_async
,
556 [IORING_OP_FALLOCATE
] = {
559 [IORING_OP_OPENAT
] = {
561 .cleanup
= io_open_cleanup
,
563 [IORING_OP_CLOSE
] = {
566 [IORING_OP_FILES_UPDATE
] = {
567 .name
= "FILES_UPDATE",
569 [IORING_OP_STATX
] = {
571 .cleanup
= io_statx_cleanup
,
574 .async_size
= sizeof(struct io_async_rw
),
578 [IORING_OP_WRITE
] = {
579 .async_size
= sizeof(struct io_async_rw
),
583 [IORING_OP_FADVISE
] = {
586 [IORING_OP_MADVISE
] = {
591 #if defined(CONFIG_NET)
592 .async_size
= sizeof(struct io_async_msghdr
),
593 .fail
= io_sendrecv_fail
,
594 .prep_async
= io_send_prep_async
,
599 #if defined(CONFIG_NET)
600 .fail
= io_sendrecv_fail
,
603 [IORING_OP_OPENAT2
] = {
605 .cleanup
= io_open_cleanup
,
607 [IORING_OP_EPOLL_CTL
] = {
610 [IORING_OP_SPLICE
] = {
613 [IORING_OP_PROVIDE_BUFFERS
] = {
614 .name
= "PROVIDE_BUFFERS",
616 [IORING_OP_REMOVE_BUFFERS
] = {
617 .name
= "REMOVE_BUFFERS",
622 [IORING_OP_SHUTDOWN
] = {
625 [IORING_OP_RENAMEAT
] = {
627 .cleanup
= io_renameat_cleanup
,
629 [IORING_OP_UNLINKAT
] = {
631 .cleanup
= io_unlinkat_cleanup
,
633 [IORING_OP_MKDIRAT
] = {
635 .cleanup
= io_mkdirat_cleanup
,
637 [IORING_OP_SYMLINKAT
] = {
639 .cleanup
= io_link_cleanup
,
641 [IORING_OP_LINKAT
] = {
643 .cleanup
= io_link_cleanup
,
645 [IORING_OP_MSG_RING
] = {
647 .cleanup
= io_msg_ring_cleanup
,
649 [IORING_OP_FSETXATTR
] = {
651 .cleanup
= io_xattr_cleanup
,
653 [IORING_OP_SETXATTR
] = {
655 .cleanup
= io_xattr_cleanup
,
657 [IORING_OP_FGETXATTR
] = {
659 .cleanup
= io_xattr_cleanup
,
661 [IORING_OP_GETXATTR
] = {
663 .cleanup
= io_xattr_cleanup
,
665 [IORING_OP_SOCKET
] = {
668 [IORING_OP_URING_CMD
] = {
670 .async_size
= 2 * sizeof(struct io_uring_sqe
),
671 .prep_async
= io_uring_cmd_prep_async
,
673 [IORING_OP_SEND_ZC
] = {
675 #if defined(CONFIG_NET)
676 .async_size
= sizeof(struct io_async_msghdr
),
677 .prep_async
= io_send_prep_async
,
678 .cleanup
= io_send_zc_cleanup
,
679 .fail
= io_sendrecv_fail
,
682 [IORING_OP_SENDMSG_ZC
] = {
683 .name
= "SENDMSG_ZC",
684 #if defined(CONFIG_NET)
685 .async_size
= sizeof(struct io_async_msghdr
),
686 .prep_async
= io_sendmsg_prep_async
,
687 .cleanup
= io_send_zc_cleanup
,
688 .fail
= io_sendrecv_fail
,
691 [IORING_OP_READ_MULTISHOT
] = {
692 .name
= "READ_MULTISHOT",
694 [IORING_OP_WAITID
] = {
696 .async_size
= sizeof(struct io_waitid_async
),
698 [IORING_OP_FUTEX_WAIT
] = {
699 .name
= "FUTEX_WAIT",
701 [IORING_OP_FUTEX_WAKE
] = {
702 .name
= "FUTEX_WAKE",
704 [IORING_OP_FUTEX_WAITV
] = {
705 .name
= "FUTEX_WAITV",
709 const char *io_uring_get_opcode(u8 opcode
)
711 if (opcode
< IORING_OP_LAST
)
712 return io_cold_defs
[opcode
].name
;
716 void __init
io_uring_optable_init(void)
720 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs
) != IORING_OP_LAST
);
721 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs
) != IORING_OP_LAST
);
723 for (i
= 0; i
< ARRAY_SIZE(io_issue_defs
); i
++) {
724 BUG_ON(!io_issue_defs
[i
].prep
);
725 if (io_issue_defs
[i
].prep
!= io_eopnotsupp_prep
)
726 BUG_ON(!io_issue_defs
[i
].issue
);
727 WARN_ON_ONCE(!io_cold_defs
[i
].name
);