1 // SPDX-License-Identifier: GPL-2.0
3 * io_uring opcode handling table
5 #include <linux/kernel.h>
6 #include <linux/errno.h>
8 #include <linux/file.h>
9 #include <linux/io_uring.h>
26 #include "openclose.h"
27 #include "uring_cmd.h"
40 static int io_no_issue(struct io_kiocb
*req
, unsigned int issue_flags
)
46 static __maybe_unused
int io_eopnotsupp_prep(struct io_kiocb
*kiocb
,
47 const struct io_uring_sqe
*sqe
)
52 const struct io_issue_def io_issue_defs
[] = {
61 .unbound_nonreg_file
= 1,
73 [IORING_OP_WRITEV
] = {
76 .unbound_nonreg_file
= 1,
90 .prep
= io_fsync_prep
,
93 [IORING_OP_READ_FIXED
] = {
95 .unbound_nonreg_file
= 1,
102 .prep
= io_prep_rw_fixed
,
105 [IORING_OP_WRITE_FIXED
] = {
108 .unbound_nonreg_file
= 1,
115 .prep
= io_prep_rw_fixed
,
118 [IORING_OP_POLL_ADD
] = {
120 .unbound_nonreg_file
= 1,
122 .prep
= io_poll_add_prep
,
123 .issue
= io_poll_add
,
125 [IORING_OP_POLL_REMOVE
] = {
127 .prep
= io_poll_remove_prep
,
128 .issue
= io_poll_remove
,
130 [IORING_OP_SYNC_FILE_RANGE
] = {
134 .issue
= io_sync_file_range
,
136 [IORING_OP_SENDMSG
] = {
138 .unbound_nonreg_file
= 1,
142 #if defined(CONFIG_NET)
143 .prep
= io_sendmsg_prep
,
146 .prep
= io_eopnotsupp_prep
,
149 [IORING_OP_RECVMSG
] = {
151 .unbound_nonreg_file
= 1,
156 #if defined(CONFIG_NET)
157 .prep
= io_recvmsg_prep
,
160 .prep
= io_eopnotsupp_prep
,
163 [IORING_OP_TIMEOUT
] = {
165 .prep
= io_timeout_prep
,
168 [IORING_OP_TIMEOUT_REMOVE
] = {
169 /* used by timeout updates' prep() */
171 .prep
= io_timeout_remove_prep
,
172 .issue
= io_timeout_remove
,
174 [IORING_OP_ACCEPT
] = {
176 .unbound_nonreg_file
= 1,
179 .ioprio
= 1, /* used for flags */
180 #if defined(CONFIG_NET)
181 .prep
= io_accept_prep
,
184 .prep
= io_eopnotsupp_prep
,
187 [IORING_OP_ASYNC_CANCEL
] = {
189 .prep
= io_async_cancel_prep
,
190 .issue
= io_async_cancel
,
192 [IORING_OP_LINK_TIMEOUT
] = {
194 .prep
= io_link_timeout_prep
,
195 .issue
= io_no_issue
,
197 [IORING_OP_CONNECT
] = {
199 .unbound_nonreg_file
= 1,
201 #if defined(CONFIG_NET)
202 .prep
= io_connect_prep
,
205 .prep
= io_eopnotsupp_prep
,
208 [IORING_OP_FALLOCATE
] = {
210 .prep
= io_fallocate_prep
,
211 .issue
= io_fallocate
,
213 [IORING_OP_OPENAT
] = {
214 .prep
= io_openat_prep
,
217 [IORING_OP_CLOSE
] = {
218 .prep
= io_close_prep
,
221 [IORING_OP_FILES_UPDATE
] = {
224 .prep
= io_files_update_prep
,
225 .issue
= io_files_update
,
227 [IORING_OP_STATX
] = {
229 .prep
= io_statx_prep
,
234 .unbound_nonreg_file
= 1,
245 [IORING_OP_WRITE
] = {
248 .unbound_nonreg_file
= 1,
258 [IORING_OP_FADVISE
] = {
261 .prep
= io_fadvise_prep
,
264 [IORING_OP_MADVISE
] = {
266 .prep
= io_madvise_prep
,
271 .unbound_nonreg_file
= 1,
276 #if defined(CONFIG_NET)
277 .prep
= io_sendmsg_prep
,
280 .prep
= io_eopnotsupp_prep
,
285 .unbound_nonreg_file
= 1,
290 #if defined(CONFIG_NET)
291 .prep
= io_recvmsg_prep
,
294 .prep
= io_eopnotsupp_prep
,
297 [IORING_OP_OPENAT2
] = {
298 .prep
= io_openat2_prep
,
301 [IORING_OP_EPOLL_CTL
] = {
302 .unbound_nonreg_file
= 1,
304 #if defined(CONFIG_EPOLL)
305 .prep
= io_epoll_ctl_prep
,
306 .issue
= io_epoll_ctl
,
308 .prep
= io_eopnotsupp_prep
,
311 [IORING_OP_SPLICE
] = {
314 .unbound_nonreg_file
= 1,
316 .prep
= io_splice_prep
,
319 [IORING_OP_PROVIDE_BUFFERS
] = {
322 .prep
= io_provide_buffers_prep
,
323 .issue
= io_provide_buffers
,
325 [IORING_OP_REMOVE_BUFFERS
] = {
328 .prep
= io_remove_buffers_prep
,
329 .issue
= io_remove_buffers
,
334 .unbound_nonreg_file
= 1,
339 [IORING_OP_SHUTDOWN
] = {
341 #if defined(CONFIG_NET)
342 .prep
= io_shutdown_prep
,
343 .issue
= io_shutdown
,
345 .prep
= io_eopnotsupp_prep
,
348 [IORING_OP_RENAMEAT
] = {
349 .prep
= io_renameat_prep
,
350 .issue
= io_renameat
,
352 [IORING_OP_UNLINKAT
] = {
353 .prep
= io_unlinkat_prep
,
354 .issue
= io_unlinkat
,
356 [IORING_OP_MKDIRAT
] = {
357 .prep
= io_mkdirat_prep
,
360 [IORING_OP_SYMLINKAT
] = {
361 .prep
= io_symlinkat_prep
,
362 .issue
= io_symlinkat
,
364 [IORING_OP_LINKAT
] = {
365 .prep
= io_linkat_prep
,
368 [IORING_OP_MSG_RING
] = {
371 .prep
= io_msg_ring_prep
,
372 .issue
= io_msg_ring
,
374 [IORING_OP_FSETXATTR
] = {
376 .prep
= io_fsetxattr_prep
,
377 .issue
= io_fsetxattr
,
379 [IORING_OP_SETXATTR
] = {
380 .prep
= io_setxattr_prep
,
381 .issue
= io_setxattr
,
383 [IORING_OP_FGETXATTR
] = {
385 .prep
= io_fgetxattr_prep
,
386 .issue
= io_fgetxattr
,
388 [IORING_OP_GETXATTR
] = {
389 .prep
= io_getxattr_prep
,
390 .issue
= io_getxattr
,
392 [IORING_OP_SOCKET
] = {
394 #if defined(CONFIG_NET)
395 .prep
= io_socket_prep
,
398 .prep
= io_eopnotsupp_prep
,
401 [IORING_OP_URING_CMD
] = {
406 .prep
= io_uring_cmd_prep
,
407 .issue
= io_uring_cmd
,
409 [IORING_OP_SEND_ZC
] = {
411 .unbound_nonreg_file
= 1,
416 #if defined(CONFIG_NET)
417 .prep
= io_send_zc_prep
,
420 .prep
= io_eopnotsupp_prep
,
423 [IORING_OP_SENDMSG_ZC
] = {
425 .unbound_nonreg_file
= 1,
429 #if defined(CONFIG_NET)
430 .prep
= io_send_zc_prep
,
431 .issue
= io_sendmsg_zc
,
433 .prep
= io_eopnotsupp_prep
,
436 [IORING_OP_READ_MULTISHOT
] = {
438 .unbound_nonreg_file
= 1,
442 .prep
= io_read_mshot_prep
,
443 .issue
= io_read_mshot
,
445 [IORING_OP_WAITID
] = {
446 .prep
= io_waitid_prep
,
449 [IORING_OP_FUTEX_WAIT
] = {
450 #if defined(CONFIG_FUTEX)
451 .prep
= io_futex_prep
,
452 .issue
= io_futex_wait
,
454 .prep
= io_eopnotsupp_prep
,
457 [IORING_OP_FUTEX_WAKE
] = {
458 #if defined(CONFIG_FUTEX)
459 .prep
= io_futex_prep
,
460 .issue
= io_futex_wake
,
462 .prep
= io_eopnotsupp_prep
,
465 [IORING_OP_FUTEX_WAITV
] = {
466 #if defined(CONFIG_FUTEX)
467 .prep
= io_futexv_prep
,
468 .issue
= io_futexv_wait
,
470 .prep
= io_eopnotsupp_prep
,
473 [IORING_OP_FIXED_FD_INSTALL
] = {
475 .prep
= io_install_fixed_fd_prep
,
476 .issue
= io_install_fixed_fd
,
478 [IORING_OP_FTRUNCATE
] = {
481 .prep
= io_ftruncate_prep
,
482 .issue
= io_ftruncate
,
486 const struct io_cold_def io_cold_defs
[] = {
490 [IORING_OP_READV
] = {
491 .async_size
= sizeof(struct io_async_rw
),
493 .prep_async
= io_readv_prep_async
,
494 .cleanup
= io_readv_writev_cleanup
,
497 [IORING_OP_WRITEV
] = {
498 .async_size
= sizeof(struct io_async_rw
),
500 .prep_async
= io_writev_prep_async
,
501 .cleanup
= io_readv_writev_cleanup
,
504 [IORING_OP_FSYNC
] = {
507 [IORING_OP_READ_FIXED
] = {
508 .async_size
= sizeof(struct io_async_rw
),
509 .name
= "READ_FIXED",
512 [IORING_OP_WRITE_FIXED
] = {
513 .async_size
= sizeof(struct io_async_rw
),
514 .name
= "WRITE_FIXED",
517 [IORING_OP_POLL_ADD
] = {
520 [IORING_OP_POLL_REMOVE
] = {
521 .name
= "POLL_REMOVE",
523 [IORING_OP_SYNC_FILE_RANGE
] = {
524 .name
= "SYNC_FILE_RANGE",
526 [IORING_OP_SENDMSG
] = {
528 #if defined(CONFIG_NET)
529 .async_size
= sizeof(struct io_async_msghdr
),
530 .prep_async
= io_sendmsg_prep_async
,
531 .cleanup
= io_sendmsg_recvmsg_cleanup
,
532 .fail
= io_sendrecv_fail
,
535 [IORING_OP_RECVMSG
] = {
537 #if defined(CONFIG_NET)
538 .async_size
= sizeof(struct io_async_msghdr
),
539 .prep_async
= io_recvmsg_prep_async
,
540 .cleanup
= io_sendmsg_recvmsg_cleanup
,
541 .fail
= io_sendrecv_fail
,
544 [IORING_OP_TIMEOUT
] = {
545 .async_size
= sizeof(struct io_timeout_data
),
548 [IORING_OP_TIMEOUT_REMOVE
] = {
549 .name
= "TIMEOUT_REMOVE",
551 [IORING_OP_ACCEPT
] = {
554 [IORING_OP_ASYNC_CANCEL
] = {
555 .name
= "ASYNC_CANCEL",
557 [IORING_OP_LINK_TIMEOUT
] = {
558 .async_size
= sizeof(struct io_timeout_data
),
559 .name
= "LINK_TIMEOUT",
561 [IORING_OP_CONNECT
] = {
563 #if defined(CONFIG_NET)
564 .async_size
= sizeof(struct io_async_connect
),
565 .prep_async
= io_connect_prep_async
,
568 [IORING_OP_FALLOCATE
] = {
571 [IORING_OP_OPENAT
] = {
573 .cleanup
= io_open_cleanup
,
575 [IORING_OP_CLOSE
] = {
578 [IORING_OP_FILES_UPDATE
] = {
579 .name
= "FILES_UPDATE",
581 [IORING_OP_STATX
] = {
583 .cleanup
= io_statx_cleanup
,
586 .async_size
= sizeof(struct io_async_rw
),
590 [IORING_OP_WRITE
] = {
591 .async_size
= sizeof(struct io_async_rw
),
595 [IORING_OP_FADVISE
] = {
598 [IORING_OP_MADVISE
] = {
603 #if defined(CONFIG_NET)
604 .async_size
= sizeof(struct io_async_msghdr
),
605 .fail
= io_sendrecv_fail
,
606 .prep_async
= io_send_prep_async
,
611 #if defined(CONFIG_NET)
612 .fail
= io_sendrecv_fail
,
615 [IORING_OP_OPENAT2
] = {
617 .cleanup
= io_open_cleanup
,
619 [IORING_OP_EPOLL_CTL
] = {
622 [IORING_OP_SPLICE
] = {
625 [IORING_OP_PROVIDE_BUFFERS
] = {
626 .name
= "PROVIDE_BUFFERS",
628 [IORING_OP_REMOVE_BUFFERS
] = {
629 .name
= "REMOVE_BUFFERS",
634 [IORING_OP_SHUTDOWN
] = {
637 [IORING_OP_RENAMEAT
] = {
639 .cleanup
= io_renameat_cleanup
,
641 [IORING_OP_UNLINKAT
] = {
643 .cleanup
= io_unlinkat_cleanup
,
645 [IORING_OP_MKDIRAT
] = {
647 .cleanup
= io_mkdirat_cleanup
,
649 [IORING_OP_SYMLINKAT
] = {
651 .cleanup
= io_link_cleanup
,
653 [IORING_OP_LINKAT
] = {
655 .cleanup
= io_link_cleanup
,
657 [IORING_OP_MSG_RING
] = {
659 .cleanup
= io_msg_ring_cleanup
,
661 [IORING_OP_FSETXATTR
] = {
663 .cleanup
= io_xattr_cleanup
,
665 [IORING_OP_SETXATTR
] = {
667 .cleanup
= io_xattr_cleanup
,
669 [IORING_OP_FGETXATTR
] = {
671 .cleanup
= io_xattr_cleanup
,
673 [IORING_OP_GETXATTR
] = {
675 .cleanup
= io_xattr_cleanup
,
677 [IORING_OP_SOCKET
] = {
680 [IORING_OP_URING_CMD
] = {
682 .async_size
= 2 * sizeof(struct io_uring_sqe
),
683 .prep_async
= io_uring_cmd_prep_async
,
685 [IORING_OP_SEND_ZC
] = {
687 #if defined(CONFIG_NET)
688 .async_size
= sizeof(struct io_async_msghdr
),
689 .prep_async
= io_send_prep_async
,
690 .cleanup
= io_send_zc_cleanup
,
691 .fail
= io_sendrecv_fail
,
694 [IORING_OP_SENDMSG_ZC
] = {
695 .name
= "SENDMSG_ZC",
696 #if defined(CONFIG_NET)
697 .async_size
= sizeof(struct io_async_msghdr
),
698 .prep_async
= io_sendmsg_prep_async
,
699 .cleanup
= io_send_zc_cleanup
,
700 .fail
= io_sendrecv_fail
,
703 [IORING_OP_READ_MULTISHOT
] = {
704 .name
= "READ_MULTISHOT",
706 [IORING_OP_WAITID
] = {
708 .async_size
= sizeof(struct io_waitid_async
),
710 [IORING_OP_FUTEX_WAIT
] = {
711 .name
= "FUTEX_WAIT",
713 [IORING_OP_FUTEX_WAKE
] = {
714 .name
= "FUTEX_WAKE",
716 [IORING_OP_FUTEX_WAITV
] = {
717 .name
= "FUTEX_WAITV",
719 [IORING_OP_FIXED_FD_INSTALL
] = {
720 .name
= "FIXED_FD_INSTALL",
722 [IORING_OP_FTRUNCATE
] = {
727 const char *io_uring_get_opcode(u8 opcode
)
729 if (opcode
< IORING_OP_LAST
)
730 return io_cold_defs
[opcode
].name
;
734 void __init
io_uring_optable_init(void)
738 BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs
) != IORING_OP_LAST
);
739 BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs
) != IORING_OP_LAST
);
741 for (i
= 0; i
< ARRAY_SIZE(io_issue_defs
); i
++) {
742 BUG_ON(!io_issue_defs
[i
].prep
);
743 if (io_issue_defs
[i
].prep
!= io_eopnotsupp_prep
)
744 BUG_ON(!io_issue_defs
[i
].issue
);
745 WARN_ON_ONCE(!io_cold_defs
[i
].name
);