io_uring/opdef.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * io_uring opcode handling table
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/errno.h>
   7 #include <linux/fs.h>
   8 #include <linux/file.h>
   9 #include <linux/io_uring.h>
  10
  11 #include "io_uring.h"
  12 #include "opdef.h"
  13 #include "refs.h"
  14 #include "tctx.h"
  15 #include "sqpoll.h"
  16 #include "fdinfo.h"
  17 #include "kbuf.h"
  18 #include "rsrc.h"
  19
  20 #include "xattr.h"
  21 #include "nop.h"
  22 #include "fs.h"
  23 #include "splice.h"
  24 #include "sync.h"
  25 #include "advise.h"
  26 #include "openclose.h"
  27 #include "uring_cmd.h"
  28 #include "epoll.h"
  29 #include "statx.h"
  30 #include "net.h"
  31 #include "msg_ring.h"
  32 #include "timeout.h"
  33 #include "poll.h"
  34 #include "cancel.h"
  35 #include "rw.h"
  36 #include "waitid.h"
  37 #include "futex.h"
  38
  39 static int io_no_issue(struct io_kiocb *req, unsigned int issue_flags)
  40 {
  41         WARN_ON_ONCE(1);
  42         return -ECANCELED;
  43 }
  44
  45 static __maybe_unused int io_eopnotsupp_prep(struct io_kiocb *kiocb,
  46                                              const struct io_uring_sqe *sqe)
  47 {
  48         return -EOPNOTSUPP;
  49 }
  50
  51 const struct io_issue_def io_issue_defs[] = {
  52         [IORING_OP_NOP] = {
  53                 .audit_skip             = 1,
  54                 .iopoll                 = 1,
  55                 .prep                   = io_nop_prep,
  56                 .issue                  = io_nop,
  57         },
  58         [IORING_OP_READV] = {
  59                 .needs_file             = 1,
  60                 .unbound_nonreg_file    = 1,
  61                 .pollin                 = 1,
  62                 .buffer_select          = 1,
  63                 .plug                   = 1,
  64                 .audit_skip             = 1,
  65                 .ioprio                 = 1,
  66                 .iopoll                 = 1,
  67                 .iopoll_queue           = 1,
  68                 .vectored               = 1,
  69                 .prep                   = io_prep_rwv,
  70                 .issue                  = io_read,
  71         },
  72         [IORING_OP_WRITEV] = {
  73                 .needs_file             = 1,
  74                 .hash_reg_file          = 1,
  75                 .unbound_nonreg_file    = 1,
  76                 .pollout                = 1,
  77                 .plug                   = 1,
  78                 .audit_skip             = 1,
  79                 .ioprio                 = 1,
  80                 .iopoll                 = 1,
  81                 .iopoll_queue           = 1,
  82                 .vectored               = 1,
  83                 .prep                   = io_prep_rwv,
  84                 .issue                  = io_write,
  85         },
  86         [IORING_OP_FSYNC] = {
  87                 .needs_file             = 1,
  88                 .audit_skip             = 1,
  89                 .prep                   = io_fsync_prep,
  90                 .issue                  = io_fsync,
  91         },
  92         [IORING_OP_READ_FIXED] = {
  93                 .needs_file             = 1,
  94                 .unbound_nonreg_file    = 1,
  95                 .pollin                 = 1,
  96                 .plug                   = 1,
  97                 .audit_skip             = 1,
  98                 .ioprio                 = 1,
  99                 .iopoll                 = 1,
 100                 .iopoll_queue           = 1,
 101                 .prep                   = io_prep_rw_fixed,
 102                 .issue                  = io_read,
 103         },
 104         [IORING_OP_WRITE_FIXED] = {
 105                 .needs_file             = 1,
 106                 .hash_reg_file          = 1,
 107                 .unbound_nonreg_file    = 1,
 108                 .pollout                = 1,
 109                 .plug                   = 1,
 110                 .audit_skip             = 1,
 111                 .ioprio                 = 1,
 112                 .iopoll                 = 1,
 113                 .iopoll_queue           = 1,
 114                 .prep                   = io_prep_rw_fixed,
 115                 .issue                  = io_write,
 116         },
 117         [IORING_OP_POLL_ADD] = {
 118                 .needs_file             = 1,
 119                 .unbound_nonreg_file    = 1,
 120                 .audit_skip             = 1,
 121                 .prep                   = io_poll_add_prep,
 122                 .issue                  = io_poll_add,
 123         },
 124         [IORING_OP_POLL_REMOVE] = {
 125                 .audit_skip             = 1,
 126                 .prep                   = io_poll_remove_prep,
 127                 .issue                  = io_poll_remove,
 128         },
 129         [IORING_OP_SYNC_FILE_RANGE] = {
 130                 .needs_file             = 1,
 131                 .audit_skip             = 1,
 132                 .prep                   = io_sfr_prep,
 133                 .issue                  = io_sync_file_range,
 134         },
 135         [IORING_OP_SENDMSG] = {
 136                 .needs_file             = 1,
 137                 .unbound_nonreg_file    = 1,
 138                 .pollout                = 1,
 139                 .ioprio                 = 1,
 140                 .manual_alloc           = 1,
 141 #if defined(CONFIG_NET)
 142                 .prep                   = io_sendmsg_prep,
 143                 .issue                  = io_sendmsg,
 144 #else
 145                 .prep                   = io_eopnotsupp_prep,
 146 #endif
 147         },
 148         [IORING_OP_RECVMSG] = {
 149                 .needs_file             = 1,
 150                 .unbound_nonreg_file    = 1,
 151                 .pollin                 = 1,
 152                 .buffer_select          = 1,
 153                 .ioprio                 = 1,
 154                 .manual_alloc           = 1,
 155 #if defined(CONFIG_NET)
 156                 .prep                   = io_recvmsg_prep,
 157                 .issue                  = io_recvmsg,
 158 #else
 159                 .prep                   = io_eopnotsupp_prep,
 160 #endif
 161         },
 162         [IORING_OP_TIMEOUT] = {
 163                 .audit_skip             = 1,
 164                 .prep                   = io_timeout_prep,
 165                 .issue                  = io_timeout,
 166         },
 167         [IORING_OP_TIMEOUT_REMOVE] = {
 168                 /* used by timeout updates' prep() */
 169                 .audit_skip             = 1,
 170                 .prep                   = io_timeout_remove_prep,
 171                 .issue                  = io_timeout_remove,
 172         },
 173         [IORING_OP_ACCEPT] = {
 174                 .needs_file             = 1,
 175                 .unbound_nonreg_file    = 1,
 176                 .pollin                 = 1,
 177                 .poll_exclusive         = 1,
 178                 .ioprio                 = 1,    /* used for flags */
 179 #if defined(CONFIG_NET)
 180                 .prep                   = io_accept_prep,
 181                 .issue                  = io_accept,
 182 #else
 183                 .prep                   = io_eopnotsupp_prep,
 184 #endif
 185         },
 186         [IORING_OP_ASYNC_CANCEL] = {
 187                 .audit_skip             = 1,
 188                 .prep                   = io_async_cancel_prep,
 189                 .issue                  = io_async_cancel,
 190         },
 191         [IORING_OP_LINK_TIMEOUT] = {
 192                 .audit_skip             = 1,
 193                 .prep                   = io_link_timeout_prep,
 194                 .issue                  = io_no_issue,
 195         },
 196         [IORING_OP_CONNECT] = {
 197                 .needs_file             = 1,
 198                 .unbound_nonreg_file    = 1,
 199                 .pollout                = 1,
 200 #if defined(CONFIG_NET)
 201                 .prep                   = io_connect_prep,
 202                 .issue                  = io_connect,
 203 #else
 204                 .prep                   = io_eopnotsupp_prep,
 205 #endif
 206         },
 207         [IORING_OP_FALLOCATE] = {
 208                 .needs_file             = 1,
 209                 .prep                   = io_fallocate_prep,
 210                 .issue                  = io_fallocate,
 211         },
 212         [IORING_OP_OPENAT] = {
 213                 .prep                   = io_openat_prep,
 214                 .issue                  = io_openat,
 215         },
 216         [IORING_OP_CLOSE] = {
 217                 .prep                   = io_close_prep,
 218                 .issue                  = io_close,
 219         },
 220         [IORING_OP_FILES_UPDATE] = {
 221                 .audit_skip             = 1,
 222                 .iopoll                 = 1,
 223                 .prep                   = io_files_update_prep,
 224                 .issue                  = io_files_update,
 225         },
 226         [IORING_OP_STATX] = {
 227                 .audit_skip             = 1,
 228                 .prep                   = io_statx_prep,
 229                 .issue                  = io_statx,
 230         },
 231         [IORING_OP_READ] = {
 232                 .needs_file             = 1,
 233                 .unbound_nonreg_file    = 1,
 234                 .pollin                 = 1,
 235                 .buffer_select          = 1,
 236                 .plug                   = 1,
 237                 .audit_skip             = 1,
 238                 .ioprio                 = 1,
 239                 .iopoll                 = 1,
 240                 .iopoll_queue           = 1,
 241                 .prep                   = io_prep_rw,
 242                 .issue                  = io_read,
 243         },
 244         [IORING_OP_WRITE] = {
 245                 .needs_file             = 1,
 246                 .hash_reg_file          = 1,
 247                 .unbound_nonreg_file    = 1,
 248                 .pollout                = 1,
 249                 .plug                   = 1,
 250                 .audit_skip             = 1,
 251                 .ioprio                 = 1,
 252                 .iopoll                 = 1,
 253                 .iopoll_queue           = 1,
 254                 .prep                   = io_prep_rw,
 255                 .issue                  = io_write,
 256         },
 257         [IORING_OP_FADVISE] = {
 258                 .needs_file             = 1,
 259                 .audit_skip             = 1,
 260                 .prep                   = io_fadvise_prep,
 261                 .issue                  = io_fadvise,
 262         },
 263         [IORING_OP_MADVISE] = {
 264                 .audit_skip             = 1,
 265                 .prep                   = io_madvise_prep,
 266                 .issue                  = io_madvise,
 267         },
 268         [IORING_OP_SEND] = {
 269                 .needs_file             = 1,
 270                 .unbound_nonreg_file    = 1,
 271                 .pollout                = 1,
 272                 .audit_skip             = 1,
 273                 .ioprio                 = 1,
 274                 .manual_alloc           = 1,
 275 #if defined(CONFIG_NET)
 276                 .prep                   = io_sendmsg_prep,
 277                 .issue                  = io_send,
 278 #else
 279                 .prep                   = io_eopnotsupp_prep,
 280 #endif
 281         },
 282         [IORING_OP_RECV] = {
 283                 .needs_file             = 1,
 284                 .unbound_nonreg_file    = 1,
 285                 .pollin                 = 1,
 286                 .buffer_select          = 1,
 287                 .audit_skip             = 1,
 288                 .ioprio                 = 1,
 289 #if defined(CONFIG_NET)
 290                 .prep                   = io_recvmsg_prep,
 291                 .issue                  = io_recv,
 292 #else
 293                 .prep                   = io_eopnotsupp_prep,
 294 #endif
 295         },
 296         [IORING_OP_OPENAT2] = {
 297                 .prep                   = io_openat2_prep,
 298                 .issue                  = io_openat2,
 299         },
 300         [IORING_OP_EPOLL_CTL] = {
 301                 .unbound_nonreg_file    = 1,
 302                 .audit_skip             = 1,
 303 #if defined(CONFIG_EPOLL)
 304                 .prep                   = io_epoll_ctl_prep,
 305                 .issue                  = io_epoll_ctl,
 306 #else
 307                 .prep                   = io_eopnotsupp_prep,
 308 #endif
 309         },
 310         [IORING_OP_SPLICE] = {
 311                 .needs_file             = 1,
 312                 .hash_reg_file          = 1,
 313                 .unbound_nonreg_file    = 1,
 314                 .audit_skip             = 1,
 315                 .prep                   = io_splice_prep,
 316                 .issue                  = io_splice,
 317         },
 318         [IORING_OP_PROVIDE_BUFFERS] = {
 319                 .audit_skip             = 1,
 320                 .iopoll                 = 1,
 321                 .prep                   = io_provide_buffers_prep,
 322                 .issue                  = io_provide_buffers,
 323         },
 324         [IORING_OP_REMOVE_BUFFERS] = {
 325                 .audit_skip             = 1,
 326                 .iopoll                 = 1,
 327                 .prep                   = io_remove_buffers_prep,
 328                 .issue                  = io_remove_buffers,
 329         },
 330         [IORING_OP_TEE] = {
 331                 .needs_file             = 1,
 332                 .hash_reg_file          = 1,
 333                 .unbound_nonreg_file    = 1,
 334                 .audit_skip             = 1,
 335                 .prep                   = io_tee_prep,
 336                 .issue                  = io_tee,
 337         },
 338         [IORING_OP_SHUTDOWN] = {
 339                 .needs_file             = 1,
 340 #if defined(CONFIG_NET)
 341                 .prep                   = io_shutdown_prep,
 342                 .issue                  = io_shutdown,
 343 #else
 344                 .prep                   = io_eopnotsupp_prep,
 345 #endif
 346         },
 347         [IORING_OP_RENAMEAT] = {
 348                 .prep                   = io_renameat_prep,
 349                 .issue                  = io_renameat,
 350         },
 351         [IORING_OP_UNLINKAT] = {
 352                 .prep                   = io_unlinkat_prep,
 353                 .issue                  = io_unlinkat,
 354         },
 355         [IORING_OP_MKDIRAT] = {
 356                 .prep                   = io_mkdirat_prep,
 357                 .issue                  = io_mkdirat,
 358         },
 359         [IORING_OP_SYMLINKAT] = {
 360                 .prep                   = io_symlinkat_prep,
 361                 .issue                  = io_symlinkat,
 362         },
 363         [IORING_OP_LINKAT] = {
 364                 .prep                   = io_linkat_prep,
 365                 .issue                  = io_linkat,
 366         },
 367         [IORING_OP_MSG_RING] = {
 368                 .needs_file             = 1,
 369                 .iopoll                 = 1,
 370                 .prep                   = io_msg_ring_prep,
 371                 .issue                  = io_msg_ring,
 372         },
 373         [IORING_OP_FSETXATTR] = {
 374                 .needs_file = 1,
 375                 .prep                   = io_fsetxattr_prep,
 376                 .issue                  = io_fsetxattr,
 377         },
 378         [IORING_OP_SETXATTR] = {
 379                 .prep                   = io_setxattr_prep,
 380                 .issue                  = io_setxattr,
 381         },
 382         [IORING_OP_FGETXATTR] = {
 383                 .needs_file = 1,
 384                 .prep                   = io_fgetxattr_prep,
 385                 .issue                  = io_fgetxattr,
 386         },
 387         [IORING_OP_GETXATTR] = {
 388                 .prep                   = io_getxattr_prep,
 389                 .issue                  = io_getxattr,
 390         },
 391         [IORING_OP_SOCKET] = {
 392                 .audit_skip             = 1,
 393 #if defined(CONFIG_NET)
 394                 .prep                   = io_socket_prep,
 395                 .issue                  = io_socket,
 396 #else
 397                 .prep                   = io_eopnotsupp_prep,
 398 #endif
 399         },
 400         [IORING_OP_URING_CMD] = {
 401                 .needs_file             = 1,
 402                 .plug                   = 1,
 403                 .iopoll                 = 1,
 404                 .iopoll_queue           = 1,
 405                 .prep                   = io_uring_cmd_prep,
 406                 .issue                  = io_uring_cmd,
 407         },
 408         [IORING_OP_SEND_ZC] = {
 409                 .needs_file             = 1,
 410                 .unbound_nonreg_file    = 1,
 411                 .pollout                = 1,
 412                 .audit_skip             = 1,
 413                 .ioprio                 = 1,
 414                 .manual_alloc           = 1,
 415 #if defined(CONFIG_NET)
 416                 .prep                   = io_send_zc_prep,
 417                 .issue                  = io_send_zc,
 418 #else
 419                 .prep                   = io_eopnotsupp_prep,
 420 #endif
 421         },
 422         [IORING_OP_SENDMSG_ZC] = {
 423                 .needs_file             = 1,
 424                 .unbound_nonreg_file    = 1,
 425                 .pollout                = 1,
 426                 .ioprio                 = 1,
 427                 .manual_alloc           = 1,
 428 #if defined(CONFIG_NET)
 429                 .prep                   = io_send_zc_prep,
 430                 .issue                  = io_sendmsg_zc,
 431 #else
 432                 .prep                   = io_eopnotsupp_prep,
 433 #endif
 434         },
 435         [IORING_OP_READ_MULTISHOT] = {
 436                 .needs_file             = 1,
 437                 .unbound_nonreg_file    = 1,
 438                 .pollin                 = 1,
 439                 .buffer_select          = 1,
 440                 .audit_skip             = 1,
 441                 .prep                   = io_read_mshot_prep,
 442                 .issue                  = io_read_mshot,
 443         },
 444         [IORING_OP_WAITID] = {
 445                 .prep                   = io_waitid_prep,
 446                 .issue                  = io_waitid,
 447         },
 448         [IORING_OP_FUTEX_WAIT] = {
 449 #if defined(CONFIG_FUTEX)
 450                 .prep                   = io_futex_prep,
 451                 .issue                  = io_futex_wait,
 452 #else
 453                 .prep                   = io_eopnotsupp_prep,
 454 #endif
 455         },
 456         [IORING_OP_FUTEX_WAKE] = {
 457 #if defined(CONFIG_FUTEX)
 458                 .prep                   = io_futex_prep,
 459                 .issue                  = io_futex_wake,
 460 #else
 461                 .prep                   = io_eopnotsupp_prep,
 462 #endif
 463         },
 464         [IORING_OP_FUTEX_WAITV] = {
 465 #if defined(CONFIG_FUTEX)
 466                 .prep                   = io_futexv_prep,
 467                 .issue                  = io_futexv_wait,
 468 #else
 469                 .prep                   = io_eopnotsupp_prep,
 470 #endif
 471         },
 472         [IORING_OP_FIXED_FD_INSTALL] = {
 473                 .needs_file             = 1,
 474                 .prep                   = io_install_fixed_fd_prep,
 475                 .issue                  = io_install_fixed_fd,
 476         },
 477 };
 478
 479 const struct io_cold_def io_cold_defs[] = {
 480         [IORING_OP_NOP] = {
 481                 .name                   = "NOP",
 482         },
 483         [IORING_OP_READV] = {
 484                 .async_size             = sizeof(struct io_async_rw),
 485                 .name                   = "READV",
 486                 .prep_async             = io_readv_prep_async,
 487                 .cleanup                = io_readv_writev_cleanup,
 488                 .fail                   = io_rw_fail,
 489         },
 490         [IORING_OP_WRITEV] = {
 491                 .async_size             = sizeof(struct io_async_rw),
 492                 .name                   = "WRITEV",
 493                 .prep_async             = io_writev_prep_async,
 494                 .cleanup                = io_readv_writev_cleanup,
 495                 .fail                   = io_rw_fail,
 496         },
 497         [IORING_OP_FSYNC] = {
 498                 .name                   = "FSYNC",
 499         },
 500         [IORING_OP_READ_FIXED] = {
 501                 .async_size             = sizeof(struct io_async_rw),
 502                 .name                   = "READ_FIXED",
 503                 .fail                   = io_rw_fail,
 504         },
 505         [IORING_OP_WRITE_FIXED] = {
 506                 .async_size             = sizeof(struct io_async_rw),
 507                 .name                   = "WRITE_FIXED",
 508                 .fail                   = io_rw_fail,
 509         },
 510         [IORING_OP_POLL_ADD] = {
 511                 .name                   = "POLL_ADD",
 512         },
 513         [IORING_OP_POLL_REMOVE] = {
 514                 .name                   = "POLL_REMOVE",
 515         },
 516         [IORING_OP_SYNC_FILE_RANGE] = {
 517                 .name                   = "SYNC_FILE_RANGE",
 518         },
 519         [IORING_OP_SENDMSG] = {
 520                 .name                   = "SENDMSG",
 521 #if defined(CONFIG_NET)
 522                 .async_size             = sizeof(struct io_async_msghdr),
 523                 .prep_async             = io_sendmsg_prep_async,
 524                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 525                 .fail                   = io_sendrecv_fail,
 526 #endif
 527         },
 528         [IORING_OP_RECVMSG] = {
 529                 .name                   = "RECVMSG",
 530 #if defined(CONFIG_NET)
 531                 .async_size             = sizeof(struct io_async_msghdr),
 532                 .prep_async             = io_recvmsg_prep_async,
 533                 .cleanup                = io_sendmsg_recvmsg_cleanup,
 534                 .fail                   = io_sendrecv_fail,
 535 #endif
 536         },
 537         [IORING_OP_TIMEOUT] = {
 538                 .async_size             = sizeof(struct io_timeout_data),
 539                 .name                   = "TIMEOUT",
 540         },
 541         [IORING_OP_TIMEOUT_REMOVE] = {
 542                 .name                   = "TIMEOUT_REMOVE",
 543         },
 544         [IORING_OP_ACCEPT] = {
 545                 .name                   = "ACCEPT",
 546         },
 547         [IORING_OP_ASYNC_CANCEL] = {
 548                 .name                   = "ASYNC_CANCEL",
 549         },
 550         [IORING_OP_LINK_TIMEOUT] = {
 551                 .async_size             = sizeof(struct io_timeout_data),
 552                 .name                   = "LINK_TIMEOUT",
 553         },
 554         [IORING_OP_CONNECT] = {
 555                 .name                   = "CONNECT",
 556 #if defined(CONFIG_NET)
 557                 .async_size             = sizeof(struct io_async_connect),
 558                 .prep_async             = io_connect_prep_async,
 559 #endif
 560         },
 561         [IORING_OP_FALLOCATE] = {
 562                 .name                   = "FALLOCATE",
 563         },
 564         [IORING_OP_OPENAT] = {
 565                 .name                   = "OPENAT",
 566                 .cleanup                = io_open_cleanup,
 567         },
 568         [IORING_OP_CLOSE] = {
 569                 .name                   = "CLOSE",
 570         },
 571         [IORING_OP_FILES_UPDATE] = {
 572                 .name                   = "FILES_UPDATE",
 573         },
 574         [IORING_OP_STATX] = {
 575                 .name                   = "STATX",
 576                 .cleanup                = io_statx_cleanup,
 577         },
 578         [IORING_OP_READ] = {
 579                 .async_size             = sizeof(struct io_async_rw),
 580                 .name                   = "READ",
 581                 .fail                   = io_rw_fail,
 582         },
 583         [IORING_OP_WRITE] = {
 584                 .async_size             = sizeof(struct io_async_rw),
 585                 .name                   = "WRITE",
 586                 .fail                   = io_rw_fail,
 587         },
 588         [IORING_OP_FADVISE] = {
 589                 .name                   = "FADVISE",
 590         },
 591         [IORING_OP_MADVISE] = {
 592                 .name                   = "MADVISE",
 593         },
 594         [IORING_OP_SEND] = {
 595                 .name                   = "SEND",
 596 #if defined(CONFIG_NET)
 597                 .async_size             = sizeof(struct io_async_msghdr),
 598                 .fail                   = io_sendrecv_fail,
 599                 .prep_async             = io_send_prep_async,
 600 #endif
 601         },
 602         [IORING_OP_RECV] = {
 603                 .name                   = "RECV",
 604 #if defined(CONFIG_NET)
 605                 .fail                   = io_sendrecv_fail,
 606 #endif
 607         },
 608         [IORING_OP_OPENAT2] = {
 609                 .name                   = "OPENAT2",
 610                 .cleanup                = io_open_cleanup,
 611         },
 612         [IORING_OP_EPOLL_CTL] = {
 613                 .name                   = "EPOLL",
 614         },
 615         [IORING_OP_SPLICE] = {
 616                 .name                   = "SPLICE",
 617         },
 618         [IORING_OP_PROVIDE_BUFFERS] = {
 619                 .name                   = "PROVIDE_BUFFERS",
 620         },
 621         [IORING_OP_REMOVE_BUFFERS] = {
 622                 .name                   = "REMOVE_BUFFERS",
 623         },
 624         [IORING_OP_TEE] = {
 625                 .name                   = "TEE",
 626         },
 627         [IORING_OP_SHUTDOWN] = {
 628                 .name                   = "SHUTDOWN",
 629         },
 630         [IORING_OP_RENAMEAT] = {
 631                 .name                   = "RENAMEAT",
 632                 .cleanup                = io_renameat_cleanup,
 633         },
 634         [IORING_OP_UNLINKAT] = {
 635                 .name                   = "UNLINKAT",
 636                 .cleanup                = io_unlinkat_cleanup,
 637         },
 638         [IORING_OP_MKDIRAT] = {
 639                 .name                   = "MKDIRAT",
 640                 .cleanup                = io_mkdirat_cleanup,
 641         },
 642         [IORING_OP_SYMLINKAT] = {
 643                 .name                   = "SYMLINKAT",
 644                 .cleanup                = io_link_cleanup,
 645         },
 646         [IORING_OP_LINKAT] = {
 647                 .name                   = "LINKAT",
 648                 .cleanup                = io_link_cleanup,
 649         },
 650         [IORING_OP_MSG_RING] = {
 651                 .name                   = "MSG_RING",
 652                 .cleanup                = io_msg_ring_cleanup,
 653         },
 654         [IORING_OP_FSETXATTR] = {
 655                 .name                   = "FSETXATTR",
 656                 .cleanup                = io_xattr_cleanup,
 657         },
 658         [IORING_OP_SETXATTR] = {
 659                 .name                   = "SETXATTR",
 660                 .cleanup                = io_xattr_cleanup,
 661         },
 662         [IORING_OP_FGETXATTR] = {
 663                 .name                   = "FGETXATTR",
 664                 .cleanup                = io_xattr_cleanup,
 665         },
 666         [IORING_OP_GETXATTR] = {
 667                 .name                   = "GETXATTR",
 668                 .cleanup                = io_xattr_cleanup,
 669         },
 670         [IORING_OP_SOCKET] = {
 671                 .name                   = "SOCKET",
 672         },
 673         [IORING_OP_URING_CMD] = {
 674                 .name                   = "URING_CMD",
 675                 .async_size             = 2 * sizeof(struct io_uring_sqe),
 676                 .prep_async             = io_uring_cmd_prep_async,
 677         },
 678         [IORING_OP_SEND_ZC] = {
 679                 .name                   = "SEND_ZC",
 680 #if defined(CONFIG_NET)
 681                 .async_size             = sizeof(struct io_async_msghdr),
 682                 .prep_async             = io_send_prep_async,
 683                 .cleanup                = io_send_zc_cleanup,
 684                 .fail                   = io_sendrecv_fail,
 685 #endif
 686         },
 687         [IORING_OP_SENDMSG_ZC] = {
 688                 .name                   = "SENDMSG_ZC",
 689 #if defined(CONFIG_NET)
 690                 .async_size             = sizeof(struct io_async_msghdr),
 691                 .prep_async             = io_sendmsg_prep_async,
 692                 .cleanup                = io_send_zc_cleanup,
 693                 .fail                   = io_sendrecv_fail,
 694 #endif
 695         },
 696         [IORING_OP_READ_MULTISHOT] = {
 697                 .name                   = "READ_MULTISHOT",
 698         },
 699         [IORING_OP_WAITID] = {
 700                 .name                   = "WAITID",
 701                 .async_size             = sizeof(struct io_waitid_async),
 702         },
 703         [IORING_OP_FUTEX_WAIT] = {
 704                 .name                   = "FUTEX_WAIT",
 705         },
 706         [IORING_OP_FUTEX_WAKE] = {
 707                 .name                   = "FUTEX_WAKE",
 708         },
 709         [IORING_OP_FUTEX_WAITV] = {
 710                 .name                   = "FUTEX_WAITV",
 711         },
 712         [IORING_OP_FIXED_FD_INSTALL] = {
 713                 .name                   = "FIXED_FD_INSTALL",
 714         },
 715 };
 716
 717 const char *io_uring_get_opcode(u8 opcode)
 718 {
 719         if (opcode < IORING_OP_LAST)
 720                 return io_cold_defs[opcode].name;
 721         return "INVALID";
 722 }
 723
 724 void __init io_uring_optable_init(void)
 725 {
 726         int i;
 727
 728         BUILD_BUG_ON(ARRAY_SIZE(io_cold_defs) != IORING_OP_LAST);
 729         BUILD_BUG_ON(ARRAY_SIZE(io_issue_defs) != IORING_OP_LAST);
 730
 731         for (i = 0; i < ARRAY_SIZE(io_issue_defs); i++) {
 732                 BUG_ON(!io_issue_defs[i].prep);
 733                 if (io_issue_defs[i].prep != io_eopnotsupp_prep)
 734                         BUG_ON(!io_issue_defs[i].issue);
 735                 WARN_ON_ONCE(!io_cold_defs[i].name);
 736         }
 737 }