include/trace/events/rpcrdma.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 /*
   3  * Copyright (c) 2017, 2018 Oracle.  All rights reserved.
   4  *
   5  * Trace point definitions for the "rpcrdma" subsystem.
   6  */
   7 #undef TRACE_SYSTEM
   8 #define TRACE_SYSTEM rpcrdma
   9
  10 #if !defined(_TRACE_RPCRDMA_H) || defined(TRACE_HEADER_MULTI_READ)
  11 #define _TRACE_RPCRDMA_H
  12
  13 #include <linux/scatterlist.h>
  14 #include <linux/tracepoint.h>
  15 #include <trace/events/rdma.h>
  16
  17 /**
  18  ** Event classes
  19  **/
  20
  21 DECLARE_EVENT_CLASS(xprtrdma_reply_event,
  22         TP_PROTO(
  23                 const struct rpcrdma_rep *rep
  24         ),
  25
  26         TP_ARGS(rep),
  27
  28         TP_STRUCT__entry(
  29                 __field(const void *, rep)
  30                 __field(const void *, r_xprt)
  31                 __field(u32, xid)
  32                 __field(u32, version)
  33                 __field(u32, proc)
  34         ),
  35
  36         TP_fast_assign(
  37                 __entry->rep = rep;
  38                 __entry->r_xprt = rep->rr_rxprt;
  39                 __entry->xid = be32_to_cpu(rep->rr_xid);
  40                 __entry->version = be32_to_cpu(rep->rr_vers);
  41                 __entry->proc = be32_to_cpu(rep->rr_proc);
  42         ),
  43
  44         TP_printk("rxprt %p xid=0x%08x rep=%p: version %u proc %u",
  45                 __entry->r_xprt, __entry->xid, __entry->rep,
  46                 __entry->version, __entry->proc
  47         )
  48 );
  49
  50 #define DEFINE_REPLY_EVENT(name)                                        \
  51                 DEFINE_EVENT(xprtrdma_reply_event, name,                \
  52                                 TP_PROTO(                               \
  53                                         const struct rpcrdma_rep *rep   \
  54                                 ),                                      \
  55                                 TP_ARGS(rep))
  56
  57 DECLARE_EVENT_CLASS(xprtrdma_rxprt,
  58         TP_PROTO(
  59                 const struct rpcrdma_xprt *r_xprt
  60         ),
  61
  62         TP_ARGS(r_xprt),
  63
  64         TP_STRUCT__entry(
  65                 __field(const void *, r_xprt)
  66                 __string(addr, rpcrdma_addrstr(r_xprt))
  67                 __string(port, rpcrdma_portstr(r_xprt))
  68         ),
  69
  70         TP_fast_assign(
  71                 __entry->r_xprt = r_xprt;
  72                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
  73                 __assign_str(port, rpcrdma_portstr(r_xprt));
  74         ),
  75
  76         TP_printk("peer=[%s]:%s r_xprt=%p",
  77                 __get_str(addr), __get_str(port), __entry->r_xprt
  78         )
  79 );
  80
  81 #define DEFINE_RXPRT_EVENT(name)                                        \
  82                 DEFINE_EVENT(xprtrdma_rxprt, name,                      \
  83                                 TP_PROTO(                               \
  84                                         const struct rpcrdma_xprt *r_xprt \
  85                                 ),                                      \
  86                                 TP_ARGS(r_xprt))
  87
  88 DECLARE_EVENT_CLASS(xprtrdma_connect_class,
  89         TP_PROTO(
  90                 const struct rpcrdma_xprt *r_xprt,
  91                 int rc
  92         ),
  93
  94         TP_ARGS(r_xprt, rc),
  95
  96         TP_STRUCT__entry(
  97                 __field(const void *, r_xprt)
  98                 __field(int, rc)
  99                 __field(int, connect_status)
 100                 __string(addr, rpcrdma_addrstr(r_xprt))
 101                 __string(port, rpcrdma_portstr(r_xprt))
 102         ),
 103
 104         TP_fast_assign(
 105                 __entry->r_xprt = r_xprt;
 106                 __entry->rc = rc;
 107                 __entry->connect_status = r_xprt->rx_ep->re_connect_status;
 108                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 109                 __assign_str(port, rpcrdma_portstr(r_xprt));
 110         ),
 111
 112         TP_printk("peer=[%s]:%s r_xprt=%p: rc=%d connection status=%d",
 113                 __get_str(addr), __get_str(port), __entry->r_xprt,
 114                 __entry->rc, __entry->connect_status
 115         )
 116 );
 117
 118 #define DEFINE_CONN_EVENT(name)                                         \
 119                 DEFINE_EVENT(xprtrdma_connect_class, xprtrdma_##name,   \
 120                                 TP_PROTO(                               \
 121                                         const struct rpcrdma_xprt *r_xprt, \
 122                                         int rc                          \
 123                                 ),                                      \
 124                                 TP_ARGS(r_xprt, rc))
 125
 126 DECLARE_EVENT_CLASS(xprtrdma_rdch_event,
 127         TP_PROTO(
 128                 const struct rpc_task *task,
 129                 unsigned int pos,
 130                 struct rpcrdma_mr *mr,
 131                 int nsegs
 132         ),
 133
 134         TP_ARGS(task, pos, mr, nsegs),
 135
 136         TP_STRUCT__entry(
 137                 __field(unsigned int, task_id)
 138                 __field(unsigned int, client_id)
 139                 __field(unsigned int, pos)
 140                 __field(int, nents)
 141                 __field(u32, handle)
 142                 __field(u32, length)
 143                 __field(u64, offset)
 144                 __field(int, nsegs)
 145         ),
 146
 147         TP_fast_assign(
 148                 __entry->task_id = task->tk_pid;
 149                 __entry->client_id = task->tk_client->cl_clid;
 150                 __entry->pos = pos;
 151                 __entry->nents = mr->mr_nents;
 152                 __entry->handle = mr->mr_handle;
 153                 __entry->length = mr->mr_length;
 154                 __entry->offset = mr->mr_offset;
 155                 __entry->nsegs = nsegs;
 156         ),
 157
 158         TP_printk("task:%u@%u pos=%u %u@0x%016llx:0x%08x (%s)",
 159                 __entry->task_id, __entry->client_id,
 160                 __entry->pos, __entry->length,
 161                 (unsigned long long)__entry->offset, __entry->handle,
 162                 __entry->nents < __entry->nsegs ? "more" : "last"
 163         )
 164 );
 165
 166 #define DEFINE_RDCH_EVENT(name)                                         \
 167                 DEFINE_EVENT(xprtrdma_rdch_event, xprtrdma_chunk_##name,\
 168                                 TP_PROTO(                               \
 169                                         const struct rpc_task *task,    \
 170                                         unsigned int pos,               \
 171                                         struct rpcrdma_mr *mr,          \
 172                                         int nsegs                       \
 173                                 ),                                      \
 174                                 TP_ARGS(task, pos, mr, nsegs))
 175
 176 DECLARE_EVENT_CLASS(xprtrdma_wrch_event,
 177         TP_PROTO(
 178                 const struct rpc_task *task,
 179                 struct rpcrdma_mr *mr,
 180                 int nsegs
 181         ),
 182
 183         TP_ARGS(task, mr, nsegs),
 184
 185         TP_STRUCT__entry(
 186                 __field(unsigned int, task_id)
 187                 __field(unsigned int, client_id)
 188                 __field(int, nents)
 189                 __field(u32, handle)
 190                 __field(u32, length)
 191                 __field(u64, offset)
 192                 __field(int, nsegs)
 193         ),
 194
 195         TP_fast_assign(
 196                 __entry->task_id = task->tk_pid;
 197                 __entry->client_id = task->tk_client->cl_clid;
 198                 __entry->nents = mr->mr_nents;
 199                 __entry->handle = mr->mr_handle;
 200                 __entry->length = mr->mr_length;
 201                 __entry->offset = mr->mr_offset;
 202                 __entry->nsegs = nsegs;
 203         ),
 204
 205         TP_printk("task:%u@%u %u@0x%016llx:0x%08x (%s)",
 206                 __entry->task_id, __entry->client_id,
 207                 __entry->length, (unsigned long long)__entry->offset,
 208                 __entry->handle,
 209                 __entry->nents < __entry->nsegs ? "more" : "last"
 210         )
 211 );
 212
 213 #define DEFINE_WRCH_EVENT(name)                                         \
 214                 DEFINE_EVENT(xprtrdma_wrch_event, xprtrdma_chunk_##name,\
 215                                 TP_PROTO(                               \
 216                                         const struct rpc_task *task,    \
 217                                         struct rpcrdma_mr *mr,          \
 218                                         int nsegs                       \
 219                                 ),                                      \
 220                                 TP_ARGS(task, mr, nsegs))
 221
 222 DECLARE_EVENT_CLASS(xprtrdma_frwr_done,
 223         TP_PROTO(
 224                 const struct ib_wc *wc,
 225                 const struct rpcrdma_frwr *frwr
 226         ),
 227
 228         TP_ARGS(wc, frwr),
 229
 230         TP_STRUCT__entry(
 231                 __field(u32, mr_id)
 232                 __field(unsigned int, status)
 233                 __field(unsigned int, vendor_err)
 234         ),
 235
 236         TP_fast_assign(
 237                 __entry->mr_id = frwr->fr_mr->res.id;
 238                 __entry->status = wc->status;
 239                 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
 240         ),
 241
 242         TP_printk(
 243                 "mr.id=%u: %s (%u/0x%x)",
 244                 __entry->mr_id, rdma_show_wc_status(__entry->status),
 245                 __entry->status, __entry->vendor_err
 246         )
 247 );
 248
 249 #define DEFINE_FRWR_DONE_EVENT(name)                                    \
 250                 DEFINE_EVENT(xprtrdma_frwr_done, name,                  \
 251                                 TP_PROTO(                               \
 252                                         const struct ib_wc *wc,         \
 253                                         const struct rpcrdma_frwr *frwr \
 254                                 ),                                      \
 255                                 TP_ARGS(wc, frwr))
 256
 257 TRACE_DEFINE_ENUM(DMA_BIDIRECTIONAL);
 258 TRACE_DEFINE_ENUM(DMA_TO_DEVICE);
 259 TRACE_DEFINE_ENUM(DMA_FROM_DEVICE);
 260 TRACE_DEFINE_ENUM(DMA_NONE);
 261
 262 #define xprtrdma_show_direction(x)                                      \
 263                 __print_symbolic(x,                                     \
 264                                 { DMA_BIDIRECTIONAL, "BIDIR" },         \
 265                                 { DMA_TO_DEVICE, "TO_DEVICE" },         \
 266                                 { DMA_FROM_DEVICE, "FROM_DEVICE" },     \
 267                                 { DMA_NONE, "NONE" })
 268
 269 DECLARE_EVENT_CLASS(xprtrdma_mr,
 270         TP_PROTO(
 271                 const struct rpcrdma_mr *mr
 272         ),
 273
 274         TP_ARGS(mr),
 275
 276         TP_STRUCT__entry(
 277                 __field(u32, mr_id)
 278                 __field(int, nents)
 279                 __field(u32, handle)
 280                 __field(u32, length)
 281                 __field(u64, offset)
 282                 __field(u32, dir)
 283         ),
 284
 285         TP_fast_assign(
 286                 __entry->mr_id  = mr->frwr.fr_mr->res.id;
 287                 __entry->nents  = mr->mr_nents;
 288                 __entry->handle = mr->mr_handle;
 289                 __entry->length = mr->mr_length;
 290                 __entry->offset = mr->mr_offset;
 291                 __entry->dir    = mr->mr_dir;
 292         ),
 293
 294         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s)",
 295                 __entry->mr_id, __entry->nents, __entry->length,
 296                 (unsigned long long)__entry->offset, __entry->handle,
 297                 xprtrdma_show_direction(__entry->dir)
 298         )
 299 );
 300
 301 #define DEFINE_MR_EVENT(name) \
 302                 DEFINE_EVENT(xprtrdma_mr, xprtrdma_mr_##name, \
 303                                 TP_PROTO( \
 304                                         const struct rpcrdma_mr *mr \
 305                                 ), \
 306                                 TP_ARGS(mr))
 307
 308 DECLARE_EVENT_CLASS(xprtrdma_cb_event,
 309         TP_PROTO(
 310                 const struct rpc_rqst *rqst
 311         ),
 312
 313         TP_ARGS(rqst),
 314
 315         TP_STRUCT__entry(
 316                 __field(const void *, rqst)
 317                 __field(const void *, rep)
 318                 __field(const void *, req)
 319                 __field(u32, xid)
 320         ),
 321
 322         TP_fast_assign(
 323                 __entry->rqst = rqst;
 324                 __entry->req = rpcr_to_rdmar(rqst);
 325                 __entry->rep = rpcr_to_rdmar(rqst)->rl_reply;
 326                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 327         ),
 328
 329         TP_printk("xid=0x%08x, rqst=%p req=%p rep=%p",
 330                 __entry->xid, __entry->rqst, __entry->req, __entry->rep
 331         )
 332 );
 333
 334 #define DEFINE_CB_EVENT(name)                                           \
 335                 DEFINE_EVENT(xprtrdma_cb_event, name,                   \
 336                                 TP_PROTO(                               \
 337                                         const struct rpc_rqst *rqst     \
 338                                 ),                                      \
 339                                 TP_ARGS(rqst))
 340
 341 /**
 342  ** Connection events
 343  **/
 344
 345 TRACE_EVENT(xprtrdma_inline_thresh,
 346         TP_PROTO(
 347                 const struct rpcrdma_ep *ep
 348         ),
 349
 350         TP_ARGS(ep),
 351
 352         TP_STRUCT__entry(
 353                 __field(unsigned int, inline_send)
 354                 __field(unsigned int, inline_recv)
 355                 __field(unsigned int, max_send)
 356                 __field(unsigned int, max_recv)
 357                 __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
 358                 __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
 359         ),
 360
 361         TP_fast_assign(
 362                 const struct rdma_cm_id *id = ep->re_id;
 363
 364                 __entry->inline_send = ep->re_inline_send;
 365                 __entry->inline_recv = ep->re_inline_recv;
 366                 __entry->max_send = ep->re_max_inline_send;
 367                 __entry->max_recv = ep->re_max_inline_recv;
 368                 memcpy(__entry->srcaddr, &id->route.addr.src_addr,
 369                        sizeof(struct sockaddr_in6));
 370                 memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
 371                        sizeof(struct sockaddr_in6));
 372         ),
 373
 374         TP_printk("%pISpc -> %pISpc neg send/recv=%u/%u, calc send/recv=%u/%u",
 375                 __entry->srcaddr, __entry->dstaddr,
 376                 __entry->inline_send, __entry->inline_recv,
 377                 __entry->max_send, __entry->max_recv
 378         )
 379 );
 380
 381 DEFINE_CONN_EVENT(connect);
 382 DEFINE_CONN_EVENT(disconnect);
 383 DEFINE_CONN_EVENT(flush_dct);
 384
 385 DEFINE_RXPRT_EVENT(xprtrdma_create);
 386 DEFINE_RXPRT_EVENT(xprtrdma_op_destroy);
 387 DEFINE_RXPRT_EVENT(xprtrdma_op_inject_dsc);
 388 DEFINE_RXPRT_EVENT(xprtrdma_op_close);
 389 DEFINE_RXPRT_EVENT(xprtrdma_op_setport);
 390
 391 TRACE_EVENT(xprtrdma_op_connect,
 392         TP_PROTO(
 393                 const struct rpcrdma_xprt *r_xprt,
 394                 unsigned long delay
 395         ),
 396
 397         TP_ARGS(r_xprt, delay),
 398
 399         TP_STRUCT__entry(
 400                 __field(const void *, r_xprt)
 401                 __field(unsigned long, delay)
 402                 __string(addr, rpcrdma_addrstr(r_xprt))
 403                 __string(port, rpcrdma_portstr(r_xprt))
 404         ),
 405
 406         TP_fast_assign(
 407                 __entry->r_xprt = r_xprt;
 408                 __entry->delay = delay;
 409                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 410                 __assign_str(port, rpcrdma_portstr(r_xprt));
 411         ),
 412
 413         TP_printk("peer=[%s]:%s r_xprt=%p delay=%lu",
 414                 __get_str(addr), __get_str(port), __entry->r_xprt,
 415                 __entry->delay
 416         )
 417 );
 418
 419
 420 TRACE_EVENT(xprtrdma_op_set_cto,
 421         TP_PROTO(
 422                 const struct rpcrdma_xprt *r_xprt,
 423                 unsigned long connect,
 424                 unsigned long reconnect
 425         ),
 426
 427         TP_ARGS(r_xprt, connect, reconnect),
 428
 429         TP_STRUCT__entry(
 430                 __field(const void *, r_xprt)
 431                 __field(unsigned long, connect)
 432                 __field(unsigned long, reconnect)
 433                 __string(addr, rpcrdma_addrstr(r_xprt))
 434                 __string(port, rpcrdma_portstr(r_xprt))
 435         ),
 436
 437         TP_fast_assign(
 438                 __entry->r_xprt = r_xprt;
 439                 __entry->connect = connect;
 440                 __entry->reconnect = reconnect;
 441                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 442                 __assign_str(port, rpcrdma_portstr(r_xprt));
 443         ),
 444
 445         TP_printk("peer=[%s]:%s r_xprt=%p: connect=%lu reconnect=%lu",
 446                 __get_str(addr), __get_str(port), __entry->r_xprt,
 447                 __entry->connect / HZ, __entry->reconnect / HZ
 448         )
 449 );
 450
 451 TRACE_EVENT(xprtrdma_qp_event,
 452         TP_PROTO(
 453                 const struct rpcrdma_ep *ep,
 454                 const struct ib_event *event
 455         ),
 456
 457         TP_ARGS(ep, event),
 458
 459         TP_STRUCT__entry(
 460                 __field(unsigned long, event)
 461                 __string(name, event->device->name)
 462                 __array(unsigned char, srcaddr, sizeof(struct sockaddr_in6))
 463                 __array(unsigned char, dstaddr, sizeof(struct sockaddr_in6))
 464         ),
 465
 466         TP_fast_assign(
 467                 const struct rdma_cm_id *id = ep->re_id;
 468
 469                 __entry->event = event->event;
 470                 __assign_str(name, event->device->name);
 471                 memcpy(__entry->srcaddr, &id->route.addr.src_addr,
 472                        sizeof(struct sockaddr_in6));
 473                 memcpy(__entry->dstaddr, &id->route.addr.dst_addr,
 474                        sizeof(struct sockaddr_in6));
 475         ),
 476
 477         TP_printk("%pISpc -> %pISpc device=%s %s (%lu)",
 478                 __entry->srcaddr, __entry->dstaddr, __get_str(name),
 479                 rdma_show_ib_event(__entry->event), __entry->event
 480         )
 481 );
 482
 483 /**
 484  ** Call events
 485  **/
 486
 487 TRACE_EVENT(xprtrdma_createmrs,
 488         TP_PROTO(
 489                 const struct rpcrdma_xprt *r_xprt,
 490                 unsigned int count
 491         ),
 492
 493         TP_ARGS(r_xprt, count),
 494
 495         TP_STRUCT__entry(
 496                 __field(const void *, r_xprt)
 497                 __string(addr, rpcrdma_addrstr(r_xprt))
 498                 __string(port, rpcrdma_portstr(r_xprt))
 499                 __field(unsigned int, count)
 500         ),
 501
 502         TP_fast_assign(
 503                 __entry->r_xprt = r_xprt;
 504                 __entry->count = count;
 505                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 506                 __assign_str(port, rpcrdma_portstr(r_xprt));
 507         ),
 508
 509         TP_printk("peer=[%s]:%s r_xprt=%p: created %u MRs",
 510                 __get_str(addr), __get_str(port), __entry->r_xprt,
 511                 __entry->count
 512         )
 513 );
 514
 515 TRACE_EVENT(xprtrdma_mr_get,
 516         TP_PROTO(
 517                 const struct rpcrdma_req *req
 518         ),
 519
 520         TP_ARGS(req),
 521
 522         TP_STRUCT__entry(
 523                 __field(const void *, req)
 524                 __field(unsigned int, task_id)
 525                 __field(unsigned int, client_id)
 526                 __field(u32, xid)
 527         ),
 528
 529         TP_fast_assign(
 530                 const struct rpc_rqst *rqst = &req->rl_slot;
 531
 532                 __entry->req = req;
 533                 __entry->task_id = rqst->rq_task->tk_pid;
 534                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 535                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 536         ),
 537
 538         TP_printk("task:%u@%u xid=0x%08x req=%p",
 539                 __entry->task_id, __entry->client_id, __entry->xid,
 540                 __entry->req
 541         )
 542 );
 543
 544 TRACE_EVENT(xprtrdma_nomrs,
 545         TP_PROTO(
 546                 const struct rpcrdma_req *req
 547         ),
 548
 549         TP_ARGS(req),
 550
 551         TP_STRUCT__entry(
 552                 __field(const void *, req)
 553                 __field(unsigned int, task_id)
 554                 __field(unsigned int, client_id)
 555                 __field(u32, xid)
 556         ),
 557
 558         TP_fast_assign(
 559                 const struct rpc_rqst *rqst = &req->rl_slot;
 560
 561                 __entry->req = req;
 562                 __entry->task_id = rqst->rq_task->tk_pid;
 563                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 564                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 565         ),
 566
 567         TP_printk("task:%u@%u xid=0x%08x req=%p",
 568                 __entry->task_id, __entry->client_id, __entry->xid,
 569                 __entry->req
 570         )
 571 );
 572
 573 DEFINE_RDCH_EVENT(read);
 574 DEFINE_WRCH_EVENT(write);
 575 DEFINE_WRCH_EVENT(reply);
 576
 577 TRACE_DEFINE_ENUM(rpcrdma_noch);
 578 TRACE_DEFINE_ENUM(rpcrdma_noch_pullup);
 579 TRACE_DEFINE_ENUM(rpcrdma_noch_mapped);
 580 TRACE_DEFINE_ENUM(rpcrdma_readch);
 581 TRACE_DEFINE_ENUM(rpcrdma_areadch);
 582 TRACE_DEFINE_ENUM(rpcrdma_writech);
 583 TRACE_DEFINE_ENUM(rpcrdma_replych);
 584
 585 #define xprtrdma_show_chunktype(x)                                      \
 586                 __print_symbolic(x,                                     \
 587                                 { rpcrdma_noch, "inline" },             \
 588                                 { rpcrdma_noch_pullup, "pullup" },      \
 589                                 { rpcrdma_noch_mapped, "mapped" },      \
 590                                 { rpcrdma_readch, "read list" },        \
 591                                 { rpcrdma_areadch, "*read list" },      \
 592                                 { rpcrdma_writech, "write list" },      \
 593                                 { rpcrdma_replych, "reply chunk" })
 594
 595 TRACE_EVENT(xprtrdma_marshal,
 596         TP_PROTO(
 597                 const struct rpcrdma_req *req,
 598                 unsigned int rtype,
 599                 unsigned int wtype
 600         ),
 601
 602         TP_ARGS(req, rtype, wtype),
 603
 604         TP_STRUCT__entry(
 605                 __field(unsigned int, task_id)
 606                 __field(unsigned int, client_id)
 607                 __field(u32, xid)
 608                 __field(unsigned int, hdrlen)
 609                 __field(unsigned int, headlen)
 610                 __field(unsigned int, pagelen)
 611                 __field(unsigned int, taillen)
 612                 __field(unsigned int, rtype)
 613                 __field(unsigned int, wtype)
 614         ),
 615
 616         TP_fast_assign(
 617                 const struct rpc_rqst *rqst = &req->rl_slot;
 618
 619                 __entry->task_id = rqst->rq_task->tk_pid;
 620                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 621                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 622                 __entry->hdrlen = req->rl_hdrbuf.len;
 623                 __entry->headlen = rqst->rq_snd_buf.head[0].iov_len;
 624                 __entry->pagelen = rqst->rq_snd_buf.page_len;
 625                 __entry->taillen = rqst->rq_snd_buf.tail[0].iov_len;
 626                 __entry->rtype = rtype;
 627                 __entry->wtype = wtype;
 628         ),
 629
 630         TP_printk("task:%u@%u xid=0x%08x: hdr=%u xdr=%u/%u/%u %s/%s",
 631                 __entry->task_id, __entry->client_id, __entry->xid,
 632                 __entry->hdrlen,
 633                 __entry->headlen, __entry->pagelen, __entry->taillen,
 634                 xprtrdma_show_chunktype(__entry->rtype),
 635                 xprtrdma_show_chunktype(__entry->wtype)
 636         )
 637 );
 638
 639 TRACE_EVENT(xprtrdma_marshal_failed,
 640         TP_PROTO(const struct rpc_rqst *rqst,
 641                  int ret
 642         ),
 643
 644         TP_ARGS(rqst, ret),
 645
 646         TP_STRUCT__entry(
 647                 __field(unsigned int, task_id)
 648                 __field(unsigned int, client_id)
 649                 __field(u32, xid)
 650                 __field(int, ret)
 651         ),
 652
 653         TP_fast_assign(
 654                 __entry->task_id = rqst->rq_task->tk_pid;
 655                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 656                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 657                 __entry->ret = ret;
 658         ),
 659
 660         TP_printk("task:%u@%u xid=0x%08x: ret=%d",
 661                 __entry->task_id, __entry->client_id, __entry->xid,
 662                 __entry->ret
 663         )
 664 );
 665
 666 TRACE_EVENT(xprtrdma_prepsend_failed,
 667         TP_PROTO(const struct rpc_rqst *rqst,
 668                  int ret
 669         ),
 670
 671         TP_ARGS(rqst, ret),
 672
 673         TP_STRUCT__entry(
 674                 __field(unsigned int, task_id)
 675                 __field(unsigned int, client_id)
 676                 __field(u32, xid)
 677                 __field(int, ret)
 678         ),
 679
 680         TP_fast_assign(
 681                 __entry->task_id = rqst->rq_task->tk_pid;
 682                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
 683                 __entry->xid = be32_to_cpu(rqst->rq_xid);
 684                 __entry->ret = ret;
 685         ),
 686
 687         TP_printk("task:%u@%u xid=0x%08x: ret=%d",
 688                 __entry->task_id, __entry->client_id, __entry->xid,
 689                 __entry->ret
 690         )
 691 );
 692
 693 TRACE_EVENT(xprtrdma_post_send,
 694         TP_PROTO(
 695                 const struct rpcrdma_req *req
 696         ),
 697
 698         TP_ARGS(req),
 699
 700         TP_STRUCT__entry(
 701                 __field(const void *, req)
 702                 __field(const void *, sc)
 703                 __field(unsigned int, task_id)
 704                 __field(unsigned int, client_id)
 705                 __field(int, num_sge)
 706                 __field(int, signaled)
 707         ),
 708
 709         TP_fast_assign(
 710                 const struct rpc_rqst *rqst = &req->rl_slot;
 711
 712                 __entry->task_id = rqst->rq_task->tk_pid;
 713                 __entry->client_id = rqst->rq_task->tk_client ?
 714                                      rqst->rq_task->tk_client->cl_clid : -1;
 715                 __entry->req = req;
 716                 __entry->sc = req->rl_sendctx;
 717                 __entry->num_sge = req->rl_wr.num_sge;
 718                 __entry->signaled = req->rl_wr.send_flags & IB_SEND_SIGNALED;
 719         ),
 720
 721         TP_printk("task:%u@%u req=%p sc=%p (%d SGE%s) %s",
 722                 __entry->task_id, __entry->client_id,
 723                 __entry->req, __entry->sc, __entry->num_sge,
 724                 (__entry->num_sge == 1 ? "" : "s"),
 725                 (__entry->signaled ? "signaled" : "")
 726         )
 727 );
 728
 729 TRACE_EVENT(xprtrdma_post_recv,
 730         TP_PROTO(
 731                 const struct rpcrdma_rep *rep
 732         ),
 733
 734         TP_ARGS(rep),
 735
 736         TP_STRUCT__entry(
 737                 __field(const void *, rep)
 738         ),
 739
 740         TP_fast_assign(
 741                 __entry->rep = rep;
 742         ),
 743
 744         TP_printk("rep=%p",
 745                 __entry->rep
 746         )
 747 );
 748
 749 TRACE_EVENT(xprtrdma_post_recvs,
 750         TP_PROTO(
 751                 const struct rpcrdma_xprt *r_xprt,
 752                 unsigned int count,
 753                 int status
 754         ),
 755
 756         TP_ARGS(r_xprt, count, status),
 757
 758         TP_STRUCT__entry(
 759                 __field(const void *, r_xprt)
 760                 __field(unsigned int, count)
 761                 __field(int, status)
 762                 __field(int, posted)
 763                 __string(addr, rpcrdma_addrstr(r_xprt))
 764                 __string(port, rpcrdma_portstr(r_xprt))
 765         ),
 766
 767         TP_fast_assign(
 768                 __entry->r_xprt = r_xprt;
 769                 __entry->count = count;
 770                 __entry->status = status;
 771                 __entry->posted = r_xprt->rx_ep->re_receive_count;
 772                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
 773                 __assign_str(port, rpcrdma_portstr(r_xprt));
 774         ),
 775
 776         TP_printk("peer=[%s]:%s r_xprt=%p: %u new recvs, %d active (rc %d)",
 777                 __get_str(addr), __get_str(port), __entry->r_xprt,
 778                 __entry->count, __entry->posted, __entry->status
 779         )
 780 );
 781
 782 TRACE_EVENT(xprtrdma_post_linv,
 783         TP_PROTO(
 784                 const struct rpcrdma_req *req,
 785                 int status
 786         ),
 787
 788         TP_ARGS(req, status),
 789
 790         TP_STRUCT__entry(
 791                 __field(const void *, req)
 792                 __field(int, status)
 793                 __field(u32, xid)
 794         ),
 795
 796         TP_fast_assign(
 797                 __entry->req = req;
 798                 __entry->status = status;
 799                 __entry->xid = be32_to_cpu(req->rl_slot.rq_xid);
 800         ),
 801
 802         TP_printk("req=%p xid=0x%08x status=%d",
 803                 __entry->req, __entry->xid, __entry->status
 804         )
 805 );
 806
 807 /**
 808  ** Completion events
 809  **/
 810
 811 TRACE_EVENT(xprtrdma_wc_send,
 812         TP_PROTO(
 813                 const struct rpcrdma_sendctx *sc,
 814                 const struct ib_wc *wc
 815         ),
 816
 817         TP_ARGS(sc, wc),
 818
 819         TP_STRUCT__entry(
 820                 __field(const void *, req)
 821                 __field(const void *, sc)
 822                 __field(unsigned int, unmap_count)
 823                 __field(unsigned int, status)
 824                 __field(unsigned int, vendor_err)
 825         ),
 826
 827         TP_fast_assign(
 828                 __entry->req = sc->sc_req;
 829                 __entry->sc = sc;
 830                 __entry->unmap_count = sc->sc_unmap_count;
 831                 __entry->status = wc->status;
 832                 __entry->vendor_err = __entry->status ? wc->vendor_err : 0;
 833         ),
 834
 835         TP_printk("req=%p sc=%p unmapped=%u: %s (%u/0x%x)",
 836                 __entry->req, __entry->sc, __entry->unmap_count,
 837                 rdma_show_wc_status(__entry->status),
 838                 __entry->status, __entry->vendor_err
 839         )
 840 );
 841
 842 TRACE_EVENT(xprtrdma_wc_receive,
 843         TP_PROTO(
 844                 const struct ib_wc *wc
 845         ),
 846
 847         TP_ARGS(wc),
 848
 849         TP_STRUCT__entry(
 850                 __field(const void *, rep)
 851                 __field(u32, byte_len)
 852                 __field(unsigned int, status)
 853                 __field(u32, vendor_err)
 854         ),
 855
 856         TP_fast_assign(
 857                 __entry->rep = container_of(wc->wr_cqe, struct rpcrdma_rep,
 858                                             rr_cqe);
 859                 __entry->status = wc->status;
 860                 if (wc->status) {
 861                         __entry->byte_len = 0;
 862                         __entry->vendor_err = wc->vendor_err;
 863                 } else {
 864                         __entry->byte_len = wc->byte_len;
 865                         __entry->vendor_err = 0;
 866                 }
 867         ),
 868
 869         TP_printk("rep=%p %u bytes: %s (%u/0x%x)",
 870                 __entry->rep, __entry->byte_len,
 871                 rdma_show_wc_status(__entry->status),
 872                 __entry->status, __entry->vendor_err
 873         )
 874 );
 875
 876 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_fastreg);
 877 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li);
 878 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_wake);
 879 DEFINE_FRWR_DONE_EVENT(xprtrdma_wc_li_done);
 880
 881 TRACE_EVENT(xprtrdma_frwr_alloc,
 882         TP_PROTO(
 883                 const struct rpcrdma_mr *mr,
 884                 int rc
 885         ),
 886
 887         TP_ARGS(mr, rc),
 888
 889         TP_STRUCT__entry(
 890                 __field(u32, mr_id)
 891                 __field(int, rc)
 892         ),
 893
 894         TP_fast_assign(
 895                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 896                 __entry->rc = rc;
 897         ),
 898
 899         TP_printk("mr.id=%u: rc=%d",
 900                 __entry->mr_id, __entry->rc
 901         )
 902 );
 903
 904 TRACE_EVENT(xprtrdma_frwr_dereg,
 905         TP_PROTO(
 906                 const struct rpcrdma_mr *mr,
 907                 int rc
 908         ),
 909
 910         TP_ARGS(mr, rc),
 911
 912         TP_STRUCT__entry(
 913                 __field(u32, mr_id)
 914                 __field(int, nents)
 915                 __field(u32, handle)
 916                 __field(u32, length)
 917                 __field(u64, offset)
 918                 __field(u32, dir)
 919                 __field(int, rc)
 920         ),
 921
 922         TP_fast_assign(
 923                 __entry->mr_id  = mr->frwr.fr_mr->res.id;
 924                 __entry->nents  = mr->mr_nents;
 925                 __entry->handle = mr->mr_handle;
 926                 __entry->length = mr->mr_length;
 927                 __entry->offset = mr->mr_offset;
 928                 __entry->dir    = mr->mr_dir;
 929                 __entry->rc     = rc;
 930         ),
 931
 932         TP_printk("mr.id=%u nents=%d %u@0x%016llx:0x%08x (%s): rc=%d",
 933                 __entry->mr_id, __entry->nents, __entry->length,
 934                 (unsigned long long)__entry->offset, __entry->handle,
 935                 xprtrdma_show_direction(__entry->dir),
 936                 __entry->rc
 937         )
 938 );
 939
 940 TRACE_EVENT(xprtrdma_frwr_sgerr,
 941         TP_PROTO(
 942                 const struct rpcrdma_mr *mr,
 943                 int sg_nents
 944         ),
 945
 946         TP_ARGS(mr, sg_nents),
 947
 948         TP_STRUCT__entry(
 949                 __field(u32, mr_id)
 950                 __field(u64, addr)
 951                 __field(u32, dir)
 952                 __field(int, nents)
 953         ),
 954
 955         TP_fast_assign(
 956                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 957                 __entry->addr = mr->mr_sg->dma_address;
 958                 __entry->dir = mr->mr_dir;
 959                 __entry->nents = sg_nents;
 960         ),
 961
 962         TP_printk("mr.id=%u DMA addr=0x%llx (%s) sg_nents=%d",
 963                 __entry->mr_id, __entry->addr,
 964                 xprtrdma_show_direction(__entry->dir),
 965                 __entry->nents
 966         )
 967 );
 968
 969 TRACE_EVENT(xprtrdma_frwr_maperr,
 970         TP_PROTO(
 971                 const struct rpcrdma_mr *mr,
 972                 int num_mapped
 973         ),
 974
 975         TP_ARGS(mr, num_mapped),
 976
 977         TP_STRUCT__entry(
 978                 __field(u32, mr_id)
 979                 __field(u64, addr)
 980                 __field(u32, dir)
 981                 __field(int, num_mapped)
 982                 __field(int, nents)
 983         ),
 984
 985         TP_fast_assign(
 986                 __entry->mr_id = mr->frwr.fr_mr->res.id;
 987                 __entry->addr = mr->mr_sg->dma_address;
 988                 __entry->dir = mr->mr_dir;
 989                 __entry->num_mapped = num_mapped;
 990                 __entry->nents = mr->mr_nents;
 991         ),
 992
 993         TP_printk("mr.id=%u DMA addr=0x%llx (%s) nents=%d of %d",
 994                 __entry->mr_id, __entry->addr,
 995                 xprtrdma_show_direction(__entry->dir),
 996                 __entry->num_mapped, __entry->nents
 997         )
 998 );
 999
1000 DEFINE_MR_EVENT(localinv);
1001 DEFINE_MR_EVENT(map);
1002 DEFINE_MR_EVENT(unmap);
1003 DEFINE_MR_EVENT(reminv);
1004 DEFINE_MR_EVENT(recycle);
1005
1006 TRACE_EVENT(xprtrdma_dma_maperr,
1007         TP_PROTO(
1008                 u64 addr
1009         ),
1010
1011         TP_ARGS(addr),
1012
1013         TP_STRUCT__entry(
1014                 __field(u64, addr)
1015         ),
1016
1017         TP_fast_assign(
1018                 __entry->addr = addr;
1019         ),
1020
1021         TP_printk("dma addr=0x%llx\n", __entry->addr)
1022 );
1023
1024 /**
1025  ** Reply events
1026  **/
1027
1028 TRACE_EVENT(xprtrdma_reply,
1029         TP_PROTO(
1030                 const struct rpc_task *task,
1031                 const struct rpcrdma_rep *rep,
1032                 const struct rpcrdma_req *req,
1033                 unsigned int credits
1034         ),
1035
1036         TP_ARGS(task, rep, req, credits),
1037
1038         TP_STRUCT__entry(
1039                 __field(unsigned int, task_id)
1040                 __field(unsigned int, client_id)
1041                 __field(const void *, rep)
1042                 __field(const void *, req)
1043                 __field(u32, xid)
1044                 __field(unsigned int, credits)
1045         ),
1046
1047         TP_fast_assign(
1048                 __entry->task_id = task->tk_pid;
1049                 __entry->client_id = task->tk_client->cl_clid;
1050                 __entry->rep = rep;
1051                 __entry->req = req;
1052                 __entry->xid = be32_to_cpu(rep->rr_xid);
1053                 __entry->credits = credits;
1054         ),
1055
1056         TP_printk("task:%u@%u xid=0x%08x, %u credits, rep=%p -> req=%p",
1057                 __entry->task_id, __entry->client_id, __entry->xid,
1058                 __entry->credits, __entry->rep, __entry->req
1059         )
1060 );
1061
1062 TRACE_EVENT(xprtrdma_defer_cmp,
1063         TP_PROTO(
1064                 const struct rpcrdma_rep *rep
1065         ),
1066
1067         TP_ARGS(rep),
1068
1069         TP_STRUCT__entry(
1070                 __field(unsigned int, task_id)
1071                 __field(unsigned int, client_id)
1072                 __field(const void *, rep)
1073                 __field(u32, xid)
1074         ),
1075
1076         TP_fast_assign(
1077                 __entry->task_id = rep->rr_rqst->rq_task->tk_pid;
1078                 __entry->client_id = rep->rr_rqst->rq_task->tk_client->cl_clid;
1079                 __entry->rep = rep;
1080                 __entry->xid = be32_to_cpu(rep->rr_xid);
1081         ),
1082
1083         TP_printk("task:%u@%u xid=0x%08x rep=%p",
1084                 __entry->task_id, __entry->client_id, __entry->xid,
1085                 __entry->rep
1086         )
1087 );
1088
1089 DEFINE_REPLY_EVENT(xprtrdma_reply_vers);
1090 DEFINE_REPLY_EVENT(xprtrdma_reply_rqst);
1091 DEFINE_REPLY_EVENT(xprtrdma_reply_short);
1092 DEFINE_REPLY_EVENT(xprtrdma_reply_hdr);
1093
1094 TRACE_EVENT(xprtrdma_fixup,
1095         TP_PROTO(
1096                 const struct rpc_rqst *rqst,
1097                 unsigned long fixup
1098         ),
1099
1100         TP_ARGS(rqst, fixup),
1101
1102         TP_STRUCT__entry(
1103                 __field(unsigned int, task_id)
1104                 __field(unsigned int, client_id)
1105                 __field(unsigned long, fixup)
1106                 __field(size_t, headlen)
1107                 __field(unsigned int, pagelen)
1108                 __field(size_t, taillen)
1109         ),
1110
1111         TP_fast_assign(
1112                 __entry->task_id = rqst->rq_task->tk_pid;
1113                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1114                 __entry->fixup = fixup;
1115                 __entry->headlen = rqst->rq_rcv_buf.head[0].iov_len;
1116                 __entry->pagelen = rqst->rq_rcv_buf.page_len;
1117                 __entry->taillen = rqst->rq_rcv_buf.tail[0].iov_len;
1118         ),
1119
1120         TP_printk("task:%u@%u fixup=%lu xdr=%zu/%u/%zu",
1121                 __entry->task_id, __entry->client_id, __entry->fixup,
1122                 __entry->headlen, __entry->pagelen, __entry->taillen
1123         )
1124 );
1125
1126 TRACE_EVENT(xprtrdma_decode_seg,
1127         TP_PROTO(
1128                 u32 handle,
1129                 u32 length,
1130                 u64 offset
1131         ),
1132
1133         TP_ARGS(handle, length, offset),
1134
1135         TP_STRUCT__entry(
1136                 __field(u32, handle)
1137                 __field(u32, length)
1138                 __field(u64, offset)
1139         ),
1140
1141         TP_fast_assign(
1142                 __entry->handle = handle;
1143                 __entry->length = length;
1144                 __entry->offset = offset;
1145         ),
1146
1147         TP_printk("%u@0x%016llx:0x%08x",
1148                 __entry->length, (unsigned long long)__entry->offset,
1149                 __entry->handle
1150         )
1151 );
1152
1153 /**
1154  ** Allocation/release of rpcrdma_reqs and rpcrdma_reps
1155  **/
1156
1157 TRACE_EVENT(xprtrdma_op_allocate,
1158         TP_PROTO(
1159                 const struct rpc_task *task,
1160                 const struct rpcrdma_req *req
1161         ),
1162
1163         TP_ARGS(task, req),
1164
1165         TP_STRUCT__entry(
1166                 __field(unsigned int, task_id)
1167                 __field(unsigned int, client_id)
1168                 __field(const void *, req)
1169                 __field(size_t, callsize)
1170                 __field(size_t, rcvsize)
1171         ),
1172
1173         TP_fast_assign(
1174                 __entry->task_id = task->tk_pid;
1175                 __entry->client_id = task->tk_client->cl_clid;
1176                 __entry->req = req;
1177                 __entry->callsize = task->tk_rqstp->rq_callsize;
1178                 __entry->rcvsize = task->tk_rqstp->rq_rcvsize;
1179         ),
1180
1181         TP_printk("task:%u@%u req=%p (%zu, %zu)",
1182                 __entry->task_id, __entry->client_id,
1183                 __entry->req, __entry->callsize, __entry->rcvsize
1184         )
1185 );
1186
1187 TRACE_EVENT(xprtrdma_op_free,
1188         TP_PROTO(
1189                 const struct rpc_task *task,
1190                 const struct rpcrdma_req *req
1191         ),
1192
1193         TP_ARGS(task, req),
1194
1195         TP_STRUCT__entry(
1196                 __field(unsigned int, task_id)
1197                 __field(unsigned int, client_id)
1198                 __field(const void *, req)
1199                 __field(const void *, rep)
1200         ),
1201
1202         TP_fast_assign(
1203                 __entry->task_id = task->tk_pid;
1204                 __entry->client_id = task->tk_client->cl_clid;
1205                 __entry->req = req;
1206                 __entry->rep = req->rl_reply;
1207         ),
1208
1209         TP_printk("task:%u@%u req=%p rep=%p",
1210                 __entry->task_id, __entry->client_id,
1211                 __entry->req, __entry->rep
1212         )
1213 );
1214
1215 /**
1216  ** Callback events
1217  **/
1218
1219 TRACE_EVENT(xprtrdma_cb_setup,
1220         TP_PROTO(
1221                 const struct rpcrdma_xprt *r_xprt,
1222                 unsigned int reqs
1223         ),
1224
1225         TP_ARGS(r_xprt, reqs),
1226
1227         TP_STRUCT__entry(
1228                 __field(const void *, r_xprt)
1229                 __field(unsigned int, reqs)
1230                 __string(addr, rpcrdma_addrstr(r_xprt))
1231                 __string(port, rpcrdma_portstr(r_xprt))
1232         ),
1233
1234         TP_fast_assign(
1235                 __entry->r_xprt = r_xprt;
1236                 __entry->reqs = reqs;
1237                 __assign_str(addr, rpcrdma_addrstr(r_xprt));
1238                 __assign_str(port, rpcrdma_portstr(r_xprt));
1239         ),
1240
1241         TP_printk("peer=[%s]:%s r_xprt=%p: %u reqs",
1242                 __get_str(addr), __get_str(port),
1243                 __entry->r_xprt, __entry->reqs
1244         )
1245 );
1246
1247 DEFINE_CB_EVENT(xprtrdma_cb_call);
1248 DEFINE_CB_EVENT(xprtrdma_cb_reply);
1249
1250 TRACE_EVENT(xprtrdma_leaked_rep,
1251         TP_PROTO(
1252                 const struct rpc_rqst *rqst,
1253                 const struct rpcrdma_rep *rep
1254         ),
1255
1256         TP_ARGS(rqst, rep),
1257
1258         TP_STRUCT__entry(
1259                 __field(unsigned int, task_id)
1260                 __field(unsigned int, client_id)
1261                 __field(u32, xid)
1262                 __field(const void *, rep)
1263         ),
1264
1265         TP_fast_assign(
1266                 __entry->task_id = rqst->rq_task->tk_pid;
1267                 __entry->client_id = rqst->rq_task->tk_client->cl_clid;
1268                 __entry->xid = be32_to_cpu(rqst->rq_xid);
1269                 __entry->rep = rep;
1270         ),
1271
1272         TP_printk("task:%u@%u xid=0x%08x rep=%p",
1273                 __entry->task_id, __entry->client_id, __entry->xid,
1274                 __entry->rep
1275         )
1276 );
1277
1278 /**
1279  ** Server-side RPC/RDMA events
1280  **/
1281
1282 DECLARE_EVENT_CLASS(svcrdma_xprt_event,
1283         TP_PROTO(
1284                 const struct svc_xprt *xprt
1285         ),
1286
1287         TP_ARGS(xprt),
1288
1289         TP_STRUCT__entry(
1290                 __field(const void *, xprt)
1291                 __string(addr, xprt->xpt_remotebuf)
1292         ),
1293
1294         TP_fast_assign(
1295                 __entry->xprt = xprt;
1296                 __assign_str(addr, xprt->xpt_remotebuf);
1297         ),
1298
1299         TP_printk("xprt=%p addr=%s",
1300                 __entry->xprt, __get_str(addr)
1301         )
1302 );
1303
1304 #define DEFINE_XPRT_EVENT(name)                                         \
1305                 DEFINE_EVENT(svcrdma_xprt_event, svcrdma_xprt_##name,   \
1306                                 TP_PROTO(                               \
1307                                         const struct svc_xprt *xprt     \
1308                                 ),                                      \
1309                                 TP_ARGS(xprt))
1310
1311 DEFINE_XPRT_EVENT(accept);
1312 DEFINE_XPRT_EVENT(fail);
1313 DEFINE_XPRT_EVENT(free);
1314
1315 TRACE_DEFINE_ENUM(RDMA_MSG);
1316 TRACE_DEFINE_ENUM(RDMA_NOMSG);
1317 TRACE_DEFINE_ENUM(RDMA_MSGP);
1318 TRACE_DEFINE_ENUM(RDMA_DONE);
1319 TRACE_DEFINE_ENUM(RDMA_ERROR);
1320
1321 #define show_rpcrdma_proc(x)                                            \
1322                 __print_symbolic(x,                                     \
1323                                 { RDMA_MSG, "RDMA_MSG" },               \
1324                                 { RDMA_NOMSG, "RDMA_NOMSG" },           \
1325                                 { RDMA_MSGP, "RDMA_MSGP" },             \
1326                                 { RDMA_DONE, "RDMA_DONE" },             \
1327                                 { RDMA_ERROR, "RDMA_ERROR" })
1328
1329 TRACE_EVENT(svcrdma_decode_rqst,
1330         TP_PROTO(
1331                 __be32 *p,
1332                 unsigned int hdrlen
1333         ),
1334
1335         TP_ARGS(p, hdrlen),
1336
1337         TP_STRUCT__entry(
1338                 __field(u32, xid)
1339                 __field(u32, vers)
1340                 __field(u32, proc)
1341                 __field(u32, credits)
1342                 __field(unsigned int, hdrlen)
1343         ),
1344
1345         TP_fast_assign(
1346                 __entry->xid = be32_to_cpup(p++);
1347                 __entry->vers = be32_to_cpup(p++);
1348                 __entry->credits = be32_to_cpup(p++);
1349                 __entry->proc = be32_to_cpup(p);
1350                 __entry->hdrlen = hdrlen;
1351         ),
1352
1353         TP_printk("xid=0x%08x vers=%u credits=%u proc=%s hdrlen=%u",
1354                 __entry->xid, __entry->vers, __entry->credits,
1355                 show_rpcrdma_proc(__entry->proc), __entry->hdrlen)
1356 );
1357
1358 TRACE_EVENT(svcrdma_decode_short,
1359         TP_PROTO(
1360                 unsigned int hdrlen
1361         ),
1362
1363         TP_ARGS(hdrlen),
1364
1365         TP_STRUCT__entry(
1366                 __field(unsigned int, hdrlen)
1367         ),
1368
1369         TP_fast_assign(
1370                 __entry->hdrlen = hdrlen;
1371         ),
1372
1373         TP_printk("hdrlen=%u", __entry->hdrlen)
1374 );
1375
1376 DECLARE_EVENT_CLASS(svcrdma_badreq_event,
1377         TP_PROTO(
1378                 __be32 *p
1379         ),
1380
1381         TP_ARGS(p),
1382
1383         TP_STRUCT__entry(
1384                 __field(u32, xid)
1385                 __field(u32, vers)
1386                 __field(u32, proc)
1387                 __field(u32, credits)
1388         ),
1389
1390         TP_fast_assign(
1391                 __entry->xid = be32_to_cpup(p++);
1392                 __entry->vers = be32_to_cpup(p++);
1393                 __entry->credits = be32_to_cpup(p++);
1394                 __entry->proc = be32_to_cpup(p);
1395         ),
1396
1397         TP_printk("xid=0x%08x vers=%u credits=%u proc=%u",
1398                 __entry->xid, __entry->vers, __entry->credits, __entry->proc)
1399 );
1400
1401 #define DEFINE_BADREQ_EVENT(name)                                       \
1402                 DEFINE_EVENT(svcrdma_badreq_event, svcrdma_decode_##name,\
1403                                 TP_PROTO(                               \
1404                                         __be32 *p                       \
1405                                 ),                                      \
1406                                 TP_ARGS(p))
1407
1408 DEFINE_BADREQ_EVENT(badvers);
1409 DEFINE_BADREQ_EVENT(drop);
1410 DEFINE_BADREQ_EVENT(badproc);
1411 DEFINE_BADREQ_EVENT(parse);
1412
1413 DECLARE_EVENT_CLASS(svcrdma_segment_event,
1414         TP_PROTO(
1415                 u32 handle,
1416                 u32 length,
1417                 u64 offset
1418         ),
1419
1420         TP_ARGS(handle, length, offset),
1421
1422         TP_STRUCT__entry(
1423                 __field(u32, handle)
1424                 __field(u32, length)
1425                 __field(u64, offset)
1426         ),
1427
1428         TP_fast_assign(
1429                 __entry->handle = handle;
1430                 __entry->length = length;
1431                 __entry->offset = offset;
1432         ),
1433
1434         TP_printk("%u@0x%016llx:0x%08x",
1435                 __entry->length, (unsigned long long)__entry->offset,
1436                 __entry->handle
1437         )
1438 );
1439
1440 #define DEFINE_SEGMENT_EVENT(name)                                      \
1441                 DEFINE_EVENT(svcrdma_segment_event, svcrdma_##name,\
1442                                 TP_PROTO(                               \
1443                                         u32 handle,                     \
1444                                         u32 length,                     \
1445                                         u64 offset                      \
1446                                 ),                                      \
1447                                 TP_ARGS(handle, length, offset))
1448
1449 DEFINE_SEGMENT_EVENT(decode_wseg);
1450 DEFINE_SEGMENT_EVENT(encode_rseg);
1451 DEFINE_SEGMENT_EVENT(send_rseg);
1452 DEFINE_SEGMENT_EVENT(encode_wseg);
1453 DEFINE_SEGMENT_EVENT(send_wseg);
1454
1455 DECLARE_EVENT_CLASS(svcrdma_chunk_event,
1456         TP_PROTO(
1457                 u32 length
1458         ),
1459
1460         TP_ARGS(length),
1461
1462         TP_STRUCT__entry(
1463                 __field(u32, length)
1464         ),
1465
1466         TP_fast_assign(
1467                 __entry->length = length;
1468         ),
1469
1470         TP_printk("length=%u",
1471                 __entry->length
1472         )
1473 );
1474
1475 #define DEFINE_CHUNK_EVENT(name)                                        \
1476                 DEFINE_EVENT(svcrdma_chunk_event, svcrdma_##name,       \
1477                                 TP_PROTO(                               \
1478                                         u32 length                      \
1479                                 ),                                      \
1480                                 TP_ARGS(length))
1481
1482 DEFINE_CHUNK_EVENT(send_pzr);
1483 DEFINE_CHUNK_EVENT(encode_write_chunk);
1484 DEFINE_CHUNK_EVENT(send_write_chunk);
1485 DEFINE_CHUNK_EVENT(encode_read_chunk);
1486 DEFINE_CHUNK_EVENT(send_reply_chunk);
1487
1488 TRACE_EVENT(svcrdma_send_read_chunk,
1489         TP_PROTO(
1490                 u32 length,
1491                 u32 position
1492         ),
1493
1494         TP_ARGS(length, position),
1495
1496         TP_STRUCT__entry(
1497                 __field(u32, length)
1498                 __field(u32, position)
1499         ),
1500
1501         TP_fast_assign(
1502                 __entry->length = length;
1503                 __entry->position = position;
1504         ),
1505
1506         TP_printk("length=%u position=%u",
1507                 __entry->length, __entry->position
1508         )
1509 );
1510
1511 DECLARE_EVENT_CLASS(svcrdma_error_event,
1512         TP_PROTO(
1513                 __be32 xid
1514         ),
1515
1516         TP_ARGS(xid),
1517
1518         TP_STRUCT__entry(
1519                 __field(u32, xid)
1520         ),
1521
1522         TP_fast_assign(
1523                 __entry->xid = be32_to_cpu(xid);
1524         ),
1525
1526         TP_printk("xid=0x%08x",
1527                 __entry->xid
1528         )
1529 );
1530
1531 #define DEFINE_ERROR_EVENT(name)                                        \
1532                 DEFINE_EVENT(svcrdma_error_event, svcrdma_err_##name,   \
1533                                 TP_PROTO(                               \
1534                                         __be32 xid                      \
1535                                 ),                                      \
1536                                 TP_ARGS(xid))
1537
1538 DEFINE_ERROR_EVENT(vers);
1539 DEFINE_ERROR_EVENT(chunk);
1540
1541 /**
1542  ** Server-side RDMA API events
1543  **/
1544
1545 DECLARE_EVENT_CLASS(svcrdma_dma_map_class,
1546         TP_PROTO(
1547                 const struct svcxprt_rdma *rdma,
1548                 u64 dma_addr,
1549                 u32 length
1550         ),
1551
1552         TP_ARGS(rdma, dma_addr, length),
1553
1554         TP_STRUCT__entry(
1555                 __field(u64, dma_addr)
1556                 __field(u32, length)
1557                 __string(device, rdma->sc_cm_id->device->name)
1558                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1559         ),
1560
1561         TP_fast_assign(
1562                 __entry->dma_addr = dma_addr;
1563                 __entry->length = length;
1564                 __assign_str(device, rdma->sc_cm_id->device->name);
1565                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1566         ),
1567
1568         TP_printk("addr=%s device=%s dma_addr=%llu length=%u",
1569                 __get_str(addr), __get_str(device),
1570                 __entry->dma_addr, __entry->length
1571         )
1572 );
1573
1574 #define DEFINE_SVC_DMA_EVENT(name)                                      \
1575                 DEFINE_EVENT(svcrdma_dma_map_class, svcrdma_##name,     \
1576                                 TP_PROTO(                               \
1577                                         const struct svcxprt_rdma *rdma,\
1578                                         u64 dma_addr,                   \
1579                                         u32 length                      \
1580                                 ),                                      \
1581                                 TP_ARGS(rdma, dma_addr, length))
1582
1583 DEFINE_SVC_DMA_EVENT(dma_map_page);
1584 DEFINE_SVC_DMA_EVENT(dma_unmap_page);
1585
1586 TRACE_EVENT(svcrdma_dma_map_rwctx,
1587         TP_PROTO(
1588                 const struct svcxprt_rdma *rdma,
1589                 int status
1590         ),
1591
1592         TP_ARGS(rdma, status),
1593
1594         TP_STRUCT__entry(
1595                 __field(int, status)
1596                 __string(device, rdma->sc_cm_id->device->name)
1597                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1598         ),
1599
1600         TP_fast_assign(
1601                 __entry->status = status;
1602                 __assign_str(device, rdma->sc_cm_id->device->name);
1603                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1604         ),
1605
1606         TP_printk("addr=%s device=%s status=%d",
1607                 __get_str(addr), __get_str(device), __entry->status
1608         )
1609 );
1610
1611 TRACE_EVENT(svcrdma_send_pullup,
1612         TP_PROTO(
1613                 unsigned int len
1614         ),
1615
1616         TP_ARGS(len),
1617
1618         TP_STRUCT__entry(
1619                 __field(unsigned int, len)
1620         ),
1621
1622         TP_fast_assign(
1623                 __entry->len = len;
1624         ),
1625
1626         TP_printk("len=%u", __entry->len)
1627 );
1628
1629 TRACE_EVENT(svcrdma_send_failed,
1630         TP_PROTO(
1631                 const struct svc_rqst *rqst,
1632                 int status
1633         ),
1634
1635         TP_ARGS(rqst, status),
1636
1637         TP_STRUCT__entry(
1638                 __field(int, status)
1639                 __field(u32, xid)
1640                 __field(const void *, xprt)
1641                 __string(addr, rqst->rq_xprt->xpt_remotebuf)
1642         ),
1643
1644         TP_fast_assign(
1645                 __entry->status = status;
1646                 __entry->xid = __be32_to_cpu(rqst->rq_xid);
1647                 __entry->xprt = rqst->rq_xprt;
1648                 __assign_str(addr, rqst->rq_xprt->xpt_remotebuf);
1649         ),
1650
1651         TP_printk("xprt=%p addr=%s xid=0x%08x status=%d",
1652                 __entry->xprt, __get_str(addr),
1653                 __entry->xid, __entry->status
1654         )
1655 );
1656
1657 DECLARE_EVENT_CLASS(svcrdma_sendcomp_event,
1658         TP_PROTO(
1659                 const struct ib_wc *wc
1660         ),
1661
1662         TP_ARGS(wc),
1663
1664         TP_STRUCT__entry(
1665                 __field(const void *, cqe)
1666                 __field(unsigned int, status)
1667                 __field(unsigned int, vendor_err)
1668         ),
1669
1670         TP_fast_assign(
1671                 __entry->cqe = wc->wr_cqe;
1672                 __entry->status = wc->status;
1673                 if (wc->status)
1674                         __entry->vendor_err = wc->vendor_err;
1675                 else
1676                         __entry->vendor_err = 0;
1677         ),
1678
1679         TP_printk("cqe=%p status=%s (%u/0x%x)",
1680                 __entry->cqe, rdma_show_wc_status(__entry->status),
1681                 __entry->status, __entry->vendor_err
1682         )
1683 );
1684
1685 #define DEFINE_SENDCOMP_EVENT(name)                                     \
1686                 DEFINE_EVENT(svcrdma_sendcomp_event, svcrdma_wc_##name, \
1687                                 TP_PROTO(                               \
1688                                         const struct ib_wc *wc          \
1689                                 ),                                      \
1690                                 TP_ARGS(wc))
1691
1692 TRACE_EVENT(svcrdma_post_send,
1693         TP_PROTO(
1694                 const struct ib_send_wr *wr
1695         ),
1696
1697         TP_ARGS(wr),
1698
1699         TP_STRUCT__entry(
1700                 __field(const void *, cqe)
1701                 __field(unsigned int, num_sge)
1702                 __field(u32, inv_rkey)
1703         ),
1704
1705         TP_fast_assign(
1706                 __entry->cqe = wr->wr_cqe;
1707                 __entry->num_sge = wr->num_sge;
1708                 __entry->inv_rkey = (wr->opcode == IB_WR_SEND_WITH_INV) ?
1709                                         wr->ex.invalidate_rkey : 0;
1710         ),
1711
1712         TP_printk("cqe=%p num_sge=%u inv_rkey=0x%08x",
1713                 __entry->cqe, __entry->num_sge,
1714                 __entry->inv_rkey
1715         )
1716 );
1717
1718 DEFINE_SENDCOMP_EVENT(send);
1719
1720 TRACE_EVENT(svcrdma_post_recv,
1721         TP_PROTO(
1722                 const struct ib_recv_wr *wr,
1723                 int status
1724         ),
1725
1726         TP_ARGS(wr, status),
1727
1728         TP_STRUCT__entry(
1729                 __field(const void *, cqe)
1730                 __field(int, status)
1731         ),
1732
1733         TP_fast_assign(
1734                 __entry->cqe = wr->wr_cqe;
1735                 __entry->status = status;
1736         ),
1737
1738         TP_printk("cqe=%p status=%d",
1739                 __entry->cqe, __entry->status
1740         )
1741 );
1742
1743 TRACE_EVENT(svcrdma_wc_receive,
1744         TP_PROTO(
1745                 const struct ib_wc *wc
1746         ),
1747
1748         TP_ARGS(wc),
1749
1750         TP_STRUCT__entry(
1751                 __field(const void *, cqe)
1752                 __field(u32, byte_len)
1753                 __field(unsigned int, status)
1754                 __field(u32, vendor_err)
1755         ),
1756
1757         TP_fast_assign(
1758                 __entry->cqe = wc->wr_cqe;
1759                 __entry->status = wc->status;
1760                 if (wc->status) {
1761                         __entry->byte_len = 0;
1762                         __entry->vendor_err = wc->vendor_err;
1763                 } else {
1764                         __entry->byte_len = wc->byte_len;
1765                         __entry->vendor_err = 0;
1766                 }
1767         ),
1768
1769         TP_printk("cqe=%p byte_len=%u status=%s (%u/0x%x)",
1770                 __entry->cqe, __entry->byte_len,
1771                 rdma_show_wc_status(__entry->status),
1772                 __entry->status, __entry->vendor_err
1773         )
1774 );
1775
1776 TRACE_EVENT(svcrdma_post_rw,
1777         TP_PROTO(
1778                 const void *cqe,
1779                 int sqecount
1780         ),
1781
1782         TP_ARGS(cqe, sqecount),
1783
1784         TP_STRUCT__entry(
1785                 __field(const void *, cqe)
1786                 __field(int, sqecount)
1787         ),
1788
1789         TP_fast_assign(
1790                 __entry->cqe = cqe;
1791                 __entry->sqecount = sqecount;
1792         ),
1793
1794         TP_printk("cqe=%p sqecount=%d",
1795                 __entry->cqe, __entry->sqecount
1796         )
1797 );
1798
1799 DEFINE_SENDCOMP_EVENT(read);
1800 DEFINE_SENDCOMP_EVENT(write);
1801
1802 TRACE_EVENT(svcrdma_qp_error,
1803         TP_PROTO(
1804                 const struct ib_event *event,
1805                 const struct sockaddr *sap
1806         ),
1807
1808         TP_ARGS(event, sap),
1809
1810         TP_STRUCT__entry(
1811                 __field(unsigned int, event)
1812                 __string(device, event->device->name)
1813                 __array(__u8, addr, INET6_ADDRSTRLEN + 10)
1814         ),
1815
1816         TP_fast_assign(
1817                 __entry->event = event->event;
1818                 __assign_str(device, event->device->name);
1819                 snprintf(__entry->addr, sizeof(__entry->addr) - 1,
1820                          "%pISpc", sap);
1821         ),
1822
1823         TP_printk("addr=%s dev=%s event=%s (%u)",
1824                 __entry->addr, __get_str(device),
1825                 rdma_show_ib_event(__entry->event), __entry->event
1826         )
1827 );
1828
1829 DECLARE_EVENT_CLASS(svcrdma_sendqueue_event,
1830         TP_PROTO(
1831                 const struct svcxprt_rdma *rdma
1832         ),
1833
1834         TP_ARGS(rdma),
1835
1836         TP_STRUCT__entry(
1837                 __field(int, avail)
1838                 __field(int, depth)
1839                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1840         ),
1841
1842         TP_fast_assign(
1843                 __entry->avail = atomic_read(&rdma->sc_sq_avail);
1844                 __entry->depth = rdma->sc_sq_depth;
1845                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1846         ),
1847
1848         TP_printk("addr=%s sc_sq_avail=%d/%d",
1849                 __get_str(addr), __entry->avail, __entry->depth
1850         )
1851 );
1852
1853 #define DEFINE_SQ_EVENT(name)                                           \
1854                 DEFINE_EVENT(svcrdma_sendqueue_event, svcrdma_sq_##name,\
1855                                 TP_PROTO(                               \
1856                                         const struct svcxprt_rdma *rdma \
1857                                 ),                                      \
1858                                 TP_ARGS(rdma))
1859
1860 DEFINE_SQ_EVENT(full);
1861 DEFINE_SQ_EVENT(retry);
1862
1863 TRACE_EVENT(svcrdma_sq_post_err,
1864         TP_PROTO(
1865                 const struct svcxprt_rdma *rdma,
1866                 int status
1867         ),
1868
1869         TP_ARGS(rdma, status),
1870
1871         TP_STRUCT__entry(
1872                 __field(int, avail)
1873                 __field(int, depth)
1874                 __field(int, status)
1875                 __string(addr, rdma->sc_xprt.xpt_remotebuf)
1876         ),
1877
1878         TP_fast_assign(
1879                 __entry->avail = atomic_read(&rdma->sc_sq_avail);
1880                 __entry->depth = rdma->sc_sq_depth;
1881                 __entry->status = status;
1882                 __assign_str(addr, rdma->sc_xprt.xpt_remotebuf);
1883         ),
1884
1885         TP_printk("addr=%s sc_sq_avail=%d/%d status=%d",
1886                 __get_str(addr), __entry->avail, __entry->depth,
1887                 __entry->status
1888         )
1889 );
1890
1891 #endif /* _TRACE_RPCRDMA_H */
1892
1893 #include <trace/define_trace.h>