tools/lib/perf/evlist.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <perf/evlist.h>
   3 #include <perf/evsel.h>
   4 #include <linux/bitops.h>
   5 #include <linux/list.h>
   6 #include <linux/hash.h>
   7 #include <sys/ioctl.h>
   8 #include <internal/evlist.h>
   9 #include <internal/evsel.h>
  10 #include <internal/xyarray.h>
  11 #include <internal/mmap.h>
  12 #include <internal/cpumap.h>
  13 #include <internal/threadmap.h>
  14 #include <internal/lib.h>
  15 #include <linux/zalloc.h>
  16 #include <stdlib.h>
  17 #include <errno.h>
  18 #include <unistd.h>
  19 #include <fcntl.h>
  20 #include <signal.h>
  21 #include <poll.h>
  22 #include <sys/mman.h>
  23 #include <perf/cpumap.h>
  24 #include <perf/threadmap.h>
  25 #include <api/fd/array.h>
  26 #include "internal.h"
  27
  28 void perf_evlist__init(struct perf_evlist *evlist)
  29 {
  30         INIT_LIST_HEAD(&evlist->entries);
  31         evlist->nr_entries = 0;
  32         fdarray__init(&evlist->pollfd, 64);
  33         perf_evlist__reset_id_hash(evlist);
  34 }
  35
  36 static void __perf_evlist__propagate_maps(struct perf_evlist *evlist,
  37                                           struct perf_evsel *evsel)
  38 {
  39         /*
  40          * We already have cpus for evsel (via PMU sysfs) so
  41          * keep it, if there's no target cpu list defined.
  42          */
  43         if (!evsel->own_cpus ||
  44             (!evsel->system_wide && evlist->has_user_cpus) ||
  45             (!evsel->system_wide &&
  46              !evsel->requires_cpu &&
  47              perf_cpu_map__empty(evlist->user_requested_cpus))) {
  48                 perf_cpu_map__put(evsel->cpus);
  49                 evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus);
  50         } else if (evsel->cpus != evsel->own_cpus) {
  51                 perf_cpu_map__put(evsel->cpus);
  52                 evsel->cpus = perf_cpu_map__get(evsel->own_cpus);
  53         }
  54
  55         if (!evsel->system_wide) {
  56                 perf_thread_map__put(evsel->threads);
  57                 evsel->threads = perf_thread_map__get(evlist->threads);
  58         }
  59
  60         evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus);
  61 }
  62
  63 static void perf_evlist__propagate_maps(struct perf_evlist *evlist)
  64 {
  65         struct perf_evsel *evsel;
  66
  67         /* Recomputing all_cpus, so start with a blank slate. */
  68         perf_cpu_map__put(evlist->all_cpus);
  69         evlist->all_cpus = NULL;
  70
  71         perf_evlist__for_each_evsel(evlist, evsel)
  72                 __perf_evlist__propagate_maps(evlist, evsel);
  73 }
  74
  75 void perf_evlist__add(struct perf_evlist *evlist,
  76                       struct perf_evsel *evsel)
  77 {
  78         evsel->idx = evlist->nr_entries;
  79         list_add_tail(&evsel->node, &evlist->entries);
  80         evlist->nr_entries += 1;
  81         __perf_evlist__propagate_maps(evlist, evsel);
  82 }
  83
  84 void perf_evlist__remove(struct perf_evlist *evlist,
  85                          struct perf_evsel *evsel)
  86 {
  87         list_del_init(&evsel->node);
  88         evlist->nr_entries -= 1;
  89 }
  90
  91 struct perf_evlist *perf_evlist__new(void)
  92 {
  93         struct perf_evlist *evlist = zalloc(sizeof(*evlist));
  94
  95         if (evlist != NULL)
  96                 perf_evlist__init(evlist);
  97
  98         return evlist;
  99 }
 100
 101 struct perf_evsel *
 102 perf_evlist__next(struct perf_evlist *evlist, struct perf_evsel *prev)
 103 {
 104         struct perf_evsel *next;
 105
 106         if (!prev) {
 107                 next = list_first_entry(&evlist->entries,
 108                                         struct perf_evsel,
 109                                         node);
 110         } else {
 111                 next = list_next_entry(prev, node);
 112         }
 113
 114         /* Empty list is noticed here so don't need checking on entry. */
 115         if (&next->node == &evlist->entries)
 116                 return NULL;
 117
 118         return next;
 119 }
 120
 121 static void perf_evlist__purge(struct perf_evlist *evlist)
 122 {
 123         struct perf_evsel *pos, *n;
 124
 125         perf_evlist__for_each_entry_safe(evlist, n, pos) {
 126                 list_del_init(&pos->node);
 127                 perf_evsel__delete(pos);
 128         }
 129
 130         evlist->nr_entries = 0;
 131 }
 132
 133 void perf_evlist__exit(struct perf_evlist *evlist)
 134 {
 135         perf_cpu_map__put(evlist->user_requested_cpus);
 136         perf_cpu_map__put(evlist->all_cpus);
 137         perf_thread_map__put(evlist->threads);
 138         evlist->user_requested_cpus = NULL;
 139         evlist->all_cpus = NULL;
 140         evlist->threads = NULL;
 141         fdarray__exit(&evlist->pollfd);
 142 }
 143
 144 void perf_evlist__delete(struct perf_evlist *evlist)
 145 {
 146         if (evlist == NULL)
 147                 return;
 148
 149         perf_evlist__munmap(evlist);
 150         perf_evlist__close(evlist);
 151         perf_evlist__purge(evlist);
 152         perf_evlist__exit(evlist);
 153         free(evlist);
 154 }
 155
 156 void perf_evlist__set_maps(struct perf_evlist *evlist,
 157                            struct perf_cpu_map *cpus,
 158                            struct perf_thread_map *threads)
 159 {
 160         /*
 161          * Allow for the possibility that one or another of the maps isn't being
 162          * changed i.e. don't put it.  Note we are assuming the maps that are
 163          * being applied are brand new and evlist is taking ownership of the
 164          * original reference count of 1.  If that is not the case it is up to
 165          * the caller to increase the reference count.
 166          */
 167         if (cpus != evlist->user_requested_cpus) {
 168                 perf_cpu_map__put(evlist->user_requested_cpus);
 169                 evlist->user_requested_cpus = perf_cpu_map__get(cpus);
 170         }
 171
 172         if (threads != evlist->threads) {
 173                 perf_thread_map__put(evlist->threads);
 174                 evlist->threads = perf_thread_map__get(threads);
 175         }
 176
 177         if (!evlist->all_cpus && cpus)
 178                 evlist->all_cpus = perf_cpu_map__get(cpus);
 179
 180         perf_evlist__propagate_maps(evlist);
 181 }
 182
 183 int perf_evlist__open(struct perf_evlist *evlist)
 184 {
 185         struct perf_evsel *evsel;
 186         int err;
 187
 188         perf_evlist__for_each_entry(evlist, evsel) {
 189                 err = perf_evsel__open(evsel, evsel->cpus, evsel->threads);
 190                 if (err < 0)
 191                         goto out_err;
 192         }
 193
 194         return 0;
 195
 196 out_err:
 197         perf_evlist__close(evlist);
 198         return err;
 199 }
 200
 201 void perf_evlist__close(struct perf_evlist *evlist)
 202 {
 203         struct perf_evsel *evsel;
 204
 205         perf_evlist__for_each_entry_reverse(evlist, evsel)
 206                 perf_evsel__close(evsel);
 207 }
 208
 209 void perf_evlist__enable(struct perf_evlist *evlist)
 210 {
 211         struct perf_evsel *evsel;
 212
 213         perf_evlist__for_each_entry(evlist, evsel)
 214                 perf_evsel__enable(evsel);
 215 }
 216
 217 void perf_evlist__disable(struct perf_evlist *evlist)
 218 {
 219         struct perf_evsel *evsel;
 220
 221         perf_evlist__for_each_entry(evlist, evsel)
 222                 perf_evsel__disable(evsel);
 223 }
 224
 225 u64 perf_evlist__read_format(struct perf_evlist *evlist)
 226 {
 227         struct perf_evsel *first = perf_evlist__first(evlist);
 228
 229         return first->attr.read_format;
 230 }
 231
 232 #define SID(e, x, y) xyarray__entry(e->sample_id, x, y)
 233
 234 static void perf_evlist__id_hash(struct perf_evlist *evlist,
 235                                  struct perf_evsel *evsel,
 236                                  int cpu, int thread, u64 id)
 237 {
 238         int hash;
 239         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 240
 241         sid->id = id;
 242         sid->evsel = evsel;
 243         hash = hash_64(sid->id, PERF_EVLIST__HLIST_BITS);
 244         hlist_add_head(&sid->node, &evlist->heads[hash]);
 245 }
 246
 247 void perf_evlist__reset_id_hash(struct perf_evlist *evlist)
 248 {
 249         int i;
 250
 251         for (i = 0; i < PERF_EVLIST__HLIST_SIZE; ++i)
 252                 INIT_HLIST_HEAD(&evlist->heads[i]);
 253 }
 254
 255 void perf_evlist__id_add(struct perf_evlist *evlist,
 256                          struct perf_evsel *evsel,
 257                          int cpu, int thread, u64 id)
 258 {
 259         perf_evlist__id_hash(evlist, evsel, cpu, thread, id);
 260         evsel->id[evsel->ids++] = id;
 261 }
 262
 263 int perf_evlist__id_add_fd(struct perf_evlist *evlist,
 264                            struct perf_evsel *evsel,
 265                            int cpu, int thread, int fd)
 266 {
 267         u64 read_data[4] = { 0, };
 268         int id_idx = 1; /* The first entry is the counter value */
 269         u64 id;
 270         int ret;
 271
 272         ret = ioctl(fd, PERF_EVENT_IOC_ID, &id);
 273         if (!ret)
 274                 goto add;
 275
 276         if (errno != ENOTTY)
 277                 return -1;
 278
 279         /* Legacy way to get event id.. All hail to old kernels! */
 280
 281         /*
 282          * This way does not work with group format read, so bail
 283          * out in that case.
 284          */
 285         if (perf_evlist__read_format(evlist) & PERF_FORMAT_GROUP)
 286                 return -1;
 287
 288         if (!(evsel->attr.read_format & PERF_FORMAT_ID) ||
 289             read(fd, &read_data, sizeof(read_data)) == -1)
 290                 return -1;
 291
 292         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED)
 293                 ++id_idx;
 294         if (evsel->attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING)
 295                 ++id_idx;
 296
 297         id = read_data[id_idx];
 298
 299 add:
 300         perf_evlist__id_add(evlist, evsel, cpu, thread, id);
 301         return 0;
 302 }
 303
 304 int perf_evlist__alloc_pollfd(struct perf_evlist *evlist)
 305 {
 306         int nr_cpus = perf_cpu_map__nr(evlist->all_cpus);
 307         int nr_threads = perf_thread_map__nr(evlist->threads);
 308         int nfds = 0;
 309         struct perf_evsel *evsel;
 310
 311         perf_evlist__for_each_entry(evlist, evsel) {
 312                 if (evsel->system_wide)
 313                         nfds += nr_cpus;
 314                 else
 315                         nfds += nr_cpus * nr_threads;
 316         }
 317
 318         if (fdarray__available_entries(&evlist->pollfd) < nfds &&
 319             fdarray__grow(&evlist->pollfd, nfds) < 0)
 320                 return -ENOMEM;
 321
 322         return 0;
 323 }
 324
 325 int perf_evlist__add_pollfd(struct perf_evlist *evlist, int fd,
 326                             void *ptr, short revent, enum fdarray_flags flags)
 327 {
 328         int pos = fdarray__add(&evlist->pollfd, fd, revent | POLLERR | POLLHUP, flags);
 329
 330         if (pos >= 0) {
 331                 evlist->pollfd.priv[pos].ptr = ptr;
 332                 fcntl(fd, F_SETFL, O_NONBLOCK);
 333         }
 334
 335         return pos;
 336 }
 337
 338 static void perf_evlist__munmap_filtered(struct fdarray *fda, int fd,
 339                                          void *arg __maybe_unused)
 340 {
 341         struct perf_mmap *map = fda->priv[fd].ptr;
 342
 343         if (map)
 344                 perf_mmap__put(map);
 345 }
 346
 347 int perf_evlist__filter_pollfd(struct perf_evlist *evlist, short revents_and_mask)
 348 {
 349         return fdarray__filter(&evlist->pollfd, revents_and_mask,
 350                                perf_evlist__munmap_filtered, NULL);
 351 }
 352
 353 int perf_evlist__poll(struct perf_evlist *evlist, int timeout)
 354 {
 355         return fdarray__poll(&evlist->pollfd, timeout);
 356 }
 357
 358 static struct perf_mmap* perf_evlist__alloc_mmap(struct perf_evlist *evlist, bool overwrite)
 359 {
 360         int i;
 361         struct perf_mmap *map;
 362
 363         map = zalloc(evlist->nr_mmaps * sizeof(struct perf_mmap));
 364         if (!map)
 365                 return NULL;
 366
 367         for (i = 0; i < evlist->nr_mmaps; i++) {
 368                 struct perf_mmap *prev = i ? &map[i - 1] : NULL;
 369
 370                 /*
 371                  * When the perf_mmap() call is made we grab one refcount, plus
 372                  * one extra to let perf_mmap__consume() get the last
 373                  * events after all real references (perf_mmap__get()) are
 374                  * dropped.
 375                  *
 376                  * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and
 377                  * thus does perf_mmap__get() on it.
 378                  */
 379                 perf_mmap__init(&map[i], prev, overwrite, NULL);
 380         }
 381
 382         return map;
 383 }
 384
 385 static void perf_evsel__set_sid_idx(struct perf_evsel *evsel, int idx, int cpu, int thread)
 386 {
 387         struct perf_sample_id *sid = SID(evsel, cpu, thread);
 388
 389         sid->idx = idx;
 390         sid->cpu = perf_cpu_map__cpu(evsel->cpus, cpu);
 391         sid->tid = perf_thread_map__pid(evsel->threads, thread);
 392 }
 393
 394 static struct perf_mmap*
 395 perf_evlist__mmap_cb_get(struct perf_evlist *evlist, bool overwrite, int idx)
 396 {
 397         struct perf_mmap *maps;
 398
 399         maps = overwrite ? evlist->mmap_ovw : evlist->mmap;
 400
 401         if (!maps) {
 402                 maps = perf_evlist__alloc_mmap(evlist, overwrite);
 403                 if (!maps)
 404                         return NULL;
 405
 406                 if (overwrite)
 407                         evlist->mmap_ovw = maps;
 408                 else
 409                         evlist->mmap = maps;
 410         }
 411
 412         return &maps[idx];
 413 }
 414
 415 #define FD(e, x, y) (*(int *) xyarray__entry(e->fd, x, y))
 416
 417 static int
 418 perf_evlist__mmap_cb_mmap(struct perf_mmap *map, struct perf_mmap_param *mp,
 419                           int output, struct perf_cpu cpu)
 420 {
 421         return perf_mmap__mmap(map, mp, output, cpu);
 422 }
 423
 424 static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_mmap *map,
 425                                         bool overwrite)
 426 {
 427         if (overwrite)
 428                 evlist->mmap_ovw_first = map;
 429         else
 430                 evlist->mmap_first = map;
 431 }
 432
 433 static int
 434 mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
 435                int idx, struct perf_mmap_param *mp, int cpu_idx,
 436                int thread, int *_output, int *_output_overwrite, int *nr_mmaps)
 437 {
 438         struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->all_cpus, cpu_idx);
 439         struct perf_evsel *evsel;
 440         int revent;
 441
 442         perf_evlist__for_each_entry(evlist, evsel) {
 443                 bool overwrite = evsel->attr.write_backward;
 444                 struct perf_mmap *map;
 445                 int *output, fd, cpu;
 446
 447                 if (evsel->system_wide && thread)
 448                         continue;
 449
 450                 cpu = perf_cpu_map__idx(evsel->cpus, evlist_cpu);
 451                 if (cpu == -1)
 452                         continue;
 453
 454                 map = ops->get(evlist, overwrite, idx);
 455                 if (map == NULL)
 456                         return -ENOMEM;
 457
 458                 if (overwrite) {
 459                         mp->prot = PROT_READ;
 460                         output   = _output_overwrite;
 461                 } else {
 462                         mp->prot = PROT_READ | PROT_WRITE;
 463                         output   = _output;
 464                 }
 465
 466                 fd = FD(evsel, cpu, thread);
 467
 468                 if (*output == -1) {
 469                         *output = fd;
 470
 471                         /*
 472                          * The last one will be done at perf_mmap__consume(), so that we
 473                          * make sure we don't prevent tools from consuming every last event in
 474                          * the ring buffer.
 475                          *
 476                          * I.e. we can get the POLLHUP meaning that the fd doesn't exist
 477                          * anymore, but the last events for it are still in the ring buffer,
 478                          * waiting to be consumed.
 479                          *
 480                          * Tools can chose to ignore this at their own discretion, but the
 481                          * evlist layer can't just drop it when filtering events in
 482                          * perf_evlist__filter_pollfd().
 483                          */
 484                         refcount_set(&map->refcnt, 2);
 485
 486                         if (ops->idx)
 487                                 ops->idx(evlist, evsel, mp, idx);
 488
 489                         if (ops->mmap(map, mp, *output, evlist_cpu) < 0)
 490                                 return -1;
 491
 492                         *nr_mmaps += 1;
 493
 494                         if (!idx)
 495                                 perf_evlist__set_mmap_first(evlist, map, overwrite);
 496                 } else {
 497                         if (ioctl(fd, PERF_EVENT_IOC_SET_OUTPUT, *output) != 0)
 498                                 return -1;
 499
 500                         perf_mmap__get(map);
 501                 }
 502
 503                 revent = !overwrite ? POLLIN : 0;
 504
 505                 if (!evsel->system_wide &&
 506                     perf_evlist__add_pollfd(evlist, fd, map, revent, fdarray_flag__default) < 0) {
 507                         perf_mmap__put(map);
 508                         return -1;
 509                 }
 510
 511                 if (evsel->attr.read_format & PERF_FORMAT_ID) {
 512                         if (perf_evlist__id_add_fd(evlist, evsel, cpu, thread,
 513                                                    fd) < 0)
 514                                 return -1;
 515                         perf_evsel__set_sid_idx(evsel, idx, cpu, thread);
 516                 }
 517         }
 518
 519         return 0;
 520 }
 521
 522 static int
 523 mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops,
 524              struct perf_mmap_param *mp)
 525 {
 526         int nr_threads = perf_thread_map__nr(evlist->threads);
 527         int nr_cpus    = perf_cpu_map__nr(evlist->all_cpus);
 528         int nr_mmaps = 0;
 529         int cpu, thread;
 530
 531         for (cpu = 0; cpu < nr_cpus; cpu++) {
 532                 int output = -1;
 533                 int output_overwrite = -1;
 534
 535                 for (thread = 0; thread < nr_threads; thread++) {
 536                         if (mmap_per_evsel(evlist, ops, cpu, mp, cpu,
 537                                            thread, &output, &output_overwrite, &nr_mmaps))
 538                                 goto out_unmap;
 539                 }
 540         }
 541
 542         if (nr_mmaps != evlist->nr_mmaps)
 543                 pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps);
 544
 545         return 0;
 546
 547 out_unmap:
 548         perf_evlist__munmap(evlist);
 549         return -1;
 550 }
 551
 552 static int perf_evlist__nr_mmaps(struct perf_evlist *evlist)
 553 {
 554         int nr_mmaps;
 555
 556         /* One for each CPU */
 557         nr_mmaps = perf_cpu_map__nr(evlist->all_cpus);
 558         if (perf_cpu_map__empty(evlist->all_cpus)) {
 559                 /* Plus one for each thread */
 560                 nr_mmaps += perf_thread_map__nr(evlist->threads);
 561                 /* Minus the per-thread CPU (-1) */
 562                 nr_mmaps -= 1;
 563         }
 564
 565         return nr_mmaps;
 566 }
 567
 568 int perf_evlist__mmap_ops(struct perf_evlist *evlist,
 569                           struct perf_evlist_mmap_ops *ops,
 570                           struct perf_mmap_param *mp)
 571 {
 572         struct perf_evsel *evsel;
 573
 574         if (!ops || !ops->get || !ops->mmap)
 575                 return -EINVAL;
 576
 577         mp->mask = evlist->mmap_len - page_size - 1;
 578
 579         evlist->nr_mmaps = perf_evlist__nr_mmaps(evlist);
 580
 581         perf_evlist__for_each_entry(evlist, evsel) {
 582                 if ((evsel->attr.read_format & PERF_FORMAT_ID) &&
 583                     evsel->sample_id == NULL &&
 584                     perf_evsel__alloc_id(evsel, evsel->fd->max_x, evsel->fd->max_y) < 0)
 585                         return -ENOMEM;
 586         }
 587
 588         if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0)
 589                 return -ENOMEM;
 590
 591         return mmap_per_cpu(evlist, ops, mp);
 592 }
 593
 594 int perf_evlist__mmap(struct perf_evlist *evlist, int pages)
 595 {
 596         struct perf_mmap_param mp;
 597         struct perf_evlist_mmap_ops ops = {
 598                 .get  = perf_evlist__mmap_cb_get,
 599                 .mmap = perf_evlist__mmap_cb_mmap,
 600         };
 601
 602         evlist->mmap_len = (pages + 1) * page_size;
 603
 604         return perf_evlist__mmap_ops(evlist, &ops, &mp);
 605 }
 606
 607 void perf_evlist__munmap(struct perf_evlist *evlist)
 608 {
 609         int i;
 610
 611         if (evlist->mmap) {
 612                 for (i = 0; i < evlist->nr_mmaps; i++)
 613                         perf_mmap__munmap(&evlist->mmap[i]);
 614         }
 615
 616         if (evlist->mmap_ovw) {
 617                 for (i = 0; i < evlist->nr_mmaps; i++)
 618                         perf_mmap__munmap(&evlist->mmap_ovw[i]);
 619         }
 620
 621         zfree(&evlist->mmap);
 622         zfree(&evlist->mmap_ovw);
 623 }
 624
 625 struct perf_mmap*
 626 perf_evlist__next_mmap(struct perf_evlist *evlist, struct perf_mmap *map,
 627                        bool overwrite)
 628 {
 629         if (map)
 630                 return map->next;
 631
 632         return overwrite ? evlist->mmap_ovw_first : evlist->mmap_first;
 633 }
 634
 635 void __perf_evlist__set_leader(struct list_head *list, struct perf_evsel *leader)
 636 {
 637         struct perf_evsel *first, *last, *evsel;
 638
 639         first = list_first_entry(list, struct perf_evsel, node);
 640         last = list_last_entry(list, struct perf_evsel, node);
 641
 642         leader->nr_members = last->idx - first->idx + 1;
 643
 644         __perf_evlist__for_each_entry(list, evsel)
 645                 evsel->leader = leader;
 646 }
 647
 648 void perf_evlist__set_leader(struct perf_evlist *evlist)
 649 {
 650         if (evlist->nr_entries) {
 651                 struct perf_evsel *first = list_entry(evlist->entries.next,
 652                                                 struct perf_evsel, node);
 653
 654                 evlist->nr_groups = evlist->nr_entries > 1 ? 1 : 0;
 655                 __perf_evlist__set_leader(&evlist->entries, first);
 656         }
 657 }