tools/testing/selftests/bpf/prog_tests/kprobe_multi_test.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #include <test_progs.h>
   3 #include "kprobe_multi.skel.h"
   4 #include "trace_helpers.h"
   5 #include "kprobe_multi_empty.skel.h"
   6 #include "kprobe_multi_override.skel.h"
   7 #include "bpf/libbpf_internal.h"
   8 #include "bpf/hashmap.h"
   9
  10 static void kprobe_multi_test_run(struct kprobe_multi *skel, bool test_return)
  11 {
  12         LIBBPF_OPTS(bpf_test_run_opts, topts);
  13         int err, prog_fd;
  14
  15         prog_fd = bpf_program__fd(skel->progs.trigger);
  16         err = bpf_prog_test_run_opts(prog_fd, &topts);
  17         ASSERT_OK(err, "test_run");
  18         ASSERT_EQ(topts.retval, 0, "test_run");
  19
  20         ASSERT_EQ(skel->bss->kprobe_test1_result, 1, "kprobe_test1_result");
  21         ASSERT_EQ(skel->bss->kprobe_test2_result, 1, "kprobe_test2_result");
  22         ASSERT_EQ(skel->bss->kprobe_test3_result, 1, "kprobe_test3_result");
  23         ASSERT_EQ(skel->bss->kprobe_test4_result, 1, "kprobe_test4_result");
  24         ASSERT_EQ(skel->bss->kprobe_test5_result, 1, "kprobe_test5_result");
  25         ASSERT_EQ(skel->bss->kprobe_test6_result, 1, "kprobe_test6_result");
  26         ASSERT_EQ(skel->bss->kprobe_test7_result, 1, "kprobe_test7_result");
  27         ASSERT_EQ(skel->bss->kprobe_test8_result, 1, "kprobe_test8_result");
  28
  29         if (test_return) {
  30                 ASSERT_EQ(skel->bss->kretprobe_test1_result, 1, "kretprobe_test1_result");
  31                 ASSERT_EQ(skel->bss->kretprobe_test2_result, 1, "kretprobe_test2_result");
  32                 ASSERT_EQ(skel->bss->kretprobe_test3_result, 1, "kretprobe_test3_result");
  33                 ASSERT_EQ(skel->bss->kretprobe_test4_result, 1, "kretprobe_test4_result");
  34                 ASSERT_EQ(skel->bss->kretprobe_test5_result, 1, "kretprobe_test5_result");
  35                 ASSERT_EQ(skel->bss->kretprobe_test6_result, 1, "kretprobe_test6_result");
  36                 ASSERT_EQ(skel->bss->kretprobe_test7_result, 1, "kretprobe_test7_result");
  37                 ASSERT_EQ(skel->bss->kretprobe_test8_result, 1, "kretprobe_test8_result");
  38         }
  39 }
  40
  41 static void test_skel_api(void)
  42 {
  43         struct kprobe_multi *skel = NULL;
  44         int err;
  45
  46         skel = kprobe_multi__open_and_load();
  47         if (!ASSERT_OK_PTR(skel, "kprobe_multi__open_and_load"))
  48                 goto cleanup;
  49
  50         skel->bss->pid = getpid();
  51         err = kprobe_multi__attach(skel);
  52         if (!ASSERT_OK(err, "kprobe_multi__attach"))
  53                 goto cleanup;
  54
  55         kprobe_multi_test_run(skel, true);
  56
  57 cleanup:
  58         kprobe_multi__destroy(skel);
  59 }
  60
  61 static void test_link_api(struct bpf_link_create_opts *opts)
  62 {
  63         int prog_fd, link1_fd = -1, link2_fd = -1;
  64         struct kprobe_multi *skel = NULL;
  65
  66         skel = kprobe_multi__open_and_load();
  67         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
  68                 goto cleanup;
  69
  70         skel->bss->pid = getpid();
  71         prog_fd = bpf_program__fd(skel->progs.test_kprobe);
  72         link1_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
  73         if (!ASSERT_GE(link1_fd, 0, "link_fd"))
  74                 goto cleanup;
  75
  76         opts->kprobe_multi.flags = BPF_F_KPROBE_MULTI_RETURN;
  77         prog_fd = bpf_program__fd(skel->progs.test_kretprobe);
  78         link2_fd = bpf_link_create(prog_fd, 0, BPF_TRACE_KPROBE_MULTI, opts);
  79         if (!ASSERT_GE(link2_fd, 0, "link_fd"))
  80                 goto cleanup;
  81
  82         kprobe_multi_test_run(skel, true);
  83
  84 cleanup:
  85         if (link1_fd != -1)
  86                 close(link1_fd);
  87         if (link2_fd != -1)
  88                 close(link2_fd);
  89         kprobe_multi__destroy(skel);
  90 }
  91
  92 #define GET_ADDR(__sym, __addr) ({                                      \
  93         __addr = ksym_get_addr(__sym);                                  \
  94         if (!ASSERT_NEQ(__addr, 0, "kallsyms load failed for " #__sym)) \
  95                 return;                                                 \
  96 })
  97
  98 static void test_link_api_addrs(void)
  99 {
 100         LIBBPF_OPTS(bpf_link_create_opts, opts);
 101         unsigned long long addrs[8];
 102
 103         GET_ADDR("bpf_fentry_test1", addrs[0]);
 104         GET_ADDR("bpf_fentry_test2", addrs[1]);
 105         GET_ADDR("bpf_fentry_test3", addrs[2]);
 106         GET_ADDR("bpf_fentry_test4", addrs[3]);
 107         GET_ADDR("bpf_fentry_test5", addrs[4]);
 108         GET_ADDR("bpf_fentry_test6", addrs[5]);
 109         GET_ADDR("bpf_fentry_test7", addrs[6]);
 110         GET_ADDR("bpf_fentry_test8", addrs[7]);
 111
 112         opts.kprobe_multi.addrs = (const unsigned long*) addrs;
 113         opts.kprobe_multi.cnt = ARRAY_SIZE(addrs);
 114         test_link_api(&opts);
 115 }
 116
 117 static void test_link_api_syms(void)
 118 {
 119         LIBBPF_OPTS(bpf_link_create_opts, opts);
 120         const char *syms[8] = {
 121                 "bpf_fentry_test1",
 122                 "bpf_fentry_test2",
 123                 "bpf_fentry_test3",
 124                 "bpf_fentry_test4",
 125                 "bpf_fentry_test5",
 126                 "bpf_fentry_test6",
 127                 "bpf_fentry_test7",
 128                 "bpf_fentry_test8",
 129         };
 130
 131         opts.kprobe_multi.syms = syms;
 132         opts.kprobe_multi.cnt = ARRAY_SIZE(syms);
 133         test_link_api(&opts);
 134 }
 135
 136 static void
 137 test_attach_api(const char *pattern, struct bpf_kprobe_multi_opts *opts)
 138 {
 139         struct bpf_link *link1 = NULL, *link2 = NULL;
 140         struct kprobe_multi *skel = NULL;
 141
 142         skel = kprobe_multi__open_and_load();
 143         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
 144                 goto cleanup;
 145
 146         skel->bss->pid = getpid();
 147         link1 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 148                                                       pattern, opts);
 149         if (!ASSERT_OK_PTR(link1, "bpf_program__attach_kprobe_multi_opts"))
 150                 goto cleanup;
 151
 152         if (opts) {
 153                 opts->retprobe = true;
 154                 link2 = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kretprobe_manual,
 155                                                               pattern, opts);
 156                 if (!ASSERT_OK_PTR(link2, "bpf_program__attach_kprobe_multi_opts"))
 157                         goto cleanup;
 158         }
 159
 160         kprobe_multi_test_run(skel, !!opts);
 161
 162 cleanup:
 163         bpf_link__destroy(link2);
 164         bpf_link__destroy(link1);
 165         kprobe_multi__destroy(skel);
 166 }
 167
 168 static void test_attach_api_pattern(void)
 169 {
 170         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 171
 172         test_attach_api("bpf_fentry_test*", &opts);
 173         test_attach_api("bpf_fentry_test?", NULL);
 174 }
 175
 176 static void test_attach_api_addrs(void)
 177 {
 178         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 179         unsigned long long addrs[8];
 180
 181         GET_ADDR("bpf_fentry_test1", addrs[0]);
 182         GET_ADDR("bpf_fentry_test2", addrs[1]);
 183         GET_ADDR("bpf_fentry_test3", addrs[2]);
 184         GET_ADDR("bpf_fentry_test4", addrs[3]);
 185         GET_ADDR("bpf_fentry_test5", addrs[4]);
 186         GET_ADDR("bpf_fentry_test6", addrs[5]);
 187         GET_ADDR("bpf_fentry_test7", addrs[6]);
 188         GET_ADDR("bpf_fentry_test8", addrs[7]);
 189
 190         opts.addrs = (const unsigned long *) addrs;
 191         opts.cnt = ARRAY_SIZE(addrs);
 192         test_attach_api(NULL, &opts);
 193 }
 194
 195 static void test_attach_api_syms(void)
 196 {
 197         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 198         const char *syms[8] = {
 199                 "bpf_fentry_test1",
 200                 "bpf_fentry_test2",
 201                 "bpf_fentry_test3",
 202                 "bpf_fentry_test4",
 203                 "bpf_fentry_test5",
 204                 "bpf_fentry_test6",
 205                 "bpf_fentry_test7",
 206                 "bpf_fentry_test8",
 207         };
 208
 209         opts.syms = syms;
 210         opts.cnt = ARRAY_SIZE(syms);
 211         test_attach_api(NULL, &opts);
 212 }
 213
 214 static void test_attach_api_fails(void)
 215 {
 216         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 217         struct kprobe_multi *skel = NULL;
 218         struct bpf_link *link = NULL;
 219         unsigned long long addrs[2];
 220         const char *syms[2] = {
 221                 "bpf_fentry_test1",
 222                 "bpf_fentry_test2",
 223         };
 224         __u64 cookies[2];
 225
 226         addrs[0] = ksym_get_addr("bpf_fentry_test1");
 227         addrs[1] = ksym_get_addr("bpf_fentry_test2");
 228
 229         if (!ASSERT_FALSE(!addrs[0] || !addrs[1], "ksym_get_addr"))
 230                 goto cleanup;
 231
 232         skel = kprobe_multi__open_and_load();
 233         if (!ASSERT_OK_PTR(skel, "fentry_raw_skel_load"))
 234                 goto cleanup;
 235
 236         skel->bss->pid = getpid();
 237
 238         /* fail_1 - pattern and opts NULL */
 239         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 240                                                      NULL, NULL);
 241         if (!ASSERT_ERR_PTR(link, "fail_1"))
 242                 goto cleanup;
 243
 244         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_1_error"))
 245                 goto cleanup;
 246
 247         /* fail_2 - both addrs and syms set */
 248         opts.addrs = (const unsigned long *) addrs;
 249         opts.syms = syms;
 250         opts.cnt = ARRAY_SIZE(syms);
 251         opts.cookies = NULL;
 252
 253         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 254                                                      NULL, &opts);
 255         if (!ASSERT_ERR_PTR(link, "fail_2"))
 256                 goto cleanup;
 257
 258         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_2_error"))
 259                 goto cleanup;
 260
 261         /* fail_3 - pattern and addrs set */
 262         opts.addrs = (const unsigned long *) addrs;
 263         opts.syms = NULL;
 264         opts.cnt = ARRAY_SIZE(syms);
 265         opts.cookies = NULL;
 266
 267         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 268                                                      "ksys_*", &opts);
 269         if (!ASSERT_ERR_PTR(link, "fail_3"))
 270                 goto cleanup;
 271
 272         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_3_error"))
 273                 goto cleanup;
 274
 275         /* fail_4 - pattern and cnt set */
 276         opts.addrs = NULL;
 277         opts.syms = NULL;
 278         opts.cnt = ARRAY_SIZE(syms);
 279         opts.cookies = NULL;
 280
 281         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 282                                                      "ksys_*", &opts);
 283         if (!ASSERT_ERR_PTR(link, "fail_4"))
 284                 goto cleanup;
 285
 286         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_4_error"))
 287                 goto cleanup;
 288
 289         /* fail_5 - pattern and cookies */
 290         opts.addrs = NULL;
 291         opts.syms = NULL;
 292         opts.cnt = 0;
 293         opts.cookies = cookies;
 294
 295         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_manual,
 296                                                      "ksys_*", &opts);
 297         if (!ASSERT_ERR_PTR(link, "fail_5"))
 298                 goto cleanup;
 299
 300         if (!ASSERT_EQ(libbpf_get_error(link), -EINVAL, "fail_5_error"))
 301                 goto cleanup;
 302
 303 cleanup:
 304         bpf_link__destroy(link);
 305         kprobe_multi__destroy(skel);
 306 }
 307
 308 static size_t symbol_hash(long key, void *ctx __maybe_unused)
 309 {
 310         return str_hash((const char *) key);
 311 }
 312
 313 static bool symbol_equal(long key1, long key2, void *ctx __maybe_unused)
 314 {
 315         return strcmp((const char *) key1, (const char *) key2) == 0;
 316 }
 317
 318 static int get_syms(char ***symsp, size_t *cntp, bool kernel)
 319 {
 320         size_t cap = 0, cnt = 0, i;
 321         char *name = NULL, **syms = NULL;
 322         struct hashmap *map;
 323         char buf[256];
 324         FILE *f;
 325         int err = 0;
 326
 327         /*
 328          * The available_filter_functions contains many duplicates,
 329          * but other than that all symbols are usable in kprobe multi
 330          * interface.
 331          * Filtering out duplicates by using hashmap__add, which won't
 332          * add existing entry.
 333          */
 334
 335         if (access("/sys/kernel/tracing/trace", F_OK) == 0)
 336                 f = fopen("/sys/kernel/tracing/available_filter_functions", "r");
 337         else
 338                 f = fopen("/sys/kernel/debug/tracing/available_filter_functions", "r");
 339
 340         if (!f)
 341                 return -EINVAL;
 342
 343         map = hashmap__new(symbol_hash, symbol_equal, NULL);
 344         if (IS_ERR(map)) {
 345                 err = libbpf_get_error(map);
 346                 goto error;
 347         }
 348
 349         while (fgets(buf, sizeof(buf), f)) {
 350                 if (kernel && strchr(buf, '['))
 351                         continue;
 352                 if (!kernel && !strchr(buf, '['))
 353                         continue;
 354
 355                 free(name);
 356                 if (sscanf(buf, "%ms$*[^\n]\n", &name) != 1)
 357                         continue;
 358                 /*
 359                  * We attach to almost all kernel functions and some of them
 360                  * will cause 'suspicious RCU usage' when fprobe is attached
 361                  * to them. Filter out the current culprits - arch_cpu_idle
 362                  * default_idle and rcu_* functions.
 363                  */
 364                 if (!strcmp(name, "arch_cpu_idle"))
 365                         continue;
 366                 if (!strcmp(name, "default_idle"))
 367                         continue;
 368                 if (!strncmp(name, "rcu_", 4))
 369                         continue;
 370                 if (!strcmp(name, "bpf_dispatcher_xdp_func"))
 371                         continue;
 372                 if (!strncmp(name, "__ftrace_invalid_address__",
 373                              sizeof("__ftrace_invalid_address__") - 1))
 374                         continue;
 375
 376                 err = hashmap__add(map, name, 0);
 377                 if (err == -EEXIST) {
 378                         err = 0;
 379                         continue;
 380                 }
 381                 if (err)
 382                         goto error;
 383
 384                 err = libbpf_ensure_mem((void **) &syms, &cap,
 385                                         sizeof(*syms), cnt + 1);
 386                 if (err)
 387                         goto error;
 388
 389                 syms[cnt++] = name;
 390                 name = NULL;
 391         }
 392
 393         *symsp = syms;
 394         *cntp = cnt;
 395
 396 error:
 397         free(name);
 398         fclose(f);
 399         hashmap__free(map);
 400         if (err) {
 401                 for (i = 0; i < cnt; i++)
 402                         free(syms[i]);
 403                 free(syms);
 404         }
 405         return err;
 406 }
 407
 408 static void test_kprobe_multi_bench_attach(bool kernel)
 409 {
 410         LIBBPF_OPTS(bpf_kprobe_multi_opts, opts);
 411         struct kprobe_multi_empty *skel = NULL;
 412         long attach_start_ns, attach_end_ns;
 413         long detach_start_ns, detach_end_ns;
 414         double attach_delta, detach_delta;
 415         struct bpf_link *link = NULL;
 416         char **syms = NULL;
 417         size_t cnt = 0, i;
 418
 419         if (!ASSERT_OK(get_syms(&syms, &cnt, kernel), "get_syms"))
 420                 return;
 421
 422         skel = kprobe_multi_empty__open_and_load();
 423         if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
 424                 goto cleanup;
 425
 426         opts.syms = (const char **) syms;
 427         opts.cnt = cnt;
 428
 429         attach_start_ns = get_time_ns();
 430         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_kprobe_empty,
 431                                                      NULL, &opts);
 432         attach_end_ns = get_time_ns();
 433
 434         if (!ASSERT_OK_PTR(link, "bpf_program__attach_kprobe_multi_opts"))
 435                 goto cleanup;
 436
 437         detach_start_ns = get_time_ns();
 438         bpf_link__destroy(link);
 439         detach_end_ns = get_time_ns();
 440
 441         attach_delta = (attach_end_ns - attach_start_ns) / 1000000000.0;
 442         detach_delta = (detach_end_ns - detach_start_ns) / 1000000000.0;
 443
 444         printf("%s: found %lu functions\n", __func__, cnt);
 445         printf("%s: attached in %7.3lfs\n", __func__, attach_delta);
 446         printf("%s: detached in %7.3lfs\n", __func__, detach_delta);
 447
 448 cleanup:
 449         kprobe_multi_empty__destroy(skel);
 450         if (syms) {
 451                 for (i = 0; i < cnt; i++)
 452                         free(syms[i]);
 453                 free(syms);
 454         }
 455 }
 456
 457 static void test_attach_override(void)
 458 {
 459         struct kprobe_multi_override *skel = NULL;
 460         struct bpf_link *link = NULL;
 461
 462         skel = kprobe_multi_override__open_and_load();
 463         if (!ASSERT_OK_PTR(skel, "kprobe_multi_empty__open_and_load"))
 464                 goto cleanup;
 465
 466         /* The test_override calls bpf_override_return so it should fail
 467          * to attach to bpf_fentry_test1 function, which is not on error
 468          * injection list.
 469          */
 470         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
 471                                                      "bpf_fentry_test1", NULL);
 472         if (!ASSERT_ERR_PTR(link, "override_attached_bpf_fentry_test1")) {
 473                 bpf_link__destroy(link);
 474                 goto cleanup;
 475         }
 476
 477         /* The should_fail_bio function is on error injection list,
 478          * attach should succeed.
 479          */
 480         link = bpf_program__attach_kprobe_multi_opts(skel->progs.test_override,
 481                                                      "should_fail_bio", NULL);
 482         if (!ASSERT_OK_PTR(link, "override_attached_should_fail_bio"))
 483                 goto cleanup;
 484
 485         bpf_link__destroy(link);
 486
 487 cleanup:
 488         kprobe_multi_override__destroy(skel);
 489 }
 490
 491 void serial_test_kprobe_multi_bench_attach(void)
 492 {
 493         if (test__start_subtest("kernel"))
 494                 test_kprobe_multi_bench_attach(true);
 495         if (test__start_subtest("modules"))
 496                 test_kprobe_multi_bench_attach(false);
 497 }
 498
 499 void test_kprobe_multi_test(void)
 500 {
 501         if (!ASSERT_OK(load_kallsyms(), "load_kallsyms"))
 502                 return;
 503
 504         if (test__start_subtest("skel_api"))
 505                 test_skel_api();
 506         if (test__start_subtest("link_api_addrs"))
 507                 test_link_api_syms();
 508         if (test__start_subtest("link_api_syms"))
 509                 test_link_api_addrs();
 510         if (test__start_subtest("attach_api_pattern"))
 511                 test_attach_api_pattern();
 512         if (test__start_subtest("attach_api_addrs"))
 513                 test_attach_api_addrs();
 514         if (test__start_subtest("attach_api_syms"))
 515                 test_attach_api_syms();
 516         if (test__start_subtest("attach_api_fails"))
 517                 test_attach_api_fails();
 518         if (test__start_subtest("attach_override"))
 519                 test_attach_override();
 520 }