tools/testing/selftests/bpf/prog_tests/sockmap_listen.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 // Copyright (c) 2020 Cloudflare
   3 /*
   4  * Test suite for SOCKMAP/SOCKHASH holding listening sockets.
   5  * Covers:
   6  *  1. BPF map operations - bpf_map_{update,lookup delete}_elem
   7  *  2. BPF redirect helpers - bpf_{sk,msg}_redirect_map
   8  *  3. BPF reuseport helper - bpf_sk_select_reuseport
   9  */
  10
  11 #include <linux/compiler.h>
  12 #include <errno.h>
  13 #include <error.h>
  14 #include <limits.h>
  15 #include <netinet/in.h>
  16 #include <pthread.h>
  17 #include <stdlib.h>
  18 #include <string.h>
  19 #include <sys/select.h>
  20 #include <unistd.h>
  21 #include <linux/vm_sockets.h>
  22
  23 #include <bpf/bpf.h>
  24 #include <bpf/libbpf.h>
  25
  26 #include "bpf_util.h"
  27 #include "test_progs.h"
  28 #include "test_sockmap_listen.skel.h"
  29
  30 #include "sockmap_helpers.h"
  31
  32 static void test_insert_invalid(struct test_sockmap_listen *skel __always_unused,
  33                                 int family, int sotype, int mapfd)
  34 {
  35         u32 key = 0;
  36         u64 value;
  37         int err;
  38
  39         value = -1;
  40         err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
  41         if (!err || errno != EINVAL)
  42                 FAIL_ERRNO("map_update: expected EINVAL");
  43
  44         value = INT_MAX;
  45         err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
  46         if (!err || errno != EBADF)
  47                 FAIL_ERRNO("map_update: expected EBADF");
  48 }
  49
  50 static void test_insert_opened(struct test_sockmap_listen *skel __always_unused,
  51                                int family, int sotype, int mapfd)
  52 {
  53         u32 key = 0;
  54         u64 value;
  55         int err, s;
  56
  57         s = xsocket(family, sotype, 0);
  58         if (s == -1)
  59                 return;
  60
  61         errno = 0;
  62         value = s;
  63         err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
  64         if (sotype == SOCK_STREAM) {
  65                 if (!err || errno != EOPNOTSUPP)
  66                         FAIL_ERRNO("map_update: expected EOPNOTSUPP");
  67         } else if (err)
  68                 FAIL_ERRNO("map_update: expected success");
  69         xclose(s);
  70 }
  71
  72 static void test_insert_bound(struct test_sockmap_listen *skel __always_unused,
  73                               int family, int sotype, int mapfd)
  74 {
  75         struct sockaddr_storage addr;
  76         socklen_t len = 0;
  77         u32 key = 0;
  78         u64 value;
  79         int err, s;
  80
  81         init_addr_loopback(family, &addr, &len);
  82
  83         s = xsocket(family, sotype, 0);
  84         if (s == -1)
  85                 return;
  86
  87         err = xbind(s, sockaddr(&addr), len);
  88         if (err)
  89                 goto close;
  90
  91         errno = 0;
  92         value = s;
  93         err = bpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
  94         if (!err || errno != EOPNOTSUPP)
  95                 FAIL_ERRNO("map_update: expected EOPNOTSUPP");
  96 close:
  97         xclose(s);
  98 }
  99
 100 static void test_insert(struct test_sockmap_listen *skel __always_unused,
 101                         int family, int sotype, int mapfd)
 102 {
 103         u64 value;
 104         u32 key;
 105         int s;
 106
 107         s = socket_loopback(family, sotype);
 108         if (s < 0)
 109                 return;
 110
 111         key = 0;
 112         value = s;
 113         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 114         xclose(s);
 115 }
 116
 117 static void test_delete_after_insert(struct test_sockmap_listen *skel __always_unused,
 118                                      int family, int sotype, int mapfd)
 119 {
 120         u64 value;
 121         u32 key;
 122         int s;
 123
 124         s = socket_loopback(family, sotype);
 125         if (s < 0)
 126                 return;
 127
 128         key = 0;
 129         value = s;
 130         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 131         xbpf_map_delete_elem(mapfd, &key);
 132         xclose(s);
 133 }
 134
 135 static void test_delete_after_close(struct test_sockmap_listen *skel __always_unused,
 136                                     int family, int sotype, int mapfd)
 137 {
 138         int err, s;
 139         u64 value;
 140         u32 key;
 141
 142         s = socket_loopback(family, sotype);
 143         if (s < 0)
 144                 return;
 145
 146         key = 0;
 147         value = s;
 148         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 149
 150         xclose(s);
 151
 152         errno = 0;
 153         err = bpf_map_delete_elem(mapfd, &key);
 154         if (!err || (errno != EINVAL && errno != ENOENT))
 155                 /* SOCKMAP and SOCKHASH return different error codes */
 156                 FAIL_ERRNO("map_delete: expected EINVAL/EINVAL");
 157 }
 158
 159 static void test_lookup_after_insert(struct test_sockmap_listen *skel __always_unused,
 160                                      int family, int sotype, int mapfd)
 161 {
 162         u64 cookie, value;
 163         socklen_t len;
 164         u32 key;
 165         int s;
 166
 167         s = socket_loopback(family, sotype);
 168         if (s < 0)
 169                 return;
 170
 171         key = 0;
 172         value = s;
 173         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 174
 175         len = sizeof(cookie);
 176         xgetsockopt(s, SOL_SOCKET, SO_COOKIE, &cookie, &len);
 177
 178         xbpf_map_lookup_elem(mapfd, &key, &value);
 179
 180         if (value != cookie) {
 181                 FAIL("map_lookup: have %#llx, want %#llx",
 182                      (unsigned long long)value, (unsigned long long)cookie);
 183         }
 184
 185         xclose(s);
 186 }
 187
 188 static void test_lookup_after_delete(struct test_sockmap_listen *skel __always_unused,
 189                                      int family, int sotype, int mapfd)
 190 {
 191         int err, s;
 192         u64 value;
 193         u32 key;
 194
 195         s = socket_loopback(family, sotype);
 196         if (s < 0)
 197                 return;
 198
 199         key = 0;
 200         value = s;
 201         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 202         xbpf_map_delete_elem(mapfd, &key);
 203
 204         errno = 0;
 205         err = bpf_map_lookup_elem(mapfd, &key, &value);
 206         if (!err || errno != ENOENT)
 207                 FAIL_ERRNO("map_lookup: expected ENOENT");
 208
 209         xclose(s);
 210 }
 211
 212 static void test_lookup_32_bit_value(struct test_sockmap_listen *skel __always_unused,
 213                                      int family, int sotype, int mapfd)
 214 {
 215         u32 key, value32;
 216         int err, s;
 217
 218         s = socket_loopback(family, sotype);
 219         if (s < 0)
 220                 return;
 221
 222         mapfd = bpf_map_create(BPF_MAP_TYPE_SOCKMAP, NULL, sizeof(key),
 223                                sizeof(value32), 1, NULL);
 224         if (mapfd < 0) {
 225                 FAIL_ERRNO("map_create");
 226                 goto close;
 227         }
 228
 229         key = 0;
 230         value32 = s;
 231         xbpf_map_update_elem(mapfd, &key, &value32, BPF_NOEXIST);
 232
 233         errno = 0;
 234         err = bpf_map_lookup_elem(mapfd, &key, &value32);
 235         if (!err || errno != ENOSPC)
 236                 FAIL_ERRNO("map_lookup: expected ENOSPC");
 237
 238         xclose(mapfd);
 239 close:
 240         xclose(s);
 241 }
 242
 243 static void test_update_existing(struct test_sockmap_listen *skel __always_unused,
 244                                  int family, int sotype, int mapfd)
 245 {
 246         int s1, s2;
 247         u64 value;
 248         u32 key;
 249
 250         s1 = socket_loopback(family, sotype);
 251         if (s1 < 0)
 252                 return;
 253
 254         s2 = socket_loopback(family, sotype);
 255         if (s2 < 0)
 256                 goto close_s1;
 257
 258         key = 0;
 259         value = s1;
 260         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 261
 262         value = s2;
 263         xbpf_map_update_elem(mapfd, &key, &value, BPF_EXIST);
 264         xclose(s2);
 265 close_s1:
 266         xclose(s1);
 267 }
 268
 269 /* Exercise the code path where we destroy child sockets that never
 270  * got accept()'ed, aka orphans, when parent socket gets closed.
 271  */
 272 static void do_destroy_orphan_child(int family, int sotype, int mapfd)
 273 {
 274         struct sockaddr_storage addr;
 275         socklen_t len;
 276         int err, s, c;
 277         u64 value;
 278         u32 key;
 279
 280         s = socket_loopback(family, sotype);
 281         if (s < 0)
 282                 return;
 283
 284         len = sizeof(addr);
 285         err = xgetsockname(s, sockaddr(&addr), &len);
 286         if (err)
 287                 goto close_srv;
 288
 289         key = 0;
 290         value = s;
 291         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 292
 293         c = xsocket(family, sotype, 0);
 294         if (c == -1)
 295                 goto close_srv;
 296
 297         xconnect(c, sockaddr(&addr), len);
 298         xclose(c);
 299 close_srv:
 300         xclose(s);
 301 }
 302
 303 static void test_destroy_orphan_child(struct test_sockmap_listen *skel,
 304                                       int family, int sotype, int mapfd)
 305 {
 306         int msg_verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
 307         int skb_verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
 308         const struct test {
 309                 int progfd;
 310                 enum bpf_attach_type atype;
 311         } tests[] = {
 312                 { -1, -1 },
 313                 { msg_verdict, BPF_SK_MSG_VERDICT },
 314                 { skb_verdict, BPF_SK_SKB_VERDICT },
 315         };
 316         const struct test *t;
 317
 318         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
 319                 if (t->progfd != -1 &&
 320                     xbpf_prog_attach(t->progfd, mapfd, t->atype, 0) != 0)
 321                         return;
 322
 323                 do_destroy_orphan_child(family, sotype, mapfd);
 324
 325                 if (t->progfd != -1)
 326                         xbpf_prog_detach2(t->progfd, mapfd, t->atype);
 327         }
 328 }
 329
 330 /* Perform a passive open after removing listening socket from SOCKMAP
 331  * to ensure that callbacks get restored properly.
 332  */
 333 static void test_clone_after_delete(struct test_sockmap_listen *skel __always_unused,
 334                                     int family, int sotype, int mapfd)
 335 {
 336         struct sockaddr_storage addr;
 337         socklen_t len;
 338         int err, s, c;
 339         u64 value;
 340         u32 key;
 341
 342         s = socket_loopback(family, sotype);
 343         if (s < 0)
 344                 return;
 345
 346         len = sizeof(addr);
 347         err = xgetsockname(s, sockaddr(&addr), &len);
 348         if (err)
 349                 goto close_srv;
 350
 351         key = 0;
 352         value = s;
 353         xbpf_map_update_elem(mapfd, &key, &value, BPF_NOEXIST);
 354         xbpf_map_delete_elem(mapfd, &key);
 355
 356         c = xsocket(family, sotype, 0);
 357         if (c < 0)
 358                 goto close_srv;
 359
 360         xconnect(c, sockaddr(&addr), len);
 361         xclose(c);
 362 close_srv:
 363         xclose(s);
 364 }
 365
 366 /* Check that child socket that got created while parent was in a
 367  * SOCKMAP, but got accept()'ed only after the parent has been removed
 368  * from SOCKMAP, gets cloned without parent psock state or callbacks.
 369  */
 370 static void test_accept_after_delete(struct test_sockmap_listen *skel __always_unused,
 371                                      int family, int sotype, int mapfd)
 372 {
 373         struct sockaddr_storage addr;
 374         const u32 zero = 0;
 375         int err, s, c, p;
 376         socklen_t len;
 377         u64 value;
 378
 379         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 380         if (s == -1)
 381                 return;
 382
 383         len = sizeof(addr);
 384         err = xgetsockname(s, sockaddr(&addr), &len);
 385         if (err)
 386                 goto close_srv;
 387
 388         value = s;
 389         err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST);
 390         if (err)
 391                 goto close_srv;
 392
 393         c = xsocket(family, sotype, 0);
 394         if (c == -1)
 395                 goto close_srv;
 396
 397         /* Create child while parent is in sockmap */
 398         err = xconnect(c, sockaddr(&addr), len);
 399         if (err)
 400                 goto close_cli;
 401
 402         /* Remove parent from sockmap */
 403         err = xbpf_map_delete_elem(mapfd, &zero);
 404         if (err)
 405                 goto close_cli;
 406
 407         p = xaccept_nonblock(s, NULL, NULL);
 408         if (p == -1)
 409                 goto close_cli;
 410
 411         /* Check that child sk_user_data is not set */
 412         value = p;
 413         xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST);
 414
 415         xclose(p);
 416 close_cli:
 417         xclose(c);
 418 close_srv:
 419         xclose(s);
 420 }
 421
 422 /* Check that child socket that got created and accepted while parent
 423  * was in a SOCKMAP is cloned without parent psock state or callbacks.
 424  */
 425 static void test_accept_before_delete(struct test_sockmap_listen *skel __always_unused,
 426                                       int family, int sotype, int mapfd)
 427 {
 428         struct sockaddr_storage addr;
 429         const u32 zero = 0, one = 1;
 430         int err, s, c, p;
 431         socklen_t len;
 432         u64 value;
 433
 434         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 435         if (s == -1)
 436                 return;
 437
 438         len = sizeof(addr);
 439         err = xgetsockname(s, sockaddr(&addr), &len);
 440         if (err)
 441                 goto close_srv;
 442
 443         value = s;
 444         err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST);
 445         if (err)
 446                 goto close_srv;
 447
 448         c = xsocket(family, sotype, 0);
 449         if (c == -1)
 450                 goto close_srv;
 451
 452         /* Create & accept child while parent is in sockmap */
 453         err = xconnect(c, sockaddr(&addr), len);
 454         if (err)
 455                 goto close_cli;
 456
 457         p = xaccept_nonblock(s, NULL, NULL);
 458         if (p == -1)
 459                 goto close_cli;
 460
 461         /* Check that child sk_user_data is not set */
 462         value = p;
 463         xbpf_map_update_elem(mapfd, &one, &value, BPF_NOEXIST);
 464
 465         xclose(p);
 466 close_cli:
 467         xclose(c);
 468 close_srv:
 469         xclose(s);
 470 }
 471
 472 struct connect_accept_ctx {
 473         int sockfd;
 474         unsigned int done;
 475         unsigned int nr_iter;
 476 };
 477
 478 static bool is_thread_done(struct connect_accept_ctx *ctx)
 479 {
 480         return READ_ONCE(ctx->done);
 481 }
 482
 483 static void *connect_accept_thread(void *arg)
 484 {
 485         struct connect_accept_ctx *ctx = arg;
 486         struct sockaddr_storage addr;
 487         int family, socktype;
 488         socklen_t len;
 489         int err, i, s;
 490
 491         s = ctx->sockfd;
 492
 493         len = sizeof(addr);
 494         err = xgetsockname(s, sockaddr(&addr), &len);
 495         if (err)
 496                 goto done;
 497
 498         len = sizeof(family);
 499         err = xgetsockopt(s, SOL_SOCKET, SO_DOMAIN, &family, &len);
 500         if (err)
 501                 goto done;
 502
 503         len = sizeof(socktype);
 504         err = xgetsockopt(s, SOL_SOCKET, SO_TYPE, &socktype, &len);
 505         if (err)
 506                 goto done;
 507
 508         for (i = 0; i < ctx->nr_iter; i++) {
 509                 int c, p;
 510
 511                 c = xsocket(family, socktype, 0);
 512                 if (c < 0)
 513                         break;
 514
 515                 err = xconnect(c, (struct sockaddr *)&addr, sizeof(addr));
 516                 if (err) {
 517                         xclose(c);
 518                         break;
 519                 }
 520
 521                 p = xaccept_nonblock(s, NULL, NULL);
 522                 if (p < 0) {
 523                         xclose(c);
 524                         break;
 525                 }
 526
 527                 xclose(p);
 528                 xclose(c);
 529         }
 530 done:
 531         WRITE_ONCE(ctx->done, 1);
 532         return NULL;
 533 }
 534
 535 static void test_syn_recv_insert_delete(struct test_sockmap_listen *skel __always_unused,
 536                                         int family, int sotype, int mapfd)
 537 {
 538         struct connect_accept_ctx ctx = { 0 };
 539         struct sockaddr_storage addr;
 540         socklen_t len;
 541         u32 zero = 0;
 542         pthread_t t;
 543         int err, s;
 544         u64 value;
 545
 546         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 547         if (s < 0)
 548                 return;
 549
 550         len = sizeof(addr);
 551         err = xgetsockname(s, sockaddr(&addr), &len);
 552         if (err)
 553                 goto close;
 554
 555         ctx.sockfd = s;
 556         ctx.nr_iter = 1000;
 557
 558         err = xpthread_create(&t, NULL, connect_accept_thread, &ctx);
 559         if (err)
 560                 goto close;
 561
 562         value = s;
 563         while (!is_thread_done(&ctx)) {
 564                 err = xbpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST);
 565                 if (err)
 566                         break;
 567
 568                 err = xbpf_map_delete_elem(mapfd, &zero);
 569                 if (err)
 570                         break;
 571         }
 572
 573         xpthread_join(t, NULL);
 574 close:
 575         xclose(s);
 576 }
 577
 578 static void *listen_thread(void *arg)
 579 {
 580         struct sockaddr unspec = { AF_UNSPEC };
 581         struct connect_accept_ctx *ctx = arg;
 582         int err, i, s;
 583
 584         s = ctx->sockfd;
 585
 586         for (i = 0; i < ctx->nr_iter; i++) {
 587                 err = xlisten(s, 1);
 588                 if (err)
 589                         break;
 590                 err = xconnect(s, &unspec, sizeof(unspec));
 591                 if (err)
 592                         break;
 593         }
 594
 595         WRITE_ONCE(ctx->done, 1);
 596         return NULL;
 597 }
 598
 599 static void test_race_insert_listen(struct test_sockmap_listen *skel __always_unused,
 600                                     int family, int socktype, int mapfd)
 601 {
 602         struct connect_accept_ctx ctx = { 0 };
 603         const u32 zero = 0;
 604         const int one = 1;
 605         pthread_t t;
 606         int err, s;
 607         u64 value;
 608
 609         s = xsocket(family, socktype, 0);
 610         if (s < 0)
 611                 return;
 612
 613         err = xsetsockopt(s, SOL_SOCKET, SO_REUSEADDR, &one, sizeof(one));
 614         if (err)
 615                 goto close;
 616
 617         ctx.sockfd = s;
 618         ctx.nr_iter = 10000;
 619
 620         err = pthread_create(&t, NULL, listen_thread, &ctx);
 621         if (err)
 622                 goto close;
 623
 624         value = s;
 625         while (!is_thread_done(&ctx)) {
 626                 err = bpf_map_update_elem(mapfd, &zero, &value, BPF_NOEXIST);
 627                 /* Expecting EOPNOTSUPP before listen() */
 628                 if (err && errno != EOPNOTSUPP) {
 629                         FAIL_ERRNO("map_update");
 630                         break;
 631                 }
 632
 633                 err = bpf_map_delete_elem(mapfd, &zero);
 634                 /* Expecting no entry after unhash on connect(AF_UNSPEC) */
 635                 if (err && errno != EINVAL && errno != ENOENT) {
 636                         FAIL_ERRNO("map_delete");
 637                         break;
 638                 }
 639         }
 640
 641         xpthread_join(t, NULL);
 642 close:
 643         xclose(s);
 644 }
 645
 646 static void zero_verdict_count(int mapfd)
 647 {
 648         unsigned int zero = 0;
 649         int key;
 650
 651         key = SK_DROP;
 652         xbpf_map_update_elem(mapfd, &key, &zero, BPF_ANY);
 653         key = SK_PASS;
 654         xbpf_map_update_elem(mapfd, &key, &zero, BPF_ANY);
 655 }
 656
 657 enum redir_mode {
 658         REDIR_INGRESS,
 659         REDIR_EGRESS,
 660 };
 661
 662 static const char *redir_mode_str(enum redir_mode mode)
 663 {
 664         switch (mode) {
 665         case REDIR_INGRESS:
 666                 return "ingress";
 667         case REDIR_EGRESS:
 668                 return "egress";
 669         default:
 670                 return "unknown";
 671         }
 672 }
 673
 674 static void redir_to_connected(int family, int sotype, int sock_mapfd,
 675                                int verd_mapfd, enum redir_mode mode)
 676 {
 677         const char *log_prefix = redir_mode_str(mode);
 678         int s, c0, c1, p0, p1;
 679         unsigned int pass;
 680         int err, n;
 681         u32 key;
 682         char b;
 683
 684         zero_verdict_count(verd_mapfd);
 685
 686         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 687         if (s < 0)
 688                 return;
 689
 690         err = create_socket_pairs(s, family, sotype, &c0, &c1, &p0, &p1);
 691         if (err)
 692                 goto close_srv;
 693
 694         err = add_to_sockmap(sock_mapfd, p0, p1);
 695         if (err)
 696                 goto close;
 697
 698         n = write(mode == REDIR_INGRESS ? c1 : p1, "a", 1);
 699         if (n < 0)
 700                 FAIL_ERRNO("%s: write", log_prefix);
 701         if (n == 0)
 702                 FAIL("%s: incomplete write", log_prefix);
 703         if (n < 1)
 704                 goto close;
 705
 706         key = SK_PASS;
 707         err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
 708         if (err)
 709                 goto close;
 710         if (pass != 1)
 711                 FAIL("%s: want pass count 1, have %d", log_prefix, pass);
 712         n = recv_timeout(c0, &b, 1, 0, IO_TIMEOUT_SEC);
 713         if (n < 0)
 714                 FAIL_ERRNO("%s: recv_timeout", log_prefix);
 715         if (n == 0)
 716                 FAIL("%s: incomplete recv", log_prefix);
 717
 718 close:
 719         xclose(p1);
 720         xclose(c1);
 721         xclose(p0);
 722         xclose(c0);
 723 close_srv:
 724         xclose(s);
 725 }
 726
 727 static void test_skb_redir_to_connected(struct test_sockmap_listen *skel,
 728                                         struct bpf_map *inner_map, int family,
 729                                         int sotype)
 730 {
 731         int verdict = bpf_program__fd(skel->progs.prog_stream_verdict);
 732         int parser = bpf_program__fd(skel->progs.prog_stream_parser);
 733         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
 734         int sock_map = bpf_map__fd(inner_map);
 735         int err;
 736
 737         err = xbpf_prog_attach(parser, sock_map, BPF_SK_SKB_STREAM_PARSER, 0);
 738         if (err)
 739                 return;
 740         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT, 0);
 741         if (err)
 742                 goto detach;
 743
 744         redir_to_connected(family, sotype, sock_map, verdict_map,
 745                            REDIR_INGRESS);
 746
 747         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT);
 748 detach:
 749         xbpf_prog_detach2(parser, sock_map, BPF_SK_SKB_STREAM_PARSER);
 750 }
 751
 752 static void test_msg_redir_to_connected(struct test_sockmap_listen *skel,
 753                                         struct bpf_map *inner_map, int family,
 754                                         int sotype)
 755 {
 756         int verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
 757         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
 758         int sock_map = bpf_map__fd(inner_map);
 759         int err;
 760
 761         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_MSG_VERDICT, 0);
 762         if (err)
 763                 return;
 764
 765         redir_to_connected(family, sotype, sock_map, verdict_map, REDIR_EGRESS);
 766
 767         xbpf_prog_detach2(verdict, sock_map, BPF_SK_MSG_VERDICT);
 768 }
 769
 770 static void redir_to_listening(int family, int sotype, int sock_mapfd,
 771                                int verd_mapfd, enum redir_mode mode)
 772 {
 773         const char *log_prefix = redir_mode_str(mode);
 774         struct sockaddr_storage addr;
 775         int s, c, p, err, n;
 776         unsigned int drop;
 777         socklen_t len;
 778         u32 key;
 779
 780         zero_verdict_count(verd_mapfd);
 781
 782         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 783         if (s < 0)
 784                 return;
 785
 786         len = sizeof(addr);
 787         err = xgetsockname(s, sockaddr(&addr), &len);
 788         if (err)
 789                 goto close_srv;
 790
 791         c = xsocket(family, sotype, 0);
 792         if (c < 0)
 793                 goto close_srv;
 794         err = xconnect(c, sockaddr(&addr), len);
 795         if (err)
 796                 goto close_cli;
 797
 798         p = xaccept_nonblock(s, NULL, NULL);
 799         if (p < 0)
 800                 goto close_cli;
 801
 802         err = add_to_sockmap(sock_mapfd, s, p);
 803         if (err)
 804                 goto close_peer;
 805
 806         n = write(mode == REDIR_INGRESS ? c : p, "a", 1);
 807         if (n < 0 && errno != EACCES)
 808                 FAIL_ERRNO("%s: write", log_prefix);
 809         if (n == 0)
 810                 FAIL("%s: incomplete write", log_prefix);
 811         if (n < 1)
 812                 goto close_peer;
 813
 814         key = SK_DROP;
 815         err = xbpf_map_lookup_elem(verd_mapfd, &key, &drop);
 816         if (err)
 817                 goto close_peer;
 818         if (drop != 1)
 819                 FAIL("%s: want drop count 1, have %d", log_prefix, drop);
 820
 821 close_peer:
 822         xclose(p);
 823 close_cli:
 824         xclose(c);
 825 close_srv:
 826         xclose(s);
 827 }
 828
 829 static void test_skb_redir_to_listening(struct test_sockmap_listen *skel,
 830                                         struct bpf_map *inner_map, int family,
 831                                         int sotype)
 832 {
 833         int verdict = bpf_program__fd(skel->progs.prog_stream_verdict);
 834         int parser = bpf_program__fd(skel->progs.prog_stream_parser);
 835         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
 836         int sock_map = bpf_map__fd(inner_map);
 837         int err;
 838
 839         err = xbpf_prog_attach(parser, sock_map, BPF_SK_SKB_STREAM_PARSER, 0);
 840         if (err)
 841                 return;
 842         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT, 0);
 843         if (err)
 844                 goto detach;
 845
 846         redir_to_listening(family, sotype, sock_map, verdict_map,
 847                            REDIR_INGRESS);
 848
 849         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT);
 850 detach:
 851         xbpf_prog_detach2(parser, sock_map, BPF_SK_SKB_STREAM_PARSER);
 852 }
 853
 854 static void test_msg_redir_to_listening(struct test_sockmap_listen *skel,
 855                                         struct bpf_map *inner_map, int family,
 856                                         int sotype)
 857 {
 858         int verdict = bpf_program__fd(skel->progs.prog_msg_verdict);
 859         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
 860         int sock_map = bpf_map__fd(inner_map);
 861         int err;
 862
 863         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_MSG_VERDICT, 0);
 864         if (err)
 865                 return;
 866
 867         redir_to_listening(family, sotype, sock_map, verdict_map, REDIR_EGRESS);
 868
 869         xbpf_prog_detach2(verdict, sock_map, BPF_SK_MSG_VERDICT);
 870 }
 871
 872 static void redir_partial(int family, int sotype, int sock_map, int parser_map)
 873 {
 874         int s, c0 = -1, c1 = -1, p0 = -1, p1 = -1;
 875         int err, n, key, value;
 876         char buf[] = "abc";
 877
 878         key = 0;
 879         value = sizeof(buf) - 1;
 880         err = xbpf_map_update_elem(parser_map, &key, &value, 0);
 881         if (err)
 882                 return;
 883
 884         s = socket_loopback(family, sotype | SOCK_NONBLOCK);
 885         if (s < 0)
 886                 goto clean_parser_map;
 887
 888         err = create_socket_pairs(s, family, sotype, &c0, &c1, &p0, &p1);
 889         if (err)
 890                 goto close_srv;
 891
 892         err = add_to_sockmap(sock_map, p0, p1);
 893         if (err)
 894                 goto close;
 895
 896         n = xsend(c1, buf, sizeof(buf), 0);
 897         if (n < sizeof(buf))
 898                 FAIL("incomplete write");
 899
 900         n = xrecv_nonblock(c0, buf, sizeof(buf), 0);
 901         if (n != sizeof(buf) - 1)
 902                 FAIL("expect %zu, received %d", sizeof(buf) - 1, n);
 903
 904 close:
 905         xclose(c0);
 906         xclose(p0);
 907         xclose(c1);
 908         xclose(p1);
 909 close_srv:
 910         xclose(s);
 911
 912 clean_parser_map:
 913         key = 0;
 914         value = 0;
 915         xbpf_map_update_elem(parser_map, &key, &value, 0);
 916 }
 917
 918 static void test_skb_redir_partial(struct test_sockmap_listen *skel,
 919                                    struct bpf_map *inner_map, int family,
 920                                    int sotype)
 921 {
 922         int verdict = bpf_program__fd(skel->progs.prog_stream_verdict);
 923         int parser = bpf_program__fd(skel->progs.prog_stream_parser);
 924         int parser_map = bpf_map__fd(skel->maps.parser_map);
 925         int sock_map = bpf_map__fd(inner_map);
 926         int err;
 927
 928         err = xbpf_prog_attach(parser, sock_map, BPF_SK_SKB_STREAM_PARSER, 0);
 929         if (err)
 930                 return;
 931
 932         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT, 0);
 933         if (err)
 934                 goto detach;
 935
 936         redir_partial(family, sotype, sock_map, parser_map);
 937
 938         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_STREAM_VERDICT);
 939 detach:
 940         xbpf_prog_detach2(parser, sock_map, BPF_SK_SKB_STREAM_PARSER);
 941 }
 942
 943 static void test_reuseport_select_listening(int family, int sotype,
 944                                             int sock_map, int verd_map,
 945                                             int reuseport_prog)
 946 {
 947         struct sockaddr_storage addr;
 948         unsigned int pass;
 949         int s, c, err;
 950         socklen_t len;
 951         u64 value;
 952         u32 key;
 953
 954         zero_verdict_count(verd_map);
 955
 956         s = socket_loopback_reuseport(family, sotype | SOCK_NONBLOCK,
 957                                       reuseport_prog);
 958         if (s < 0)
 959                 return;
 960
 961         len = sizeof(addr);
 962         err = xgetsockname(s, sockaddr(&addr), &len);
 963         if (err)
 964                 goto close_srv;
 965
 966         key = 0;
 967         value = s;
 968         err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST);
 969         if (err)
 970                 goto close_srv;
 971
 972         c = xsocket(family, sotype, 0);
 973         if (c < 0)
 974                 goto close_srv;
 975         err = xconnect(c, sockaddr(&addr), len);
 976         if (err)
 977                 goto close_cli;
 978
 979         if (sotype == SOCK_STREAM) {
 980                 int p;
 981
 982                 p = xaccept_nonblock(s, NULL, NULL);
 983                 if (p < 0)
 984                         goto close_cli;
 985                 xclose(p);
 986         } else {
 987                 char b = 'a';
 988                 ssize_t n;
 989
 990                 n = xsend(c, &b, sizeof(b), 0);
 991                 if (n == -1)
 992                         goto close_cli;
 993
 994                 n = xrecv_nonblock(s, &b, sizeof(b), 0);
 995                 if (n == -1)
 996                         goto close_cli;
 997         }
 998
 999         key = SK_PASS;
1000         err = xbpf_map_lookup_elem(verd_map, &key, &pass);
1001         if (err)
1002                 goto close_cli;
1003         if (pass != 1)
1004                 FAIL("want pass count 1, have %d", pass);
1005
1006 close_cli:
1007         xclose(c);
1008 close_srv:
1009         xclose(s);
1010 }
1011
1012 static void test_reuseport_select_connected(int family, int sotype,
1013                                             int sock_map, int verd_map,
1014                                             int reuseport_prog)
1015 {
1016         struct sockaddr_storage addr;
1017         int s, c0, c1, p0, err;
1018         unsigned int drop;
1019         socklen_t len;
1020         u64 value;
1021         u32 key;
1022
1023         zero_verdict_count(verd_map);
1024
1025         s = socket_loopback_reuseport(family, sotype, reuseport_prog);
1026         if (s < 0)
1027                 return;
1028
1029         /* Populate sock_map[0] to avoid ENOENT on first connection */
1030         key = 0;
1031         value = s;
1032         err = xbpf_map_update_elem(sock_map, &key, &value, BPF_NOEXIST);
1033         if (err)
1034                 goto close_srv;
1035
1036         len = sizeof(addr);
1037         err = xgetsockname(s, sockaddr(&addr), &len);
1038         if (err)
1039                 goto close_srv;
1040
1041         c0 = xsocket(family, sotype, 0);
1042         if (c0 < 0)
1043                 goto close_srv;
1044
1045         err = xconnect(c0, sockaddr(&addr), len);
1046         if (err)
1047                 goto close_cli0;
1048
1049         if (sotype == SOCK_STREAM) {
1050                 p0 = xaccept_nonblock(s, NULL, NULL);
1051                 if (p0 < 0)
1052                         goto close_cli0;
1053         } else {
1054                 p0 = xsocket(family, sotype, 0);
1055                 if (p0 < 0)
1056                         goto close_cli0;
1057
1058                 len = sizeof(addr);
1059                 err = xgetsockname(c0, sockaddr(&addr), &len);
1060                 if (err)
1061                         goto close_cli0;
1062
1063                 err = xconnect(p0, sockaddr(&addr), len);
1064                 if (err)
1065                         goto close_cli0;
1066         }
1067
1068         /* Update sock_map[0] to redirect to a connected socket */
1069         key = 0;
1070         value = p0;
1071         err = xbpf_map_update_elem(sock_map, &key, &value, BPF_EXIST);
1072         if (err)
1073                 goto close_peer0;
1074
1075         c1 = xsocket(family, sotype, 0);
1076         if (c1 < 0)
1077                 goto close_peer0;
1078
1079         len = sizeof(addr);
1080         err = xgetsockname(s, sockaddr(&addr), &len);
1081         if (err)
1082                 goto close_srv;
1083
1084         errno = 0;
1085         err = connect(c1, sockaddr(&addr), len);
1086         if (sotype == SOCK_DGRAM) {
1087                 char b = 'a';
1088                 ssize_t n;
1089
1090                 n = xsend(c1, &b, sizeof(b), 0);
1091                 if (n == -1)
1092                         goto close_cli1;
1093
1094                 n = recv_timeout(c1, &b, sizeof(b), 0, IO_TIMEOUT_SEC);
1095                 err = n == -1;
1096         }
1097         if (!err || errno != ECONNREFUSED)
1098                 FAIL_ERRNO("connect: expected ECONNREFUSED");
1099
1100         key = SK_DROP;
1101         err = xbpf_map_lookup_elem(verd_map, &key, &drop);
1102         if (err)
1103                 goto close_cli1;
1104         if (drop != 1)
1105                 FAIL("want drop count 1, have %d", drop);
1106
1107 close_cli1:
1108         xclose(c1);
1109 close_peer0:
1110         xclose(p0);
1111 close_cli0:
1112         xclose(c0);
1113 close_srv:
1114         xclose(s);
1115 }
1116
1117 /* Check that redirecting across reuseport groups is not allowed. */
1118 static void test_reuseport_mixed_groups(int family, int sotype, int sock_map,
1119                                         int verd_map, int reuseport_prog)
1120 {
1121         struct sockaddr_storage addr;
1122         int s1, s2, c, err;
1123         unsigned int drop;
1124         socklen_t len;
1125         u32 key;
1126
1127         zero_verdict_count(verd_map);
1128
1129         /* Create two listeners, each in its own reuseport group */
1130         s1 = socket_loopback_reuseport(family, sotype, reuseport_prog);
1131         if (s1 < 0)
1132                 return;
1133
1134         s2 = socket_loopback_reuseport(family, sotype, reuseport_prog);
1135         if (s2 < 0)
1136                 goto close_srv1;
1137
1138         err = add_to_sockmap(sock_map, s1, s2);
1139         if (err)
1140                 goto close_srv2;
1141
1142         /* Connect to s2, reuseport BPF selects s1 via sock_map[0] */
1143         len = sizeof(addr);
1144         err = xgetsockname(s2, sockaddr(&addr), &len);
1145         if (err)
1146                 goto close_srv2;
1147
1148         c = xsocket(family, sotype, 0);
1149         if (c < 0)
1150                 goto close_srv2;
1151
1152         err = connect(c, sockaddr(&addr), len);
1153         if (sotype == SOCK_DGRAM) {
1154                 char b = 'a';
1155                 ssize_t n;
1156
1157                 n = xsend(c, &b, sizeof(b), 0);
1158                 if (n == -1)
1159                         goto close_cli;
1160
1161                 n = recv_timeout(c, &b, sizeof(b), 0, IO_TIMEOUT_SEC);
1162                 err = n == -1;
1163         }
1164         if (!err || errno != ECONNREFUSED) {
1165                 FAIL_ERRNO("connect: expected ECONNREFUSED");
1166                 goto close_cli;
1167         }
1168
1169         /* Expect drop, can't redirect outside of reuseport group */
1170         key = SK_DROP;
1171         err = xbpf_map_lookup_elem(verd_map, &key, &drop);
1172         if (err)
1173                 goto close_cli;
1174         if (drop != 1)
1175                 FAIL("want drop count 1, have %d", drop);
1176
1177 close_cli:
1178         xclose(c);
1179 close_srv2:
1180         xclose(s2);
1181 close_srv1:
1182         xclose(s1);
1183 }
1184
1185 #define TEST(fn, ...)                                                          \
1186         {                                                                      \
1187                 fn, #fn, __VA_ARGS__                                           \
1188         }
1189
1190 static void test_ops_cleanup(const struct bpf_map *map)
1191 {
1192         int err, mapfd;
1193         u32 key;
1194
1195         mapfd = bpf_map__fd(map);
1196
1197         for (key = 0; key < bpf_map__max_entries(map); key++) {
1198                 err = bpf_map_delete_elem(mapfd, &key);
1199                 if (err && errno != EINVAL && errno != ENOENT)
1200                         FAIL_ERRNO("map_delete: expected EINVAL/ENOENT");
1201         }
1202 }
1203
1204 static const char *family_str(sa_family_t family)
1205 {
1206         switch (family) {
1207         case AF_INET:
1208                 return "IPv4";
1209         case AF_INET6:
1210                 return "IPv6";
1211         case AF_UNIX:
1212                 return "Unix";
1213         case AF_VSOCK:
1214                 return "VSOCK";
1215         default:
1216                 return "unknown";
1217         }
1218 }
1219
1220 static const char *map_type_str(const struct bpf_map *map)
1221 {
1222         int type;
1223
1224         if (!map)
1225                 return "invalid";
1226         type = bpf_map__type(map);
1227
1228         switch (type) {
1229         case BPF_MAP_TYPE_SOCKMAP:
1230                 return "sockmap";
1231         case BPF_MAP_TYPE_SOCKHASH:
1232                 return "sockhash";
1233         default:
1234                 return "unknown";
1235         }
1236 }
1237
1238 static const char *sotype_str(int sotype)
1239 {
1240         switch (sotype) {
1241         case SOCK_DGRAM:
1242                 return "UDP";
1243         case SOCK_STREAM:
1244                 return "TCP";
1245         default:
1246                 return "unknown";
1247         }
1248 }
1249
1250 static void test_ops(struct test_sockmap_listen *skel, struct bpf_map *map,
1251                      int family, int sotype)
1252 {
1253         const struct op_test {
1254                 void (*fn)(struct test_sockmap_listen *skel,
1255                            int family, int sotype, int mapfd);
1256                 const char *name;
1257                 int sotype;
1258         } tests[] = {
1259                 /* insert */
1260                 TEST(test_insert_invalid),
1261                 TEST(test_insert_opened),
1262                 TEST(test_insert_bound, SOCK_STREAM),
1263                 TEST(test_insert),
1264                 /* delete */
1265                 TEST(test_delete_after_insert),
1266                 TEST(test_delete_after_close),
1267                 /* lookup */
1268                 TEST(test_lookup_after_insert),
1269                 TEST(test_lookup_after_delete),
1270                 TEST(test_lookup_32_bit_value),
1271                 /* update */
1272                 TEST(test_update_existing),
1273                 /* races with insert/delete */
1274                 TEST(test_destroy_orphan_child, SOCK_STREAM),
1275                 TEST(test_syn_recv_insert_delete, SOCK_STREAM),
1276                 TEST(test_race_insert_listen, SOCK_STREAM),
1277                 /* child clone */
1278                 TEST(test_clone_after_delete, SOCK_STREAM),
1279                 TEST(test_accept_after_delete, SOCK_STREAM),
1280                 TEST(test_accept_before_delete, SOCK_STREAM),
1281         };
1282         const char *family_name, *map_name, *sotype_name;
1283         const struct op_test *t;
1284         char s[MAX_TEST_NAME];
1285         int map_fd;
1286
1287         family_name = family_str(family);
1288         map_name = map_type_str(map);
1289         sotype_name = sotype_str(sotype);
1290         map_fd = bpf_map__fd(map);
1291
1292         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
1293                 snprintf(s, sizeof(s), "%s %s %s %s", map_name, family_name,
1294                          sotype_name, t->name);
1295
1296                 if (t->sotype != 0 && t->sotype != sotype)
1297                         continue;
1298
1299                 if (!test__start_subtest(s))
1300                         continue;
1301
1302                 t->fn(skel, family, sotype, map_fd);
1303                 test_ops_cleanup(map);
1304         }
1305 }
1306
1307 static void test_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1308                        int family, int sotype)
1309 {
1310         const struct redir_test {
1311                 void (*fn)(struct test_sockmap_listen *skel,
1312                            struct bpf_map *map, int family, int sotype);
1313                 const char *name;
1314         } tests[] = {
1315                 TEST(test_skb_redir_to_connected),
1316                 TEST(test_skb_redir_to_listening),
1317                 TEST(test_skb_redir_partial),
1318                 TEST(test_msg_redir_to_connected),
1319                 TEST(test_msg_redir_to_listening),
1320         };
1321         const char *family_name, *map_name;
1322         const struct redir_test *t;
1323         char s[MAX_TEST_NAME];
1324
1325         family_name = family_str(family);
1326         map_name = map_type_str(map);
1327
1328         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
1329                 snprintf(s, sizeof(s), "%s %s %s", map_name, family_name,
1330                          t->name);
1331
1332                 if (!test__start_subtest(s))
1333                         continue;
1334
1335                 t->fn(skel, map, family, sotype);
1336         }
1337 }
1338
1339 static void pairs_redir_to_connected(int cli0, int peer0, int cli1, int peer1,
1340                                      int sock_mapfd, int verd_mapfd, enum redir_mode mode)
1341 {
1342         const char *log_prefix = redir_mode_str(mode);
1343         unsigned int pass;
1344         int err, n;
1345         u32 key;
1346         char b;
1347
1348         zero_verdict_count(verd_mapfd);
1349
1350         err = add_to_sockmap(sock_mapfd, peer0, peer1);
1351         if (err)
1352                 return;
1353
1354         n = write(cli1, "a", 1);
1355         if (n < 0)
1356                 FAIL_ERRNO("%s: write", log_prefix);
1357         if (n == 0)
1358                 FAIL("%s: incomplete write", log_prefix);
1359         if (n < 1)
1360                 return;
1361
1362         key = SK_PASS;
1363         err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1364         if (err)
1365                 return;
1366         if (pass != 1)
1367                 FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1368
1369         n = recv_timeout(mode == REDIR_INGRESS ? peer0 : cli0, &b, 1, 0, IO_TIMEOUT_SEC);
1370         if (n < 0)
1371                 FAIL_ERRNO("%s: recv_timeout", log_prefix);
1372         if (n == 0)
1373                 FAIL("%s: incomplete recv", log_prefix);
1374 }
1375
1376 static void unix_redir_to_connected(int sotype, int sock_mapfd,
1377                                int verd_mapfd, enum redir_mode mode)
1378 {
1379         int c0, c1, p0, p1;
1380         int sfd[2];
1381
1382         if (socketpair(AF_UNIX, sotype | SOCK_NONBLOCK, 0, sfd))
1383                 return;
1384         c0 = sfd[0], p0 = sfd[1];
1385
1386         if (socketpair(AF_UNIX, sotype | SOCK_NONBLOCK, 0, sfd))
1387                 goto close0;
1388         c1 = sfd[0], p1 = sfd[1];
1389
1390         pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, verd_mapfd, mode);
1391
1392         xclose(c1);
1393         xclose(p1);
1394 close0:
1395         xclose(c0);
1396         xclose(p0);
1397 }
1398
1399 static void unix_skb_redir_to_connected(struct test_sockmap_listen *skel,
1400                                         struct bpf_map *inner_map, int sotype)
1401 {
1402         int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1403         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1404         int sock_map = bpf_map__fd(inner_map);
1405         int err;
1406
1407         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1408         if (err)
1409                 return;
1410
1411         skel->bss->test_ingress = false;
1412         unix_redir_to_connected(sotype, sock_map, verdict_map, REDIR_EGRESS);
1413         skel->bss->test_ingress = true;
1414         unix_redir_to_connected(sotype, sock_map, verdict_map, REDIR_INGRESS);
1415
1416         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1417 }
1418
1419 static void test_unix_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1420                             int sotype)
1421 {
1422         const char *family_name, *map_name;
1423         char s[MAX_TEST_NAME];
1424
1425         family_name = family_str(AF_UNIX);
1426         map_name = map_type_str(map);
1427         snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1428         if (!test__start_subtest(s))
1429                 return;
1430         unix_skb_redir_to_connected(skel, map, sotype);
1431 }
1432
1433 /* Returns two connected loopback vsock sockets */
1434 static int vsock_socketpair_connectible(int sotype, int *v0, int *v1)
1435 {
1436         struct sockaddr_storage addr;
1437         socklen_t len = sizeof(addr);
1438         int s, p, c;
1439
1440         s = socket_loopback(AF_VSOCK, sotype);
1441         if (s < 0)
1442                 return -1;
1443
1444         c = xsocket(AF_VSOCK, sotype | SOCK_NONBLOCK, 0);
1445         if (c == -1)
1446                 goto close_srv;
1447
1448         if (getsockname(s, sockaddr(&addr), &len) < 0)
1449                 goto close_cli;
1450
1451         if (connect(c, sockaddr(&addr), len) < 0 && errno != EINPROGRESS) {
1452                 FAIL_ERRNO("connect");
1453                 goto close_cli;
1454         }
1455
1456         len = sizeof(addr);
1457         p = accept_timeout(s, sockaddr(&addr), &len, IO_TIMEOUT_SEC);
1458         if (p < 0)
1459                 goto close_cli;
1460
1461         if (poll_connect(c, IO_TIMEOUT_SEC) < 0) {
1462                 FAIL_ERRNO("poll_connect");
1463                 goto close_acc;
1464         }
1465
1466         *v0 = p;
1467         *v1 = c;
1468
1469         return 0;
1470
1471 close_acc:
1472         close(p);
1473 close_cli:
1474         close(c);
1475 close_srv:
1476         close(s);
1477
1478         return -1;
1479 }
1480
1481 static void vsock_unix_redir_connectible(int sock_mapfd, int verd_mapfd,
1482                                          enum redir_mode mode, int sotype)
1483 {
1484         const char *log_prefix = redir_mode_str(mode);
1485         char a = 'a', b = 'b';
1486         int u0, u1, v0, v1;
1487         int sfd[2];
1488         unsigned int pass;
1489         int err, n;
1490         u32 key;
1491
1492         zero_verdict_count(verd_mapfd);
1493
1494         if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_NONBLOCK, 0, sfd))
1495                 return;
1496
1497         u0 = sfd[0];
1498         u1 = sfd[1];
1499
1500         err = vsock_socketpair_connectible(sotype, &v0, &v1);
1501         if (err) {
1502                 FAIL("vsock_socketpair_connectible() failed");
1503                 goto close_uds;
1504         }
1505
1506         err = add_to_sockmap(sock_mapfd, u0, v0);
1507         if (err) {
1508                 FAIL("add_to_sockmap failed");
1509                 goto close_vsock;
1510         }
1511
1512         n = write(v1, &a, sizeof(a));
1513         if (n < 0)
1514                 FAIL_ERRNO("%s: write", log_prefix);
1515         if (n == 0)
1516                 FAIL("%s: incomplete write", log_prefix);
1517         if (n < 1)
1518                 goto out;
1519
1520         n = xrecv_nonblock(mode == REDIR_INGRESS ? u0 : u1, &b, sizeof(b), 0);
1521         if (n < 0)
1522                 FAIL("%s: recv() err, errno=%d", log_prefix, errno);
1523         if (n == 0)
1524                 FAIL("%s: incomplete recv", log_prefix);
1525         if (b != a)
1526                 FAIL("%s: vsock socket map failed, %c != %c", log_prefix, a, b);
1527
1528         key = SK_PASS;
1529         err = xbpf_map_lookup_elem(verd_mapfd, &key, &pass);
1530         if (err)
1531                 goto out;
1532         if (pass != 1)
1533                 FAIL("%s: want pass count 1, have %d", log_prefix, pass);
1534 out:
1535         key = 0;
1536         bpf_map_delete_elem(sock_mapfd, &key);
1537         key = 1;
1538         bpf_map_delete_elem(sock_mapfd, &key);
1539
1540 close_vsock:
1541         close(v0);
1542         close(v1);
1543
1544 close_uds:
1545         close(u0);
1546         close(u1);
1547 }
1548
1549 static void vsock_unix_skb_redir_connectible(struct test_sockmap_listen *skel,
1550                                              struct bpf_map *inner_map,
1551                                              int sotype)
1552 {
1553         int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1554         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1555         int sock_map = bpf_map__fd(inner_map);
1556         int err;
1557
1558         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1559         if (err)
1560                 return;
1561
1562         skel->bss->test_ingress = false;
1563         vsock_unix_redir_connectible(sock_map, verdict_map, REDIR_EGRESS, sotype);
1564         skel->bss->test_ingress = true;
1565         vsock_unix_redir_connectible(sock_map, verdict_map, REDIR_INGRESS, sotype);
1566
1567         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1568 }
1569
1570 static void test_vsock_redir(struct test_sockmap_listen *skel, struct bpf_map *map)
1571 {
1572         const char *family_name, *map_name;
1573         char s[MAX_TEST_NAME];
1574
1575         family_name = family_str(AF_VSOCK);
1576         map_name = map_type_str(map);
1577         snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1578         if (!test__start_subtest(s))
1579                 return;
1580
1581         vsock_unix_skb_redir_connectible(skel, map, SOCK_STREAM);
1582         vsock_unix_skb_redir_connectible(skel, map, SOCK_SEQPACKET);
1583 }
1584
1585 static void test_reuseport(struct test_sockmap_listen *skel,
1586                            struct bpf_map *map, int family, int sotype)
1587 {
1588         const struct reuseport_test {
1589                 void (*fn)(int family, int sotype, int socket_map,
1590                            int verdict_map, int reuseport_prog);
1591                 const char *name;
1592                 int sotype;
1593         } tests[] = {
1594                 TEST(test_reuseport_select_listening),
1595                 TEST(test_reuseport_select_connected),
1596                 TEST(test_reuseport_mixed_groups),
1597         };
1598         int socket_map, verdict_map, reuseport_prog;
1599         const char *family_name, *map_name, *sotype_name;
1600         const struct reuseport_test *t;
1601         char s[MAX_TEST_NAME];
1602
1603         family_name = family_str(family);
1604         map_name = map_type_str(map);
1605         sotype_name = sotype_str(sotype);
1606
1607         socket_map = bpf_map__fd(map);
1608         verdict_map = bpf_map__fd(skel->maps.verdict_map);
1609         reuseport_prog = bpf_program__fd(skel->progs.prog_reuseport);
1610
1611         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
1612                 snprintf(s, sizeof(s), "%s %s %s %s", map_name, family_name,
1613                          sotype_name, t->name);
1614
1615                 if (t->sotype != 0 && t->sotype != sotype)
1616                         continue;
1617
1618                 if (!test__start_subtest(s))
1619                         continue;
1620
1621                 t->fn(family, sotype, socket_map, verdict_map, reuseport_prog);
1622         }
1623 }
1624
1625 static int inet_socketpair(int family, int type, int *s, int *c)
1626 {
1627         struct sockaddr_storage addr;
1628         socklen_t len;
1629         int p0, c0;
1630         int err;
1631
1632         p0 = socket_loopback(family, type | SOCK_NONBLOCK);
1633         if (p0 < 0)
1634                 return p0;
1635
1636         len = sizeof(addr);
1637         err = xgetsockname(p0, sockaddr(&addr), &len);
1638         if (err)
1639                 goto close_peer0;
1640
1641         c0 = xsocket(family, type | SOCK_NONBLOCK, 0);
1642         if (c0 < 0) {
1643                 err = c0;
1644                 goto close_peer0;
1645         }
1646         err = xconnect(c0, sockaddr(&addr), len);
1647         if (err)
1648                 goto close_cli0;
1649         err = xgetsockname(c0, sockaddr(&addr), &len);
1650         if (err)
1651                 goto close_cli0;
1652         err = xconnect(p0, sockaddr(&addr), len);
1653         if (err)
1654                 goto close_cli0;
1655
1656         *s = p0;
1657         *c = c0;
1658         return 0;
1659
1660 close_cli0:
1661         xclose(c0);
1662 close_peer0:
1663         xclose(p0);
1664         return err;
1665 }
1666
1667 static void udp_redir_to_connected(int family, int sock_mapfd, int verd_mapfd,
1668                                    enum redir_mode mode)
1669 {
1670         int c0, c1, p0, p1;
1671         int err;
1672
1673         err = inet_socketpair(family, SOCK_DGRAM, &p0, &c0);
1674         if (err)
1675                 return;
1676         err = inet_socketpair(family, SOCK_DGRAM, &p1, &c1);
1677         if (err)
1678                 goto close_cli0;
1679
1680         pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, verd_mapfd, mode);
1681
1682         xclose(c1);
1683         xclose(p1);
1684 close_cli0:
1685         xclose(c0);
1686         xclose(p0);
1687 }
1688
1689 static void udp_skb_redir_to_connected(struct test_sockmap_listen *skel,
1690                                        struct bpf_map *inner_map, int family)
1691 {
1692         int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1693         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1694         int sock_map = bpf_map__fd(inner_map);
1695         int err;
1696
1697         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1698         if (err)
1699                 return;
1700
1701         skel->bss->test_ingress = false;
1702         udp_redir_to_connected(family, sock_map, verdict_map, REDIR_EGRESS);
1703         skel->bss->test_ingress = true;
1704         udp_redir_to_connected(family, sock_map, verdict_map, REDIR_INGRESS);
1705
1706         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1707 }
1708
1709 static void test_udp_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1710                            int family)
1711 {
1712         const char *family_name, *map_name;
1713         char s[MAX_TEST_NAME];
1714
1715         family_name = family_str(family);
1716         map_name = map_type_str(map);
1717         snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1718         if (!test__start_subtest(s))
1719                 return;
1720         udp_skb_redir_to_connected(skel, map, family);
1721 }
1722
1723 static void inet_unix_redir_to_connected(int family, int type, int sock_mapfd,
1724                                         int verd_mapfd, enum redir_mode mode)
1725 {
1726         int c0, c1, p0, p1;
1727         int sfd[2];
1728         int err;
1729
1730         if (socketpair(AF_UNIX, SOCK_DGRAM | SOCK_NONBLOCK, 0, sfd))
1731                 return;
1732         c0 = sfd[0], p0 = sfd[1];
1733
1734         err = inet_socketpair(family, SOCK_DGRAM, &p1, &c1);
1735         if (err)
1736                 goto close;
1737
1738         pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, verd_mapfd, mode);
1739
1740         xclose(c1);
1741         xclose(p1);
1742 close:
1743         xclose(c0);
1744         xclose(p0);
1745 }
1746
1747 static void inet_unix_skb_redir_to_connected(struct test_sockmap_listen *skel,
1748                                             struct bpf_map *inner_map, int family)
1749 {
1750         int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1751         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1752         int sock_map = bpf_map__fd(inner_map);
1753         int err;
1754
1755         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1756         if (err)
1757                 return;
1758
1759         skel->bss->test_ingress = false;
1760         inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1761                                     REDIR_EGRESS);
1762         inet_unix_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1763                                     REDIR_EGRESS);
1764         skel->bss->test_ingress = true;
1765         inet_unix_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1766                                     REDIR_INGRESS);
1767         inet_unix_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1768                                     REDIR_INGRESS);
1769
1770         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1771 }
1772
1773 static void unix_inet_redir_to_connected(int family, int type, int sock_mapfd,
1774                                         int verd_mapfd, enum redir_mode mode)
1775 {
1776         int c0, c1, p0, p1;
1777         int sfd[2];
1778         int err;
1779
1780         err = inet_socketpair(family, SOCK_DGRAM, &p0, &c0);
1781         if (err)
1782                 return;
1783
1784         if (socketpair(AF_UNIX, SOCK_DGRAM | SOCK_NONBLOCK, 0, sfd))
1785                 goto close_cli0;
1786         c1 = sfd[0], p1 = sfd[1];
1787
1788         pairs_redir_to_connected(c0, p0, c1, p1, sock_mapfd, verd_mapfd, mode);
1789
1790         xclose(c1);
1791         xclose(p1);
1792 close_cli0:
1793         xclose(c0);
1794         xclose(p0);
1795
1796 }
1797
1798 static void unix_inet_skb_redir_to_connected(struct test_sockmap_listen *skel,
1799                                             struct bpf_map *inner_map, int family)
1800 {
1801         int verdict = bpf_program__fd(skel->progs.prog_skb_verdict);
1802         int verdict_map = bpf_map__fd(skel->maps.verdict_map);
1803         int sock_map = bpf_map__fd(inner_map);
1804         int err;
1805
1806         err = xbpf_prog_attach(verdict, sock_map, BPF_SK_SKB_VERDICT, 0);
1807         if (err)
1808                 return;
1809
1810         skel->bss->test_ingress = false;
1811         unix_inet_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1812                                      REDIR_EGRESS);
1813         unix_inet_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1814                                      REDIR_EGRESS);
1815         skel->bss->test_ingress = true;
1816         unix_inet_redir_to_connected(family, SOCK_DGRAM, sock_map, verdict_map,
1817                                      REDIR_INGRESS);
1818         unix_inet_redir_to_connected(family, SOCK_STREAM, sock_map, verdict_map,
1819                                      REDIR_INGRESS);
1820
1821         xbpf_prog_detach2(verdict, sock_map, BPF_SK_SKB_VERDICT);
1822 }
1823
1824 static void test_udp_unix_redir(struct test_sockmap_listen *skel, struct bpf_map *map,
1825                                 int family)
1826 {
1827         const char *family_name, *map_name;
1828         char s[MAX_TEST_NAME];
1829
1830         family_name = family_str(family);
1831         map_name = map_type_str(map);
1832         snprintf(s, sizeof(s), "%s %s %s", map_name, family_name, __func__);
1833         if (!test__start_subtest(s))
1834                 return;
1835         inet_unix_skb_redir_to_connected(skel, map, family);
1836         unix_inet_skb_redir_to_connected(skel, map, family);
1837 }
1838
1839 static void run_tests(struct test_sockmap_listen *skel, struct bpf_map *map,
1840                       int family)
1841 {
1842         test_ops(skel, map, family, SOCK_STREAM);
1843         test_ops(skel, map, family, SOCK_DGRAM);
1844         test_redir(skel, map, family, SOCK_STREAM);
1845         test_reuseport(skel, map, family, SOCK_STREAM);
1846         test_reuseport(skel, map, family, SOCK_DGRAM);
1847         test_udp_redir(skel, map, family);
1848         test_udp_unix_redir(skel, map, family);
1849 }
1850
1851 void serial_test_sockmap_listen(void)
1852 {
1853         struct test_sockmap_listen *skel;
1854
1855         skel = test_sockmap_listen__open_and_load();
1856         if (!skel) {
1857                 FAIL("skeleton open/load failed");
1858                 return;
1859         }
1860
1861         skel->bss->test_sockmap = true;
1862         run_tests(skel, skel->maps.sock_map, AF_INET);
1863         run_tests(skel, skel->maps.sock_map, AF_INET6);
1864         test_unix_redir(skel, skel->maps.sock_map, SOCK_DGRAM);
1865         test_unix_redir(skel, skel->maps.sock_map, SOCK_STREAM);
1866         test_vsock_redir(skel, skel->maps.sock_map);
1867
1868         skel->bss->test_sockmap = false;
1869         run_tests(skel, skel->maps.sock_hash, AF_INET);
1870         run_tests(skel, skel->maps.sock_hash, AF_INET6);
1871         test_unix_redir(skel, skel->maps.sock_hash, SOCK_DGRAM);
1872         test_unix_redir(skel, skel->maps.sock_hash, SOCK_STREAM);
1873         test_vsock_redir(skel, skel->maps.sock_hash);
1874
1875         test_sockmap_listen__destroy(skel);
1876 }