kernel/bpf/offload.c

   1 /*
   2  * Copyright (C) 2017-2018 Netronome Systems, Inc.
   3  *
   4  * This software is licensed under the GNU General License Version 2,
   5  * June 1991 as shown in the file COPYING in the top-level directory of this
   6  * source tree.
   7  *
   8  * THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS"
   9  * WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
  10  * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  11  * FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE
  12  * OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME
  13  * THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
  14  */
  15
  16 #include <linux/bpf.h>
  17 #include <linux/bpf_verifier.h>
  18 #include <linux/bug.h>
  19 #include <linux/kdev_t.h>
  20 #include <linux/list.h>
  21 #include <linux/lockdep.h>
  22 #include <linux/netdevice.h>
  23 #include <linux/printk.h>
  24 #include <linux/proc_ns.h>
  25 #include <linux/rhashtable.h>
  26 #include <linux/rtnetlink.h>
  27 #include <linux/rwsem.h>
  28 #include <net/xdp.h>
  29
  30 /* Protects offdevs, members of bpf_offload_netdev and offload members
  31  * of all progs.
  32  * RTNL lock cannot be taken when holding this lock.
  33  */
  34 static DECLARE_RWSEM(bpf_devs_lock);
  35
  36 struct bpf_offload_dev {
  37         const struct bpf_prog_offload_ops *ops;
  38         struct list_head netdevs;
  39         void *priv;
  40 };
  41
  42 struct bpf_offload_netdev {
  43         struct rhash_head l;
  44         struct net_device *netdev;
  45         struct bpf_offload_dev *offdev; /* NULL when bound-only */
  46         struct list_head progs;
  47         struct list_head maps;
  48         struct list_head offdev_netdevs;
  49 };
  50
  51 static const struct rhashtable_params offdevs_params = {
  52         .nelem_hint             = 4,
  53         .key_len                = sizeof(struct net_device *),
  54         .key_offset             = offsetof(struct bpf_offload_netdev, netdev),
  55         .head_offset            = offsetof(struct bpf_offload_netdev, l),
  56         .automatic_shrinking    = true,
  57 };
  58
  59 static struct rhashtable offdevs;
  60
  61 static int bpf_dev_offload_check(struct net_device *netdev)
  62 {
  63         if (!netdev)
  64                 return -EINVAL;
  65         if (!netdev->netdev_ops->ndo_bpf)
  66                 return -EOPNOTSUPP;
  67         return 0;
  68 }
  69
  70 static struct bpf_offload_netdev *
  71 bpf_offload_find_netdev(struct net_device *netdev)
  72 {
  73         lockdep_assert_held(&bpf_devs_lock);
  74
  75         return rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
  76 }
  77
  78 static int __bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
  79                                              struct net_device *netdev)
  80 {
  81         struct bpf_offload_netdev *ondev;
  82         int err;
  83
  84         ondev = kzalloc(sizeof(*ondev), GFP_KERNEL);
  85         if (!ondev)
  86                 return -ENOMEM;
  87
  88         ondev->netdev = netdev;
  89         ondev->offdev = offdev;
  90         INIT_LIST_HEAD(&ondev->progs);
  91         INIT_LIST_HEAD(&ondev->maps);
  92
  93         err = rhashtable_insert_fast(&offdevs, &ondev->l, offdevs_params);
  94         if (err) {
  95                 netdev_warn(netdev, "failed to register for BPF offload\n");
  96                 goto err_free;
  97         }
  98
  99         if (offdev)
 100                 list_add(&ondev->offdev_netdevs, &offdev->netdevs);
 101         return 0;
 102
 103 err_free:
 104         kfree(ondev);
 105         return err;
 106 }
 107
 108 static void __bpf_prog_offload_destroy(struct bpf_prog *prog)
 109 {
 110         struct bpf_prog_offload *offload = prog->aux->offload;
 111
 112         if (offload->dev_state)
 113                 offload->offdev->ops->destroy(prog);
 114
 115         list_del_init(&offload->offloads);
 116         kfree(offload);
 117         prog->aux->offload = NULL;
 118 }
 119
 120 static int bpf_map_offload_ndo(struct bpf_offloaded_map *offmap,
 121                                enum bpf_netdev_command cmd)
 122 {
 123         struct netdev_bpf data = {};
 124         struct net_device *netdev;
 125
 126         ASSERT_RTNL();
 127
 128         data.command = cmd;
 129         data.offmap = offmap;
 130         /* Caller must make sure netdev is valid */
 131         netdev = offmap->netdev;
 132
 133         return netdev->netdev_ops->ndo_bpf(netdev, &data);
 134 }
 135
 136 static void __bpf_map_offload_destroy(struct bpf_offloaded_map *offmap)
 137 {
 138         WARN_ON(bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_FREE));
 139         /* Make sure BPF_MAP_GET_NEXT_ID can't find this dead map */
 140         bpf_map_free_id(&offmap->map);
 141         list_del_init(&offmap->offloads);
 142         offmap->netdev = NULL;
 143 }
 144
 145 static void __bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
 146                                                 struct net_device *netdev)
 147 {
 148         struct bpf_offload_netdev *ondev, *altdev = NULL;
 149         struct bpf_offloaded_map *offmap, *mtmp;
 150         struct bpf_prog_offload *offload, *ptmp;
 151
 152         ASSERT_RTNL();
 153
 154         ondev = rhashtable_lookup_fast(&offdevs, &netdev, offdevs_params);
 155         if (WARN_ON(!ondev))
 156                 return;
 157
 158         WARN_ON(rhashtable_remove_fast(&offdevs, &ondev->l, offdevs_params));
 159
 160         /* Try to move the objects to another netdev of the device */
 161         if (offdev) {
 162                 list_del(&ondev->offdev_netdevs);
 163                 altdev = list_first_entry_or_null(&offdev->netdevs,
 164                                                   struct bpf_offload_netdev,
 165                                                   offdev_netdevs);
 166         }
 167
 168         if (altdev) {
 169                 list_for_each_entry(offload, &ondev->progs, offloads)
 170                         offload->netdev = altdev->netdev;
 171                 list_splice_init(&ondev->progs, &altdev->progs);
 172
 173                 list_for_each_entry(offmap, &ondev->maps, offloads)
 174                         offmap->netdev = altdev->netdev;
 175                 list_splice_init(&ondev->maps, &altdev->maps);
 176         } else {
 177                 list_for_each_entry_safe(offload, ptmp, &ondev->progs, offloads)
 178                         __bpf_prog_offload_destroy(offload->prog);
 179                 list_for_each_entry_safe(offmap, mtmp, &ondev->maps, offloads)
 180                         __bpf_map_offload_destroy(offmap);
 181         }
 182
 183         WARN_ON(!list_empty(&ondev->progs));
 184         WARN_ON(!list_empty(&ondev->maps));
 185         kfree(ondev);
 186 }
 187
 188 static int __bpf_prog_dev_bound_init(struct bpf_prog *prog, struct net_device *netdev)
 189 {
 190         struct bpf_offload_netdev *ondev;
 191         struct bpf_prog_offload *offload;
 192         int err;
 193
 194         offload = kzalloc(sizeof(*offload), GFP_USER);
 195         if (!offload)
 196                 return -ENOMEM;
 197
 198         offload->prog = prog;
 199         offload->netdev = netdev;
 200
 201         ondev = bpf_offload_find_netdev(offload->netdev);
 202         if (!ondev) {
 203                 if (bpf_prog_is_offloaded(prog->aux)) {
 204                         err = -EINVAL;
 205                         goto err_free;
 206                 }
 207
 208                 /* When only binding to the device, explicitly
 209                  * create an entry in the hashtable.
 210                  */
 211                 err = __bpf_offload_dev_netdev_register(NULL, offload->netdev);
 212                 if (err)
 213                         goto err_free;
 214                 ondev = bpf_offload_find_netdev(offload->netdev);
 215         }
 216         offload->offdev = ondev->offdev;
 217         prog->aux->offload = offload;
 218         list_add_tail(&offload->offloads, &ondev->progs);
 219
 220         return 0;
 221 err_free:
 222         kfree(offload);
 223         return err;
 224 }
 225
 226 int bpf_prog_dev_bound_init(struct bpf_prog *prog, union bpf_attr *attr)
 227 {
 228         struct net_device *netdev;
 229         int err;
 230
 231         if (attr->prog_type != BPF_PROG_TYPE_SCHED_CLS &&
 232             attr->prog_type != BPF_PROG_TYPE_XDP)
 233                 return -EINVAL;
 234
 235         if (attr->prog_flags & ~BPF_F_XDP_DEV_BOUND_ONLY)
 236                 return -EINVAL;
 237
 238         if (attr->prog_type == BPF_PROG_TYPE_SCHED_CLS &&
 239             attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY)
 240                 return -EINVAL;
 241
 242         netdev = dev_get_by_index(current->nsproxy->net_ns, attr->prog_ifindex);
 243         if (!netdev)
 244                 return -EINVAL;
 245
 246         err = bpf_dev_offload_check(netdev);
 247         if (err)
 248                 goto out;
 249
 250         prog->aux->offload_requested = !(attr->prog_flags & BPF_F_XDP_DEV_BOUND_ONLY);
 251
 252         down_write(&bpf_devs_lock);
 253         err = __bpf_prog_dev_bound_init(prog, netdev);
 254         up_write(&bpf_devs_lock);
 255
 256 out:
 257         dev_put(netdev);
 258         return err;
 259 }
 260
 261 int bpf_prog_dev_bound_inherit(struct bpf_prog *new_prog, struct bpf_prog *old_prog)
 262 {
 263         int err;
 264
 265         if (!bpf_prog_is_dev_bound(old_prog->aux))
 266                 return 0;
 267
 268         if (bpf_prog_is_offloaded(old_prog->aux))
 269                 return -EINVAL;
 270
 271         new_prog->aux->dev_bound = old_prog->aux->dev_bound;
 272         new_prog->aux->offload_requested = old_prog->aux->offload_requested;
 273
 274         down_write(&bpf_devs_lock);
 275         if (!old_prog->aux->offload) {
 276                 err = -EINVAL;
 277                 goto out;
 278         }
 279
 280         err = __bpf_prog_dev_bound_init(new_prog, old_prog->aux->offload->netdev);
 281
 282 out:
 283         up_write(&bpf_devs_lock);
 284         return err;
 285 }
 286
 287 int bpf_prog_offload_verifier_prep(struct bpf_prog *prog)
 288 {
 289         struct bpf_prog_offload *offload;
 290         int ret = -ENODEV;
 291
 292         down_read(&bpf_devs_lock);
 293         offload = prog->aux->offload;
 294         if (offload) {
 295                 ret = offload->offdev->ops->prepare(prog);
 296                 offload->dev_state = !ret;
 297         }
 298         up_read(&bpf_devs_lock);
 299
 300         return ret;
 301 }
 302
 303 int bpf_prog_offload_verify_insn(struct bpf_verifier_env *env,
 304                                  int insn_idx, int prev_insn_idx)
 305 {
 306         struct bpf_prog_offload *offload;
 307         int ret = -ENODEV;
 308
 309         down_read(&bpf_devs_lock);
 310         offload = env->prog->aux->offload;
 311         if (offload)
 312                 ret = offload->offdev->ops->insn_hook(env, insn_idx,
 313                                                       prev_insn_idx);
 314         up_read(&bpf_devs_lock);
 315
 316         return ret;
 317 }
 318
 319 int bpf_prog_offload_finalize(struct bpf_verifier_env *env)
 320 {
 321         struct bpf_prog_offload *offload;
 322         int ret = -ENODEV;
 323
 324         down_read(&bpf_devs_lock);
 325         offload = env->prog->aux->offload;
 326         if (offload) {
 327                 if (offload->offdev->ops->finalize)
 328                         ret = offload->offdev->ops->finalize(env);
 329                 else
 330                         ret = 0;
 331         }
 332         up_read(&bpf_devs_lock);
 333
 334         return ret;
 335 }
 336
 337 void
 338 bpf_prog_offload_replace_insn(struct bpf_verifier_env *env, u32 off,
 339                               struct bpf_insn *insn)
 340 {
 341         const struct bpf_prog_offload_ops *ops;
 342         struct bpf_prog_offload *offload;
 343         int ret = -EOPNOTSUPP;
 344
 345         down_read(&bpf_devs_lock);
 346         offload = env->prog->aux->offload;
 347         if (offload) {
 348                 ops = offload->offdev->ops;
 349                 if (!offload->opt_failed && ops->replace_insn)
 350                         ret = ops->replace_insn(env, off, insn);
 351                 offload->opt_failed |= ret;
 352         }
 353         up_read(&bpf_devs_lock);
 354 }
 355
 356 void
 357 bpf_prog_offload_remove_insns(struct bpf_verifier_env *env, u32 off, u32 cnt)
 358 {
 359         struct bpf_prog_offload *offload;
 360         int ret = -EOPNOTSUPP;
 361
 362         down_read(&bpf_devs_lock);
 363         offload = env->prog->aux->offload;
 364         if (offload) {
 365                 if (!offload->opt_failed && offload->offdev->ops->remove_insns)
 366                         ret = offload->offdev->ops->remove_insns(env, off, cnt);
 367                 offload->opt_failed |= ret;
 368         }
 369         up_read(&bpf_devs_lock);
 370 }
 371
 372 void bpf_prog_dev_bound_destroy(struct bpf_prog *prog)
 373 {
 374         struct bpf_offload_netdev *ondev;
 375         struct net_device *netdev;
 376
 377         rtnl_lock();
 378         down_write(&bpf_devs_lock);
 379         if (prog->aux->offload) {
 380                 list_del_init(&prog->aux->offload->offloads);
 381
 382                 netdev = prog->aux->offload->netdev;
 383                 __bpf_prog_offload_destroy(prog);
 384
 385                 ondev = bpf_offload_find_netdev(netdev);
 386                 if (!ondev->offdev && list_empty(&ondev->progs))
 387                         __bpf_offload_dev_netdev_unregister(NULL, netdev);
 388         }
 389         up_write(&bpf_devs_lock);
 390         rtnl_unlock();
 391 }
 392
 393 static int bpf_prog_offload_translate(struct bpf_prog *prog)
 394 {
 395         struct bpf_prog_offload *offload;
 396         int ret = -ENODEV;
 397
 398         down_read(&bpf_devs_lock);
 399         offload = prog->aux->offload;
 400         if (offload)
 401                 ret = offload->offdev->ops->translate(prog);
 402         up_read(&bpf_devs_lock);
 403
 404         return ret;
 405 }
 406
 407 static unsigned int bpf_prog_warn_on_exec(const void *ctx,
 408                                           const struct bpf_insn *insn)
 409 {
 410         WARN(1, "attempt to execute device eBPF program on the host!");
 411         return 0;
 412 }
 413
 414 int bpf_prog_offload_compile(struct bpf_prog *prog)
 415 {
 416         prog->bpf_func = bpf_prog_warn_on_exec;
 417
 418         return bpf_prog_offload_translate(prog);
 419 }
 420
 421 struct ns_get_path_bpf_prog_args {
 422         struct bpf_prog *prog;
 423         struct bpf_prog_info *info;
 424 };
 425
 426 static struct ns_common *bpf_prog_offload_info_fill_ns(void *private_data)
 427 {
 428         struct ns_get_path_bpf_prog_args *args = private_data;
 429         struct bpf_prog_aux *aux = args->prog->aux;
 430         struct ns_common *ns;
 431         struct net *net;
 432
 433         rtnl_lock();
 434         down_read(&bpf_devs_lock);
 435
 436         if (aux->offload) {
 437                 args->info->ifindex = aux->offload->netdev->ifindex;
 438                 net = dev_net(aux->offload->netdev);
 439                 get_net(net);
 440                 ns = &net->ns;
 441         } else {
 442                 args->info->ifindex = 0;
 443                 ns = NULL;
 444         }
 445
 446         up_read(&bpf_devs_lock);
 447         rtnl_unlock();
 448
 449         return ns;
 450 }
 451
 452 int bpf_prog_offload_info_fill(struct bpf_prog_info *info,
 453                                struct bpf_prog *prog)
 454 {
 455         struct ns_get_path_bpf_prog_args args = {
 456                 .prog   = prog,
 457                 .info   = info,
 458         };
 459         struct bpf_prog_aux *aux = prog->aux;
 460         struct inode *ns_inode;
 461         struct path ns_path;
 462         char __user *uinsns;
 463         int res;
 464         u32 ulen;
 465
 466         res = ns_get_path_cb(&ns_path, bpf_prog_offload_info_fill_ns, &args);
 467         if (res) {
 468                 if (!info->ifindex)
 469                         return -ENODEV;
 470                 return res;
 471         }
 472
 473         down_read(&bpf_devs_lock);
 474
 475         if (!aux->offload) {
 476                 up_read(&bpf_devs_lock);
 477                 return -ENODEV;
 478         }
 479
 480         ulen = info->jited_prog_len;
 481         info->jited_prog_len = aux->offload->jited_len;
 482         if (info->jited_prog_len && ulen) {
 483                 uinsns = u64_to_user_ptr(info->jited_prog_insns);
 484                 ulen = min_t(u32, info->jited_prog_len, ulen);
 485                 if (copy_to_user(uinsns, aux->offload->jited_image, ulen)) {
 486                         up_read(&bpf_devs_lock);
 487                         return -EFAULT;
 488                 }
 489         }
 490
 491         up_read(&bpf_devs_lock);
 492
 493         ns_inode = ns_path.dentry->d_inode;
 494         info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
 495         info->netns_ino = ns_inode->i_ino;
 496         path_put(&ns_path);
 497
 498         return 0;
 499 }
 500
 501 const struct bpf_prog_ops bpf_offload_prog_ops = {
 502 };
 503
 504 struct bpf_map *bpf_map_offload_map_alloc(union bpf_attr *attr)
 505 {
 506         struct net *net = current->nsproxy->net_ns;
 507         struct bpf_offload_netdev *ondev;
 508         struct bpf_offloaded_map *offmap;
 509         int err;
 510
 511         if (!capable(CAP_SYS_ADMIN))
 512                 return ERR_PTR(-EPERM);
 513         if (attr->map_type != BPF_MAP_TYPE_ARRAY &&
 514             attr->map_type != BPF_MAP_TYPE_HASH)
 515                 return ERR_PTR(-EINVAL);
 516
 517         offmap = bpf_map_area_alloc(sizeof(*offmap), NUMA_NO_NODE);
 518         if (!offmap)
 519                 return ERR_PTR(-ENOMEM);
 520
 521         bpf_map_init_from_attr(&offmap->map, attr);
 522
 523         rtnl_lock();
 524         down_write(&bpf_devs_lock);
 525         offmap->netdev = __dev_get_by_index(net, attr->map_ifindex);
 526         err = bpf_dev_offload_check(offmap->netdev);
 527         if (err)
 528                 goto err_unlock;
 529
 530         ondev = bpf_offload_find_netdev(offmap->netdev);
 531         if (!ondev) {
 532                 err = -EINVAL;
 533                 goto err_unlock;
 534         }
 535
 536         err = bpf_map_offload_ndo(offmap, BPF_OFFLOAD_MAP_ALLOC);
 537         if (err)
 538                 goto err_unlock;
 539
 540         list_add_tail(&offmap->offloads, &ondev->maps);
 541         up_write(&bpf_devs_lock);
 542         rtnl_unlock();
 543
 544         return &offmap->map;
 545
 546 err_unlock:
 547         up_write(&bpf_devs_lock);
 548         rtnl_unlock();
 549         bpf_map_area_free(offmap);
 550         return ERR_PTR(err);
 551 }
 552
 553 void bpf_map_offload_map_free(struct bpf_map *map)
 554 {
 555         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 556
 557         rtnl_lock();
 558         down_write(&bpf_devs_lock);
 559         if (offmap->netdev)
 560                 __bpf_map_offload_destroy(offmap);
 561         up_write(&bpf_devs_lock);
 562         rtnl_unlock();
 563
 564         bpf_map_area_free(offmap);
 565 }
 566
 567 u64 bpf_map_offload_map_mem_usage(const struct bpf_map *map)
 568 {
 569         /* The memory dynamically allocated in netdev dev_ops is not counted */
 570         return sizeof(struct bpf_offloaded_map);
 571 }
 572
 573 int bpf_map_offload_lookup_elem(struct bpf_map *map, void *key, void *value)
 574 {
 575         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 576         int ret = -ENODEV;
 577
 578         down_read(&bpf_devs_lock);
 579         if (offmap->netdev)
 580                 ret = offmap->dev_ops->map_lookup_elem(offmap, key, value);
 581         up_read(&bpf_devs_lock);
 582
 583         return ret;
 584 }
 585
 586 int bpf_map_offload_update_elem(struct bpf_map *map,
 587                                 void *key, void *value, u64 flags)
 588 {
 589         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 590         int ret = -ENODEV;
 591
 592         if (unlikely(flags > BPF_EXIST))
 593                 return -EINVAL;
 594
 595         down_read(&bpf_devs_lock);
 596         if (offmap->netdev)
 597                 ret = offmap->dev_ops->map_update_elem(offmap, key, value,
 598                                                        flags);
 599         up_read(&bpf_devs_lock);
 600
 601         return ret;
 602 }
 603
 604 int bpf_map_offload_delete_elem(struct bpf_map *map, void *key)
 605 {
 606         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 607         int ret = -ENODEV;
 608
 609         down_read(&bpf_devs_lock);
 610         if (offmap->netdev)
 611                 ret = offmap->dev_ops->map_delete_elem(offmap, key);
 612         up_read(&bpf_devs_lock);
 613
 614         return ret;
 615 }
 616
 617 int bpf_map_offload_get_next_key(struct bpf_map *map, void *key, void *next_key)
 618 {
 619         struct bpf_offloaded_map *offmap = map_to_offmap(map);
 620         int ret = -ENODEV;
 621
 622         down_read(&bpf_devs_lock);
 623         if (offmap->netdev)
 624                 ret = offmap->dev_ops->map_get_next_key(offmap, key, next_key);
 625         up_read(&bpf_devs_lock);
 626
 627         return ret;
 628 }
 629
 630 struct ns_get_path_bpf_map_args {
 631         struct bpf_offloaded_map *offmap;
 632         struct bpf_map_info *info;
 633 };
 634
 635 static struct ns_common *bpf_map_offload_info_fill_ns(void *private_data)
 636 {
 637         struct ns_get_path_bpf_map_args *args = private_data;
 638         struct ns_common *ns;
 639         struct net *net;
 640
 641         rtnl_lock();
 642         down_read(&bpf_devs_lock);
 643
 644         if (args->offmap->netdev) {
 645                 args->info->ifindex = args->offmap->netdev->ifindex;
 646                 net = dev_net(args->offmap->netdev);
 647                 get_net(net);
 648                 ns = &net->ns;
 649         } else {
 650                 args->info->ifindex = 0;
 651                 ns = NULL;
 652         }
 653
 654         up_read(&bpf_devs_lock);
 655         rtnl_unlock();
 656
 657         return ns;
 658 }
 659
 660 int bpf_map_offload_info_fill(struct bpf_map_info *info, struct bpf_map *map)
 661 {
 662         struct ns_get_path_bpf_map_args args = {
 663                 .offmap = map_to_offmap(map),
 664                 .info   = info,
 665         };
 666         struct inode *ns_inode;
 667         struct path ns_path;
 668         int res;
 669
 670         res = ns_get_path_cb(&ns_path, bpf_map_offload_info_fill_ns, &args);
 671         if (res) {
 672                 if (!info->ifindex)
 673                         return -ENODEV;
 674                 return res;
 675         }
 676
 677         ns_inode = ns_path.dentry->d_inode;
 678         info->netns_dev = new_encode_dev(ns_inode->i_sb->s_dev);
 679         info->netns_ino = ns_inode->i_ino;
 680         path_put(&ns_path);
 681
 682         return 0;
 683 }
 684
 685 static bool __bpf_offload_dev_match(struct bpf_prog *prog,
 686                                     struct net_device *netdev)
 687 {
 688         struct bpf_offload_netdev *ondev1, *ondev2;
 689         struct bpf_prog_offload *offload;
 690
 691         if (!bpf_prog_is_dev_bound(prog->aux))
 692                 return false;
 693
 694         offload = prog->aux->offload;
 695         if (!offload)
 696                 return false;
 697         if (offload->netdev == netdev)
 698                 return true;
 699
 700         ondev1 = bpf_offload_find_netdev(offload->netdev);
 701         ondev2 = bpf_offload_find_netdev(netdev);
 702
 703         return ondev1 && ondev2 && ondev1->offdev == ondev2->offdev;
 704 }
 705
 706 bool bpf_offload_dev_match(struct bpf_prog *prog, struct net_device *netdev)
 707 {
 708         bool ret;
 709
 710         down_read(&bpf_devs_lock);
 711         ret = __bpf_offload_dev_match(prog, netdev);
 712         up_read(&bpf_devs_lock);
 713
 714         return ret;
 715 }
 716 EXPORT_SYMBOL_GPL(bpf_offload_dev_match);
 717
 718 bool bpf_prog_dev_bound_match(const struct bpf_prog *lhs, const struct bpf_prog *rhs)
 719 {
 720         bool ret;
 721
 722         if (bpf_prog_is_offloaded(lhs->aux) != bpf_prog_is_offloaded(rhs->aux))
 723                 return false;
 724
 725         down_read(&bpf_devs_lock);
 726         ret = lhs->aux->offload && rhs->aux->offload &&
 727               lhs->aux->offload->netdev &&
 728               lhs->aux->offload->netdev == rhs->aux->offload->netdev;
 729         up_read(&bpf_devs_lock);
 730
 731         return ret;
 732 }
 733
 734 bool bpf_offload_prog_map_match(struct bpf_prog *prog, struct bpf_map *map)
 735 {
 736         struct bpf_offloaded_map *offmap;
 737         bool ret;
 738
 739         if (!bpf_map_is_offloaded(map))
 740                 return bpf_map_offload_neutral(map);
 741         offmap = map_to_offmap(map);
 742
 743         down_read(&bpf_devs_lock);
 744         ret = __bpf_offload_dev_match(prog, offmap->netdev);
 745         up_read(&bpf_devs_lock);
 746
 747         return ret;
 748 }
 749
 750 int bpf_offload_dev_netdev_register(struct bpf_offload_dev *offdev,
 751                                     struct net_device *netdev)
 752 {
 753         int err;
 754
 755         down_write(&bpf_devs_lock);
 756         err = __bpf_offload_dev_netdev_register(offdev, netdev);
 757         up_write(&bpf_devs_lock);
 758         return err;
 759 }
 760 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_register);
 761
 762 void bpf_offload_dev_netdev_unregister(struct bpf_offload_dev *offdev,
 763                                        struct net_device *netdev)
 764 {
 765         down_write(&bpf_devs_lock);
 766         __bpf_offload_dev_netdev_unregister(offdev, netdev);
 767         up_write(&bpf_devs_lock);
 768 }
 769 EXPORT_SYMBOL_GPL(bpf_offload_dev_netdev_unregister);
 770
 771 struct bpf_offload_dev *
 772 bpf_offload_dev_create(const struct bpf_prog_offload_ops *ops, void *priv)
 773 {
 774         struct bpf_offload_dev *offdev;
 775
 776         offdev = kzalloc(sizeof(*offdev), GFP_KERNEL);
 777         if (!offdev)
 778                 return ERR_PTR(-ENOMEM);
 779
 780         offdev->ops = ops;
 781         offdev->priv = priv;
 782         INIT_LIST_HEAD(&offdev->netdevs);
 783
 784         return offdev;
 785 }
 786 EXPORT_SYMBOL_GPL(bpf_offload_dev_create);
 787
 788 void bpf_offload_dev_destroy(struct bpf_offload_dev *offdev)
 789 {
 790         WARN_ON(!list_empty(&offdev->netdevs));
 791         kfree(offdev);
 792 }
 793 EXPORT_SYMBOL_GPL(bpf_offload_dev_destroy);
 794
 795 void *bpf_offload_dev_priv(struct bpf_offload_dev *offdev)
 796 {
 797         return offdev->priv;
 798 }
 799 EXPORT_SYMBOL_GPL(bpf_offload_dev_priv);
 800
 801 void bpf_dev_bound_netdev_unregister(struct net_device *dev)
 802 {
 803         struct bpf_offload_netdev *ondev;
 804
 805         ASSERT_RTNL();
 806
 807         down_write(&bpf_devs_lock);
 808         ondev = bpf_offload_find_netdev(dev);
 809         if (ondev && !ondev->offdev)
 810                 __bpf_offload_dev_netdev_unregister(NULL, ondev->netdev);
 811         up_write(&bpf_devs_lock);
 812 }
 813
 814 int bpf_dev_bound_kfunc_check(struct bpf_verifier_log *log,
 815                               struct bpf_prog_aux *prog_aux)
 816 {
 817         if (!bpf_prog_is_dev_bound(prog_aux)) {
 818                 bpf_log(log, "metadata kfuncs require device-bound program\n");
 819                 return -EINVAL;
 820         }
 821
 822         if (bpf_prog_is_offloaded(prog_aux)) {
 823                 bpf_log(log, "metadata kfuncs can't be offloaded\n");
 824                 return -EINVAL;
 825         }
 826
 827         return 0;
 828 }
 829
 830 void *bpf_dev_bound_resolve_kfunc(struct bpf_prog *prog, u32 func_id)
 831 {
 832         const struct xdp_metadata_ops *ops;
 833         void *p = NULL;
 834
 835         /* We don't hold bpf_devs_lock while resolving several
 836          * kfuncs and can race with the unregister_netdevice().
 837          * We rely on bpf_dev_bound_match() check at attach
 838          * to render this program unusable.
 839          */
 840         down_read(&bpf_devs_lock);
 841         if (!prog->aux->offload)
 842                 goto out;
 843
 844         ops = prog->aux->offload->netdev->xdp_metadata_ops;
 845         if (!ops)
 846                 goto out;
 847
 848         if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_TIMESTAMP))
 849                 p = ops->xmo_rx_timestamp;
 850         else if (func_id == bpf_xdp_metadata_kfunc_id(XDP_METADATA_KFUNC_RX_HASH))
 851                 p = ops->xmo_rx_hash;
 852 out:
 853         up_read(&bpf_devs_lock);
 854
 855         return p;
 856 }
 857
 858 static int __init bpf_offload_init(void)
 859 {
 860         return rhashtable_init(&offdevs, &offdevs_params);
 861 }
 862
 863 core_initcall(bpf_offload_init);