RDMA/core: introduce rdma_restrict_node_type()

For smbdirect it required to use different ports depending
on the RDMA protocol. E.g. for iWarp 5445 is needed
(as tcp port 445 already used by the raw tcp transport for SMB),
while InfiniBand, RoCEv1 and RoCEv2 use port 445, as they
use an independent port range (even for RoCEv2, which uses udp
port 4791 itself).

Currently ksmbd is not able to function correctly at
all if the system has iWarp (RDMA_NODE_RNIC) interface(s)
and any InfiniBand, RoCEv1 and/or RoCEv2 interface(s)
at the same time.

And cifs.ko uses 5445 with a fallback to 445, which
means depending on the available interfaces, it tries
5445 in the RoCE range or may tries iWarp with 445
as a fallback. This leads to strange error messages
and strange network captures.

To avoid these problems they will be able to
use rdma_restrict_node_type(RDMA_NODE_RNIC) before
trying port 5445 and rdma_restrict_node_type(RDMA_NODE_IB_CA)
before trying port 445. It means we'll get early
-ENODEV early from rdma_resolve_addr() without any
network traffic and timeouts.

This is designed to be called before calling any
of rdma_bind_addr(), rdma_resolve_addr() or rdma_listen().

Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: Steve French <smfrench@gmail.com>
Cc: Tom Talpey <tom@talpey.com>
Cc: Long Li <longli@microsoft.com>
Cc: linux-rdma@vger.kernel.org
Cc: linux-cifs@vger.kernel.org
Cc: samba-technical@lists.samba.org
Acked-by: Leon Romanovsky <leon@kernel.org>
Acked-by: Namjae Jeon <linkinjeon@kernel.org>
Signed-off-by: Stefan Metzmacher <metze@samba.org>
Signed-off-by: Steve French <stfrench@microsoft.com>

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index f00f1d3fbd9c534a0033b7f578e8d4b4bdf87388..0ee855a71ed4f4ac109930d4540a0fd7cd3c6c9e 100644 (file)
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -793,6 +793,9 @@ static int cma_acquire_dev_by_src_ip(struct rdma_id_private *id_priv)
 
        mutex_lock(&lock);
        list_for_each_entry(cma_dev, &dev_list, list) {
+               if (id_priv->restricted_node_type != RDMA_NODE_UNSPECIFIED &&
+                   id_priv->restricted_node_type != cma_dev->device->node_type)
+                       continue;
                rdma_for_each_port (cma_dev->device, port) {
                        gidp = rdma_protocol_roce(cma_dev->device, port) ?
                               &iboe_gid : &gid;
@@ -1015,6 +1018,7 @@ __rdma_create_id(struct net *net, rdma_cm_event_handler event_handler,
                return ERR_PTR(-ENOMEM);
 
        id_priv->state = RDMA_CM_IDLE;
+       id_priv->restricted_node_type = RDMA_NODE_UNSPECIFIED;
        id_priv->id.context = context;
        id_priv->id.event_handler = event_handler;
        id_priv->id.ps = ps;
@@ -4177,6 +4181,32 @@ err:
 }
 EXPORT_SYMBOL(rdma_resolve_addr);
 
+int rdma_restrict_node_type(struct rdma_cm_id *id, u8 node_type)
+{
+       struct rdma_id_private *id_priv =
+               container_of(id, struct rdma_id_private, id);
+       int ret = 0;
+
+       switch (node_type) {
+       case RDMA_NODE_UNSPECIFIED:
+       case RDMA_NODE_IB_CA:
+       case RDMA_NODE_RNIC:
+               break;
+       default:
+               return -EINVAL;
+       }
+
+       mutex_lock(&lock);
+       if (id_priv->cma_dev)
+               ret = -EALREADY;
+       else
+               id_priv->restricted_node_type = node_type;
+       mutex_unlock(&lock);
+
+       return ret;
+}
+EXPORT_SYMBOL(rdma_restrict_node_type);
+
 int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
 {
        struct rdma_id_private *id_priv =

diff --git a/drivers/infiniband/core/cma_priv.h b/drivers/infiniband/core/cma_priv.h
index c604b601f4d92929d44225634c70b3fc7d8d1ccc..04332eb8266882d7d7cf252e683ae4371467bf79 100644 (file)
--- a/drivers/infiniband/core/cma_priv.h
+++ b/drivers/infiniband/core/cma_priv.h
@@ -72,6 +72,7 @@ struct rdma_id_private {
 
        int                     internal_id;
        enum rdma_cm_state      state;
+       u8                      restricted_node_type;
        spinlock_t              lock;
        struct mutex            qp_mutex;
 

diff --git a/include/rdma/rdma_cm.h b/include/rdma/rdma_cm.h
index 9bd930a83e6e39d6b23709369f513f8c93a7a09f..6de6fd8bd15eec7a486938031f34216d363c55ff 100644 (file)
--- a/include/rdma/rdma_cm.h
+++ b/include/rdma/rdma_cm.h
@@ -168,6 +168,23 @@ struct rdma_cm_id *rdma_create_user_id(rdma_cm_event_handler event_handler,
   */
 void rdma_destroy_id(struct rdma_cm_id *id);
 
+/**
+ * rdma_restrict_node_type - Restrict an RDMA identifier to specific
+ *   RDMA device node type.
+ *
+ * @id: RDMA identifier.
+ * @node_type: The device node type. Only RDMA_NODE_UNSPECIFIED (default),
+ *   RDMA_NODE_RNIC and RDMA_NODE_IB_CA are allowed
+ *
+ * This allows the caller to restrict the possible devices
+ * used to iWarp (RDMA_NODE_RNIC) or InfiniBand/RoCEv1/RoCEv2 (RDMA_NODE_IB_CA).
+ *
+ * It needs to be called before the RDMA identifier is bound
+ * to an device, which mean it should be called before
+ * rdma_bind_addr(), rdma_bind_addr() and rdma_listen().
+ */
+int rdma_restrict_node_type(struct rdma_cm_id *id, u8 node_type);
+
 /**
  * rdma_bind_addr - Bind an RDMA identifier to a source address and
  *   associated RDMA device, if needed.