From: Lukas Sismis <lsismis@oisf.net>
Date: Wed, 30 Nov 2022 10:45:32 +0000 (+0100)
Subject: dpdk: decrease intensity of warnings related to NUMA placement
X-Git-Tag: suricata-7.0.0-rc1~268
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=563f8bb56b64f841b3b27f6023d779803224b5a0;p=thirdparty%2Fsuricata.git

dpdk: decrease intensity of warnings related to NUMA placement

Ticket: #5617
---

diff --git a/src/runmode-dpdk.c b/src/runmode-dpdk.c
index a0d1d2871d..acb9258404 100644
--- a/src/runmode-dpdk.c
+++ b/src/runmode-dpdk.c
@@ -1358,6 +1358,7 @@ static void *ParseDpdkConfigAndConfigureDevice(const char *iface)
     (void)SC_ATOMIC_ADD(iconf->ref, iconf->threads);
     // This counter is increased by worker threads that individually pick queue IDs.
     SC_ATOMIC_RESET(iconf->queue_id);
+    SC_ATOMIC_RESET(iconf->inconsitent_numa_cnt);
     return iconf;
 }
 
diff --git a/src/source-dpdk.c b/src/source-dpdk.c
index 207d56d701..0fe83ac3de 100644
--- a/src/source-dpdk.c
+++ b/src/source-dpdk.c
@@ -232,6 +232,19 @@ static int GetNumaNode(void)
     return node;
 }
 
+static int GetCPU(void)
+{
+    int cpu = -1;
+
+#if defined(__linux__)
+    cpu = sched_getcpu();
+#else
+    SCLogWarning(SC_ERR_TM_THREADS_ERROR, "CPU id retrieval is not supported on this OS.");
+#endif
+
+    return cpu;
+}
+
 /**
  * \brief Registration Function for ReceiveDPDK.
  * \todo Unit tests are needed for this module.
@@ -451,12 +464,20 @@ static TmEcode ReceiveDPDKThreadInit(ThreadVars *tv, const void *initdata, void
     ptv->threads = dpdk_config->threads;
     ptv->port_id = dpdk_config->port_id;
     ptv->out_port_id = dpdk_config->out_port_id;
-    uint16_t queue_id = SC_ATOMIC_ADD(dpdk_config->queue_id, 1);
-    ptv->queue_id = queue_id;
     // pass the pointer to the mempool and then forget about it. Mempool is freed in thread deinit.
     ptv->pkt_mempool = dpdk_config->pkt_mempool;
     dpdk_config->pkt_mempool = NULL;
 
+    thread_numa = GetNumaNode();
+    if (thread_numa >= 0 && thread_numa != rte_eth_dev_socket_id(ptv->port_id)) {
+        SC_ATOMIC_ADD(dpdk_config->inconsitent_numa_cnt, 1);
+        SCLogInfo("NIC on NUMA %d but thread id %d on NUMA %d. Decreased performance expected",
+                rte_eth_dev_socket_id(ptv->port_id), GetCPU(), thread_numa);
+    }
+
+    uint16_t queue_id = SC_ATOMIC_ADD(dpdk_config->queue_id, 1);
+    ptv->queue_id = queue_id;
+
     // the last thread starts the device
     if (queue_id == dpdk_config->threads - 1) {
         retval = rte_eth_dev_start(ptv->port_id);
@@ -476,13 +497,15 @@ static TmEcode ReceiveDPDKThreadInit(ThreadVars *tv, const void *initdata, void
 
         // some PMDs requires additional actions only after the device has started
         DevicePostStartPMDSpecificActions(ptv, dev_info.driver_name);
-    }
 
-    thread_numa = GetNumaNode();
-    if (thread_numa >= 0 && thread_numa != rte_eth_dev_socket_id(ptv->port_id)) {
-        SCLogWarning(SC_WARN_DPDK_CONF,
-                "NIC on NUMA %d but thread on NUMA %d. Decreased performance expected",
-                rte_eth_dev_socket_id(ptv->port_id), thread_numa);
+        uint16_t inconsist_numa_map = SC_ATOMIC_GET(dpdk_config->inconsitent_numa_cnt);
+        if (inconsist_numa_map > 0) {
+            SCLogWarning(SC_WARN_DPDK_CONF,
+                    "Expect decreased performance due to interface (%s) being on NUMA node %d "
+                    "but is used by %d threads on distinct NUMA nodes. "
+                    "Run with verbose logging to determine individual thread ids",
+                    dpdk_config->iface, rte_eth_dev_socket_id(ptv->port_id), inconsist_numa_map);
+        }
     }
 
     *data = (void *)ptv;
diff --git a/src/source-dpdk.h b/src/source-dpdk.h
index ae85525313..9a7d6500e4 100644
--- a/src/source-dpdk.h
+++ b/src/source-dpdk.h
@@ -69,6 +69,7 @@ typedef struct DPDKIfaceConfig_ {
     SC_ATOMIC_DECLARE(unsigned int, ref);
     /* threads bind queue id one by one */
     SC_ATOMIC_DECLARE(uint16_t, queue_id);
+    SC_ATOMIC_DECLARE(uint16_t, inconsitent_numa_cnt);
     void (*DerefFunc)(void *);
 
     struct rte_flow *flow[100];