From: Ensar Sarajčić <dev@ensarsarajcic.com>
Date: Thu, 12 Dec 2024 08:04:56 +0000 (+0100)
Subject: dnsdist: add support for labels on custom metrics
X-Git-Tag: dnsdist-2.0.0-alpha0^2~15
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=b51f86701152b8972cc4637cbcbcf52ec4b73811;p=thirdparty%2Fpdns.git

dnsdist: add support for labels on custom metrics

This adds support for prometheus labels on custom metrics. Changes the
metrics maps to hold a map of labels to metric values for each metric.
Metrics without labels have an empty string label combination value.

Adds an optional options table as the last parameter of `incMetric`,
`decMetric`, `setMetric` and `getMetric`, which can be used to set
labels by using the `label` key or to set the previously used parameter
(either `step` or `value`).

Closes: #13359
---

diff --git a/pdns/dnsdistdist/dnsdist-lua-ffi.cc b/pdns/dnsdistdist/dnsdist-lua-ffi.cc
index 192e8f1cfd..c7a0c055f6 100644
--- a/pdns/dnsdistdist/dnsdist-lua-ffi.cc
+++ b/pdns/dnsdistdist/dnsdist-lua-ffi.cc
@@ -1811,7 +1811,8 @@ bool dnsdist_ffi_metric_declare(const char* name, size_t nameLen, const char* ty
 
 void dnsdist_ffi_metric_inc(const char* metricName, size_t metricNameLen)
 {
-  auto result = dnsdist::metrics::incrementCustomCounter(std::string_view(metricName, metricNameLen), 1U);
+  // TODO: add labels?
+  auto result = dnsdist::metrics::incrementCustomCounter(std::string_view(metricName, metricNameLen), 1U, {});
   if (std::get_if<dnsdist::metrics::Error>(&result) != nullptr) {
     return;
   }
@@ -1819,7 +1820,8 @@ void dnsdist_ffi_metric_inc(const char* metricName, size_t metricNameLen)
 
 void dnsdist_ffi_metric_inc_by(const char* metricName, size_t metricNameLen, uint64_t value)
 {
-  auto result = dnsdist::metrics::incrementCustomCounter(std::string_view(metricName, metricNameLen), value);
+  // TODO: add labels?
+  auto result = dnsdist::metrics::incrementCustomCounter(std::string_view(metricName, metricNameLen), value, {});
   if (std::get_if<dnsdist::metrics::Error>(&result) != nullptr) {
     return;
   }
@@ -1827,7 +1829,8 @@ void dnsdist_ffi_metric_inc_by(const char* metricName, size_t metricNameLen, uin
 
 void dnsdist_ffi_metric_dec(const char* metricName, size_t metricNameLen)
 {
-  auto result = dnsdist::metrics::decrementCustomCounter(std::string_view(metricName, metricNameLen), 1U);
+  // TODO: add labels?
+  auto result = dnsdist::metrics::decrementCustomCounter(std::string_view(metricName, metricNameLen), 1U, {});
   if (std::get_if<dnsdist::metrics::Error>(&result) != nullptr) {
     return;
   }
@@ -1835,7 +1838,8 @@ void dnsdist_ffi_metric_dec(const char* metricName, size_t metricNameLen)
 
 void dnsdist_ffi_metric_set(const char* metricName, size_t metricNameLen, double value)
 {
-  auto result = dnsdist::metrics::setCustomGauge(std::string_view(metricName, metricNameLen), value);
+  // TODO: add labels?
+  auto result = dnsdist::metrics::setCustomGauge(std::string_view(metricName, metricNameLen), value, {});
   if (std::get_if<dnsdist::metrics::Error>(&result) != nullptr) {
     return;
   }
@@ -1843,7 +1847,8 @@ void dnsdist_ffi_metric_set(const char* metricName, size_t metricNameLen, double
 
 double dnsdist_ffi_metric_get(const char* metricName, size_t metricNameLen, bool isCounter)
 {
-  auto result = dnsdist::metrics::getCustomMetric(std::string_view(metricName, metricNameLen));
+  // TODO: add labels?
+  auto result = dnsdist::metrics::getCustomMetric(std::string_view(metricName, metricNameLen), {});
   if (std::get_if<dnsdist::metrics::Error>(&result) != nullptr) {
     return 0.;
   }
diff --git a/pdns/dnsdistdist/dnsdist-lua.cc b/pdns/dnsdistdist/dnsdist-lua.cc
index 09818d9821..53e1038c13 100644
--- a/pdns/dnsdistdist/dnsdist-lua.cc
+++ b/pdns/dnsdistdist/dnsdist-lua.cc
@@ -85,6 +85,8 @@
 
 using std::thread;
 
+using update_metric_opts_t = LuaAssociativeTable<boost::variant<uint64_t, double, LuaAssociativeTable<std::string>>>;
+
 static boost::tribool s_noLuaSideEffect;
 
 /* this is a best effort way to prevent logging calls with no side-effects in the output of delta()
@@ -3419,8 +3421,20 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     }
     return true;
   });
-  luaCtx.writeFunction("incMetric", [](const std::string& name, boost::optional<uint64_t> step) {
-    auto result = dnsdist::metrics::incrementCustomCounter(name, step ? *step : 1);
+  luaCtx.writeFunction("incMetric", [](const std::string& name, boost::optional<boost::variant<uint64_t, boost::optional<update_metric_opts_t>>> opts) {
+    auto incOpts = opts.get_value_or(1);
+    uint64_t step = 1;
+    std::unordered_map<std::string, std::string> labels;
+    if (auto* custom_step = boost::get<uint64_t>(&incOpts)) {
+      step = *custom_step;
+    }
+    else {
+      boost::optional<update_metric_opts_t> vars = boost::get<boost::optional<update_metric_opts_t>>(incOpts);
+      getOptionalValue<uint64_t>(vars, "step", step);
+      getOptionalValue<LuaAssociativeTable<std::string>>(vars, "labels", labels);
+      checkAllParametersConsumed("incMetric", vars);
+    }
+    auto result = dnsdist::metrics::incrementCustomCounter(name, step, labels);
     if (const auto* errorStr = std::get_if<dnsdist::metrics::Error>(&result)) {
       g_outputBuffer = *errorStr + "'\n";
       errlog("Error in incMetric: %s", *errorStr);
@@ -3428,8 +3442,20 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     }
     return std::get<uint64_t>(result);
   });
-  luaCtx.writeFunction("decMetric", [](const std::string& name, boost::optional<uint64_t> step) {
-    auto result = dnsdist::metrics::decrementCustomCounter(name, step ? *step : 1);
+  luaCtx.writeFunction("decMetric", [](const std::string& name, boost::optional<boost::variant<uint64_t, boost::optional<update_metric_opts_t>>> opts) {
+    auto decOpts = opts.get_value_or(1);
+    uint64_t step = 1;
+    std::unordered_map<std::string, std::string> labels;
+    if (auto custom_step = boost::get<uint64_t>(&decOpts)) {
+      step = *custom_step;
+    }
+    else {
+      boost::optional<update_metric_opts_t> vars = boost::get<boost::optional<update_metric_opts_t>>(decOpts);
+      getOptionalValue<uint64_t>(vars, "step", step);
+      getOptionalValue<LuaAssociativeTable<std::string>>(vars, "labels", labels);
+      checkAllParametersConsumed("decMetric", vars);
+    }
+    auto result = dnsdist::metrics::decrementCustomCounter(name, step, labels);
     if (const auto* errorStr = std::get_if<dnsdist::metrics::Error>(&result)) {
       g_outputBuffer = *errorStr + "'\n";
       errlog("Error in decMetric: %s", *errorStr);
@@ -3437,8 +3463,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     }
     return std::get<uint64_t>(result);
   });
-  luaCtx.writeFunction("setMetric", [](const std::string& name, const double value) -> double {
-    auto result = dnsdist::metrics::setCustomGauge(name, value);
+  luaCtx.writeFunction("setMetric", [](const std::string& name, const double value, boost::optional<update_metric_opts_t> opts) -> double {
+    std::unordered_map<std::string, std::string> labels;
+    if (opts) {
+      getOptionalValue<LuaAssociativeTable<std::string>>(opts, "labels", labels);
+    }
+    checkAllParametersConsumed("setMetric", opts);
+    auto result = dnsdist::metrics::setCustomGauge(name, value, labels);
     if (const auto* errorStr = std::get_if<dnsdist::metrics::Error>(&result)) {
       g_outputBuffer = *errorStr + "'\n";
       errlog("Error in setMetric: %s", *errorStr);
@@ -3446,8 +3477,13 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck)
     }
     return std::get<double>(result);
   });
-  luaCtx.writeFunction("getMetric", [](const std::string& name) {
-    auto result = dnsdist::metrics::getCustomMetric(name);
+  luaCtx.writeFunction("getMetric", [](const std::string& name, boost::optional<update_metric_opts_t> opts) {
+    std::unordered_map<std::string, std::string> labels;
+    if (opts) {
+      getOptionalValue<LuaAssociativeTable<std::string>>(opts, "labels", labels);
+    }
+    checkAllParametersConsumed("getMetric", opts);
+    auto result = dnsdist::metrics::getCustomMetric(name, labels);
     if (const auto* errorStr = std::get_if<dnsdist::metrics::Error>(&result)) {
       g_outputBuffer = *errorStr + "'\n";
       errlog("Error in getMetric: %s", *errorStr);
diff --git a/pdns/dnsdistdist/dnsdist-metrics.cc b/pdns/dnsdistdist/dnsdist-metrics.cc
index 434088df4e..039b1ffc89 100644
--- a/pdns/dnsdistdist/dnsdist-metrics.cc
+++ b/pdns/dnsdistdist/dnsdist-metrics.cc
@@ -19,6 +19,8 @@
  * along with this program; if not, write to the Free Software
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  */
+#include <boost/algorithm/string/join.hpp>
+#include <numeric>
 #include <regex>
 
 #include "dnsdist-metrics.hh"
@@ -67,100 +69,100 @@ struct MutableGauge
   mutable pdns::stat_double_t d_value{0};
 };
 
-static SharedLockGuarded<std::map<std::string, MutableCounter, std::less<>>> s_customCounters;
-static SharedLockGuarded<std::map<std::string, MutableGauge, std::less<>>> s_customGauges;
+static SharedLockGuarded<std::map<std::string, std::map<std::string, MutableCounter>, std::less<>>> s_customCounters;
+static SharedLockGuarded<std::map<std::string, std::map<std::string, MutableGauge>, std::less<>>> s_customGauges;
 
 Stats::Stats() :
-  entries{std::vector<EntryPair>{
-    {"responses", &responses},
-    {"servfail-responses", &servfailResponses},
-    {"queries", &queries},
-    {"frontend-nxdomain", &frontendNXDomain},
-    {"frontend-servfail", &frontendServFail},
-    {"frontend-noerror", &frontendNoError},
-    {"acl-drops", &aclDrops},
-    {"rule-drop", &ruleDrop},
-    {"rule-nxdomain", &ruleNXDomain},
-    {"rule-refused", &ruleRefused},
-    {"rule-servfail", &ruleServFail},
-    {"rule-truncated", &ruleTruncated},
-    {"self-answered", &selfAnswered},
-    {"downstream-timeouts", &downstreamTimeouts},
-    {"downstream-send-errors", &downstreamSendErrors},
-    {"trunc-failures", &truncFail},
-    {"no-policy", &noPolicy},
-    {"latency0-1", &latency0_1},
-    {"latency1-10", &latency1_10},
-    {"latency10-50", &latency10_50},
-    {"latency50-100", &latency50_100},
-    {"latency100-1000", &latency100_1000},
-    {"latency-slow", &latencySlow},
-    {"latency-avg100", &latencyAvg100},
-    {"latency-avg1000", &latencyAvg1000},
-    {"latency-avg10000", &latencyAvg10000},
-    {"latency-avg1000000", &latencyAvg1000000},
-    {"latency-tcp-avg100", &latencyTCPAvg100},
-    {"latency-tcp-avg1000", &latencyTCPAvg1000},
-    {"latency-tcp-avg10000", &latencyTCPAvg10000},
-    {"latency-tcp-avg1000000", &latencyTCPAvg1000000},
-    {"latency-dot-avg100", &latencyDoTAvg100},
-    {"latency-dot-avg1000", &latencyDoTAvg1000},
-    {"latency-dot-avg10000", &latencyDoTAvg10000},
-    {"latency-dot-avg1000000", &latencyDoTAvg1000000},
-    {"latency-doh-avg100", &latencyDoHAvg100},
-    {"latency-doh-avg1000", &latencyDoHAvg1000},
-    {"latency-doh-avg10000", &latencyDoHAvg10000},
-    {"latency-doh-avg1000000", &latencyDoHAvg1000000},
-    {"latency-doq-avg100", &latencyDoQAvg100},
-    {"latency-doq-avg1000", &latencyDoQAvg1000},
-    {"latency-doq-avg10000", &latencyDoQAvg10000},
-    {"latency-doq-avg1000000", &latencyDoQAvg1000000},
-    {"latency-doh3-avg100", &latencyDoH3Avg100},
-    {"latency-doh3-avg1000", &latencyDoH3Avg1000},
-    {"latency-doh3-avg10000", &latencyDoH3Avg10000},
-    {"latency-doh3-avg1000000", &latencyDoH3Avg1000000},
-    {"uptime", uptimeOfProcess},
-    {"real-memory-usage", getRealMemoryUsage},
-    {"special-memory-usage", getSpecialMemoryUsage},
-    {"udp-in-errors", [](const std::string&) { return udpErrorStats("udp-in-errors"); }},
-    {"udp-noport-errors", [](const std::string&) { return udpErrorStats("udp-noport-errors"); }},
-    {"udp-recvbuf-errors", [](const std::string&) { return udpErrorStats("udp-recvbuf-errors"); }},
-    {"udp-sndbuf-errors", [](const std::string&) { return udpErrorStats("udp-sndbuf-errors"); }},
-    {"udp-in-csum-errors", [](const std::string&) { return udpErrorStats("udp-in-csum-errors"); }},
-    {"udp6-in-errors", [](const std::string&) { return udp6ErrorStats("udp6-in-errors"); }},
-    {"udp6-recvbuf-errors", [](const std::string&) { return udp6ErrorStats("udp6-recvbuf-errors"); }},
-    {"udp6-sndbuf-errors", [](const std::string&) { return udp6ErrorStats("udp6-sndbuf-errors"); }},
-    {"udp6-noport-errors", [](const std::string&) { return udp6ErrorStats("udp6-noport-errors"); }},
-    {"udp6-in-csum-errors", [](const std::string&) { return udp6ErrorStats("udp6-in-csum-errors"); }},
-    {"tcp-listen-overflows", [](const std::string&) { return tcpErrorStats("ListenOverflows"); }},
-    {"noncompliant-queries", &nonCompliantQueries},
-    {"noncompliant-responses", &nonCompliantResponses},
-    {"proxy-protocol-invalid", &proxyProtocolInvalid},
-    {"rdqueries", &rdQueries},
-    {"empty-queries", &emptyQueries},
-    {"cache-hits", &cacheHits},
-    {"cache-misses", &cacheMisses},
-    {"cpu-iowait", getCPUIOWait},
-    {"cpu-steal", getCPUSteal},
-    {"cpu-sys-msec", getCPUTimeSystem},
-    {"cpu-user-msec", getCPUTimeUser},
-    {"fd-usage", getOpenFileDescriptors},
-    {"dyn-blocked", &dynBlocked},
+  entries{std::vector<EntryTriple>{
+    {"responses", "", &responses},
+    {"servfail-responses", "", &servfailResponses},
+    {"queries", "", &queries},
+    {"frontend-nxdomain", "", &frontendNXDomain},
+    {"frontend-servfail", "", &frontendServFail},
+    {"frontend-noerror", "", &frontendNoError},
+    {"acl-drops", "", &aclDrops},
+    {"rule-drop", "", &ruleDrop},
+    {"rule-nxdomain", "", &ruleNXDomain},
+    {"rule-refused", "", &ruleRefused},
+    {"rule-servfail", "", &ruleServFail},
+    {"rule-truncated", "", &ruleTruncated},
+    {"self-answered", "", &selfAnswered},
+    {"downstream-timeouts", "", &downstreamTimeouts},
+    {"downstream-send-errors", "", &downstreamSendErrors},
+    {"trunc-failures", "", &truncFail},
+    {"no-policy", "", &noPolicy},
+    {"latency0-1", "", &latency0_1},
+    {"latency1-10", "", &latency1_10},
+    {"latency10-50", "", &latency10_50},
+    {"latency50-100", "", &latency50_100},
+    {"latency100-1000", "", &latency100_1000},
+    {"latency-slow", "", &latencySlow},
+    {"latency-avg100", "", &latencyAvg100},
+    {"latency-avg1000", "", &latencyAvg1000},
+    {"latency-avg10000", "", &latencyAvg10000},
+    {"latency-avg1000000", "", &latencyAvg1000000},
+    {"latency-tcp-avg100", "", &latencyTCPAvg100},
+    {"latency-tcp-avg1000", "", &latencyTCPAvg1000},
+    {"latency-tcp-avg10000", "", &latencyTCPAvg10000},
+    {"latency-tcp-avg1000000", "", &latencyTCPAvg1000000},
+    {"latency-dot-avg100", "", &latencyDoTAvg100},
+    {"latency-dot-avg1000", "", &latencyDoTAvg1000},
+    {"latency-dot-avg10000", "", &latencyDoTAvg10000},
+    {"latency-dot-avg1000000", "", &latencyDoTAvg1000000},
+    {"latency-doh-avg100", "", &latencyDoHAvg100},
+    {"latency-doh-avg1000", "", &latencyDoHAvg1000},
+    {"latency-doh-avg10000", "", &latencyDoHAvg10000},
+    {"latency-doh-avg1000000", "", &latencyDoHAvg1000000},
+    {"latency-doq-avg100", "", &latencyDoQAvg100},
+    {"latency-doq-avg1000", "", &latencyDoQAvg1000},
+    {"latency-doq-avg10000", "", &latencyDoQAvg10000},
+    {"latency-doq-avg1000000", "", &latencyDoQAvg1000000},
+    {"latency-doh3-avg100", "", &latencyDoH3Avg100},
+    {"latency-doh3-avg1000", "", &latencyDoH3Avg1000},
+    {"latency-doh3-avg10000", "", &latencyDoH3Avg10000},
+    {"latency-doh3-avg1000000", "", &latencyDoH3Avg1000000},
+    {"uptime", "", uptimeOfProcess},
+    {"real-memory-usage", "", getRealMemoryUsage},
+    {"special-memory-usage", "", getSpecialMemoryUsage},
+    {"udp-in-errors", "", [](const std::string&) { return udpErrorStats("udp-in-errors"); }},
+    {"udp-noport-errors", "", [](const std::string&) { return udpErrorStats("udp-noport-errors"); }},
+    {"udp-recvbuf-errors", "", [](const std::string&) { return udpErrorStats("udp-recvbuf-errors"); }},
+    {"udp-sndbuf-errors", "", [](const std::string&) { return udpErrorStats("udp-sndbuf-errors"); }},
+    {"udp-in-csum-errors", "", [](const std::string&) { return udpErrorStats("udp-in-csum-errors"); }},
+    {"udp6-in-errors", "", [](const std::string&) { return udp6ErrorStats("udp6-in-errors"); }},
+    {"udp6-recvbuf-errors", "", [](const std::string&) { return udp6ErrorStats("udp6-recvbuf-errors"); }},
+    {"udp6-sndbuf-errors", "", [](const std::string&) { return udp6ErrorStats("udp6-sndbuf-errors"); }},
+    {"udp6-noport-errors", "", [](const std::string&) { return udp6ErrorStats("udp6-noport-errors"); }},
+    {"udp6-in-csum-errors", "", [](const std::string&) { return udp6ErrorStats("udp6-in-csum-errors"); }},
+    {"tcp-listen-overflows", "", [](const std::string&) { return tcpErrorStats("ListenOverflows"); }},
+    {"noncompliant-queries", "", &nonCompliantQueries},
+    {"noncompliant-responses", "", &nonCompliantResponses},
+    {"proxy-protocol-invalid", "", &proxyProtocolInvalid},
+    {"rdqueries", "", &rdQueries},
+    {"empty-queries", "", &emptyQueries},
+    {"cache-hits", "", &cacheHits},
+    {"cache-misses", "", &cacheMisses},
+    {"cpu-iowait", "", getCPUIOWait},
+    {"cpu-steal", "", getCPUSteal},
+    {"cpu-sys-msec", "", getCPUTimeSystem},
+    {"cpu-user-msec", "", getCPUTimeUser},
+    {"fd-usage", "", getOpenFileDescriptors},
+    {"dyn-blocked", "", &dynBlocked},
 #ifndef DISABLE_DYNBLOCKS
-    {"dyn-block-nmg-size", [](const std::string&) { return dnsdist::DynamicBlocks::getClientAddressDynamicRules().size(); }},
+    {"dyn-block-nmg-size", "", [](const std::string&) { return dnsdist::DynamicBlocks::getClientAddressDynamicRules().size(); }},
 #endif /* DISABLE_DYNBLOCKS */
-    {"security-status", &securityStatus},
-    {"doh-query-pipe-full", &dohQueryPipeFull},
-    {"doh-response-pipe-full", &dohResponsePipeFull},
-    {"doq-response-pipe-full", &doqResponsePipeFull},
-    {"doh3-response-pipe-full", &doh3ResponsePipeFull},
-    {"outgoing-doh-query-pipe-full", &outgoingDoHQueryPipeFull},
-    {"tcp-query-pipe-full", &tcpQueryPipeFull},
-    {"tcp-cross-protocol-query-pipe-full", &tcpCrossProtocolQueryPipeFull},
-    {"tcp-cross-protocol-response-pipe-full", &tcpCrossProtocolResponsePipeFull},
+    {"security-status", "", &securityStatus},
+    {"doh-query-pipe-full", "", &dohQueryPipeFull},
+    {"doh-response-pipe-full", "", &dohResponsePipeFull},
+    {"doq-response-pipe-full", "", &doqResponsePipeFull},
+    {"doh3-response-pipe-full", "", &doh3ResponsePipeFull},
+    {"outgoing-doh-query-pipe-full", "", &outgoingDoHQueryPipeFull},
+    {"tcp-query-pipe-full", "", &tcpQueryPipeFull},
+    {"tcp-cross-protocol-query-pipe-full", "", &tcpCrossProtocolQueryPipeFull},
+    {"tcp-cross-protocol-response-pipe-full", "", &tcpCrossProtocolResponsePipeFull},
     // Latency histogram
-    {"latency-sum", &latencySum},
-    {"latency-count", &latencyCount},
+    {"latency-sum", "", &latencySum},
+    {"latency-count", "", &latencyCount},
   }}
 {
 }
@@ -176,18 +178,16 @@ std::optional<std::string> declareCustomMetric(const std::string& name, const st
   const std::string finalCustomName(customName ? *customName : "");
   if (type == "counter") {
     auto customCounters = s_customCounters.write_lock();
-    auto itp = customCounters->insert({name, MutableCounter()});
+    auto itp = customCounters->emplace(name, std::map<std::string, MutableCounter>());
     if (itp.second) {
-      g_stats.entries.write_lock()->emplace_back(Stats::EntryPair{name, &(*customCounters)[name].d_value});
       dnsdist::prometheus::PrometheusMetricDefinition def{name, type, description, finalCustomName};
       dnsdist::webserver::addMetricDefinition(def);
     }
   }
   else if (type == "gauge") {
     auto customGauges = s_customGauges.write_lock();
-    auto itp = customGauges->insert({name, MutableGauge()});
+    auto itp = customGauges->emplace(name, std::map<std::string, MutableGauge>());
     if (itp.second) {
-      g_stats.entries.write_lock()->emplace_back(Stats::EntryPair{name, &(*customGauges)[name].d_value});
       dnsdist::prometheus::PrometheusMetricDefinition def{name, type, description, finalCustomName};
       dnsdist::webserver::addMetricDefinition(def);
     }
@@ -198,57 +198,108 @@ std::optional<std::string> declareCustomMetric(const std::string& name, const st
   return std::nullopt;
 }
 
-std::variant<uint64_t, Error> incrementCustomCounter(const std::string_view& name, uint64_t step)
+static string prometheusLabelValueEscape(const string& value)
 {
-  auto customCounters = s_customCounters.read_lock();
+  string ret;
+
+  for (char i : value) {
+    if (i == '"' || i == '\\') {
+      ret += '\\';
+      ret += i;
+    }
+    else if (i == '\n') {
+      ret += '\\';
+      ret += 'n';
+    }
+    else
+      ret += i;
+  }
+  return ret;
+}
+
+static std::string generateCombinationOfLabels(const std::unordered_map<std::string, std::string>& labels)
+{
+  return std::accumulate(labels.begin(), labels.end(), std::string(), [](const std::string& acc, const std::pair<std::string, std::string>& l) {
+    return acc + (acc.empty() ? std::string() : ",") + l.first + "=" + "\"" + prometheusLabelValueEscape(l.second) + "\"";
+  });
+}
+
+std::variant<uint64_t, Error> incrementCustomCounter(const std::string_view& name, uint64_t step, const std::unordered_map<std::string, std::string>& labels)
+{
+  auto customCounters = s_customCounters.write_lock();
   auto metric = customCounters->find(name);
   if (metric != customCounters->end()) {
-    metric->second.d_value += step;
-    return metric->second.d_value.load();
+    auto combinationOfLabels = generateCombinationOfLabels(labels);
+    auto metricEntry = metric->second.find(combinationOfLabels);
+    if (metricEntry == metric->second.end()) {
+      metricEntry = metric->second.emplace(combinationOfLabels, MutableCounter()).first;
+      g_stats.entries.write_lock()->emplace_back(Stats::EntryTriple{std::string(name), combinationOfLabels, &metricEntry->second.d_value});
+    }
+    metricEntry->second.d_value += step;
+    return metricEntry->second.d_value.load();
   }
   return std::string("Unable to increment custom metric '") + std::string(name) + "': no such counter";
 }
 
-std::variant<uint64_t, Error> decrementCustomCounter(const std::string_view& name, uint64_t step)
+std::variant<uint64_t, Error> decrementCustomCounter(const std::string_view& name, uint64_t step, const std::unordered_map<std::string, std::string>& labels)
 {
-  auto customCounters = s_customCounters.read_lock();
+  auto customCounters = s_customCounters.write_lock();
   auto metric = customCounters->find(name);
   if (metric != customCounters->end()) {
-    metric->second.d_value -= step;
-    return metric->second.d_value.load();
+    auto combinationOfLabels = generateCombinationOfLabels(labels);
+    auto metricEntry = metric->second.find(combinationOfLabels);
+    if (metricEntry == metric->second.end()) {
+      metricEntry = metric->second.emplace(combinationOfLabels, MutableCounter()).first;
+      g_stats.entries.write_lock()->emplace_back(Stats::EntryTriple{std::string(name), combinationOfLabels, &metricEntry->second.d_value});
+    }
+    metricEntry->second.d_value -= step;
+    return metricEntry->second.d_value.load();
   }
   return std::string("Unable to decrement custom metric '") + std::string(name) + "': no such counter";
 }
 
-std::variant<double, Error> setCustomGauge(const std::string_view& name, const double value)
+std::variant<double, Error> setCustomGauge(const std::string_view& name, const double value, const std::unordered_map<std::string, std::string>& labels)
 {
-  auto customGauges = s_customGauges.read_lock();
+  auto customGauges = s_customGauges.write_lock();
   auto metric = customGauges->find(name);
   if (metric != customGauges->end()) {
-    metric->second.d_value = value;
+    auto combinationOfLabels = generateCombinationOfLabels(labels);
+    auto metricEntry = metric->second.find(combinationOfLabels);
+    if (metricEntry == metric->second.end()) {
+      metricEntry = metric->second.emplace(combinationOfLabels, MutableGauge()).first;
+      g_stats.entries.write_lock()->emplace_back(Stats::EntryTriple{std::string(name), combinationOfLabels, &metricEntry->second.d_value});
+    }
+    metricEntry->second.d_value = value;
     return value;
   }
 
   return std::string("Unable to set metric '") + std::string(name) + "': no such gauge";
 }
 
-std::variant<double, Error> getCustomMetric(const std::string_view& name)
+std::variant<double, Error> getCustomMetric(const std::string_view& name, const std::unordered_map<std::string, std::string>& labels)
 {
   {
     auto customCounters = s_customCounters.read_lock();
     auto counter = customCounters->find(name);
     if (counter != customCounters->end()) {
-      return static_cast<double>(counter->second.d_value.load());
+      auto combinationOfLabels = generateCombinationOfLabels(labels);
+      auto metricEntry = counter->second.find(combinationOfLabels);
+      if (metricEntry != counter->second.end()) {
+        return static_cast<double>(metricEntry->second.d_value.load());
+      }
     }
   }
   {
     auto customGauges = s_customGauges.read_lock();
     auto gauge = customGauges->find(name);
     if (gauge != customGauges->end()) {
-      return gauge->second.d_value.load();
+      auto combinationOfLabels = generateCombinationOfLabels(labels);
+      auto metricEntry = gauge->second.find(combinationOfLabels);
+      if (metricEntry != gauge->second.end()) {
+        return metricEntry->second.d_value.load();
+      }
     }
   }
   return std::string("Unable to get metric '") + std::string(name) + "': no such metric";
 }
-
 }
diff --git a/pdns/dnsdistdist/dnsdist-metrics.hh b/pdns/dnsdistdist/dnsdist-metrics.hh
index 47a3fb8407..8afb67509d 100644
--- a/pdns/dnsdistdist/dnsdist-metrics.hh
+++ b/pdns/dnsdistdist/dnsdist-metrics.hh
@@ -25,6 +25,7 @@
 #include <optional>
 #include <string>
 #include <string_view>
+#include <unordered_map>
 #include <variant>
 
 #include "lock.hh"
@@ -35,10 +36,10 @@ namespace dnsdist::metrics
 using Error = std::string;
 
 [[nodiscard]] std::optional<Error> declareCustomMetric(const std::string& name, const std::string& type, const std::string& description, std::optional<std::string> customName);
-[[nodiscard]] std::variant<uint64_t, Error> incrementCustomCounter(const std::string_view& name, uint64_t step);
-[[nodiscard]] std::variant<uint64_t, Error> decrementCustomCounter(const std::string_view& name, uint64_t step);
-[[nodiscard]] std::variant<double, Error> setCustomGauge(const std::string_view& name, const double value);
-[[nodiscard]] std::variant<double, Error> getCustomMetric(const std::string_view& name);
+[[nodiscard]] std::variant<uint64_t, Error> incrementCustomCounter(const std::string_view& name, uint64_t step, const std::unordered_map<std::string, std::string>& labels);
+[[nodiscard]] std::variant<uint64_t, Error> decrementCustomCounter(const std::string_view& name, uint64_t step, const std::unordered_map<std::string, std::string>& labels);
+[[nodiscard]] std::variant<double, Error> setCustomGauge(const std::string_view& name, const double value, const std::unordered_map<std::string, std::string>& labels);
+[[nodiscard]] std::variant<double, Error> getCustomMetric(const std::string_view& name, const std::unordered_map<std::string, std::string>& labels);
 
 using pdns::stat_t;
 
@@ -89,13 +90,14 @@ struct Stats
   pdns::stat_double_t latencyDoH3Avg100{0}, latencyDoH3Avg1000{0}, latencyDoH3Avg10000{0}, latencyDoH3Avg1000000{0};
   using statfunction_t = std::function<uint64_t(const std::string&)>;
   using entry_t = std::variant<stat_t*, pdns::stat_double_t*, statfunction_t>;
-  struct EntryPair
+  struct EntryTriple
   {
     std::string d_name;
+    std::string d_labels;
     entry_t d_value;
   };
 
-  SharedLockGuarded<std::vector<EntryPair>> entries;
+  SharedLockGuarded<std::vector<EntryTriple>> entries;
 };
 
 extern struct Stats g_stats;
diff --git a/pdns/dnsdistdist/dnsdist-web.cc b/pdns/dnsdistdist/dnsdist-web.cc
index 5cdfead09c..1f38516e8d 100644
--- a/pdns/dnsdistdist/dnsdist-web.cc
+++ b/pdns/dnsdistdist/dnsdist-web.cc
@@ -500,6 +500,10 @@ static void handlePrometheus(const YaHTTP::Request& req, YaHTTP::Response& resp)
         prometheusMetricName = metricDetails.customName;
       }
 
+      if (!entry.d_labels.empty()) {
+        prometheusMetricName += "{" + entry.d_labels + "}";
+      }
+
       // for these we have the help and types encoded in the sources
       // but we need to be careful about labels in custom metrics
       std::string helpName = prometheusMetricName.substr(0, prometheusMetricName.find('{'));