From: Charles-Henri Bruyand Date: Wed, 1 Jun 2022 08:01:16 +0000 (+0200) Subject: dnsdist: add support for user defined metrics X-Git-Tag: auth-4.8.0-alpha0~79^2~5 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6211164a3b554d2803202ce2f5efe67e54dfd6fc;p=thirdparty%2Fpdns.git dnsdist: add support for user defined metrics --- diff --git a/pdns/dnsdist-carbon.cc b/pdns/dnsdist-carbon.cc index cdad9dd69a..b405c13b11 100644 --- a/pdns/dnsdist-carbon.cc +++ b/pdns/dnsdist-carbon.cc @@ -75,6 +75,8 @@ void carbonDumpThread() str<(&e.second)) str<<(*val)->load(); + else if(const auto& adval = boost::get*>(&e.second)) + str<<(*adval)->load(); else if (const auto& dval = boost::get(&e.second)) str<<**dval; else diff --git a/pdns/dnsdist-lua.cc b/pdns/dnsdist-lua.cc index a2f2fac7c6..7df6abe007 100644 --- a/pdns/dnsdist-lua.cc +++ b/pdns/dnsdist-lua.cc @@ -29,6 +29,7 @@ #include #include +#include #include #include #include @@ -2861,6 +2862,67 @@ static void setupLuaConfig(LuaContext& luaCtx, bool client, bool configCheck) std::thread newThread(LuaThread, code); newThread.detach(); }); + + luaCtx.writeFunction("declareMetric", [](const std::string& name, const std::string& type) { + if (g_configurationDone) { + g_outputBuffer = "declareMetric cannot be used at runtime!\n"; + return false; + } + if (!std::regex_match(name, std::regex("^[a-z0-9-]+$"))) { + return false; + } + if (type == "counter") { + auto itp = g_stats.customCounters.emplace(name, 0); + if (itp.second) { + g_stats.entries.emplace_back(name, &g_stats.customCounters[name]); + } + } else if (type == "gauge") { + auto itp = g_stats.customGauges.emplace(name, 0.); + if (itp.second) { + g_stats.entries.emplace_back(name, &g_stats.customGauges[name]); + } + } else { + g_outputBuffer = "declareMetric unknown type '" + type + "'\n"; + errlog("Unable to declareMetric '%s': no such type '%s'", name, type); + return false; + } + return true; + }); + luaCtx.writeFunction("incMetric", [](const std::string& name) { + if (g_stats.customCounters.count(name) > 0) { + return ++g_stats.customCounters[name]; + } + g_outputBuffer = "incMetric no such metric '" + name + "'\n"; + errlog("Unable to incMetric: no such name '%s'", name); + return (uint64_t)0; + }); + luaCtx.writeFunction("decMetric", [](const std::string& name) { + if (g_stats.customCounters.count(name) > 0) { + return --g_stats.customCounters[name]; + } + g_outputBuffer = "decMetric no such metric '" + name + "'\n"; + errlog("Unable to decMetric: no such name '%s'", name); + return (uint64_t)0; + }); + luaCtx.writeFunction("setMetric", [](const std::string& name, const double& value) { + if (g_stats.customGauges.count(name) > 0) { + g_stats.customGauges[name] = value; + return value; + } + g_outputBuffer = "setMetric no such metric '" + name + "'\n"; + errlog("Unable to setMetric: no such name '%s'", name); + return 0.; + }); + luaCtx.writeFunction("getMetric", [](const std::string& name) { + if (g_stats.customCounters.count(name) > 0) { + return (double)g_stats.customCounters[name].load(); + } else if (g_stats.customGauges.count(name) > 0) { + return g_stats.customGauges[name].load(); + } + g_outputBuffer = "getMetric no such metric '" + name + "'\n"; + errlog("Unable to getMetric: no such name '%s'", name); + return 0.; + }); } vector> setupLua(LuaContext& luaCtx, bool client, bool configCheck, const std::string& config) diff --git a/pdns/dnsdist-web.cc b/pdns/dnsdist-web.cc index a60adaf216..9407c5523b 100644 --- a/pdns/dnsdist-web.cc +++ b/pdns/dnsdist-web.cc @@ -465,6 +465,8 @@ static void handlePrometheus(const YaHTTP::Request& req, YaHTTP::Response& resp) if (const auto& val = boost::get(&std::get<1>(e))) output << (*val)->load(); + else if (const auto& adval = boost::get*>(&std::get<1>(e))) + output << (*adval)->load(); else if (const auto& dval = boost::get(&std::get<1>(e))) output << **dval; else @@ -851,12 +853,15 @@ static void handleJSONStats(const YaHTTP::Request& req, YaHTTP::Response& resp) for (const auto& e : g_stats.entries) { if (e.first == "special-memory-usage") continue; // Too expensive for get-all - if(const auto& val = boost::get(&e.second)) + if (const auto& val = boost::get(&e.second)) { obj.insert({e.first, (double)(*val)->load()}); - else if (const auto& dval = boost::get(&e.second)) + } else if (const auto& adval = boost::get*>(&e.second)) { + obj.insert({e.first, (*adval)->load()}); + } else if (const auto& dval = boost::get(&e.second)) { obj.insert({e.first, (**dval)}); - else + } else { obj.insert({e.first, (double)(*boost::get(&e.second))(e.first)}); + } } Json my_json = obj; resp.body = my_json.dump(); @@ -1228,13 +1233,20 @@ static void handleStatsOnly(const YaHTTP::Request& req, YaHTTP::Response& resp) if (item.first == "special-memory-usage") continue; // Too expensive for get-all - if(const auto& val = boost::get(&item.second)) { + if (const auto& val = boost::get(&item.second)) { doc.push_back(Json::object { { "type", "StatisticItem" }, { "name", item.first }, { "value", (double)(*val)->load() } }); } + else if(const auto& adval = boost::get*>(&item.second)) { + doc.push_back(Json::object { + { "type", "StatisticItem" }, + { "name", item.first }, + { "value", (*adval)->load() } + }); + } else if (const auto& dval = boost::get(&item.second)) { doc.push_back(Json::object { { "type", "StatisticItem" }, diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh index feb383e5b5..cad4862369 100644 --- a/pdns/dnsdist.hh +++ b/pdns/dnsdist.hh @@ -365,10 +365,10 @@ struct DNSDistStats stat_t tcpQueryPipeFull{0}; stat_t tcpCrossProtocolQueryPipeFull{0}; stat_t tcpCrossProtocolResponsePipeFull{0}; - double latencyAvg100{0}, latencyAvg1000{0}, latencyAvg10000{0}, latencyAvg1000000{0}; typedef std::function statfunction_t; - typedef boost::variant entry_t; + typedef boost::variant*, double*, statfunction_t> entry_t; + std::vector> entries{ {"responses", &responses}, {"servfail-responses", &servfailResponses}, @@ -436,6 +436,8 @@ struct DNSDistStats {"latency-sum", &latencySum}, {"latency-count", &latencyCount}, }; + std::map customCounters; + std::map > customGauges; }; extern struct DNSDistStats g_stats; diff --git a/pdns/dnsdistdist/docs/reference/custommetrics.rst b/pdns/dnsdistdist/docs/reference/custommetrics.rst new file mode 100644 index 0000000000..22e0071606 --- /dev/null +++ b/pdns/dnsdistdist/docs/reference/custommetrics.rst @@ -0,0 +1,58 @@ +Custom Metrics +===================================== + +You can define at configuration time your own metrics that can be updated using lua. + +The first step is to declare a new metric using :func:`declareMetric`. + +Then you can update those at runtime using the following functions, depending on the metric type: + + * manipulate counters using :func:`incMetric` and :func:`decMetric` + * update a gauge using :func:`setMetric` + +.. function:: declareMetric(name, type) -> bool + + .. versionadded:: 1.x + + Return true if declaration was successful + + :param str name: The name of the metric, lowercase alnum characters only + :param str type: The desired type in ``gauge`` or ``counter`` + +.. function:: incMetric(name) -> int + + .. versionadded:: 1.x + + Increment counter by one, will issue an error if the metric is not declared or not a ``counter`` + Return the new value + + :param str name: The name of the metric + +.. function:: decMetric(name) -> int + + .. versionadded:: 1.x + + Decrement counter by one, will issue an error if the metric is not declared or not a ``counter`` + Return the new value + + :param str name: The name of the metric + +.. function:: getMetric(name) -> double + + .. versionadded:: 1.x + + Get metric value + + :param str name: The name of the metric + +.. function:: setMetric(name, value) -> double + + .. versionadded:: 1.x + + Decrement counter by one, will issue an error if the metric is not declared or not a ``counter`` + Return the new value + + :param str name: The name of the metric + + + diff --git a/pdns/dnsdistdist/docs/reference/index.rst b/pdns/dnsdistdist/docs/reference/index.rst index 9bccb195c1..e30237e83a 100755 --- a/pdns/dnsdistdist/docs/reference/index.rst +++ b/pdns/dnsdistdist/docs/reference/index.rst @@ -24,4 +24,5 @@ These chapters contain extensive information on all functions and object availab kvs logging web - svc \ No newline at end of file + svc + custommetrics diff --git a/regression-tests.dnsdist/test_API.py b/regression-tests.dnsdist/test_API.py index d58dd7d80f..4b001f2edf 100644 --- a/regression-tests.dnsdist/test_API.py +++ b/regression-tests.dnsdist/test_API.py @@ -771,3 +771,36 @@ class TestWebConcurrentConnections(APITestsBase): r = requests.get(url, auth=('whatever', self._webServerBasicAuthPassword), timeout=self._webTimeout) self.assertTrue(r) self.assertEqual(r.status_code, 200) + +class TestAPICustomStatistics(APITestsBase): + __test__ = True + _maxConns = 2 + + _config_params = ['_testServerPort', '_webServerPort', '_webServerBasicAuthPasswordHashed', '_webServerAPIKeyHashed'] + _config_template = """ + newServer{address="127.0.0.1:%s"} + webserver("127.0.0.1:%s") + declareMetric("my-custom-metric", "counter") + declareMetric("my-other-metric", "counter") + declareMetric("my-gauge", "gauge") + setWebserverConfig({password="%s", apiKey="%s"}) + """ + + def testCustomStats(self): + """ + API: /jsonstat?command=stats + Test custom statistics are exposed + """ + headers = {'x-api-key': self._webServerAPIKey} + url = 'http://127.0.0.1:' + str(self._webServerPort) + '/jsonstat?command=stats' + r = requests.get(url, headers=headers, timeout=self._webTimeout) + self.assertTrue(r) + self.assertEqual(r.status_code, 200) + self.assertTrue(r.json()) + content = r.json() + + expected = ['my-custom-metric', 'my-other-metric', 'my-gauge'] + + for key in expected: + self.assertIn(key, content) + self.assertTrue(content[key] >= 0) diff --git a/regression-tests.dnsdist/test_Advanced.py b/regression-tests.dnsdist/test_Advanced.py index 2425c736c3..a01119e5d4 100644 --- a/regression-tests.dnsdist/test_Advanced.py +++ b/regression-tests.dnsdist/test_Advanced.py @@ -454,3 +454,66 @@ class TestProtocols(DNSDistTest): receivedQuery.id = query.id self.assertEqual(receivedQuery, query) self.assertEqual(receivedResponse, response) + +class TestCustomMetrics(DNSDistTest): + _config_template = """ + function custommetrics(dq) + initialCounter = getMetric("my-custom-counter") + initialGauge = getMetric("my-custom-counter") + incMetric("my-custom-counter") + setMetric("my-custom-gauge", initialGauge + 1.3) + if getMetric("my-custom-counter") ~= (initialCounter + 1) or getMetric("my-custom-gauge") ~= (initialGauge + 1.3) then + return DNSAction.Spoof, '1.2.3.5' + end + return DNSAction.Spoof, '4.3.2.1' + end + + function declareNewMetric(dq) + if declareMetric("new-runtime-metric", "counter") then + return DNSAction.Spoof, '1.2.3.4' + end + return DNSAction.None + end + + declareMetric("my-custom-counter", "counter") + declareMetric("my-custom-gauge", "gauge") + addAction("declare.metric.advanced.tests.powerdns.com.", LuaAction(declareNewMetric)) + addAction("operations.metric.advanced.tests.powerdns.com.", LuaAction(custommetrics)) + newServer{address="127.0.0.1:%s"} + """ + + def testDeclareAfterConfig(self): + """ + Advanced: Test custom metric declaration after config done + """ + name = 'declare.metric.advanced.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + response = dns.message.make_response(query) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (receivedQuery, receivedResponse) = sender(query, response) + receivedQuery.id = query.id + self.assertEqual(receivedQuery, query) + self.assertEqual(receivedResponse, response) + + def testMetricOperations(self): + """ + Advanced: Test basic operations on custom metrics + """ + name = 'operations.metric.advanced.tests.powerdns.com.' + query = dns.message.make_query(name, 'A', 'IN') + # dnsdist set RA = RD for spoofed responses + query.flags &= ~dns.flags.RD + response = dns.message.make_response(query) + rrset = dns.rrset.from_text(name, + 60, + dns.rdataclass.IN, + dns.rdatatype.A, + '4.3.2.1') + response.answer.append(rrset) + + for method in ("sendUDPQuery", "sendTCPQuery"): + sender = getattr(self, method) + (_, receivedResponse) = sender(query, response=None, useQueue=False) + self.assertEqual(receivedResponse, response)