From: Remi Gacogne Date: Fri, 21 Feb 2025 09:09:13 +0000 (+0100) Subject: rec: Fix a race reported by TSAN around the secpoll status X-Git-Tag: dnsdist-2.0.0-alpha1~67^2~1 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=6f3b477f0a865bbfd276430d080013064757b0b3;p=thirdparty%2Fpdns.git rec: Fix a race reported by TSAN around the secpoll status We used to store the security polling status in a regular integer, and the status can be updated from the `rec/web+stat` thread while being read from a Rust-based web-server thread, which is correctly reported by TSAN as a data race: ``` WARNING: ThreadSanitizer: data race (pid=2006) Write of size 4 at 0x55f19579db54 by thread T5: #0 doSecPoll(long*, std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/secpoll-recursor.cc:84:23 (pdns_recursor+0x814e3c) #1 SyncRes::doResolveAt(std::unordered_map >, bool>, std::hash, std::equal_to, std::allocator >, bool> > > >&, DNSName, bool, DNSName const&, QType, std::vector >&, unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::set, std::allocator >&, SyncRes::Context&, SyncRes::StopAtDelegation*, std::map >, std::less, std::allocator > > > >*) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/syncres.cc:6040:25 (pdns_recursor+0x84be60) #2 SyncRes::doResolveNoQNameMinimization(DNSName const&, QType, std::vector >&, unsigned int, std::set, std::allocator >&, SyncRes::Context&, bool*, SyncRes::StopAtDelegation*) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/syncres.cc:2099:11 (pdns_recursor+0x838903) #3 SyncRes::doResolve(DNSName const&, QType, std::vector >&, unsigned int, std::set, std::allocator >&, SyncRes::Context&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/syncres.cc:1835:13 (pdns_recursor+0x825337) #4 SyncRes::beginResolve(DNSName const&, QType, QClass, std::vector >&, unsigned int) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/syncres.cc:797:13 (pdns_recursor+0x828974) #5 doSecPoll(long*, std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/secpoll-recursor.cc:55:22 (pdns_recursor+0x814039) #6 houseKeepingWork(std::shared_ptr const&)::$_18::operator()() const /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2651:9 (pdns_recursor+0x68abb3) #7 void std::__invoke_impl const&)::$_18&>(std::__invoke_other, houseKeepingWork(std::shared_ptr const&)::$_18&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:61:14 (pdns_recursor+0x68abb3) #8 std::enable_if const&)::$_18&>, void>::type std::__invoke_r const&)::$_18&>(houseKeepingWork(std::shared_ptr const&)::$_18&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:111:2 (pdns_recursor+0x68abb3) #9 std::_Function_handler const&)::$_18>::_M_invoke(std::_Any_data const&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:290:9 (pdns_recursor+0x68abb3) #10 std::function::operator()() const /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:591:9 (pdns_recursor+0x688561) #11 PeriodicTask::runIfDue(timeval&, std::function const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2490:7 (pdns_recursor+0x688561) #12 houseKeepingWork(std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2649:17 (pdns_recursor+0x688561) #13 houseKeeping(void*) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2715:5 (pdns_recursor+0x688561) #14 MTasker, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()::operator()() const /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/./mtasker.hh:397:5 (pdns_recursor+0x5f04b5) #15 void std::__invoke_impl, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()&>(std::__invoke_other, MTasker, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:61:14 (pdns_recursor+0x5f0191) #16 std::enable_if, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()&>, void>::type std::__invoke_r, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()&>(MTasker, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:111:2 (pdns_recursor+0x5f0191) #17 std::_Function_handler, std::vector > >, PacketIDCompare>::makeThread(void (*)(void*), void*)::'lambda'()>::_M_invoke(std::_Any_data const&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:290:9 (pdns_recursor+0x5f0191) #18 std::function::operator()() const /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:591:9 (pdns_recursor+0x591fdf) #19 threadWrapper(boost::context::detail::transfer_t) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/mtasker_context.cc:163:7 (pdns_recursor+0x591fdf) #20 MTasker, std::vector > >, PacketIDCompare>::schedule(timeval const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/./mtasker.hh:426:7 (pdns_recursor+0x6ade0b) #21 recLoop() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2764:29 (pdns_recursor+0x666577) #22 recursorThread() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2959:5 (pdns_recursor+0x666577) #23 recLoop() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2773:24 (pdns_recursor+0x66664d) #24 recursorThread() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2959:5 (pdns_recursor+0x66664d) #25 recLoop() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2806:26 (pdns_recursor+0x668385) #26 recursorThread() /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2959:5 (pdns_recursor+0x668385) #27 RecThreadInfo::start(unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0::operator()() const /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:241:5 (pdns_recursor+0x69caf7) #28 void std::__invoke_impl, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0>(std::__invoke_other, RecThreadInfo::start(unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:61:14 (pdns_recursor+0x69caf7) #29 std::__invoke_result, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0>::type std::__invoke, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0>(RecThreadInfo::start(unsigned int, std::__cxx11::basic_string, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:96:14 (pdns_recursor+0x69caf7) #30 void std::thread::_Invoker, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0> >::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_thread.h:252:13 (pdns_recursor+0x69caf7) #31 std::thread::_Invoker, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0> >::operator()() /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_thread.h:259:11 (pdns_recursor+0x69caf7) #32 std::thread::_State_impl, std::allocator > const&, std::map, std::allocator >, std::less, std::allocator, std::allocator > > > > const&, std::shared_ptr const&)::$_0> > >::_M_run() /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_thread.h:210:13 (pdns_recursor+0x69caf7) #33 (libstdc++.so.6+0xd44a2) Previous read of size 4 at 0x55f19579db54 by thread T6: #0 getAllStatsMap[abi:cxx11](StatComponent) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec_channel_rec.cc:231:101 (pdns_recursor+0x760f2e) #1 productServerStatisticsFetch(std::map, std::allocator >, std::__cxx11::basic_string, std::allocator >, std::less, std::allocator > >, std::allocator, std::allocator > const, std::__cxx11::basic_string, std::allocator > > > >&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/ws-recursor.cc:55:16 (pdns_recursor+0x8fcd70) #2 apiServerStatistics(HttpRequest*, HttpResponse*) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/ws-api.cc:189:3 (pdns_recursor+0x8f5f71) #3 void std::__invoke_impl(std::__invoke_other, void (*&)(HttpRequest*, HttpResponse*), HttpRequest*&&, HttpResponse*&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:61:14 (pdns_recursor+0x918cc8) #4 std::enable_if, void>::type std::__invoke_r(void (*&)(HttpRequest*, HttpResponse*), HttpRequest*&&, HttpResponse*&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/invoke.h:111:2 (pdns_recursor+0x918cc8) #5 std::_Function_handler::_M_invoke(std::_Any_data const&, HttpRequest*&&, HttpResponse*&&) /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:290:9 (pdns_recursor+0x918cc8) #6 std::function::operator()(HttpRequest*, HttpResponse*) const /usr/bin/../lib/gcc/x86_64-linux-gnu/12/../../../../include/c++/12/bits/std_function.h:591:9 (pdns_recursor+0x8ff173) #7 rustWrapper(std::function const&, pdns::rust::web::rec::Request const&, pdns::rust::web::rec::Response&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/ws-recursor.cc:1067:5 (pdns_recursor+0x8ff173) #8 pdns::rust::web::rec::apiServerStatistics(pdns::rust::web::rec::Request const&, pdns::rust::web::rec::Response&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/ws-recursor.cc:1102:1 (pdns_recursor+0x907b80) #9 pdns$rust$web$rec$cxxbridge1$apiServerStatistics (pdns_recursor+0x9984b2) Location is global 'g_security_status' of size 4 at 0x55f19579db54 (pdns_recursor+0x000004479b54) Thread T5 'rec/web+stat' (tid=2012, running) created by main thread at: #0 pthread_create (pdns_recursor+0x1fe42d) #1 std::thread::_M_start_thread(std::unique_ptr >, void (*)()) (libstdc++.so.6+0xd4578) #2 RecThreadInfo::runThreads(std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:355:10 (pdns_recursor+0x663382) #3 serviceMain(std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2402:9 (pdns_recursor+0x67fdaf) #4 main /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:3316:11 (pdns_recursor+0x678755) Thread T6 (tid=2013, running) created by main thread at: #0 pthread_create (pdns_recursor+0x1fe42d) #1 std::sys::pal::unix::thread::Thread::new::he1793c71df66b318 /rustc/e71f9a9a98b0faf423844bf0ba7438f29dc27d58/library/std/src/sys/pal/unix/thread.rs:84:19 (pdns_recursor+0xc1ef11) #2 RecThreadInfo::runThreads(std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:358:7 (pdns_recursor+0x663417) #3 serviceMain(std::shared_ptr const&) /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:2402:9 (pdns_recursor+0x67fdaf) #4 main /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/rec-main.cc:3316:11 (pdns_recursor+0x678755) SUMMARY: ThreadSanitizer: data race /__w/pdns/pdns/pdns/recursordist/pdns-recursor-0.0.0-git1/secpoll-recursor.cc:84:23 in doSecPoll(long*, std::shared_ptr const&) ``` This commit switches to an atomic type to store the security polling status and clarify that the security polling message is not actually shared outside of the security polling function. It appears that the security polling status was the last metric stored in a `uint32_t` type so we can get rid of some now unused code in the process. --- diff --git a/pdns/recursordist/metrics_table.py b/pdns/recursordist/metrics_table.py index 93e6e3ba9a..f1cf57a8b8 100644 --- a/pdns/recursordist/metrics_table.py +++ b/pdns/recursordist/metrics_table.py @@ -1,4 +1,4 @@ -# From this table all metrics related files are generated by the metric.py script: +# From this table all metrics related files are generated by the metrics.py script: # # - RECURSOR-MIB.txt # - docs/rec-metrics-gen.rst @@ -569,7 +569,7 @@ 'name': 'noping-outqueries', # XXX obsolete? 'lambda': '[] { return g_Counters.sum(rec::Counter::noPingOutQueries); }', 'snmp': 73, - 'desc': 'Number of outgoing queries without ping', + 'desc': 'Number of outgoing queries without ping', }, { 'name': 'noedns-outqueries', diff --git a/pdns/recursordist/rec_channel_rec.cc b/pdns/recursordist/rec_channel_rec.cc index 949e46e148..d8ef379b2a 100644 --- a/pdns/recursordist/rec_channel_rec.cc +++ b/pdns/recursordist/rec_channel_rec.cc @@ -90,7 +90,6 @@ bool PrefixDashNumberCompare::operator()(const std::string& a, const std::string return aa < bb; } -static map d_get32bitpointers; static map d_getatomics; static map> d_get64bitmembers; static map> d_getmultimembers; @@ -125,14 +124,6 @@ void disableStats(StatComponent component, const string& stats) } } -static void addGetStat(const string& name, const uint32_t* place) -{ - if (!d_get32bitpointers.emplace(name, place).second) { - cerr << "addGetStat: double def " << name << endl; - _exit(1); - } -} - static void addGetStat(const string& name, const pdns::stat_t* place) { if (!d_getatomics.emplace(name, place).second) { @@ -190,12 +181,12 @@ static std::optional get(const string& name) { std::optional ret; - if (d_get32bitpointers.count(name)) - return *d_get32bitpointers.find(name)->second; - if (d_getatomics.count(name)) + if (d_getatomics.count(name)) { return d_getatomics.find(name)->second->load(); - if (d_get64bitmembers.count(name)) + } + if (d_get64bitmembers.count(name)) { return d_get64bitmembers.find(name)->second(); + } { auto dm = d_dynmetrics.lock(); @@ -226,11 +217,6 @@ StatsMap getAllStatsMap(StatComponent component) StatsMap ret; const auto& disabledlistMap = s_disabledStats.at(component); - for (const auto& the32bits : d_get32bitpointers) { - if (disabledlistMap.count(the32bits.first) == 0) { - ret.emplace(the32bits.first, StatsMapEntry{getPrometheusName(the32bits.first), std::to_string(*the32bits.second)}); - } - } for (const auto& atomic : d_getatomics) { if (disabledlistMap.count(atomic.first) == 0) { ret.emplace(atomic.first, StatsMapEntry{getPrometheusName(atomic.first), std::to_string(atomic.second->load())}); diff --git a/pdns/recursordist/secpoll-recursor.cc b/pdns/recursordist/secpoll-recursor.cc index d6c4558d63..db8e6fe121 100644 --- a/pdns/recursordist/secpoll-recursor.cc +++ b/pdns/recursordist/secpoll-recursor.cc @@ -14,8 +14,7 @@ #define PACKAGEVERSION getPDNSVersion() #endif -uint32_t g_security_status; -string g_security_message; +pdns::stat_t g_security_status; void doSecPoll(time_t* last_secpoll, Logr::log_t log) { @@ -87,19 +86,17 @@ void doSecPoll(time_t* last_secpoll, Logr::log_t log) return; } - g_security_message = std::move(security_message); - - auto rlog = vlog->withValues("securitymessage", Logging::Loggable(g_security_message), "status", Logging::Loggable(security_status)); + auto rlog = vlog->withValues("securitymessage", Logging::Loggable(security_message), "status", Logging::Loggable(security_status)); if (g_security_status != 1 && security_status == 1) { - SLOG(g_log << Logger::Warning << "Polled security status of version " << pkgv << ", no known issues reported: " << g_security_message << endl, + SLOG(g_log << Logger::Warning << "Polled security status of version " << pkgv << ", no known issues reported: " << security_message << endl, rlog->info(Logr::Notice, "Polled security status of version, no known issues reported")); } if (security_status == 2) { - SLOG(g_log << Logger::Error << "PowerDNS Security Update Recommended: " << g_security_message << endl, + SLOG(g_log << Logger::Error << "PowerDNS Security Update Recommended: " << security_message << endl, rlog->info(Logr::Error, "PowerDNS Security Update Recommended")); } if (security_status == 3) { - SLOG(g_log << Logger::Error << "PowerDNS Security Update Mandatory: " << g_security_message << endl, + SLOG(g_log << Logger::Error << "PowerDNS Security Update Mandatory: " << security_message << endl, rlog->info(Logr::Error, "PowerDNS Security Update Mandatory")); } diff --git a/pdns/recursordist/secpoll-recursor.hh b/pdns/recursordist/secpoll-recursor.hh index 8fc6f4074d..be23e8e36b 100644 --- a/pdns/recursordist/secpoll-recursor.hh +++ b/pdns/recursordist/secpoll-recursor.hh @@ -23,8 +23,8 @@ #include #include "namespaces.hh" #include "logr.hh" +#include "stat_t.hh" #include void doSecPoll(time_t*, Logr::log_t); -extern uint32_t g_security_status; -extern std::string g_security_message; +extern pdns::stat_t g_security_status;