From: Remi Gacogne Date: Thu, 16 Mar 2023 12:43:00 +0000 (+0100) Subject: auth: Prevent a race during the processing of SVC auto-hints X-Git-Tag: auth-4.8.0-alpha1~3^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=refs%2Fpull%2F12659%2Fhead;p=thirdparty%2Fpdns.git auth: Prevent a race during the processing of SVC auto-hints When `svc-autohints` is enabled, the content of SVCB and HTTPS records is modified in `PacketHandler::doAdditionalProcessing()` to expand the IPv4 and IPv6 with their actual values. This causes an issue because the content of these records might be shared between threads, via the record cache, and one thread could be trying to read from the internal `std::set` while a second thread is altering it, leading to a data race and possibly to memory corruption and a crash. This is correctly detected by TSAN: ``` WARNING: ThreadSanitizer: data race (pid=102795) Write of size 8 at 0x7b3400010350 by thread T33: #0 operator delete(void*) (pdns_server+0x211b7c) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #1 std::__new_allocator>::deallocate(std::_Rb_tree_node*, unsigned long) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/new_allocator.h:158:2 (pdns_server+0x33fc78) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #2 std::allocator_traits>>::deallocate(std::allocator>&, std::_Rb_tree_node*, unsigned long) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/alloc_traits.h:496:13 (pdns_server+0x33fc78) #3 std::_Rb_tree, std::less, std::allocator>::_M_put_node(std::_Rb_tree_node*) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/stl_tree.h:565:9 (pdns_server+0x33fc78) #4 std::_Rb_tree, std::less, std::allocator>::_M_drop_node(std::_Rb_tree_node*) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/stl_tree.h:632:2 (pdns_server+0x33fc78) #5 std::_Rb_tree, std::less, std::allocator>::_M_erase_aux(std::_Rb_tree_const_iterator) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/stl_tree.h:2495:7 (pdns_server+0x33fc78) #6 std::_Rb_tree, std::less, std::allocator>::erase[abi:cxx11](std::_Rb_tree_const_iterator) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/stl_tree.h:1197:2 (pdns_server+0x33fc78) #7 std::set, std::allocator>::erase[abi:cxx11](std::_Rb_tree_const_iterator) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/stl_set.h:655:21 (pdns_server+0x33fc78) #8 SVCBBaseRecordContent::setHints(SvcParam::SvcParamKey const&, std::vector> const&) /work/pdns/pdns/dnsrecords.cc:768:14 (pdns_server+0x33fc78) #9 PacketHandler::doAdditionalProcessing(DNSPacket&, std::unique_ptr>&) /work/pdns/pdns/packethandler.cc:565:16 (pdns_server+0x4ed330) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #10 PacketHandler::doQuestion(DNSPacket&) /work/pdns/pdns/packethandler.cc:1794:5 (pdns_server+0x4f79b4) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #11 PacketHandler::question(DNSPacket&) /work/pdns/pdns/packethandler.cc:1175:10 (pdns_server+0x4f649a) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #12 MultiThreadDistributor::distribute(int) /work/pdns/pdns/./distributor.hh:220:14 (pdns_server+0x260f70) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #13 MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()::operator()() const /work/pdns/pdns/./distributor.hh:179:25 (pdns_server+0x260b31) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #14 void std::__invoke_impl::MultiThreadDistributor(int)::'lambda'()>(std::__invoke_other, MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:61:14 (pdns_server+0x260b31) #15 std::__invoke_result::MultiThreadDistributor(int)::'lambda'()>::type std::__invoke::MultiThreadDistributor(int)::'lambda'()>(MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:96:14 (pdns_server+0x260b31) #16 void std::thread::_Invoker::MultiThreadDistributor(int)::'lambda'()>>::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:258:13 (pdns_server+0x260b31) #17 std::thread::_Invoker::MultiThreadDistributor(int)::'lambda'()>>::operator()() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:265:11 (pdns_server+0x260b31) #18 std::thread::_State_impl::MultiThreadDistributor(int)::'lambda'()>>>::_M_run() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:210:13 (pdns_server+0x260b31) #19 execute_native_thread_routine /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:82:18 (libstdc++.so.6+0xd72c2) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) Previous read of size 2 at 0x7b3400010350 by thread T39: #0 SvcParam::getKey() const /work/pdns/pdns/./svc-records.hh:80:12 (pdns_server+0x3721f3) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #1 GenericDNSPacketWriter>>::xfrSvcParamKeyVals(std::set, std::allocator> const&) /work/pdns/pdns/dnswriter.cc:404:23 (pdns_server+0x3721f3) #2 void HTTPSRecordContent::xfrPacket>>>(GenericDNSPacketWriter>>&, bool) /work/pdns/pdns/dnsrecords.cc:348:1 (pdns_server+0x3349bd) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #3 HTTPSRecordContent::toPacket(GenericDNSPacketWriter>>&) /work/pdns/pdns/dnsrecords.cc:348:1 (pdns_server+0x3349bd) #4 DNSRecordContent::serialize[abi:cxx11](DNSName const&, bool, bool) /work/pdns/pdns/./dnsparser.hh:215:11 (pdns_server+0x311140) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #5 DNSPacket::addRecord(DNSZoneRecord&&) /work/pdns/pdns/dnspacket.cc:177:68 (pdns_server+0x2fa894) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #6 PacketHandler::doAdditionalProcessing(DNSPacket&, std::unique_ptr>&) /work/pdns/pdns/packethandler.cc:542:8 (pdns_server+0x4eccf2) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #7 PacketHandler::doQuestion(DNSPacket&) /work/pdns/pdns/packethandler.cc:1794:5 (pdns_server+0x4f79b4) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #8 PacketHandler::question(DNSPacket&) /work/pdns/pdns/packethandler.cc:1175:10 (pdns_server+0x4f649a) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #9 MultiThreadDistributor::distribute(int) /work/pdns/pdns/./distributor.hh:220:14 (pdns_server+0x260f70) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #10 MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()::operator()() const /work/pdns/pdns/./distributor.hh:179:25 (pdns_server+0x260b31) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #11 void std::__invoke_impl::MultiThreadDistributor(int)::'lambda'()>(std::__invoke_other, MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:61:14 (pdns_server+0x260b31) #12 std::__invoke_result::MultiThreadDistributor(int)::'lambda'()>::type std::__invoke::MultiThreadDistributor(int)::'lambda'()>(MultiThreadDistributor::MultiThreadDistributor(int)::'lambda'()&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:96:14 (pdns_server+0x260b31) #13 void std::thread::_Invoker::MultiThreadDistributor(int)::'lambda'()>>::_M_invoke<0ul>(std::_Index_tuple<0ul>) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:258:13 (pdns_server+0x260b31) #14 std::thread::_Invoker::MultiThreadDistributor(int)::'lambda'()>>::operator()() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:265:11 (pdns_server+0x260b31) #15 std::thread::_State_impl::MultiThreadDistributor(int)::'lambda'()>>>::_M_run() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:210:13 (pdns_server+0x260b31) #16 execute_native_thread_routine /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:82:18 (libstdc++.so.6+0xd72c2) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) Thread T33 'pdns/distributo' (tid=102833, running) created by thread T17 at: #0 pthread_create (pdns_server+0x1904e6) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #1 __gthread_create /usr/src/debug/gcc/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:663:35 (libstdc++.so.6+0xd73a9) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) #2 std::thread::_M_start_thread(std::unique_ptr>, void (*)()) /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:147:37 (libstdc++.so.6+0xd73a9) #3 Distributor::Create(int) /work/pdns/pdns/./distributor.hh:134:18 (pdns_server+0x256d23) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #4 qthread(unsigned int) /work/pdns/pdns/auth-main.cc:536:25 (pdns_server+0x256d23) #5 void std::__invoke_impl(std::__invoke_other, void (*&&)(unsigned int), unsigned int&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:61:14 (pdns_server+0x2635f0) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #6 std::__invoke_result::type std::__invoke(void (*&&)(unsigned int), unsigned int&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:96:14 (pdns_server+0x2635f0) #7 void std::thread::_Invoker>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:258:13 (pdns_server+0x2635f0) #8 std::thread::_Invoker>::operator()() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:265:11 (pdns_server+0x2635f0) #9 std::thread::_State_impl>>::_M_run() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:210:13 (pdns_server+0x2635f0) #10 execute_native_thread_routine /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:82:18 (libstdc++.so.6+0xd72c2) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) Thread T39 'pdns/distributo' (tid=102837, running) created by thread T19 at: #0 pthread_create (pdns_server+0x1904e6) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #1 __gthread_create /usr/src/debug/gcc/gcc-build/x86_64-pc-linux-gnu/libstdc++-v3/include/x86_64-pc-linux-gnu/bits/gthr-default.h:663:35 (libstdc++.so.6+0xd73a9) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) #2 std::thread::_M_start_thread(std::unique_ptr>, void (*)()) /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:147:37 (libstdc++.so.6+0xd73a9) #3 Distributor::Create(int) /work/pdns/pdns/./distributor.hh:134:18 (pdns_server+0x256d23) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #4 qthread(unsigned int) /work/pdns/pdns/auth-main.cc:536:25 (pdns_server+0x256d23) #5 void std::__invoke_impl(std::__invoke_other, void (*&&)(unsigned int), unsigned int&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:61:14 (pdns_server+0x2635f0) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) #6 std::__invoke_result::type std::__invoke(void (*&&)(unsigned int), unsigned int&&) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/invoke.h:96:14 (pdns_server+0x2635f0) #7 void std::thread::_Invoker>::_M_invoke<0ul, 1ul>(std::_Index_tuple<0ul, 1ul>) /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:258:13 (pdns_server+0x2635f0) #8 std::thread::_Invoker>::operator()() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:265:11 (pdns_server+0x2635f0) #9 std::thread::_State_impl>>::_M_run() /usr/bin/../lib64/gcc/x86_64-pc-linux-gnu/12.2.1/../../../../include/c++/12.2.1/bits/std_thread.h:210:13 (pdns_server+0x2635f0) #10 execute_native_thread_routine /usr/src/debug/gcc/gcc/libstdc++-v3/src/c++11/thread.cc:82:18 (libstdc++.so.6+0xd72c2) (BuildId: 6fe66a2d539a78c993bd2d377e00fad389220963) SUMMARY: ThreadSanitizer: data race (/work/pdns-rgacogne/pdns/pdns_server+0x211b7c) (BuildId: 384adc19a67695435bd5e89d0a77f562561f4593) in operator delete(void*) ``` To prevent this issue, this commit wraps the internal `std::set` in a mutex. In theory this will cause a performance impact, but in practice I did not find it to be noticeable. If we ever do, a different solution would be to duplicate the content of the SVCB/HTTPS records before modifying them. --- diff --git a/pdns/dnsrecords.cc b/pdns/dnsrecords.cc index aab8577513..475f12cfa5 100644 --- a/pdns/dnsrecords.cc +++ b/pdns/dnsrecords.cc @@ -341,7 +341,7 @@ boilerplate_conv(SVCB, conv.xfr16BitInt(d_priority); conv.xfrName(d_target, false, true); if (d_priority != 0) { - conv.xfrSvcParamKeyVals(d_params); + conv.xfrSvcParamKeyVals(*d_params.lock()); } ) @@ -349,7 +349,7 @@ boilerplate_conv(HTTPS, conv.xfr16BitInt(d_priority); conv.xfrName(d_target, false, true); if (d_priority != 0) { - conv.xfrSvcParamKeyVals(d_params); + conv.xfrSvcParamKeyVals(*d_params.lock()); } ) @@ -747,58 +747,64 @@ string APLRecordContent::getZoneRepresentation(bool /* noDot */) const { /* APL end */ /* SVCB start */ -bool SVCBBaseRecordContent::autoHint(const SvcParam::SvcParamKey &key) const { - auto p = getParamIt(key); - if (p == d_params.end()) { +bool SVCBBaseRecordContent::autoHint(const SvcParam::SvcParamKey &key) { + auto params = d_params.lock(); + auto p = getParamIt(key, *params); + if (p == params->end()) { return false; } return p->getAutoHint(); } void SVCBBaseRecordContent::setHints(const SvcParam::SvcParamKey &key, const std::vector &addresses) { - auto p = getParamIt(key); - if (p == d_params.end()) { - return; - } std::vector h; h.reserve(h.size() + addresses.size()); h.insert(h.end(), addresses.begin(), addresses.end()); + + auto params = d_params.lock(); + auto p = getParamIt(key, *params); + if (p == params->end()) { + return; + } try { auto newParam = SvcParam(key, std::move(h)); - d_params.erase(p); - d_params.insert(newParam); - } catch(...) { + params->erase(p); + params->insert(newParam); + } catch (...) { // XXX maybe we should SERVFAIL instead? return; } } void SVCBBaseRecordContent::removeParam(const SvcParam::SvcParamKey &key) { - auto p = getParamIt(key); - if (p == d_params.end()) { + auto params = d_params.lock(); + auto p = getParamIt(key, *params); + if (p == params->end()) { return; } - d_params.erase(p); + params->erase(p); } -bool SVCBBaseRecordContent::hasParams() const { - return !d_params.empty(); +bool SVCBBaseRecordContent::hasParams() { + return !d_params.lock()->empty(); } -bool SVCBBaseRecordContent::hasParam(const SvcParam::SvcParamKey &key) const { - return getParamIt(key) != d_params.end(); +bool SVCBBaseRecordContent::hasParam(const SvcParam::SvcParamKey &key) { + auto params = d_params.lock(); + return getParamIt(key, *params) != params->end(); } -SvcParam SVCBBaseRecordContent::getParam(const SvcParam::SvcParamKey &key) const { - auto p = getParamIt(key); - if (p == d_params.end()) { +SvcParam SVCBBaseRecordContent::getParam(const SvcParam::SvcParamKey &key) { + auto params = d_params.lock(); + auto p = getParamIt(key, *params); + if (p == params->end()) { throw std::out_of_range("No param with key " + SvcParam::keyToString(key)); } return *p; } -set::const_iterator SVCBBaseRecordContent::getParamIt(const SvcParam::SvcParamKey &key) const { - auto p = std::find_if(d_params.begin(), d_params.end(), +set::const_iterator SVCBBaseRecordContent::getParamIt(const SvcParam::SvcParamKey &key, std::set& params) { + auto p = std::find_if(params.begin(), params.end(), [&key](const SvcParam ¶m) { return param.getKey() == key; }); diff --git a/pdns/dnsrecords.hh b/pdns/dnsrecords.hh index 3974084b4e..245d8e8d4e 100644 --- a/pdns/dnsrecords.hh +++ b/pdns/dnsrecords.hh @@ -26,6 +26,7 @@ #include "dnsparser.hh" #include "dnswriter.hh" +#include "lock.hh" #include "rcpgenerator.hh" #include #include @@ -516,25 +517,25 @@ class SVCBBaseRecordContent : public DNSRecordContent const DNSName& getTarget() const {return d_target;} uint16_t getPriority() const {return d_priority;} // Returns true if a value for |key| was set to 'auto' - bool autoHint(const SvcParam::SvcParamKey &key) const; + bool autoHint(const SvcParam::SvcParamKey &key); // Sets the |addresses| to the existing hints for |key| void setHints(const SvcParam::SvcParamKey &key, const std::vector &addresses); // Removes the parameter for |key| from d_params void removeParam(const SvcParam::SvcParamKey &key); // Whether or not there are any parameter - bool hasParams() const; + bool hasParams(); // Whether or not the param of |key| exists - bool hasParam(const SvcParam::SvcParamKey &key) const; + bool hasParam(const SvcParam::SvcParamKey &key); // Get the parameter with |key|, will throw out_of_range if param isn't there - SvcParam getParam(const SvcParam::SvcParamKey &key) const; + SvcParam getParam(const SvcParam::SvcParamKey &key); protected: - uint16_t d_priority; + LockGuarded> d_params; DNSName d_target; - set d_params; + uint16_t d_priority; // Get the iterator to parameter with |key|, return value can be d_params::end - set::const_iterator getParamIt(const SvcParam::SvcParamKey &key) const; + static set::const_iterator getParamIt(const SvcParam::SvcParamKey &key, set& params); }; class SVCBRecordContent : public SVCBBaseRecordContent