fix(dnsdist): don't assert on OT SpanID mismatch

Spans from the backend receive threads *could* be opened before the
ones from the frontend receive threads are closed. This commit no longer
asserts but cleans up nicely.

It also adds an `abort` when sanitizers are enabled to catch this issue
when it happens inside CI.

diff --git a/pdns/dnsdistdist/dnsdist-opentelemetry.cc b/pdns/dnsdistdist/dnsdist-opentelemetry.cc
index 45e76d9a61ae01f118fadc2074fda6799a7978fc..44de1c4e861840889c142a2b8917d8645833ebf3 100644 (file)
--- a/pdns/dnsdistdist/dnsdist-opentelemetry.cc
+++ b/pdns/dnsdistdist/dnsdist-opentelemetry.cc
@@ -22,6 +22,7 @@
 
 #include "dnsdist-opentelemetry.hh"
 #include "dnsdist-ecs.hh"
+#include "sanitizer.hh"
 
 #include <memory>
 #include <vector>
@@ -173,8 +174,28 @@ void Tracer::closeSpan([[maybe_unused]] const SpanID& spanID)
 
     // Only closers are allowed, so this can never happen
     assert(!data->d_spanIDStack.empty());
-    assert(data->d_spanIDStack.back() == spanID);
-    data->d_spanIDStack.pop_back();
+
+#if defined(__SANITIZE_THREAD__) || defined(__SANITIZE_ADDRESS__)
+    if (data->d_spanIDStack.back() != spanID) {
+      std::cout << "data->d_spanIDStack.back() != spanID " << std::endl;
+      std::cout << "SpanID: " << spanID.toLogString() << std::endl;
+      std::cout << "SpanID stack:" << std::endl;
+      for (const auto& sanitzerSpanID : data->d_spanIDStack) {
+        auto sanitizer_spanIt = std::find_if(
+          spans.rbegin(),
+          spans.rend(),
+          [sanitzerSpanID](const miniSpan& span) { return span.span_id == sanitzerSpanID; });
+        std::cout << "  " << sanitzerSpanID.toLogString() << "(" << sanitizer_spanIt->name << ")" << std::endl;
+      }
+      abort();
+    }
+#endif
+
+    // XXX: This assert should always pass, but there are some timing issues
+    //   assert(data->d_spanIDStack.back() == spanID);
+    //   data->d_spanIDStack.pop_back();
+    // So we'll clean up the stack for now.
+    data->d_spanIDStack.erase(std::find(data->d_spanIDStack.begin(), data->d_spanIDStack.end(), spanID));
   }
 #endif
 }