From: Remi Gacogne <remi.gacogne@powerdns.com>
Date: Thu, 14 Oct 2021 08:41:58 +0000 (+0200)
Subject: dnsdist: Replace shared by unique ptrs, reduce structs size
X-Git-Tag: dnsdist-1.7.0-alpha2~5^2~1
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=73571c03409ab29bacba4dbb52720a549da241a5;p=thirdparty%2Fpdns.git

dnsdist: Replace shared by unique ptrs, reduce structs size

This commit replaces the DNSCrypt and QTaq shared pointers by unique ones,
since these are not actually shared. This should improve performance a
bit since we no longer need to clear a cache line for the reference
counter. It also reduces the size of the IDState and DNSQuestion structures
by a few bytes.
It also removes a few fields in the DOHUnit structure that are redundant
with ones in the embedded IDState structure.
The rest of the commit re-order fields in various structures to prevent
wasted space due to padding, reducing the memory footprint of these structs.

On x86_64 (in bytes):

IDState 344 -> 328
DOHUnit 712 -> 632
TCPQuery 416 -> 400
IncomingTCPConnectionState 968 -> 944
TCPConnectionToBackend 728 -> 712
DownstreamState 2368 -> 2240
---

diff --git a/pdns/dnsdist-dnscrypt.cc b/pdns/dnsdist-dnscrypt.cc
index fe18169db9..99301445c3 100644
--- a/pdns/dnsdist-dnscrypt.cc
+++ b/pdns/dnsdist-dnscrypt.cc
@@ -24,17 +24,17 @@
 #include "dnscrypt.hh"
 
 #ifdef HAVE_DNSCRYPT
-int handleDNSCryptQuery(PacketBuffer& packet, std::shared_ptr<DNSCryptQuery>& query, bool tcp, time_t now, PacketBuffer& response)
+int handleDNSCryptQuery(PacketBuffer& packet, DNSCryptQuery& query, bool tcp, time_t now, PacketBuffer& response)
 {
-  query->parsePacket(packet, tcp, now);
+  query.parsePacket(packet, tcp, now);
 
-  if (query->isValid() == false) {
+  if (query.isValid() == false) {
     vinfolog("Dropping DNSCrypt invalid query");
     return false;
   }
 
-  if (query->isEncrypted() == false) {
-    query->getCertificateResponse(now, response);
+  if (query.isEncrypted() == false) {
+    query.getCertificateResponse(now, response);
 
     return false;
   }
diff --git a/pdns/dnsdist-idstate.hh b/pdns/dnsdist-idstate.hh
index 5d991042fd..3caf1353a1 100644
--- a/pdns/dnsdist-idstate.hh
+++ b/pdns/dnsdist-idstate.hh
@@ -44,9 +44,7 @@ struct StopWatch
 
   void start()
   {
-    if (gettime(&d_start, d_needRealTime) < 0) {
-      unixDie("Getting timestamp");
-    }
+    d_start = getCurrentTime();
   }
 
   void set(const struct timespec& from)
@@ -56,33 +54,33 @@ struct StopWatch
 
   double udiff() const
   {
-    struct timespec now;
-    if (gettime(&now, d_needRealTime) < 0) {
-      unixDie("Getting timestamp");
-    }
-
+    struct timespec now = getCurrentTime();
     return 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0;
   }
 
   double udiffAndSet()
+  {
+    struct timespec now = getCurrentTime();
+    auto ret = 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0;
+    d_start = now;
+    return ret;
+  }
+
+  struct timespec getCurrentTime() const
   {
     struct timespec now;
     if (gettime(&now, d_needRealTime) < 0) {
       unixDie("Getting timestamp");
     }
-
-    auto ret = 1000000.0 * (now.tv_sec - d_start.tv_sec) + (now.tv_nsec - d_start.tv_nsec) / 1000.0;
-    d_start = now;
-    return ret;
+    return now;
   }
 
   struct timespec d_start
   {
     0, 0
   };
-
 private:
-  bool d_needRealTime{false};
+  bool d_needRealTime;
 };
 
 /* g++ defines __SANITIZE_THREAD__
@@ -100,7 +98,7 @@ struct IDState
     sentTime(true), tempFailureTTL(boost::none) { origDest.sin4.sin_family = 0; }
   IDState(const IDState& orig) = delete;
   IDState(IDState&& rhs) :
-    subnet(rhs.subnet), origRemote(rhs.origRemote), origDest(rhs.origDest), hopRemote(rhs.hopRemote), hopLocal(rhs.hopLocal), qname(std::move(rhs.qname)), sentTime(rhs.sentTime), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), packetCache(std::move(rhs.packetCache)), qTag(std::move(rhs.qTag)), tempFailureTTL(rhs.tempFailureTTL), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), cacheKeyUDP(rhs.cacheKeyUDP), origFD(rhs.origFD), delayMsec(rhs.delayMsec), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), cacheFlags(rhs.cacheFlags), protocol(rhs.protocol), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope)
+    subnet(rhs.subnet), origRemote(rhs.origRemote), origDest(rhs.origDest), hopRemote(rhs.hopRemote), hopLocal(rhs.hopLocal), qname(std::move(rhs.qname)), sentTime(rhs.sentTime), packetCache(std::move(rhs.packetCache)), dnsCryptQuery(std::move(rhs.dnsCryptQuery)), qTag(std::move(rhs.qTag)), tempFailureTTL(rhs.tempFailureTTL), cs(rhs.cs), du(std::move(rhs.du)), cacheKey(rhs.cacheKey), cacheKeyNoECS(rhs.cacheKeyNoECS), cacheKeyUDP(rhs.cacheKeyUDP), origFD(rhs.origFD), delayMsec(rhs.delayMsec), qtype(rhs.qtype), qclass(rhs.qclass), origID(rhs.origID), origFlags(rhs.origFlags), cacheFlags(rhs.cacheFlags), protocol(rhs.protocol), ednsAdded(rhs.ednsAdded), ecsAdded(rhs.ecsAdded), skipCache(rhs.skipCache), destHarvested(rhs.destHarvested), dnssecOK(rhs.dnssecOK), useZeroScope(rhs.useZeroScope)
   {
     if (rhs.isInUse()) {
       throw std::runtime_error("Trying to move an in-use IDState");
@@ -237,9 +235,9 @@ struct IDState
   ComboAddress hopLocal;
   DNSName qname; // 24
   StopWatch sentTime; // 16
-  std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr}; // 16
   std::shared_ptr<DNSDistPacketCache> packetCache{nullptr}; // 16
-  std::shared_ptr<QTag> qTag{nullptr}; // 16
+  std::unique_ptr<DNSCryptQuery> dnsCryptQuery{nullptr}; // 8
+  std::unique_ptr<QTag> qTag{nullptr}; // 8
   boost::optional<uint32_t> tempFailureTTL; // 8
   const ClientState* cs{nullptr}; // 8
   DOHUnit* du{nullptr}; // 8
diff --git a/pdns/dnsdist-tcp.cc b/pdns/dnsdist-tcp.cc
index 9f87070df0..d70f7692e5 100644
--- a/pdns/dnsdist-tcp.cc
+++ b/pdns/dnsdist-tcp.cc
@@ -607,7 +607,7 @@ static void handleQuery(std::shared_ptr<IncomingTCPConnectionState>& state, cons
   struct timespec queryRealTime;
   gettime(&queryRealTime, true);
 
-  std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
+  std::unique_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
   auto dnsCryptResponse = checkDNSCryptQuery(*state->d_ci.cs, state->d_buffer, dnsCryptQuery, queryRealTime.tv_sec, true);
   if (dnsCryptResponse) {
     TCPResponse response;
diff --git a/pdns/dnsdist.cc b/pdns/dnsdist.cc
index 306446f814..94286c507d 100644
--- a/pdns/dnsdist.cc
+++ b/pdns/dnsdist.cc
@@ -390,7 +390,7 @@ static bool fixUpResponse(PacketBuffer& response, const DNSName& qname, uint16_t
 }
 
 #ifdef HAVE_DNSCRYPT
-static bool encryptResponse(PacketBuffer& response, size_t maximumSize, bool tcp, std::shared_ptr<DNSCryptQuery> dnsCryptQuery)
+static bool encryptResponse(PacketBuffer& response, size_t maximumSize, bool tcp, std::unique_ptr<DNSCryptQuery>& dnsCryptQuery)
 {
   if (dnsCryptQuery) {
     int res = dnsCryptQuery->encryptResponse(response, maximumSize, tcp);
@@ -1063,14 +1063,14 @@ static bool isUDPQueryAcceptable(ClientState& cs, LocalHolders& holders, const s
   return true;
 }
 
-bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp)
+bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp)
 {
   if (cs.dnscryptCtx) {
 #ifdef HAVE_DNSCRYPT
     PacketBuffer response;
-    dnsCryptQuery = std::make_shared<DNSCryptQuery>(cs.dnscryptCtx);
+    dnsCryptQuery = std::make_unique<DNSCryptQuery>(cs.dnscryptCtx);
 
-    bool decrypted = handleDNSCryptQuery(query, dnsCryptQuery, tcp, now, response);
+    bool decrypted = handleDNSCryptQuery(query, *dnsCryptQuery, tcp, now, response);
 
     if (!decrypted) {
       if (response.size() > 0) {
@@ -1126,7 +1126,7 @@ static bool prepareOutgoingResponse(LocalHolders& holders, ClientState& cs, DNSQ
   DNSResponse dr(dq.qname, dq.qtype, dq.qclass, dq.local, dq.remote, dq.getMutableData(), dq.protocol, dq.queryTime);
 
   dr.uniqueId = dq.uniqueId;
-  dr.qTag = dq.qTag;
+  dr.qTag = std::move(dq.qTag);
   dr.delayMsec = dq.delayMsec;
 
   if (!applyRulesToResponse(cacheHit ? holders.cacheHitRespRuleactions : holders.selfAnsweredRespRuleactions, dr)) {
@@ -1432,7 +1432,7 @@ static void processUDPQuery(ClientState& cs, LocalHolders& holders, const struct
     struct timespec queryRealTime;
     gettime(&queryRealTime, true);
 
-    std::shared_ptr<DNSCryptQuery> dnsCryptQuery = nullptr;
+    std::unique_ptr<DNSCryptQuery> dnsCryptQuery = nullptr;
     auto dnsCryptResponse = checkDNSCryptQuery(cs, query, dnsCryptQuery, queryRealTime.tv_sec, false);
     if (dnsCryptResponse) {
       sendUDPResponse(cs.udpFD, query, 0, dest, remote);
diff --git a/pdns/dnsdist.hh b/pdns/dnsdist.hh
index f58f83cfa6..08d44ee849 100644
--- a/pdns/dnsdist.hh
+++ b/pdns/dnsdist.hh
@@ -123,14 +123,14 @@ struct DNSQuestion
 
   void setTag(const std::string& key, std::string&& value) {
     if (!qTag) {
-      qTag = std::make_shared<QTag>();
+      qTag = std::make_unique<QTag>();
     }
     qTag->insert_or_assign(key, std::move(value));
   }
 
   void setTag(const std::string& key, const std::string& value) {
     if (!qTag) {
-      qTag = std::make_shared<QTag>();
+      qTag = std::make_unique<QTag>();
     }
     qTag->insert_or_assign(key, value);
   }
@@ -144,6 +144,8 @@ public:
   boost::optional<Netmask> subnet;
   std::string sni; /* Server Name Indication, if any (DoT or DoH) */
   std::string poolname;
+  mutable std::shared_ptr<std::map<uint16_t, EDNSOptionView> > ednsOptions;
+  std::shared_ptr<DNSDistPacketCache> packetCache{nullptr};
   const DNSName* qname{nullptr};
   const ComboAddress* local{nullptr};
   const ComboAddress* remote{nullptr};
@@ -152,11 +154,9 @@ public:
      is enabled */
   const ComboAddress* hopLocal{nullptr};  /* the address dnsdist received the packet from, see above */
   const ComboAddress* hopRemote{nullptr};
-  std::shared_ptr<QTag> qTag{nullptr};
+  std::unique_ptr<QTag> qTag{nullptr};
   std::unique_ptr<std::vector<ProxyProtocolValue>> proxyProtocolValues{nullptr};
-  mutable std::shared_ptr<std::map<uint16_t, EDNSOptionView> > ednsOptions;
-  std::shared_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
-  std::shared_ptr<DNSDistPacketCache> packetCache{nullptr};
+  std::unique_ptr<DNSCryptQuery> dnsCryptQuery{nullptr};
   const struct timespec* queryTime{nullptr};
   struct DOHUnit* du{nullptr};
   int delayMsec{0};
@@ -565,16 +565,10 @@ extern QueryCount g_qcount;
 
 struct ClientState
 {
-  ClientState(const ComboAddress& local_, bool isTCP_, bool doReusePort, int fastOpenQueue, const std::string& itfName, const std::set<int>& cpus_): cpus(cpus_), local(local_), interface(itfName), fastOpenQueueSize(fastOpenQueue), tcp(isTCP_), reuseport(doReusePort)
+  ClientState(const ComboAddress& local_, bool isTCP_, bool doReusePort, int fastOpenQueue, const std::string& itfName, const std::set<int>& cpus_): cpus(cpus_), interface(itfName), local(local_), fastOpenQueueSize(fastOpenQueue), tcp(isTCP_), reuseport(doReusePort)
   {
   }
 
-  std::set<int> cpus;
-  ComboAddress local;
-  std::shared_ptr<DNSCryptContext> dnscryptCtx{nullptr};
-  std::shared_ptr<TLSFrontend> tlsFrontend{nullptr};
-  std::shared_ptr<DOHFrontend> dohFrontend{nullptr};
-  std::string interface;
   stat_t queries{0};
   mutable stat_t responses{0};
   mutable stat_t tcpDiedReadingQuery{0};
@@ -598,6 +592,13 @@ struct ClientState
   pdns::stat_t_trait<double> tcpAvgQueriesPerConnection{0.0};
   /* in ms */
   pdns::stat_t_trait<double> tcpAvgConnectionDuration{0.0};
+  std::set<int> cpus;
+  std::string interface;
+  ComboAddress local;
+  std::shared_ptr<DNSCryptContext> dnscryptCtx{nullptr};
+  std::shared_ptr<TLSFrontend> tlsFrontend{nullptr};
+  std::shared_ptr<DOHFrontend> dohFrontend{nullptr};
+  std::shared_ptr<BPFFilter> d_filter{nullptr};
   size_t d_maxInFlightQueriesPerConn{1};
   size_t d_tcpConcurrentConnectionsLimit{0};
   int udpFD{-1};
@@ -656,8 +657,6 @@ struct ClientState
     return result;
   }
 
-  shared_ptr<BPFFilter> d_filter;
-
   void detachFilter()
   {
     if (d_filter) {
@@ -691,26 +690,6 @@ struct DownstreamState
   DownstreamState(const ComboAddress& remote_): DownstreamState(remote_, ComboAddress(), 0, std::string(), 1, true) {}
   ~DownstreamState();
 
-  boost::uuids::uuid id;
-  SharedLockGuarded<std::vector<unsigned int>> hashes;
-  std::vector<int> sockets;
-  const std::string sourceItfName;
-  std::string d_tlsSubjectName;
-  std::string d_dohPath;
-  std::mutex connectLock;
-  LockGuarded<std::unique_ptr<FDMultiplexer>> mplexer{nullptr};
-  std::shared_ptr<TLSCtx> d_tlsCtx{nullptr};
-  std::thread tid;
-  const ComboAddress remote;
-  QPSLimiter qps;
-  vector<IDState> idStates;
-  const ComboAddress sourceAddr;
-  checkfunc_t checkFunction;
-  DNSName checkName{"a.root-servers.net."};
-  QType checkType{QType::A};
-  uint16_t checkClass{QClass::IN};
-  std::atomic<uint64_t> idOffset{0};
-  std::atomic<bool> hashesComputed{false};
   stat_t sendErrors{0};
   stat_t outstanding{0};
   stat_t reuseds{0};
@@ -737,11 +716,34 @@ struct DownstreamState
   pdns::stat_t_trait<double> tcpAvgQueriesPerConnection{0.0};
   /* in ms */
   pdns::stat_t_trait<double> tcpAvgConnectionDuration{0.0};
+  pdns::stat_t_trait<double> queryLoad{0.0};
+  pdns::stat_t_trait<double> dropRate{0.0};
+  boost::uuids::uuid id;
+  const ComboAddress remote;
+  const ComboAddress sourceAddr;
+  SharedLockGuarded<std::vector<unsigned int>> hashes;
+  LockGuarded<std::unique_ptr<FDMultiplexer>> mplexer{nullptr};
+  const std::string sourceItfName;
+  std::string d_tlsSubjectName;
+  std::string d_dohPath;
+private:
+  std::string name;
+  std::string nameWithAddr;
+public:
+  std::shared_ptr<TLSCtx> d_tlsCtx{nullptr};
+  std::vector<int> sockets;
+  vector<IDState> idStates;
+  set<string> pools;
+  std::mutex connectLock;
+  std::thread tid;
+  checkfunc_t checkFunction;
+  DNSName checkName{"a.root-servers.net."};
+  StopWatch sw;
+  QPSLimiter qps;
+  std::atomic<uint64_t> idOffset{0};
   size_t socketsOffset{0};
   size_t d_maxInFlightQueriesPerConn{1};
   size_t d_tcpConcurrentConnectionsLimit{0};
-  pdns::stat_t_trait<double> queryLoad{0.0};
-  pdns::stat_t_trait<double> dropRate{0.0};
   double latencyUsec{0.0};
   double latencyUsecTCP{0.0};
   int order{1};
@@ -752,6 +754,8 @@ struct DownstreamState
   unsigned int checkInterval{1};
   unsigned int lastCheck{0};
   const unsigned int sourceItf{0};
+  QType checkType{QType::A};
+  uint16_t checkClass{QClass::IN};
   uint16_t d_retries{5};
   uint16_t xpfRRCode{0};
   uint16_t checkTimeout{1000}; /* in milliseconds */
@@ -759,9 +763,11 @@ struct DownstreamState
   uint8_t consecutiveSuccessfulChecks{0};
   uint8_t maxCheckFailures{1};
   uint8_t minRiseSuccesses{1};
-  StopWatch sw;
-  set<string> pools;
   enum class Availability : uint8_t { Up, Down, Auto} availability{Availability::Auto};
+private:
+  bool d_stopped{false};
+public:
+  std::atomic<bool> hashesComputed{false};
   bool mustResolve{false};
   bool upStatus{false};
   bool useECS{false};
@@ -878,10 +884,6 @@ struct DownstreamState
     }
     return dnsdist::Protocol::DoUDP;
   }
-private:
-  std::string name;
-  std::string nameWithAddr;
-  bool d_stopped{false};
 };
 using servers_t =vector<std::shared_ptr<DownstreamState>>;
 
@@ -1049,8 +1051,8 @@ bool processRulesResult(const DNSAction::Action& action, DNSQuestion& dq, std::s
 bool checkQueryHeaders(const struct dnsheader* dh);
 
 extern std::vector<std::shared_ptr<DNSCryptContext>> g_dnsCryptLocals;
-int handleDNSCryptQuery(PacketBuffer& packet, std::shared_ptr<DNSCryptQuery>& query, bool tcp, time_t now, PacketBuffer& response);
-bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp);
+int handleDNSCryptQuery(PacketBuffer& packet, DNSCryptQuery& query, bool tcp, time_t now, PacketBuffer& response);
+bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp);
 
 uint16_t getRandomDNSID();
 
diff --git a/pdns/dnsdistdist/dnsdist-backend.cc b/pdns/dnsdistdist/dnsdist-backend.cc
index d72605794b..3139cda675 100644
--- a/pdns/dnsdistdist/dnsdist-backend.cc
+++ b/pdns/dnsdistdist/dnsdist-backend.cc
@@ -164,7 +164,7 @@ void DownstreamState::setWeight(int newWeight)
   }
 }
 
-DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_, size_t numberOfSockets, bool connect): sourceItfName(sourceItfName_), remote(remote_), idStates(connect ? g_maxOutstanding : 0), sourceAddr(sourceAddr_), sourceItf(sourceItf_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort())
+DownstreamState::DownstreamState(const ComboAddress& remote_, const ComboAddress& sourceAddr_, unsigned int sourceItf_, const std::string& sourceItfName_, size_t numberOfSockets, bool connect): remote(remote_), sourceAddr(sourceAddr_), sourceItfName(sourceItfName_), name(remote_.toStringWithPort()), nameWithAddr(remote_.toStringWithPort()), idStates(connect ? g_maxOutstanding : 0), sourceItf(sourceItf_)
 {
   id = getUniqueID();
   threadStarted.clear();
diff --git a/pdns/dnsdistdist/dnsdist-idstate.cc b/pdns/dnsdistdist/dnsdist-idstate.cc
index 0f3eeb247b..286808c765 100644
--- a/pdns/dnsdistdist/dnsdist-idstate.cc
+++ b/pdns/dnsdistdist/dnsdist-idstate.cc
@@ -53,7 +53,7 @@ void setIDStateFromDNSQuestion(IDState& ids, DNSQuestion& dq, DNSName&& qname)
   ids.ednsAdded = dq.ednsAdded;
   ids.ecsAdded = dq.ecsAdded;
   ids.useZeroScope = dq.useZeroScope;
-  ids.qTag = dq.qTag;
+  ids.qTag = std::move(dq.qTag);
   ids.dnssecOK = dq.dnssecOK;
   ids.uniqueId = std::move(dq.uniqueId);
 
diff --git a/pdns/dnsdistdist/dnsdist-nghttp2.cc b/pdns/dnsdistdist/dnsdist-nghttp2.cc
index 69c88dfd9b..c97180f174 100644
--- a/pdns/dnsdistdist/dnsdist-nghttp2.cc
+++ b/pdns/dnsdistdist/dnsdist-nghttp2.cc
@@ -109,11 +109,11 @@ private:
 
   static const std::unordered_map<std::string, std::string> s_constants;
 
-  std::unique_ptr<nghttp2_session, void (*)(nghttp2_session*)> d_session{nullptr, nghttp2_session_del};
   std::unordered_map<int32_t, PendingRequest> d_currentStreams;
+  std::string d_proxyProtocolPayload;
   PacketBuffer d_out;
   PacketBuffer d_in;
-  std::string d_proxyProtocolPayload;
+  std::unique_ptr<nghttp2_session, void (*)(nghttp2_session*)> d_session{nullptr, nghttp2_session_del};
   size_t d_outPos{0};
   size_t d_inPos{0};
   uint32_t d_highestStreamID{0};
diff --git a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
index 249596b1b8..c05c1264df 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-downstream.hh
@@ -10,7 +10,7 @@
 class TCPConnectionToBackend : public std::enable_shared_from_this<TCPConnectionToBackend>
 {
 public:
-  TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, std::unique_ptr<FDMultiplexer>& mplexer, const struct timeval& now): d_responseBuffer(s_maxPacketCacheEntrySize), d_mplexer(mplexer), d_ds(ds), d_connectionStartTime(now), d_lastDataReceivedTime(now), d_enableFastOpen(ds->tcpFastOpen)
+  TCPConnectionToBackend(std::shared_ptr<DownstreamState>& ds, std::unique_ptr<FDMultiplexer>& mplexer, const struct timeval& now): d_connectionStartTime(now), d_lastDataReceivedTime(now), d_ds(ds), d_responseBuffer(s_maxPacketCacheEntrySize), d_mplexer(mplexer), d_enableFastOpen(ds->tcpFastOpen)
   {
     reconnect();
   }
@@ -206,18 +206,18 @@ protected:
     return res;
   }
 
-  PacketBuffer d_responseBuffer;
+  TCPQuery d_currentQuery;
   std::deque<TCPQuery> d_pendingQueries;
   std::unordered_map<uint16_t, TCPQuery> d_pendingResponses;
+  struct timeval d_connectionStartTime;
+  struct timeval d_lastDataReceivedTime;
+  std::shared_ptr<DownstreamState> d_ds{nullptr};
+  std::shared_ptr<TCPQuerySender> d_sender{nullptr};
+  PacketBuffer d_responseBuffer;
   std::unique_ptr<FDMultiplexer>& d_mplexer;
   std::unique_ptr<std::vector<ProxyProtocolValue>> d_proxyProtocolValuesSent{nullptr};
   std::unique_ptr<TCPIOHandler> d_handler{nullptr};
   std::unique_ptr<IOStateHandler> d_ioState{nullptr};
-  std::shared_ptr<DownstreamState> d_ds{nullptr};
-  std::shared_ptr<TCPQuerySender> d_sender{nullptr};
-  TCPQuery d_currentQuery;
-  struct timeval d_connectionStartTime;
-  struct timeval d_lastDataReceivedTime;
   size_t d_currentPos{0};
   uint64_t d_queries{0};
   uint64_t d_downstreamFailures{0};
diff --git a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
index e42c8acf74..99e933aa2c 100644
--- a/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp-upstream.hh
@@ -19,7 +19,7 @@ public:
 class IncomingTCPConnectionState : public TCPQuerySender, public std::enable_shared_from_this<IncomingTCPConnectionState>
 {
 public:
-  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_threadData(threadData), d_ci(std::move(ci)), d_handler(d_ci.fd, timeval{g_tcpRecvTimeout,0}, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_ioState(make_unique<IOStateHandler>(*threadData.mplexer, d_ci.fd)), d_connectionStartTime(now)
+  IncomingTCPConnectionState(ConnectionInfo&& ci, TCPClientThreadData& threadData, const struct timeval& now): d_buffer(s_maxPacketCacheEntrySize), d_ci(std::move(ci)), d_handler(d_ci.fd, timeval{g_tcpRecvTimeout,0}, d_ci.cs->tlsFrontend ? d_ci.cs->tlsFrontend->getContext() : nullptr, now.tv_sec), d_connectionStartTime(now), d_ioState(make_unique<IOStateHandler>(*threadData.mplexer, d_ci.fd)), d_threadData(threadData)
   {
     d_origDest.reset();
     d_origDest.sin4.sin_family = d_ci.remote.sin4.sin_family;
@@ -145,25 +145,25 @@ static void handleTimeout(std::shared_ptr<IncomingTCPConnectionState>& state, bo
     return o.str();
   }
 
-  enum class State { doingHandshake, readingProxyProtocolHeader, waitingForQuery, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
+  enum class State : uint8_t { doingHandshake, readingProxyProtocolHeader, waitingForQuery, readingQuerySize, readingQuery, sendingResponse, idle /* in case of XFR, we stop processing queries */ };
 
+  TCPResponse d_currentResponse;
   std::map<std::shared_ptr<DownstreamState>, std::deque<std::shared_ptr<TCPConnectionToBackend>>> d_activeConnectionsToBackend;
-  PacketBuffer d_buffer;
   std::deque<TCPResponse> d_queuedResponses;
-  TCPClientThreadData& d_threadData;
-  TCPResponse d_currentResponse;
+  PacketBuffer d_buffer;
   ConnectionInfo d_ci;
   ComboAddress d_origDest;
   ComboAddress d_proxiedRemote;
   ComboAddress d_proxiedDestination;
   TCPIOHandler d_handler;
-  std::unique_ptr<IOStateHandler> d_ioState{nullptr};
-  std::unique_ptr<std::vector<ProxyProtocolValue>> d_proxyProtocolValues{nullptr};
   struct timeval d_connectionStartTime;
   struct timeval d_handshakeDoneTime;
   struct timeval d_firstQuerySizeReadTime;
   struct timeval d_querySizeReadTime;
   struct timeval d_queryReadTime;
+  std::unique_ptr<IOStateHandler> d_ioState{nullptr};
+  std::unique_ptr<std::vector<ProxyProtocolValue>> d_proxyProtocolValues{nullptr};
+  TCPClientThreadData& d_threadData;
   size_t d_currentPos{0};
   size_t d_proxyProtocolNeed{0};
   size_t d_queriesCount{0};
diff --git a/pdns/dnsdistdist/dnsdist-tcp.hh b/pdns/dnsdistdist/dnsdist-tcp.hh
index 5294988065..b9c86c916f 100644
--- a/pdns/dnsdistdist/dnsdist-tcp.hh
+++ b/pdns/dnsdistdist/dnsdist-tcp.hh
@@ -84,7 +84,7 @@ struct InternalQuery
   }
 
   InternalQuery(InternalQuery&& rhs) :
-    d_idstate(std::move(rhs.d_idstate)), d_buffer(std::move(rhs.d_buffer)), d_proxyProtocolPayload(std::move(rhs.d_proxyProtocolPayload)), d_xfrMasterSerial(rhs.d_xfrMasterSerial), d_xfrSerialCount(rhs.d_xfrSerialCount), d_downstreamFailures(rhs.d_downstreamFailures), d_xfrMasterSerialCount(rhs.d_xfrMasterSerialCount), d_proxyProtocolPayloadAdded(rhs.d_proxyProtocolPayloadAdded)
+    d_idstate(std::move(rhs.d_idstate)), d_proxyProtocolPayload(std::move(rhs.d_proxyProtocolPayload)), d_buffer(std::move(rhs.d_buffer)), d_xfrMasterSerial(rhs.d_xfrMasterSerial), d_xfrSerialCount(rhs.d_xfrSerialCount), d_downstreamFailures(rhs.d_downstreamFailures), d_xfrMasterSerialCount(rhs.d_xfrMasterSerialCount), d_proxyProtocolPayloadAdded(rhs.d_proxyProtocolPayloadAdded)
   {
   }
   InternalQuery& operator=(InternalQuery&& rhs)
@@ -109,8 +109,8 @@ struct InternalQuery
   }
 
   IDState d_idstate;
-  PacketBuffer d_buffer;
   std::string d_proxyProtocolPayload;
+  PacketBuffer d_buffer;
   uint32_t d_xfrMasterSerial{0};
   uint32_t d_xfrSerialCount{0};
   uint32_t d_downstreamFailures{0};
diff --git a/pdns/dnsdistdist/doh.cc b/pdns/dnsdistdist/doh.cc
index f87dba226a..863ee78c56 100644
--- a/pdns/dnsdistdist/doh.cc
+++ b/pdns/dnsdistdist/doh.cc
@@ -554,7 +554,7 @@ static int processDOHQuery(DOHUnit* du)
       // outside of the main DoH thread
       return -1;
     }
-    remote = du->remote;
+    remote = du->ids.origRemote;
     DOHServerConfig* dsc = du->dsc;
     auto& holders = dsc->holders;
     ClientState& cs = *dsc->cs;
@@ -599,8 +599,8 @@ static int processDOHQuery(DOHUnit* du)
     uint16_t qtype, qclass;
     unsigned int qnameWireLength = 0;
     DNSName qname(reinterpret_cast<const char*>(du->query.data()), du->query.size(), sizeof(dnsheader), false, &qtype, &qclass, &qnameWireLength);
-    DNSQuestion dq(&qname, qtype, qclass, &du->dest, &du->remote, du->query, dnsdist::Protocol::DoH, &queryRealTime);
-    dq.ednsAdded = du->ednsAdded;
+    DNSQuestion dq(&qname, qtype, qclass, &du->ids.origDest, &du->ids.origRemote, du->query, dnsdist::Protocol::DoH, &queryRealTime);
+    dq.ednsAdded = du->ids.ednsAdded;
     dq.du = du;
     dq.sni = std::move(du->sni);
 
@@ -657,7 +657,7 @@ static int processDOHQuery(DOHUnit* du)
       }
     }
 
-    ComboAddress dest = du->dest;
+    ComboAddress dest = du->ids.origDest;
     unsigned int idOffset = (du->downstream->idOffset++) % du->downstream->idStates.size();
     IDState* ids = &du->downstream->idStates[idOffset];
     ids->age = 0;
@@ -826,8 +826,8 @@ static void doh_dispatch_query(DOHServerConfig* dsc, h2o_handler_t* self, h2o_re
     auto du = std::unique_ptr<DOHUnit>(new DOHUnit);
     du->dsc = dsc;
     du->req = req;
-    du->dest = local;
-    du->remote = remote;
+    du->ids.origDest = local;
+    du->ids.origRemote = remote;
     du->rsock = dsc->dohresponsepair[0];
     du->query = std::move(query);
     du->path = std::move(path);
@@ -1260,7 +1260,7 @@ static void dnsdistclient(int qsock)
         if (generateOptRR(std::string(), du->query, 4096, 4096, 0, false)) {
           dh = const_cast<struct dnsheader*>(reinterpret_cast<const struct dnsheader*>(du->query.data())); // may have reallocated
           dh->arcount = htons(1);
-          du->ednsAdded = true;
+          du->ids.ednsAdded = true;
         }
       }
       else {
diff --git a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc
index e100ff5193..b17b1944c7 100644
--- a/pdns/dnsdistdist/test-dnsdisttcp_cc.cc
+++ b/pdns/dnsdistdist/test-dnsdisttcp_cc.cc
@@ -41,7 +41,7 @@ GlobalStateHolder<servers_t> g_dstates;
 
 QueryCount g_qcount;
 
-bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::shared_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp)
+bool checkDNSCryptQuery(const ClientState& cs, PacketBuffer& query, std::unique_ptr<DNSCryptQuery>& dnsCryptQuery, time_t now, bool tcp)
 {
   return false;
 }
diff --git a/pdns/doh.hh b/pdns/doh.hh
index e9cc9df5a8..16210d9493 100644
--- a/pdns/doh.hh
+++ b/pdns/doh.hh
@@ -187,6 +187,7 @@ struct DOHUnit
 {
   DOHUnit()
   {
+    ids.ednsAdded = false;
   }
   DOHUnit(const DOHUnit&) = delete;
   DOHUnit& operator=(const DOHUnit&) = delete;
@@ -209,21 +210,19 @@ struct DOHUnit
 
   void handleUDPResponse(PacketBuffer&& response, IDState&& state);
 
-  std::vector<std::pair<std::string, std::string>> headers;
-  PacketBuffer query;
-  PacketBuffer response;
   IDState ids;
   std::string sni;
   std::string path;
   std::string scheme;
   std::string host;
-  ComboAddress remote;
-  ComboAddress dest;
+  std::string contentType;
+  std::vector<std::pair<std::string, std::string>> headers;
+  PacketBuffer query;
+  PacketBuffer response;
+  std::shared_ptr<DownstreamState> downstream{nullptr};
   st_h2o_req_t* req{nullptr};
   DOHUnit** self{nullptr};
   DOHServerConfig* dsc{nullptr};
-  std::shared_ptr<DownstreamState> downstream{nullptr};
-  std::string contentType;
   std::atomic<uint64_t> d_refcnt{1};
   size_t query_at{0};
   size_t proxyProtocolPayloadSize{0};
@@ -236,7 +235,6 @@ struct DOHUnit
      the main DoH thread.
   */
   uint16_t status_code{200};
-  bool ednsAdded{false};
   /* whether the query was re-sent to the backend over
      TCP after receiving a truncated answer over UDP */
   bool tcp{false};