From 2a17f6c6c25fb3bc6eabf4542421850652ea92b9 Mon Sep 17 00:00:00 2001 From: Otto Date: Mon, 3 May 2021 14:34:29 +0200 Subject: [PATCH] Start using tcpiohandler for real --- pdns/lua-recursor4.cc | 3 +- pdns/lua-recursor4.hh | 3 +- pdns/lwres.cc | 40 ++++--- pdns/lwres.hh | 3 +- pdns/pdns_recursor.cc | 181 +++++++++++++++++++++++++----- pdns/rec-carbon.cc | 5 +- pdns/recursordist/Makefile.am | 2 + pdns/recursordist/libssl.cc | 1 + pdns/recursordist/libssl.hh | 1 + pdns/recursordist/tcpiohandler.hh | 1 + pdns/syncres.cc | 15 ++- pdns/syncres.hh | 14 ++- pdns/ws-recursor.cc | 10 +- 13 files changed, 214 insertions(+), 65 deletions(-) create mode 120000 pdns/recursordist/libssl.cc create mode 120000 pdns/recursordist/libssl.hh create mode 120000 pdns/recursordist/tcpiohandler.hh diff --git a/pdns/lua-recursor4.cc b/pdns/lua-recursor4.cc index 14313aaf43..5aef9e9f33 100644 --- a/pdns/lua-recursor4.cc +++ b/pdns/lua-recursor4.cc @@ -675,7 +675,8 @@ loop:; ret=getFakePTRRecords(dq.followupName, dq.records); } else if(dq.followupFunction=="udpQueryResponse") { - dq.udpAnswer = GenUDPQueryResponse(dq.udpQueryDest, dq.udpQuery); + PacketBuffer p = GenUDPQueryResponse(dq.udpQueryDest, dq.udpQuery); + dq.udpAnswer = std::string(reinterpret_cast(p.data()), p.size()); auto cbFunc = d_lw->readVariable>(dq.udpCallback).get_value_or(0); if(!cbFunc) { g_log< #include "lua-recursor4-ffi.hh" -string GenUDPQueryResponse(const ComboAddress& dest, const string& query); +PacketBuffer GenUDPQueryResponse(const ComboAddress& dest, const string& query); unsigned int getRecursorThreadId(); // pdns_ffi_param_t is a lightuserdata diff --git a/pdns/lwres.cc b/pdns/lwres.cc index 483efd42fc..8c3e431b7a 100644 --- a/pdns/lwres.cc +++ b/pdns/lwres.cc @@ -48,6 +48,7 @@ #include "validate-recursor.hh" #include "ednssubnet.hh" #include "query-local-address.hh" +#include "tcpiohandler.hh" #include "rec-protozero.hh" #include "uuid-utils.hh" @@ -100,7 +101,7 @@ static bool isEnabledForResponses(const std::shared_ptr>>& fstreamLoggers, const ComboAddress&localip, const ComboAddress& ip, bool doTCP, boost::optional auth, const std::string& packet, const struct timeval& queryTime, const struct timeval& replyTime) +static void logFstreamResponse(const std::shared_ptr>>& fstreamLoggers, const ComboAddress&localip, const ComboAddress& ip, bool doTCP, boost::optional auth, const PacketBuffer& packet, const struct timeval& queryTime, const struct timeval& replyTime) { if (fstreamLoggers == nullptr) return; @@ -109,7 +110,7 @@ static void logFstreamResponse(const std::shared_ptr(&*packet.begin()), packet.size(), &ts1, &ts2, auth); + DnstapMessage message(str, DnstapMessage::MessageType::resolver_response, SyncRes::s_serverID, &localip, &ip, doTCP, reinterpret_cast(packet.data()), packet.size(), &ts1, &ts2, auth); for (auto& logger : *fstreamLoggers) { logger->queueData(str); @@ -235,7 +236,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int { size_t len; size_t bufsize=g_outgoingEDNSBufsize; - std::string buf; + PacketBuffer buf; buf.resize(bufsize); vector vpacket; // string mapped0x20=dns0x20(domain); @@ -340,28 +341,25 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int } else { try { - Socket s(ip.sin4.sin_family, SOCK_STREAM); + const int timeout = g_networkTimeoutMsec / 1000; // XXX tcpiohandler's unit is seconds + Socket s(ip.sin4.sin_family, SOCK_STREAM); s.setNonBlocking(); - if (SyncRes::s_tcp_fast_open_connect) { - try { - s.setFastOpenConnect(); - } - catch (const NetworkError& e) { - // Ignore error, we did a pre-check in pdns_recursor.cc:checkTFOconnect() - } - } - localip = pdns::getQueryLocalAddress(ip.sin4.sin_family, 0); s.bind(localip); - s.connect(ip); + std::shared_ptr tlsCtx{nullptr}; + TCPIOHandler handler("", s.releaseHandle(), timeout, tlsCtx, now->tv_sec); + IOState state = handler.tryConnect(SyncRes::s_tcp_fast_open_connect, ip); uint16_t tlen=htons(vpacket.size()); char *lenP=(char*)&tlen; const char *msgP=(const char*)&*vpacket.begin(); - string packet=string(lenP, lenP+2)+string(msgP, msgP+vpacket.size()); - ret = asendtcp(packet, &s); + PacketBuffer packet; + packet.reserve(2 + vpacket.size()); + packet.insert(packet.end(), lenP, lenP+2); + packet.insert(packet.end(), msgP, msgP+vpacket.size()); + ret = asendtcp(packet, handler); if (ret != LWResult::Result::Success) { return ret; } @@ -373,21 +371,21 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int #endif /* HAVE_FSTRM */ packet.clear(); - ret = arecvtcp(packet, 2, &s, false); + ret = arecvtcp(packet, 2, handler, false); if (ret != LWResult::Result::Success) { return ret; } - memcpy(&tlen, packet.c_str(), sizeof(tlen)); + memcpy(&tlen, packet.data(), sizeof(tlen)); len=ntohs(tlen); // switch to the 'len' shared with the rest of the function - ret = arecvtcp(packet, len, &s, false); + ret = arecvtcp(packet, len, handler, false); if (ret != LWResult::Result::Success) { return ret; } buf.resize(len); - memcpy(const_cast(buf.data()), packet.c_str(), len); + memcpy(buf.data(), packet.data(), len); ret = LWResult::Result::Success; } @@ -417,7 +415,7 @@ LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int lwr->d_records.clear(); try { lwr->d_tcbit=0; - MOADNSParser mdp(false, buf); + MOADNSParser mdp(false, reinterpret_cast(buf.data()), buf.size()); lwr->d_aabit=mdp.d_header.aa; lwr->d_tcbit=mdp.d_header.tc; lwr->d_rcode=mdp.d_header.rcode; diff --git a/pdns/lwres.hh b/pdns/lwres.hh index c27c57c2d9..ea039bbf2a 100644 --- a/pdns/lwres.hh +++ b/pdns/lwres.hh @@ -42,6 +42,7 @@ #include "remote_logger.hh" #include "fstrm_logger.hh" #include "resolve-context.hh" +#include "noinitvector.hh" class LWResException : public PDNSException { @@ -67,7 +68,7 @@ public: LWResult::Result asendto(const char *data, size_t len, int flags, const ComboAddress& ip, uint16_t id, const DNSName& domain, uint16_t qtype, int* fd); -LWResult::Result arecvfrom(std::string& packet, int flags, const ComboAddress& ip, size_t *d_len, uint16_t id, +LWResult::Result arecvfrom(PacketBuffer& packet, int flags, const ComboAddress& ip, size_t *d_len, uint16_t id, const DNSName& domain, uint16_t qtype, int fd, const struct timeval* now); LWResult::Result asyncresolve(const ComboAddress& ip, const DNSName& domain, int type, bool doTCP, bool sendRDQuery, int EDNS0Level, struct timeval* now, boost::optional& srcmask, boost::optional context, const std::shared_ptr>>& outgoingLoggers, const std::shared_ptr>>& fstrmLoggers, const std::set& exportTypes, LWResult* res, bool* chained); diff --git a/pdns/pdns_recursor.cc b/pdns/pdns_recursor.cc index 8da5d8112d..bc482caa92 100644 --- a/pdns/pdns_recursor.cc +++ b/pdns/pdns_recursor.cc @@ -398,14 +398,14 @@ static bool isHandlerThread() static void handleTCPClientWritable(int fd, FDMultiplexer::funcparam_t& var); -LWResult::Result asendtcp(const string& data, Socket* sock) +LWResult::Result asendtcp(const PacketBuffer& data, Socket* sock) { PacketID pident; pident.tcpsock=sock->getHandle(); - pident.outMSG=data; + pident.outMSG = data; t_fdm->addWriteFD(sock->getHandle(), handleTCPClientWritable, pident); - string packet; + PacketBuffer packet; int ret = MT->waitEvent(pident, &packet, g_networkTimeoutMsec); if (ret == 0) { //timeout @@ -423,9 +423,37 @@ LWResult::Result asendtcp(const string& data, Socket* sock) return LWResult::Result::Success; } +static void TCPIOHandlerWritable(int fd, FDMultiplexer::funcparam_t& var); + +LWResult::Result asendtcp(const PacketBuffer& data, TCPIOHandler& handler) +{ + PacketID pident; + pident.tcphandler = &handler; + pident.tcpsock = handler.getDescriptor(); + pident.outMSG = data; + + t_fdm->addWriteFD(handler.getDescriptor(), TCPIOHandlerWritable, pident); + PacketBuffer packet; + + int ret = MT->waitEvent(pident, &packet, g_networkTimeoutMsec); + if (ret == 0) { //timeout + t_fdm->removeWriteFD(handler.getDescriptor()); + return LWResult::Result::Timeout; + } + else if (ret == -1) { // error + t_fdm->removeWriteFD(handler.getDescriptor()); + return LWResult::Result::PermanentError; + } + else if (packet.size() != data.size()) { // main loop tells us what it sent out, or empty in case of an error + return LWResult::Result::PermanentError; + } + + return LWResult::Result::Success; +} + static void handleTCPClientReadable(int fd, FDMultiplexer::funcparam_t& var); -LWResult::Result arecvtcp(string& data, const size_t len, Socket* sock, const bool incompleteOkay) +LWResult::Result arecvtcp(PacketBuffer& data, const size_t len, Socket* sock, const bool incompleteOkay) { data.clear(); PacketID pident; @@ -434,7 +462,7 @@ LWResult::Result arecvtcp(string& data, const size_t len, Socket* sock, const bo pident.inIncompleteOkay=incompleteOkay; t_fdm->addReadFD(sock->getHandle(), handleTCPClientReadable, pident); - int ret = MT->waitEvent(pident,&data, g_networkTimeoutMsec); + int ret = MT->waitEvent(pident, &data, g_networkTimeoutMsec); if (ret == 0) { t_fdm->removeReadFD(sock->getHandle()); return LWResult::Result::Timeout; @@ -450,14 +478,44 @@ LWResult::Result arecvtcp(string& data, const size_t len, Socket* sock, const bo return LWResult::Result::Success; } +static void TCPIOHandlerReadable(int fd, FDMultiplexer::funcparam_t& var); + +LWResult::Result arecvtcp(PacketBuffer& data, const size_t len, TCPIOHandler& handler, const bool incompleteOkay) +{ + data.clear(); + + PacketID pident; + pident.tcphandler = &handler; + pident.tcpsock = handler.getDescriptor(); + pident.inNeeded = len; + pident.inIncompleteOkay = incompleteOkay; + t_fdm->addReadFD(handler.getDescriptor(), TCPIOHandlerReadable, pident); + + int ret = MT->waitEvent(pident, &data, g_networkTimeoutMsec); + if (ret == 0) { + t_fdm->removeReadFD(handler.getDescriptor()); + return LWResult::Result::Timeout; + } + else if (ret == -1) { + t_fdm->removeWriteFD(handler.getDescriptor()); + return LWResult::Result::PermanentError; + } + else if (data.empty()) {// error, EOF or other + return LWResult::Result::PermanentError; + } + + return LWResult::Result::Success; +} + static void handleGenUDPQueryResponse(int fd, FDMultiplexer::funcparam_t& var) { - PacketID pident=*boost::any_cast(&var); - char resp[512]; + PacketID pident = *boost::any_cast(&var); + PacketBuffer resp; + resp.resize(512); ComboAddress fromaddr; - socklen_t addrlen=sizeof(fromaddr); + socklen_t addrlen = sizeof(fromaddr); - ssize_t ret=recvfrom(fd, resp, sizeof(resp), 0, (sockaddr *)&fromaddr, &addrlen); + ssize_t ret = recvfrom(fd, resp.data(), resp.size(), 0, (sockaddr *)&fromaddr, &addrlen); if (fromaddr != pident.remote) { g_log<removeReadFD(fd); if(ret >= 0) { - string data(resp, (size_t) ret); - MT->sendEvent(pident, &data); + MT->sendEvent(pident, &resp); } else { - string empty; + PacketBuffer empty; MT->sendEvent(pident, &empty); // cerr<<"Had some kind of error: "<addReadFD(s.getHandle(), handleGenUDPQueryResponse, pident); - string data; + PacketBuffer data; - int ret=MT->waitEvent(pident,&data, g_networkTimeoutMsec); + int ret=MT->waitEvent(pident, &data, g_networkTimeoutMsec); if(!ret || ret==-1) { // timeout t_fdm->removeReadFD(s.getHandle()); @@ -698,7 +755,7 @@ LWResult::Result asendto(const char *data, size_t len, int flags, return LWResult::Result::Success; } -LWResult::Result arecvfrom(std::string& packet, int flags, const ComboAddress& fromaddr, size_t *d_len, +LWResult::Result arecvfrom(PacketBuffer& packet, int flags, const ComboAddress& fromaddr, size_t *d_len, uint16_t id, const DNSName& domain, uint16_t qtype, int fd, const struct timeval* now) { static const unsigned int nearMissLimit = ::arg().asNum("spoof-nearmiss-max"); @@ -4017,12 +4074,12 @@ static void handleTCPClientReadable(int fd, FDMultiplexer::funcparam_t& var) ssize_t ret=recv(fd, buffer.get(), pident->inNeeded,0); if(ret > 0) { - pident->inMSG.append(&buffer[0], &buffer[ret]); - pident->inNeeded-=(size_t)ret; + pident->inMSG.insert(pident->inMSG.end(), &buffer[0], &buffer[ret]); + pident->inNeeded -= (size_t)ret; if(!pident->inNeeded || pident->inIncompleteOkay) { // cerr<<"Got entire load of "<inMSG.size()<<" bytes"<inMSG; + PacketBuffer msg = pident->inMSG; t_fdm->removeReadFD(fd); MT->sendEvent(pid, &msg); @@ -4034,7 +4091,46 @@ static void handleTCPClientReadable(int fd, FDMultiplexer::funcparam_t& var) else { PacketID tmp=*pident; t_fdm->removeReadFD(fd); // pident might now be invalid (it isn't, but still) - string empty; + PacketBuffer empty; + MT->sendEvent(tmp, &empty); // this conveys error status + } +} + +static void TCPIOHandlerReadable(int fd, FDMultiplexer::funcparam_t& var) +{ + PacketID* pident=boost::any_cast(&var); + assert(pident->tcphandler != nullptr); + assert(fd == pident->tcphandler->getDescriptor()); + // XXX Likely it is possible to directly write into inMSG + // To be able to do that, inMSG has to be resized before. Afer the read we may have to resize again + // taking into account the actual bytes read. Wondering if this is worth the trouble... + PacketBuffer buffer; + buffer.resize(pident->inNeeded); + + try { + size_t pos = 0; + IOState state = pident->tcphandler->tryRead(buffer, pos, pident->inNeeded); + switch (state) { + case IOState::Done: + case IOState::NeedRead: + pident->inMSG.insert(pident->inMSG.end(), buffer.data(), buffer.data() + pos); + pident->inNeeded -= pos; + if (pident->inNeeded == 0 || pident->inIncompleteOkay) { + PacketID pid = *pident; + PacketBuffer msg = pident->inMSG; + t_fdm->removeReadFD(fd); + MT->sendEvent(pid, &msg); + } + break; + case IOState::NeedWrite: + // What to do? + break; + } + } + catch (const std::runtime_error& e) { + PacketID tmp = *pident; + t_fdm->removeReadFD(fd); // pident might now be invalid (it isn't, but still) + PacketBuffer empty; MT->sendEvent(tmp, &empty); // this conveys error status } } @@ -4042,7 +4138,7 @@ static void handleTCPClientReadable(int fd, FDMultiplexer::funcparam_t& var) static void handleTCPClientWritable(int fd, FDMultiplexer::funcparam_t& var) { PacketID* pid = boost::any_cast(&var); - ssize_t ret = send(fd, pid->outMSG.c_str() + pid->outPos, pid->outMSG.size() - pid->outPos,0); + ssize_t ret = send(fd, pid->outMSG.data() + pid->outPos, pid->outMSG.size() - pid->outPos,0); if (ret > 0) { pid->outPos += (ssize_t)ret; if (pid->outPos == pid->outMSG.size()) { @@ -4054,13 +4150,44 @@ static void handleTCPClientWritable(int fd, FDMultiplexer::funcparam_t& var) else { // error or EOF PacketID tmp(*pid); t_fdm->removeWriteFD(fd); - string sent; + PacketBuffer sent; + MT->sendEvent(tmp, &sent); // we convey error status by sending empty string + } +} + +static void TCPIOHandlerWritable(int fd, FDMultiplexer::funcparam_t& var) +{ + PacketID* pid = boost::any_cast(&var); + assert(pid->tcphandler != nullptr); + assert(fd == pid->tcphandler->getDescriptor()); + + try { + IOState state = pid->tcphandler->tryWrite(pid->outMSG, pid->outPos, pid->outMSG.size()); + switch (state) { + case IOState::Done: { + PacketID tmp = *pid; + t_fdm->removeWriteFD(fd); + MT->sendEvent(tmp, &tmp.outMSG); // send back what we sent to convey everything is ok + break; + } + case IOState::NeedWrite: + // We'll get back later + break; + case IOState::NeedRead: + // What to do? + break; + } + } + catch (const std::runtime_error& e) { + PacketID tmp = *pid; + t_fdm->removeWriteFD(fd); + PacketBuffer sent; MT->sendEvent(tmp, &sent); // we convey error status by sending empty string } } // resend event to everybody chained onto it -static void doResends(MT_t::waiters_t::iterator& iter, PacketID resend, const string& content) +static void doResends(MT_t::waiters_t::iterator& iter, PacketID resend, const PacketBuffer& content) { // We close the chain for new entries, since they won't be processed anyway iter->key.closed = true; @@ -4082,7 +4209,7 @@ static void handleUDPServerResponse(int fd, FDMultiplexer::funcparam_t& var) { PacketID pid=boost::any_cast(var); ssize_t len; - std::string packet; + PacketBuffer packet; packet.resize(g_outgoingEDNSBufsize); ComboAddress fromaddr; socklen_t addrlen=sizeof(fromaddr); @@ -4100,7 +4227,7 @@ static void handleUDPServerResponse(int fd, FDMultiplexer::funcparam_t& var) } t_udpclientsocks->returnSocket(fd); - string empty; + PacketBuffer empty; MT_t::waiters_t::iterator iter=MT->d_waiters.find(pid); if(iter != MT->d_waiters.end()) @@ -4131,7 +4258,7 @@ static void handleUDPServerResponse(int fd, FDMultiplexer::funcparam_t& var) else { try { if(len > 12) - pident.domain=DNSName(&packet.at(0), len, 12, false, &pident.type); // don't copy this from above - we need to do the actual read + pident.domain=DNSName(reinterpret_cast(packet.data()), len, 12, false, &pident.type); // don't copy this from above - we need to do the actual read } catch(std::exception& e) { g_stats.serverParseError++; // won't be fed to lwres.cc, so we have to increment @@ -5237,7 +5364,7 @@ try t_bogusqueryring = std::unique_ptr > >(new boost::circular_buffer >()); t_bogusqueryring->set_capacity(ringsize); } - MT=std::unique_ptr >(new MTasker(::arg().asNum("stack-size"))); + MT=std::unique_ptr >(new MTasker(::arg().asNum("stack-size"))); threadInfo.mt = MT.get(); /* start protobuf export threads if needed */ diff --git a/pdns/rec-carbon.cc b/pdns/rec-carbon.cc index f141c765a0..07d117bdbc 100644 --- a/pdns/rec-carbon.cc +++ b/pdns/rec-carbon.cc @@ -45,7 +45,7 @@ try } registerAllStats(); - string msg; + PacketBuffer msg; for(const auto& carbonServer: carbonServers) { ComboAddress remote(carbonServer, 2003); Socket s(remote.sin4.sin_family, SOCK_STREAM); @@ -62,7 +62,8 @@ try for(const auto& val : all) { str<first, *remoteIP, false, truncated, spoofed); - if (spoofed || (gotAnswer && truncated) ) { + bool gotAnswer = false; + +// Option below is for debugging purposes ony +#define USE_TCP_ONLY 0 + +#if !USE_TCP_ONLY + gotAnswer = doResolveAtThisIP(prefix, qname, qtype, lwr, ednsmask, auth, sendRDQuery, wasForwarded, + tns->first, *remoteIP, false, truncated, spoofed); + if (spoofed || (gotAnswer && truncated)) { +#else + { +#endif /* retry, over TCP this time */ gotAnswer = doResolveAtThisIP(prefix, qname, qtype, lwr, ednsmask, auth, sendRDQuery, wasForwarded, tns->first, *remoteIP, true, truncated, spoofed); diff --git a/pdns/syncres.hh b/pdns/syncres.hh index f4d8f2790a..e272394090 100644 --- a/pdns/syncres.hh +++ b/pdns/syncres.hh @@ -52,6 +52,7 @@ #include "proxy-protocol.hh" #include "sholder.hh" #include "histogram.hh" +#include "tcpiohandler.hh" #ifdef HAVE_CONFIG_H #include "config.h" @@ -920,8 +921,10 @@ private: class Socket; /* external functions, opaque to us */ -LWResult::Result asendtcp(const string& data, Socket* sock); -LWResult::Result arecvtcp(string& data, size_t len, Socket* sock, bool incompleteOkay); +LWResult::Result asendtcp(const PacketBuffer& data, Socket* sock); +LWResult::Result arecvtcp(PacketBuffer& data, size_t len, Socket* sock, bool incompleteOkay); +LWResult::Result asendtcp(const PacketBuffer& data, TCPIOHandler&); +LWResult::Result arecvtcp(PacketBuffer& data, size_t len, TCPIOHandler&, bool incompleteOkay); struct PacketID { @@ -933,11 +936,12 @@ struct PacketID ComboAddress remote; // this is the remote DNSName domain; // this is the question - string inMSG; // they'll go here - string outMSG; // the outgoing message that needs to be sent + PacketBuffer inMSG; // they'll go here + PacketBuffer outMSG; // the outgoing message that needs to be sent typedef set chain_t; mutable chain_t chain; + TCPIOHandler *tcphandler{nullptr}; size_t inNeeded{0}; // if this is set, we'll read until inNeeded bytes are read string::size_type outPos{0}; // how far we are along in the outMSG mutable uint32_t nearMisses{0}; // number of near misses - host correct, id wrong @@ -981,7 +985,7 @@ struct PacketIDBirthdayCompare: public std::binary_function g_recCache; extern thread_local std::unique_ptr t_packetCache; -typedef MTasker MT_t; +typedef MTasker MT_t; MT_t* getMT(); struct RecursorStats diff --git a/pdns/ws-recursor.cc b/pdns/ws-recursor.cc index be56ca78cb..3c8c699371 100644 --- a/pdns/ws-recursor.cc +++ b/pdns/ws-recursor.cc @@ -1242,19 +1242,20 @@ void AsyncWebServer::serveConnection(std::shared_ptr client) const { HttpRequest req(logprefix); HttpResponse resp; ComboAddress remote; - string reply; + PacketBuffer reply; try { YaHTTP::AsyncRequestLoader yarl; yarl.initialize(&req); client->setNonBlocking(); - string data; + PacketBuffer data; try { while(!req.complete) { auto ret = arecvtcp(data, 16384, client.get(), true); if (ret == LWResult::Result::Success) { - req.complete = yarl.feed(data); + string str(reinterpret_cast(data.data()), data.size()); + req.complete = yarl.feed(str); } else { // read error OR EOF break; @@ -1275,7 +1276,8 @@ void AsyncWebServer::serveConnection(std::shared_ptr client) const { WebServer::handleRequest(req, resp); ostringstream ss; resp.write(ss); - reply = ss.str(); + const string &s = ss.str(); + reply.insert(reply.end(), s.cbegin(), s.cend()); logResponse(resp, remote, logprefix); -- 2.47.2