From: Alex Rousskov Date: Mon, 4 Aug 2014 21:44:31 +0000 (-0600) Subject: Major source layout change: Moved FTP code into servers/, clients/, and ftp/. X-Git-Tag: SQUID_3_5_0_1~117^2~10 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=92ae4c86c14b2c6346ee32550d472eb57fd1caea;p=thirdparty%2Fsquid.git Major source layout change: Moved FTP code into servers/, clients/, and ftp/. src/servers/FtpServer.* # new FTP server, relaying FTP src/servers/HttpServer.* # old ConnStateData parts conflicting w/ FtpServer src/clients/FtpClient.* # code shared by old and new FTP clients src/clients/FtpGateway.* # old FTP client, translating back to HTTP src/clients/FtpNative.* # new FTP client, relaying FTP src/ftp/* # FTP stuff shared by clients and servers This change attempts to preserve code in moved files to the extent possible: Only copied or added code was polished. Future changes will polish moved code. Unfortunately, bzr does not track code changes across file splits, so the code moved from ConnStateData (client_side.cc) into HttpServer and FtpServer classes appears as removed and added in bzr diff. If you want to see the branch log for the FtpServer or HttpServer code added in this revision, look in the previous branch revision of client_side.cc. --- diff --git a/configure.ac b/configure.ac index ec1d0089d4..60b18b08fb 100644 --- a/configure.ac +++ b/configure.ac @@ -3457,8 +3457,11 @@ AC_CONFIG_FILES([ scripts/Makefile src/Makefile src/anyp/Makefile + src/ftp/Makefile src/base/Makefile src/acl/Makefile + src/clients/Makefile + src/servers/Makefile src/fs/Makefile src/repl/Makefile src/auth/Makefile diff --git a/src/FtpGatewayServer.h b/src/FtpGatewayServer.h deleted file mode 100644 index bbfe99d4ae..0000000000 --- a/src/FtpGatewayServer.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * DEBUG: section 09 File Transfer Protocol (FTP) - * - */ - -#ifndef SQUID_FTP_GATEWAY_SERVER_H -#define SQUID_FTP_GATEWAY_SERVER_H - -class FwdState; - -void ftpGatewayServerStart(FwdState *const); - -#endif /* SQUID_FTP_GATEWAY_SERVER_H */ diff --git a/src/FwdState.cc b/src/FwdState.cc index 935837da22..081f4326ab 100644 --- a/src/FwdState.cc +++ b/src/FwdState.cc @@ -39,6 +39,7 @@ #include "CacheManager.h" #include "CachePeer.h" #include "client_side.h" +#include "clients/forward.h" #include "comm/Connection.h" #include "comm/ConnOpener.h" #include "comm/Loops.h" @@ -47,8 +48,6 @@ #include "event.h" #include "fd.h" #include "fde.h" -#include "ftp.h" -#include "FtpGatewayServer.h" #include "FwdState.h" #include "globals.h" #include "gopher.h" @@ -720,15 +719,9 @@ FwdState::connectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, in } #endif - const CbcPointer &clientConnState = - request->clientConnectionManager; - if (clientConnState.valid() && clientConnState->isFtp) { - // this is not an idle connection, so we do not want I/O monitoring - const bool monitor = false; - clientConnState->pinConnection(serverConnection(), request, - serverConnection()->getPeer(), false, - monitor); - } + // should reach ConnStateData before the dispatched Client job starts + CallJobHere1(17, 4, request->clientConnectionManager, ConnStateData, + ConnStateData::notePeerConnection, serverConnection()); dispatch(); } @@ -1007,7 +1000,7 @@ FwdState::dispatch() break; case AnyP::PROTO_FTP: - if (request->clientConnectionManager->isFtp) + if (request->flags.ftpNative) ftpGatewayServerStart(this); else ftpStart(this); diff --git a/src/Makefile.am b/src/Makefile.am index af3d86a5d6..0fa15d2f07 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -46,8 +46,8 @@ LOADABLE_MODULES_SOURCES = \ LoadableModules.h \ LoadableModules.cc -SUBDIRS = base anyp parser comm eui acl format fs repl -DIST_SUBDIRS = base anyp parser comm eui acl format fs repl +SUBDIRS = base anyp ftp parser comm eui acl format clients servers fs repl +DIST_SUBDIRS = base anyp ftp parser comm eui acl format clients servers fs repl if ENABLE_AUTH SUBDIRS += auth @@ -312,6 +312,7 @@ squid_SOURCES = \ ClientRequestContext.h \ clientStream.cc \ clientStream.h \ + clientStreamForward.h \ CollapsedForwarding.cc \ CollapsedForwarding.h \ CompletionDispatcher.cc \ @@ -363,12 +364,6 @@ squid_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ Generic.h \ @@ -640,6 +635,9 @@ squid_LDADD = \ icmp/libicmp.la icmp/libicmp-core.la \ log/liblog.la \ format/libformat.la \ + clients/libclients.la \ + servers/libservers.la \ + ftp/libftp.la \ $(XTRA_OBJS) \ $(DISK_LINKOBJS) \ $(REPL_OBJS) \ @@ -675,6 +673,9 @@ squid_DEPENDENCIES = \ $(AUTH_LIBS) \ acl/libapi.la \ base/libbase.la \ + ftp/libftp.la \ + clients/libclients.la \ + servers/libservers.la \ libsquid.la \ ip/libip.la \ fs/libfs.la \ @@ -1469,12 +1470,6 @@ tests_testCacheManager_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -1612,7 +1607,10 @@ nodist_tests_testCacheManager_SOURCES = \ $(DISKIO_GEN_SOURCE) # comm.cc only requires comm/libcomm.la until fdc_table is dead. tests_testCacheManager_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ http/libsquid-http.la \ + ftp/libftp.la \ ident/libident.la \ acl/libacls.la \ acl/libstate.la \ @@ -1892,12 +1890,6 @@ tests_testEvent_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -2050,7 +2042,10 @@ nodist_tests_testEvent_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_testEvent_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ http/libsquid-http.la \ + ftp/libftp.la \ ident/libident.la \ acl/libacls.la \ acl/libstate.la \ @@ -2146,12 +2141,6 @@ tests_testEventLoop_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -2304,7 +2293,10 @@ nodist_tests_testEventLoop_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_testEventLoop_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ http/libsquid-http.la \ + ftp/libftp.la \ ident/libident.la \ acl/libacls.la \ acl/libstate.la \ @@ -2396,12 +2388,6 @@ tests_test_http_range_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -2551,7 +2537,10 @@ nodist_tests_test_http_range_SOURCES = \ $(BUILT_SOURCES) \ $(DISKIO_GEN_SOURCE) tests_test_http_range_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ http/libsquid-http.la \ + ftp/libftp.la \ ident/libident.la \ acl/libacls.la \ acl/libstate.la \ @@ -2708,12 +2697,6 @@ tests_testHttpRequest_SOURCES = \ fde.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -2847,6 +2830,9 @@ tests_testHttpRequest_SOURCES = \ nodist_tests_testHttpRequest_SOURCES = \ $(BUILT_SOURCES) tests_testHttpRequest_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ + ftp/libftp.la \ ident/libident.la \ acl/libacls.la \ acl/libstate.la \ @@ -3532,12 +3518,6 @@ tests_testURL_SOURCES = \ filemap.cc \ fqdncache.h \ fqdncache.cc \ - ftp.h \ - ftp.cc \ - FtpGatewayServer.h \ - FtpGatewayServer.cc \ - FtpServer.h \ - FtpServer.cc \ FwdState.cc \ FwdState.h \ gopher.h \ @@ -3692,7 +3672,10 @@ tests_testURL_SOURCES = \ nodist_tests_testURL_SOURCES = \ $(BUILT_SOURCES) tests_testURL_LDADD = \ + clients/libclients.la \ + servers/libservers.la \ http/libsquid-http.la \ + ftp/libftp.la \ anyp/libanyp.la \ ident/libident.la \ acl/libacls.la \ diff --git a/src/RequestFlags.h b/src/RequestFlags.h index f8ab00f958..ebf84fc8b6 100644 --- a/src/RequestFlags.h +++ b/src/RequestFlags.h @@ -127,6 +127,8 @@ public: bool done_follow_x_forwarded_for :1; /** set for ssl-bumped requests */ bool sslBumped :1; + /// carries a representation of an FTP command [received on ftp_port] + bool ftpNative :1; bool destinationIpLookedUp:1; /** request to reset the TCP stream */ bool resetTcp:1; diff --git a/src/acl/forward.h b/src/acl/forward.h index 02f32a0b87..c6dbec2a8a 100644 --- a/src/acl/forward.h +++ b/src/acl/forward.h @@ -21,6 +21,9 @@ class Tree; } // namespace Acl +class allow_t; +typedef void ACLCB(allow_t, void *); + #define ACL_NAME_SZ 64 // TODO: Consider renaming all users and removing. Cons: hides the difference diff --git a/src/clientStream.h b/src/clientStream.h index ce9b252c8a..8cce69a1a5 100644 --- a/src/clientStream.h +++ b/src/clientStream.h @@ -34,6 +34,7 @@ #include "base/RefCount.h" #include "dlink.h" +#include "clientStreamForward.h" #include "StoreIOBuffer.h" /** @@ -95,28 +96,6 @@ \li Because of the callback nature of squid, every node would have to keep these parameters in their context anyway, so this reduces programmer overhead. */ -/// \ingroup ClientStreamAPI -typedef RefCount ClientStreamData; - -class clientStreamNode; -class ClientHttpRequest; -class HttpReply; - -/* client stream read callback */ -/// \ingroup ClientStreamAPI -typedef void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer); - -/* client stream read */ -/// \ingroup ClientStreamAPI -typedef void CSR(clientStreamNode *, ClientHttpRequest *); - -/* client stream detach */ -/// \ingroup ClientStreamAPI -typedef void CSD(clientStreamNode *, ClientHttpRequest *); - -/// \ingroup ClientStreamAPI -typedef clientStream_status_t CSS(clientStreamNode *, ClientHttpRequest *); - /// \ingroup ClientStreamAPI class clientStreamNode { diff --git a/src/clientStreamForward.h b/src/clientStreamForward.h new file mode 100644 index 0000000000..fdc094b010 --- /dev/null +++ b/src/clientStreamForward.h @@ -0,0 +1,34 @@ +#ifndef SQUID_CLIENTSTREAM_FORWARD_H +#define SQUID_CLIENTSTREAM_FORWARD_H + +#include "enums.h" + +class Lock; +template class RefCount; + +/// \ingroup ClientStreamAPI +typedef RefCount ClientStreamData; + +/* Callbacks for ClientStreams API */ + +class clientStreamNode; +class ClientHttpRequest; +class HttpReply; +class StoreIOBuffer; + +/* client stream read callback */ +/// \ingroup ClientStreamAPI +typedef void CSCB(clientStreamNode *, ClientHttpRequest *, HttpReply *, StoreIOBuffer); + +/* client stream read */ +/// \ingroup ClientStreamAPI +typedef void CSR(clientStreamNode *, ClientHttpRequest *); + +/* client stream detach */ +/// \ingroup ClientStreamAPI +typedef void CSD(clientStreamNode *, ClientHttpRequest *); + +/// \ingroup ClientStreamAPI +typedef clientStream_status_t CSS(clientStreamNode *, ClientHttpRequest *); + +#endif /* SQUID_CLIENTSTREAM_FORWARD_H */ diff --git a/src/client_side.cc b/src/client_side.cc index 51fae83636..7d044d1c60 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -102,7 +102,6 @@ #include "errorpage.h" #include "fd.h" #include "fde.h" -#include "FtpServer.h" #include "fqdncache.h" #include "FwdState.h" #include "globals.h" @@ -125,6 +124,7 @@ #include "mime_header.h" #include "profiler/Profiler.h" #include "rfc1738.h" +#include "servers/forward.h" #include "SquidConfig.h" #include "SquidTime.h" #include "StatCounters.h" @@ -200,16 +200,11 @@ static IOACB httpAccept; #if USE_OPENSSL static IOACB httpsAccept; #endif -static IOACB ftpAccept; static CTCB clientLifetimeTimeout; static ClientSocketContext *parseHttpRequestAbort(ConnStateData * conn, const char *uri); -static ClientSocketContext *parseHttpRequest(ConnStateData *, HttpParser *, HttpRequestMethod *, Http::ProtocolVersion *); #if USE_IDENT static IDCB clientIdentDone; #endif -static CSCB clientSocketRecipient; -static CSD clientSocketDetach; -static void clientSetKeepaliveFlag(ClientHttpRequest *); static int clientIsContentLengthValid(HttpRequest * r); static int clientIsRequestBodyTooLargeForPolicy(int64_t bodyLength); @@ -221,63 +216,12 @@ void prepareLogWithRequestDetails(HttpRequest *, AccessLogEntry::Pointer &); #ifndef PURIFY static bool connIsUsable(ConnStateData * conn); #endif -static int responseFinishedOrFailed(HttpReply * rep, StoreIOBuffer const &receivedData); static void ClientSocketContextPushDeferredIfNeeded(ClientSocketContext::Pointer deferredRequest, ConnStateData * conn); static void clientUpdateSocketStats(LogTags logType, size_t size); char *skipLeadingSpace(char *aString); static void connNoteUseOfBuffer(ConnStateData* conn, size_t byteCount); -static void FtpChangeState(ConnStateData *connState, const ConnStateData::FtpState newState, const char *reason); -static IOACB FtpAcceptDataConnection; -static void FtpCloseDataConnection(ConnStateData *conn); -static ClientSocketContext *FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::ProtocolVersion *http_ver); -static bool FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms); -static CNCB FtpHandleConnectDone; - -static void FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer data); -typedef void FtpReplyHandler(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data); -static FtpReplyHandler FtpHandleFeatReply; -static FtpReplyHandler FtpHandlePasvReply; -static FtpReplyHandler FtpHandlePortReply; -static FtpReplyHandler FtpHandleErrorReply; -static FtpReplyHandler FtpHandleDataReply; -static FtpReplyHandler FtpHandleUploadReply; -static FtpReplyHandler FtpHandleEprtReply; -static FtpReplyHandler FtpHandleEpsvReply; - -static void FtpWriteEarlyReply(ConnStateData *conn, const int code, const char *msg); -static void FtpWriteReply(ClientSocketContext *context, MemBuf &mb); -static void FtpWriteCustomReply(ClientSocketContext *context, const int code, const char *msg, const HttpReply *reply = NULL); -static void FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply); -static void FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply, AsyncCall::Pointer call); -static void FtpWriteErrorReply(ClientSocketContext *context, const HttpReply *reply, const int status); - -static void FtpPrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = ""); -static IOCB FtpWroteEarlyReply; -static IOCB FtpWroteReply; -static IOCB FtpWroteReplyData; - -typedef bool FtpRequestHandler(ClientSocketContext *context, String &cmd, String ¶ms); -static FtpRequestHandler FtpHandleRequest; -static FtpRequestHandler FtpHandleFeatRequest; -static FtpRequestHandler FtpHandlePasvRequest; -static FtpRequestHandler FtpHandlePortRequest; -static FtpRequestHandler FtpHandleDataRequest; -static FtpRequestHandler FtpHandleUploadRequest; -static FtpRequestHandler FtpHandleEprtRequest; -static FtpRequestHandler FtpHandleEpsvRequest; -static FtpRequestHandler FtpHandleCwdRequest; -static FtpRequestHandler FtpHandlePassRequest; -static FtpRequestHandler FtpHandleCdupRequest; - -static bool FtpCheckDataConnPre(ClientSocketContext *context); -static bool FtpCheckDataConnPost(ClientSocketContext *context); -static void FtpSetDataCommand(ClientSocketContext *context); -static void FtpSetReply(ClientSocketContext *context, const int code, const char *msg); -static bool FtpSupportedCommand(const String &name); - - clientStreamNode * ClientSocketContext::getTail() const { @@ -319,25 +263,6 @@ ConnStateData::readSomeData() Comm::Read(clientConnection, reader); } -void -ConnStateData::readSomeFtpData() -{ - if (ftp.reader != NULL) - return; - - const size_t availSpace = sizeof(ftp.uploadBuf) - ftp.uploadAvailSize; - if (availSpace <= 0) - return; - - debugs(33, 4, HERE << ftp.dataConn << ": reading FTP data..."); - - typedef CommCbMemFunT Dialer; - ftp.reader = JobCallback(33, 5, Dialer, this, - ConnStateData::clientReadFtpData); - comm_read(ftp.dataConn, ftp.uploadBuf + ftp.uploadAvailSize, availSpace, - ftp.reader); -} - void ClientSocketContext::removeFromConnectionList(ConnStateData * conn) { @@ -430,7 +355,7 @@ ClientSocketContext::ClientSocketContext(const Comm::ConnectionPointer &aConn, C void ClientSocketContext::writeControlMsg(HttpControlMsg &msg) { - const HttpReply::Pointer rep(msg.reply); + HttpReply::Pointer rep(msg.reply); Must(rep != NULL); // remember the callback @@ -439,25 +364,7 @@ ClientSocketContext::writeControlMsg(HttpControlMsg &msg) AsyncCall::Pointer call = commCbCall(33, 5, "ClientSocketContext::wroteControlMsg", CommIoCbPtrFun(&WroteControlMsg, this)); - if (getConn()->isFtp) { - FtpWriteForwardedReply(this, rep.getRaw(), call); - return; - } - - // apply selected clientReplyContext::buildReplyHeader() mods - // it is not clear what headers are required for control messages - rep->header.removeHopByHopEntries(); - rep->header.putStr(HDR_CONNECTION, "keep-alive"); - httpHdrMangleList(&rep->header, http->request, ROR_REPLY); - - MemBuf *mb = rep->pack(); - - debugs(11, 2, "HTTP Client " << clientConnection); - debugs(11, 2, "HTTP Client CONTROL MSG:\n---------\n" << mb->buf << "\n----------"); - - Comm::Write(clientConnection, mb, call); - - delete mb; + getConn()->writeControlMsgAndCall(this, rep.getRaw(), call); } /// called when we wrote the 1xx response @@ -479,6 +386,9 @@ ClientSocketContext::wroteControlMsg(const Comm::ConnectionPointer &conn, char * // close on 1xx errors to be conservative and to simplify the code // (if we do not close, we must notify the source of a failure!) conn->close(); + + // XXX: writeControlMsgAndCall() should handle writer-specific writing + // results, including errors and then call us with success/failure outcome. } /// wroteControlMsg() wrapper: ClientSocketContext is not an AsyncJob @@ -959,8 +869,6 @@ ConnStateData::~ConnStateData() assert(this != NULL); debugs(33, 3, HERE << clientConnection); - FtpCloseDataConnection(this); - if (isOpen()) debugs(33, DBG_IMPORTANT, "BUG: ConnStateData did not close " << clientConnection); @@ -981,7 +889,7 @@ ConnStateData::~ConnStateData() * to set this relatively early in the request processing * to handle hacks for broken servers and clients. */ -static void +void clientSetKeepaliveFlag(ClientHttpRequest * http) { HttpRequest *request = http->request; @@ -1054,15 +962,6 @@ ClientSocketContext::deferRecipientForLater(clientStreamNode * node, HttpReply * return; } -int -responseFinishedOrFailed(HttpReply * rep, StoreIOBuffer const & receivedData) -{ - if (rep == NULL && receivedData.data == NULL && receivedData.length == 0) - return 1; - - return 0; -} - bool ClientSocketContext::startOfOutput() const { @@ -1559,7 +1458,7 @@ ClientSocketContext::sendStartOfMessage(HttpReply * rep, StoreIOBuffer bodyData) * data context is not NULL * There are no more entries in the stream chain. */ -static void +void clientSocketRecipient(clientStreamNode * node, ClientHttpRequest * http, HttpReply * rep, StoreIOBuffer receivedData) { @@ -1580,37 +1479,10 @@ clientSocketRecipient(clientStreamNode * node, ClientHttpRequest * http, /* TODO: check offset is what we asked for */ - if (context != http->getConn()->getCurrentContext()) { + if (context != http->getConn()->getCurrentContext()) context->deferRecipientForLater(node, rep, receivedData); - PROF_stop(clientSocketRecipient); - return; - } - - if (http->getConn()->isFtp) { - assert(context->http == http); - FtpHandleReply(context.getRaw(), rep, receivedData); - PROF_stop(clientSocketRecipient); - return; - } - - // After sending Transfer-Encoding: chunked (at least), always send - // the last-chunk if there was no error, ignoring responseFinishedOrFailed. - const bool mustSendLastChunk = http->request->flags.chunkedReply && - !http->request->flags.streamError && !context->startOfOutput(); - if (responseFinishedOrFailed(rep, receivedData) && !mustSendLastChunk) { - context->writeComplete(context->clientConnection, NULL, 0, Comm::OK); - PROF_stop(clientSocketRecipient); - return; - } - - if (!context->startOfOutput()) - context->sendBody(rep, receivedData); - else { - assert(rep); - http->al->reply = rep; - HTTPMSGLOCK(http->al->reply); - context->sendStartOfMessage(rep, receivedData); - } + else + http->getConn()->handleReply(rep, receivedData); PROF_stop(clientSocketRecipient); } @@ -1660,8 +1532,7 @@ ConnStateData::readNextRequest() typedef CommCbMemFunT TimeoutDialer; AsyncCall::Pointer timeoutCall = JobCallback(33, 5, TimeoutDialer, this, ConnStateData::requestTimeout); - const int timeout = isFtp ? Config.Timeout.ftpClientIdle : - Config.Timeout.clientIdlePconn; + const int timeout = idleTimeout(); commSetConnTimeout(clientConnection, timeout, timeoutCall); readSomeData(); @@ -2041,10 +1912,6 @@ ClientSocketContext::writeComplete(const Comm::ConnectionPointer &conn, char *bu } } -SQUIDCEXTERN CSR clientGetMoreData; -SQUIDCEXTERN CSS clientReplyStatus; -SQUIDCEXTERN CSD clientReplyDetach; - static ClientSocketContext * parseHttpRequestAbort(ConnStateData * csd, const char *uri) { @@ -2286,7 +2153,7 @@ prepareTransparentURL(ConnStateData * conn, ClientHttpRequest *http, char *url, * \return NULL on incomplete requests, * a ClientSocketContext structure on success or failure. */ -static ClientSocketContext * +ClientSocketContext * parseHttpRequest(ConnStateData *csd, HttpParser *hp, HttpRequestMethod * method_p, Http::ProtocolVersion *http_ver) { char *req_hdr = NULL; @@ -2539,12 +2406,19 @@ ConnStateData::connFinishedWithConn(int size) return 0; } +void +ConnStateData::consumeInput(const size_t byteCount) +{ + assert(byteCount > 0 && byteCount <= in.buf.length()); + in.buf.consume(byteCount); + debugs(33, 5, "in.buf has " << in.buf.length() << " unused bytes"); +} + +// TODO: Remove when renaming ConnStateData void connNoteUseOfBuffer(ConnStateData* conn, size_t byteCount) { - assert(byteCount > 0 && byteCount <= conn->in.buf.length()); - conn->in.buf.consume(byteCount); - debugs(33, 5, "conn->in.buf has " << conn->in.buf.length() << " bytes unused."); + conn->consumeInput(byteCount); } /// respond with ERR_TOO_BIG if request header exceeds request_header_max_size @@ -2680,7 +2554,7 @@ bool ConnStateData::serveDelayedError(ClientSocketContext *context) } #endif // USE_OPENSSL -static void +void clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *context, const HttpRequestMethod& method, Http::ProtocolVersion http_ver) { ClientHttpRequest *http = context->http; @@ -2691,12 +2565,19 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c bool unsupportedTe = false; bool expectBody = false; - /* We have an initial client stream in place should it be needed */ - /* setup our private context */ - if (!conn->isFtp) - context->registerWithConn(); + // temporary hack to avoid splitting this huge function with sensitive code + const bool isFtp = !hp; + if (isFtp) { + // In FTP, case, we already have the request parsed and checked, so we + // only need to go through the final body/conn setup to doCallouts(). + assert(http->request); + request = http->request; + notedUseOfBuffer = true; + goto doFtpAndHttp; + } if (context->flags.parsed_ok == 0) { + assert(hp); clientStreamNode *node = context->getClientReplyContext(); debugs(33, 2, "clientProcessRequest: Invalid Request"); conn->quitAfterError(NULL); @@ -2721,11 +2602,6 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c goto finish; } - if (conn->isFtp) { - assert(http->request); - request = http->request; - notedUseOfBuffer = true; - } else if ((request = HttpRequest::CreateFromUrlAndMethod(http->uri, method)) == NULL) { clientStreamNode *node = context->getClientReplyContext(); debugs(33, 5, "Invalid URL: " << http->uri); @@ -2762,7 +2638,7 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c /* compile headers */ /* we should skip request line! */ /* XXX should actually know the damned buffer size here */ - if (!conn->isFtp && http_ver.major >= 1 && + if (http_ver.major >= 1 && !request->parseHeader(HttpParserHdrBuf(hp), HttpParserHdrSz(hp))) { clientStreamNode *node = context->getClientReplyContext(); debugs(33, 5, "Failed to parse request headers:\n" << HttpParserHdrBuf(hp)); @@ -2777,6 +2653,11 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c goto finish; } +doFtpAndHttp: + // Some blobs below are still HTTP-specific, but we would have to rewrite + // this entire function to remove them from the FTP code path. Connection + // setup and body_pipe preparation blobs are needed for FTP. + request->clientConnectionManager = conn; request->flags.accelerated = http->flags.accel; @@ -2896,12 +2777,12 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c } } - if (!conn->isFtp) { + if (!isFtp) { http->request = request.getRaw(); HTTPMSGLOCK(http->request); } - clientSetKeepaliveFlag(http); + clientSetKeepaliveFlag(http); // Let tunneling code be fully responsible for CONNECT requests if (http->request->method == Http::METHOD_CONNECT) { context->mayUseConnection(true); @@ -2944,7 +2825,7 @@ clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *c goto finish; } - if (!conn->isFtp) { + if (!isFtp) { // We may stop producing, comm_close, and/or call setReplyToError() // below, so quit on errors to avoid http->doCallouts() if (!conn->handleRequestBodyData()) @@ -2980,46 +2861,6 @@ finish: } } -void -ConnStateData::processFtpRequest(ClientSocketContext *const context) -{ - ClientHttpRequest *const http = context->http; - assert(http != NULL); - HttpRequest *const request = http->request; - assert(request != NULL); - debugs(33, 9, request); - - HttpHeader &header = request->header; - assert(header.has(HDR_FTP_COMMAND)); - String &cmd = header.findEntry(HDR_FTP_COMMAND)->value; - assert(header.has(HDR_FTP_ARGUMENTS)); - String ¶ms = header.findEntry(HDR_FTP_ARGUMENTS)->value; - - const bool fwd = !http->storeEntry() && - FtpHandleRequest(context, cmd, params); - - if (http->storeEntry() != NULL) { - debugs(33, 4, "got an immediate response"); - assert(http->storeEntry() != NULL); - clientSetKeepaliveFlag(http); - context->pullData(); - } else if (fwd) { - debugs(33, 4, "forwarding request to server side"); - assert(http->storeEntry() == NULL); - clientProcessRequest(this, &parser_, context, request->method, - request->http_ver); - } else { - debugs(33, 4, "will resume processing later"); - } -} - -void -ConnStateData::resumeFtpRequest(ClientSocketContext *const context) -{ - debugs(33, 4, "resuming"); - processFtpRequest(context); -} - static void connStripBufferWhitespace (ConnStateData * conn) { @@ -3029,6 +2870,12 @@ connStripBufferWhitespace (ConnStateData * conn) } } +int +ConnStateData::pipelinePrefetchMax() const +{ + return Config.pipeline_max_prefetch; +} + /** * Limit the number of concurrent requests. * \return true when there are available position(s) in the pipeline queue for another request. @@ -3041,7 +2888,7 @@ ConnStateData::concurrentRequestQueueFilled() const // default to the configured pipeline size. // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue - const int concurrentRequestLimit = (isFtp ? 0 : Config.pipeline_max_prefetch) + 1; + const int concurrentRequestLimit = pipelinePrefetchMax() + 1; // when queue filled already we cant add more. if (existingRequestCount >= concurrentRequestLimit) { @@ -3061,7 +2908,6 @@ ConnStateData::concurrentRequestQueueFilled() const bool ConnStateData::clientParseRequests() { - HttpRequestMethod method; bool parsed_req = false; debugs(33, 5, HERE << clientConnection << ": attempting to parse"); @@ -3079,18 +2925,8 @@ ConnStateData::clientParseRequests() if (concurrentRequestQueueFilled()) break; - ClientSocketContext *context = NULL; Http::ProtocolVersion http_ver; - if (!isFtp) { - /* Begin the parsing */ - PROF_start(parseHttpRequest); - HttpParserInit(&parser_, in.buf.c_str(), in.buf.length()); - - /* Process request */ - context = parseHttpRequest(this, &parser_, &method, &http_ver); - PROF_stop(parseHttpRequest); - } else - context = FtpParseRequest(this, &method, &http_ver); + ClientSocketContext *context = parseOneRequest(http_ver); /* partial or incomplete request */ if (!context) { @@ -3107,14 +2943,7 @@ ConnStateData::clientParseRequests() CommTimeoutCbPtrFun(clientLifetimeTimeout, context->http)); commSetConnTimeout(clientConnection, Config.Timeout.lifetime, timeoutCall); - if (!isFtp) - clientProcessRequest(this, &parser_, context, method, http_ver); - else { - // Process FTP request asynchronously to make sure FTP - // data connection accept callback is fired first. - CallJobHere1(33, 4, CbcPointer(this), - ConnStateData, ConnStateData::processFtpRequest, context); - } + processParsedRequest(context, http_ver); parsed_req = true; // XXX: do we really need to parse everything right NOW ? @@ -3227,59 +3056,6 @@ ConnStateData::clientReadRequest(const CommIoCbParams &io) clientAfterReadingRequests(); } -void -ConnStateData::clientReadFtpData(const CommIoCbParams &io) -{ - debugs(33,5,HERE << io.conn << " size " << io.size); - Must(ftp.reader != NULL); - ftp.reader = NULL; - - assert(Comm::IsConnOpen(ftp.dataConn)); - assert(io.conn->fd == ftp.dataConn->fd); - - if (io.flag == Comm::OK && bodyPipe != NULL) { - if (io.size > 0) { - kb_incr(&(statCounter.client_http.kbytes_in), io.size); - - char *const current_buf = ftp.uploadBuf + ftp.uploadAvailSize; - if (io.buf != current_buf) - memmove(current_buf, io.buf, io.size); - ftp.uploadAvailSize += io.size; - handleFtpRequestData(); - } else if (io.size == 0) { - debugs(33, 5, HERE << io.conn << " closed"); - FtpCloseDataConnection(this); - if (ftp.uploadAvailSize <= 0) - finishDechunkingRequest(true); - } - } else { // not Comm::Flags::OK or unexpected read - debugs(33, 5, HERE << io.conn << " closed"); - FtpCloseDataConnection(this); - finishDechunkingRequest(false); - } - -} - -void -ConnStateData::handleFtpRequestData() -{ - assert(bodyPipe != NULL); - - debugs(33,5, HERE << "handling FTP request data for " << clientConnection); - const size_t putSize = bodyPipe->putMoreData(ftp.uploadBuf, - ftp.uploadAvailSize); - if (putSize > 0) { - ftp.uploadAvailSize -= putSize; - if (ftp.uploadAvailSize > 0) - memmove(ftp.uploadBuf, ftp.uploadBuf + putSize, ftp.uploadAvailSize); - } - - if (Comm::IsConnOpen(ftp.dataConn)) - readSomeFtpData(); - else if (ftp.uploadAvailSize <= 0) - finishDechunkingRequest(true); -} - /** * called when new request data has been read from the socket * @@ -3423,24 +3199,6 @@ ConnStateData::abortChunkedRequestBody(const err_type error) flags.readMore = false; } -void -ConnStateData::noteMoreBodySpaceAvailable(BodyPipe::Pointer ) -{ - if (isFtp) { - handleFtpRequestData(); - return; - } - - if (!handleRequestBodyData()) - return; - - // too late to read more body - if (!isOpen() || stoppedReceiving()) - return; - - readSomeData(); -} - void ConnStateData::noteBodyConsumerAborted(BodyPipe::Pointer ) { @@ -3448,12 +3206,7 @@ ConnStateData::noteBodyConsumerAborted(BodyPipe::Pointer ) if (bodyPipe != NULL) bodyPipe->enableAutoConsumption(); - if (isFtp) { - FtpCloseDataConnection(this); - return; - } - - stopReceiving("virgin request body consumer aborted"); // closes ASAP + // kids extend } /** general lifetime handler for HTTP requests */ @@ -3484,8 +3237,7 @@ clientLifetimeTimeout(const CommTimeoutCbParams &io) } ConnStateData::ConnStateData(const MasterXaction::Pointer &xact): - AsyncJob("ConnStateData"), - isFtp(xact->squidPort->transport.protocol == AnyP::PROTO_FTP), // TODO: convert into a method? + AsyncJob("ConnStateData"), // kids overwrite #if USE_OPENSSL sslBumpMode(Ssl::bumpEnd), switchedToHttps_(false), @@ -3507,6 +3259,15 @@ ConnStateData::ConnStateData(const MasterXaction::Pointer &xact): log_addr = xact->tcpClient->remote; log_addr.applyMask(Config.Addrs.client_netmask); + flags.readMore = true; // kids may overwrite +} + +void +ConnStateData::start() +{ + BodyProducer::start(); + HttpControlMsgSink::start(); + // ensure a buffer is present for this connection in.maybeMakeSpaceAvailable(); @@ -3536,61 +3297,17 @@ ConnStateData::ConnStateData(const MasterXaction::Pointer &xact): #if USE_IDENT if (Ident::TheConfig.identLookup) { ACLFilledChecklist identChecklist(Ident::TheConfig.identLookup, NULL, NULL); - identChecklist.src_addr = xact->tcpClient->remote; - identChecklist.my_addr = xact->tcpClient->local; + identChecklist.src_addr = clientConnection->remote; + identChecklist.my_addr = clientConnection->local; if (identChecklist.fastCheck() == ACCESS_ALLOWED) - Ident::Start(xact->tcpClient, clientIdentDone, this); + Ident::Start(clientConnection, clientIdentDone, this); } #endif clientdbEstablished(clientConnection->remote, 1); - flags.readMore = !isFtp; - - if (isFtp) { - ftp.gotEpsvAll = false; - ftp.readGreeting = false; - ftp.state = FTP_BEGIN; - ftp.uploadAvailSize = 0; - } -} - -/** Handle a new connection on HTTP socket. */ -void -httpAccept(const CommAcceptCbParams ¶ms) -{ - MasterXaction::Pointer xact = params.xaction; - AnyP::PortCfgPointer s = xact->squidPort; - - // NP: it is possible the port was reconfigured when the call or accept() was queued. - - if (params.flag != Comm::OK) { - // Its possible the call was still queued when the client disconnected - debugs(33, 2, "httpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); - return; - } - - debugs(33, 4, HERE << params.conn << ": accepted"); - fd_note(params.conn->fd, "client http connect"); - - if (s->tcp_keepalive.enabled) { - commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout); - } - - ++ incoming_sockets_accepted; - - // Socket is ready, setup the connection manager to start using it - ConnStateData *connState = new ConnStateData(xact); - - typedef CommCbMemFunT TimeoutDialer; - AsyncCall::Pointer timeoutCall = JobCallback(33, 5, - TimeoutDialer, connState, ConnStateData::requestTimeout); - commSetConnTimeout(params.conn, Config.Timeout.request, timeoutCall); - - connState->readSomeData(); - #if USE_DELAY_POOLS - fd_table[params.conn->fd].clientInfo = NULL; + fd_table[clientConnection->fd].clientInfo = NULL; if (Config.onoff.client_db) { /* it was said several times that client write limiter does not work if client_db is disabled */ @@ -3601,8 +3318,8 @@ httpAccept(const CommAcceptCbParams ¶ms) // TODO: we check early to limit error response bandwith but we // should recheck when we can honor delay_pool_uses_indirect // TODO: we should also pass the port details for myportname here. - ch.src_addr = params.conn->remote; - ch.my_addr = params.conn->local; + ch.src_addr = clientConnection->remote; + ch.my_addr = clientConnection->local; for (unsigned int pool = 0; pool < pools.size(); ++pool) { @@ -3614,11 +3331,11 @@ httpAccept(const CommAcceptCbParams ¶ms) /* request client information from db after we did all checks this will save hash lookup if client failed checks */ - ClientInfo * cli = clientdbGetInfo(params.conn->remote); + ClientInfo * cli = clientdbGetInfo(clientConnection->remote); assert(cli); /* put client info in FDE */ - fd_table[params.conn->fd].clientInfo = cli; + fd_table[clientConnection->fd].clientInfo = cli; /* setup write limiter for this request */ const double burst = floor(0.5 + @@ -3632,11 +3349,13 @@ httpAccept(const CommAcceptCbParams ¶ms) } } #endif + + // kids must extend to actually start doing something (e.g., reading) } -/** handle a new FTP connection */ -static void -ftpAccept(const CommAcceptCbParams ¶ms) +/** Handle a new connection on HTTP socket. */ +void +httpAccept(const CommAcceptCbParams ¶ms) { MasterXaction::Pointer xact = params.xaction; AnyP::PortCfgPointer s = xact->squidPort; @@ -3645,33 +3364,22 @@ ftpAccept(const CommAcceptCbParams ¶ms) if (params.flag != Comm::OK) { // Its possible the call was still queued when the client disconnected - debugs(33, 2, "ftpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); + debugs(33, 2, "httpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); return; } debugs(33, 4, HERE << params.conn << ": accepted"); - fd_note(params.conn->fd, "client ftp connect"); + fd_note(params.conn->fd, "client http connect"); if (s->tcp_keepalive.enabled) { commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout); } - ++incoming_sockets_accepted; + ++ incoming_sockets_accepted; // Socket is ready, setup the connection manager to start using it - ConnStateData *connState = new ConnStateData(xact); - - if (connState->transparent()) { - char buf[MAX_IPSTRLEN]; - connState->clientConnection->local.toUrl(buf, MAX_IPSTRLEN); - connState->ftp.host = buf; - const char *uri = connState->ftpBuildUri(); - debugs(33, 5, HERE << "FTP transparent URL: " << uri); - } - - FtpWriteEarlyReply(connState, 220, "Service ready"); - - // TODO: Merge common httpAccept() parts, applying USE_DELAY_POOLS to FTP. + ConnStateData *connState = Http::NewServer(xact); + AsyncJob::Start(connState); } #if USE_OPENSSL @@ -3934,10 +3642,19 @@ httpsAccept(const CommAcceptCbParams ¶ms) ++incoming_sockets_accepted; // Socket is ready, setup the connection manager to start using it - ConnStateData *connState = new ConnStateData(xact); + ConnStateData *connState = Https::NewServer(xact); + AsyncJob::Start(connState); // will eventually call postHttpsAccept() +} + +void +ConnStateData::postHttpsAccept() +{ + // XXX: Remove these change-minimizing variables before commit + ConnStateData *connState = this; + const AnyP::PortCfgPointer s = port; if (s->flags.tunnelSslBumping) { - debugs(33, 5, "httpsAccept: accept transparent connection: " << params.conn); + debugs(33, 5, "httpsAccept: accept transparent connection: " << clientConnection); if (!Config.accessList.ssl_bump) { httpsSslBumpAccessCheckDone(ACCESS_DENIED, connState); @@ -3948,13 +3665,13 @@ httpsAccept(const CommAcceptCbParams ¶ms) // using tproxy/intercept provided destination IP and port. HttpRequest *request = new HttpRequest(); static char ip[MAX_IPSTRLEN]; - assert(params.conn->flags & (COMM_TRANSPARENT | COMM_INTERCEPTION)); - request->SetHost(params.conn->local.toStr(ip, sizeof(ip))); - request->port = params.conn->local.port(); + assert(clientConnection->flags & (COMM_TRANSPARENT | COMM_INTERCEPTION)); + request->SetHost(clientConnection->local.toStr(ip, sizeof(ip))); + request->port = clientConnection->local.port(); request->myportname = s->name; ACLFilledChecklist *acl_checklist = new ACLFilledChecklist(Config.accessList.ssl_bump, request, NULL); - acl_checklist->src_addr = params.conn->remote; + acl_checklist->src_addr = clientConnection->remote; acl_checklist->my_addr = s->s; acl_checklist->nonBlockingCheck(httpsSslBumpAccessCheckDone, connState); return; @@ -4401,34 +4118,25 @@ clientHttpsConnectionsOpen(void) } #endif -static void -clientFtpConnectionsOpen(void) -{ - for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) { - if (MAXTCPLISTENPORTS == NHttpSockets) { - debugs(1, DBG_IMPORTANT, "Ignoring 'ftp_port' lines exceeding the limit."); - debugs(1, DBG_IMPORTANT, "The limit is " << MAXTCPLISTENPORTS << " FTP ports."); - continue; - } - - // Fill out a Comm::Connection which IPC will open as a listener for us - s->listenConn = new Comm::Connection; - s->listenConn->local = s->s; - s->listenConn->flags = COMM_NONBLOCKING | (s->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) | - (s->flags.natIntercept ? COMM_INTERCEPTION : 0); - - // setup the subscriptions such that new connections accepted by listenConn are handled by FTP - typedef CommCbFunPtrCallT AcceptCall; - RefCount subCall = commCbCall(5, 5, "ftpAccept", CommAcceptCbPtrFun(ftpAccept, CommAcceptCbParams(NULL))); - Subscription::Pointer sub = new CallSubscription(subCall); +void +clientStartListeningOn(AnyP::PortCfgPointer &port, const RefCount< CommCbFunPtrCallT > &subCall, const Ipc::FdNoteId fdNote) { + // Fill out a Comm::Connection which IPC will open as a listener for us + port->listenConn = new Comm::Connection; + port->listenConn->local = port->s; + port->listenConn->flags = COMM_NONBLOCKING | (port->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) | + (port->flags.natIntercept ? COMM_INTERCEPTION : 0); + + // route new connections to subCall + typedef CommCbFunPtrCallT AcceptCall; + Subscription::Pointer sub = new CallSubscription(subCall); + AsyncCall::Pointer listenCall = asyncCall(33, 2, "clientListenerConnectionOpened", + ListeningStartedDialer(&clientListenerConnectionOpened, + port, fdNote, sub)); + Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, port->listenConn, fdNote, listenCall); - AsyncCall::Pointer listenCall = asyncCall(33, 2, "clientListenerConnectionOpened", - ListeningStartedDialer(&clientListenerConnectionOpened, - s, Ipc::fdnFtpSocket, sub)); - Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnFtpSocket, listenCall); - HttpSockets[NHttpSockets] = -1; - ++NHttpSockets; - } + assert(NHttpSockets < MAXTCPLISTENPORTS); + HttpSockets[NHttpSockets] = -1; + ++NHttpSockets; } /// process clientHttpConnectionsOpen result @@ -4463,7 +4171,7 @@ clientOpenListenSockets(void) #if USE_OPENSSL clientHttpsConnectionsOpen(); #endif - clientFtpConnectionsOpen(); + Ftp::StartListening(); if (NHttpSockets < 1) fatal("No HTTP, HTTPS or FTP ports configured"); @@ -4490,13 +4198,7 @@ clientConnectionsClose(void) } #endif - for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) { - if (s->listenConn != NULL) { - debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local); - s->listenConn->close(); - s->listenConn = NULL; - } - } + Ftp::StopListening(); // TODO see if we can drop HttpSockets array entirely */ for (int i = 0; i < NHttpSockets; ++i) { @@ -4584,8 +4286,6 @@ clientAclChecklistCreate(const acl_access * acl, ClientHttpRequest * http) return ch; } -CBDATA_CLASS_INIT(ConnStateData); - bool ConnStateData::transparent() const { @@ -4737,16 +4437,6 @@ ConnStateData::clientPinnedConnectionClosed(const CommCloseCbParams &io) const bool sawZeroReply = pinning.zeroReply; // reset when unpinning unpinConnection(false); - if (isFtp) { - // if the server control connection is gone, reset state to login again - // TODO: merge with similar code in FtpHandleUserRequest() - debugs(33, 5, "will need to re-login due to FTP server closure"); - ftp.readGreeting = false; - FtpChangeState(this, ConnStateData::FTP_BEGIN, "server closure"); - // XXX: Not enough. Gateway::ServerStateData::sendCommand() will not - // re-login because clientState() is not ConnStateData::FTP_CONNECTED. - } - if (sawZeroReply && clientConnection != NULL) { debugs(33, 3, "Closing client connection on pinned zero reply."); clientConnection->close(); @@ -4920,1314 +4610,3 @@ ConnStateData::unpinConnection(const bool andClose) /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host * connection has gone away */ } - -const char * -ConnStateData::ftpBuildUri(const char *file) -{ - ftp.uri = "ftp://"; - ftp.uri.append(ftp.host); - if (port->ftp_track_dirs && ftp.workingDir.size()) { - if (ftp.workingDir[0] != '/') - ftp.uri.append("/"); - ftp.uri.append(ftp.workingDir); - } - - if (ftp.uri[ftp.uri.size() - 1] != '/') - ftp.uri.append("/"); - - if (port->ftp_track_dirs && file) { - //remove any '/' from the beginning of path - while (*file == '/') - ++file; - ftp.uri.append(file); - } - - return ftp.uri.termedBuf(); -} - -void -ConnStateData::ftpSetWorkingDir(const char *dir) -{ - ftp.workingDir = dir; -} - -static void -FtpAcceptDataConnection(const CommAcceptCbParams ¶ms) -{ - ConnStateData *connState = static_cast(params.data); - - if (params.flag != Comm::OK) { - // Its possible the call was still queued when the client disconnected - debugs(33, 2, HERE << connState->ftp.dataListenConn << ": accept " - "failure: " << xstrerr(params.xerrno)); - return; - } - - debugs(33, 4, "accepted " << params.conn); - fd_note(params.conn->fd, "passive client ftp data"); - ++incoming_sockets_accepted; - - if (!connState->clientConnection) { - debugs(33, 5, "late data connection?"); - FtpCloseDataConnection(connState); // in case we are still listening - params.conn->close(); - } else - if (params.conn->remote != connState->clientConnection->remote) { - debugs(33, 2, "rogue data conn? ctrl: " << connState->clientConnection->remote); - params.conn->close(); - // Some FTP servers close control connection here, but it may make - // things worse from DoS p.o.v. and no better from data stealing p.o.v. - } else { - FtpCloseDataConnection(connState); - connState->ftp.dataConn = params.conn; - connState->ftp.uploadAvailSize = 0; - debugs(33, 7, "ready for data"); - if (connState->ftp.onDataAcceptCall != NULL) { - AsyncCall::Pointer call = connState->ftp.onDataAcceptCall; - connState->ftp.onDataAcceptCall = NULL; - // If we got an upload request, start reading data from the client. - if (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) - connState->readSomeFtpData(); - else - Must(connState->ftp.state == ConnStateData::FTP_HANDLE_DATA_REQUEST); - MemBuf mb; - mb.init(); - mb.Printf("150 Data connection opened.\r\n"); - Comm::Write(connState->clientConnection, &mb, call); - } - } -} - -static void -FtpCloseDataConnection(ConnStateData *conn) -{ - if (conn->ftp.listener != NULL) { - conn->ftp.listener->cancel("no longer needed"); - conn->ftp.listener = NULL; - } - - if (Comm::IsConnOpen(conn->ftp.dataListenConn)) { - debugs(33, 5, HERE << "FTP closing client data listen socket: " << - *conn->ftp.dataListenConn); - conn->ftp.dataListenConn->close(); - } - conn->ftp.dataListenConn = NULL; - - if (conn->ftp.reader != NULL) { - // Comm::ReadCancel can deal with negative FDs - Comm::ReadCancel(conn->ftp.dataConn->fd, conn->ftp.reader); - conn->ftp.reader = NULL; - } - - if (Comm::IsConnOpen(conn->ftp.dataConn)) { - debugs(33, 5, HERE << "FTP closing client data connection: " << - *conn->ftp.dataConn); - conn->ftp.dataConn->close(); - } - conn->ftp.dataConn = NULL; -} - -/// Writes FTP [error] response before we fully parsed the FTP request and -/// created the corresponding HTTP request wrapper for that FTP request. -static void -FtpWriteEarlyReply(ConnStateData *connState, const int code, const char *msg) -{ - debugs(33, 7, HERE << code << ' ' << msg); - assert(99 < code && code < 1000); - - MemBuf mb; - mb.init(); - mb.Printf("%i %s\r\n", code, msg); - - AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteEarlyReply", - CommIoCbPtrFun(&FtpWroteEarlyReply, connState)); - Comm::Write(connState->clientConnection, &mb, call); - - connState->flags.readMore = false; - - // TODO: Create master transaction. Log it in FtpWroteEarlyReply. -} - -static void -FtpWriteReply(ClientSocketContext *context, MemBuf &mb) -{ - debugs(11, 2, "FTP Client " << context->clientConnection); - debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << - "\n----------"); - - AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", - CommIoCbPtrFun(&FtpWroteReply, context)); - Comm::Write(context->clientConnection, &mb, call); -} - -static void -FtpWriteCustomReply(ClientSocketContext *context, const int code, const char *msg, const HttpReply *reply) -{ - debugs(33, 7, HERE << code << ' ' << msg); - assert(99 < code && code < 1000); - - const bool sendDetails = reply != NULL && - reply->header.has(HDR_FTP_STATUS) && reply->header.has(HDR_FTP_REASON); - - MemBuf mb; - mb.init(); - if (sendDetails) { - mb.Printf("%i-%s\r\n", code, msg); - mb.Printf(" Server reply:\r\n"); - FtpPrintReply(mb, reply, " "); - mb.Printf("%i \r\n", code); - } else - mb.Printf("%i %s\r\n", code, msg); - - FtpWriteReply(context, mb); -} - -static void -FtpChangeState(ConnStateData *connState, const ConnStateData::FtpState newState, const char *reason) -{ - assert(connState); - if (connState->ftp.state == newState) { - debugs(33, 3, "client state unchanged at " << connState->ftp.state << - " because " << reason); - connState->ftp.state = newState; - } else { - debugs(33, 3, "client state was " << connState->ftp.state << - ", now " << newState << " because " << reason); - connState->ftp.state = newState; - } -} - -/** Parse an FTP request - * - * \note Sets result->flags.parsed_ok to 0 if failed to parse the request, - * to 1 if the request was correctly parsed. - * \param[in] connState a ConnStateData. The caller must make sure it is not null - * \param[out] mehtod_p will be set as a side-effect of the parsing. - * Pointed-to value will be set to Http::METHOD_NONE in case of - * parsing failure - * \param[out] http_ver will be set as a side-effect of the parsing - * \return NULL on incomplete requests, - * a ClientSocketContext structure on success or failure. - */ -static ClientSocketContext * -FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::ProtocolVersion *http_ver) -{ - *http_ver = Http::ProtocolVersion(1, 1); - - // TODO: Use tokenizer for parsing instead of raw pointer manipulation. - const char *inBuf = connState->in.buf.rawContent(); - - const char *const eor = - static_cast(memchr(inBuf, '\n', - min(static_cast(connState->in.buf.length()), Config.maxRequestHeaderSize))); - - if (eor == NULL && connState->in.buf.length() >= Config.maxRequestHeaderSize) { - FtpChangeState(connState, ConnStateData::FTP_ERROR, "huge req"); - FtpWriteEarlyReply(connState, 421, "Too large request"); - return NULL; - } - - if (eor == NULL) { - debugs(33, 5, HERE << "Incomplete request, waiting for end of request"); - return NULL; - } - - const size_t req_sz = eor + 1 - inBuf; - - // skip leading whitespaces - const char *boc = inBuf; // beginning of command - while (boc < eor && isspace(*boc)) ++boc; - if (boc >= eor) { - debugs(33, 5, HERE << "Empty request, ignoring"); - connNoteUseOfBuffer(connState, req_sz); - return NULL; - } - - const char *eoc = boc; // end of command - while (eoc < eor && !isspace(*eoc)) ++eoc; - connState->in.buf.setAt(eoc - inBuf, '\0'); - - const char *bop = eoc + 1; // beginning of parameter - while (bop < eor && isspace(*bop)) ++bop; - if (bop < eor) { - const char *eop = eor - 1; - while (isspace(*eop)) --eop; - assert(eop >= bop); - connState->in.buf.setAt(eop + 1 - inBuf, '\0'); - } else - bop = NULL; - - debugs(33, 7, HERE << "Parsed FTP command " << boc << " with " << - (bop == NULL ? "no " : "") << "parameters" << - (bop != NULL ? ": " : "") << bop); - - // TODO: Use SBuf instead of String - const String cmd = boc; - String params = bop; - - connNoteUseOfBuffer(connState, req_sz); - - if (!connState->ftp.readGreeting) { - // the first command must be USER - if (!connState->pinning.pinned && cmd.caseCmp("USER") != 0) { - FtpWriteEarlyReply(connState, 530, "Must login first"); - return NULL; - } - } - - // We need to process USER request now because it sets ftp server Hostname. - if (cmd.caseCmp("USER") == 0 && - !FtpHandleUserRequest(connState, cmd, params)) - return NULL; - - if (!FtpSupportedCommand(cmd)) { - FtpWriteEarlyReply(connState, 502, "Unknown or unsupported command"); - return NULL; - } - - *method_p = !cmd.caseCmp("APPE") || !cmd.caseCmp("STOR") || - !cmd.caseCmp("STOU") ? Http::METHOD_PUT : Http::METHOD_GET; - - char *uri; - const char *aPath = params.size() > 0 && Ftp::hasPathParameter(cmd)? - params.termedBuf() : NULL; - uri = xstrdup(connState->ftpBuildUri(aPath)); - HttpRequest *const request = - HttpRequest::CreateFromUrlAndMethod(uri, *method_p); - if (request == NULL) { - debugs(33, 5, HERE << "Invalid FTP URL: " << connState->ftp.uri); - FtpWriteEarlyReply(connState, 501, "Invalid host"); - connState->ftp.uri.clean(); - safe_free(uri); - return NULL; - } - - request->http_ver = *http_ver; - - // Our fake Request-URIs are not distinctive enough for caching to work - request->flags.cachable = false; // XXX: reset later by maybeCacheable() - request->flags.noCache = true; - - request->header.putStr(HDR_FTP_COMMAND, cmd.termedBuf()); - request->header.putStr(HDR_FTP_ARGUMENTS, params.termedBuf() != NULL ? - params.termedBuf() : ""); - if (*method_p == Http::METHOD_PUT) { - request->header.putStr(HDR_EXPECT, "100-continue"); - request->header.putStr(HDR_TRANSFER_ENCODING, "chunked"); - } - - ClientHttpRequest *const http = new ClientHttpRequest(connState); - http->request = request; - HTTPMSGLOCK(http->request); - http->req_sz = req_sz; - http->uri = uri; - - ClientSocketContext *const result = - new ClientSocketContext(connState->clientConnection, http); - - StoreIOBuffer tempBuffer; - tempBuffer.data = result->reqbuf; - tempBuffer.length = HTTP_REQBUF_SZ; - - ClientStreamData newServer = new clientReplyContext(http); - ClientStreamData newClient = result; - clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, - clientReplyStatus, newServer, clientSocketRecipient, - clientSocketDetach, newClient, tempBuffer); - - result->registerWithConn(); - result->flags.parsed_ok = 1; - connState->flags.readMore = false; - return result; -} - -static void -FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer data) -{ - if (context->http && context->http->al != NULL && - !context->http->al->reply && reply) { - context->http->al->reply = reply; - HTTPMSGLOCK(context->http->al->reply); - } - - static FtpReplyHandler *handlers[] = { - NULL, // FTP_BEGIN - NULL, // FTP_CONNECTED - FtpHandleFeatReply, // FTP_HANDLE_FEAT - FtpHandlePasvReply, // FTP_HANDLE_PASV - FtpHandlePortReply, // FTP_HANDLE_PORT - FtpHandleDataReply, // FTP_HANDLE_DATA_REQUEST - FtpHandleUploadReply, // FTP_HANDLE_UPLOAD_REQUEST - FtpHandleEprtReply,// FTP_HANDLE_EPRT - FtpHandleEpsvReply,// FTP_HANDLE_EPSV - NULL, // FTP_HANDLE_CWD - NULL, //FTP_HANDLE_PASS - NULL, // FTP_HANDLE_CDUP - FtpHandleErrorReply // FTP_ERROR - }; - const ConnStateData::FtpState state = context->getConn()->ftp.state; - FtpReplyHandler *const handler = handlers[state]; - if (handler) - (*handler)(context, reply, data); - else - FtpWriteForwardedReply(context, reply); -} - -static void -FtpHandleFeatReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - if (context->http->request->errType != ERR_NONE) { - FtpWriteCustomReply(context, 502, "Server does not support FEAT", reply); - return; - } - - HttpReply *filteredReply = reply->clone(); - HttpHeader &filteredHeader = filteredReply->header; - - // Remove all unsupported commands from the response wrapper. - int deletedCount = 0; - HttpHeaderPos pos = HttpHeaderInitPos; - bool hasEPRT = false; - bool hasEPSV = false; - int prependSpaces = 1; - while (const HttpHeaderEntry *e = filteredHeader.getEntry(&pos)) { - if (e->id == HDR_FTP_PRE) { - // assume RFC 2389 FEAT response format, quoted by Squid: - // <"> SP NAME [SP PARAMS] <"> - // but accommodate MS servers sending four SPs before NAME - if (e->value.size() < 4) - continue; - const char *raw = e->value.termedBuf(); - if (raw[0] != '"' || raw[1] != ' ') - continue; - const char *beg = raw + 1 + strspn(raw + 1, " "); // after quote and spaces - // command name ends with (SP parameter) or quote - const char *end = beg + strcspn(beg, " \""); - - if (end <= beg) - continue; - - // compute the number of spaces before the command - prependSpaces = beg - raw - 1; - - const String cmd = e->value.substr(beg-raw, end-raw); - - if (!FtpSupportedCommand(cmd)) - filteredHeader.delAt(pos, deletedCount); - - if (cmd == "EPRT") - hasEPRT = true; - else if (cmd == "EPSV") - hasEPSV = true; - } - } - - char buf[256]; - int insertedCount = 0; - if (!hasEPRT) { - snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT"); - filteredHeader.putStr(HDR_FTP_PRE, buf); - ++insertedCount; - } - if (!hasEPSV) { - snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV"); - filteredHeader.putStr(HDR_FTP_PRE, buf); - ++insertedCount; - } - - if (deletedCount || insertedCount) { - filteredHeader.refreshMask(); - debugs(33, 5, "deleted " << deletedCount << " inserted " << insertedCount); - } - - FtpWriteForwardedReply(context, filteredReply); -} - -static void -FtpHandlePasvReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - if (context->http->request->errType != ERR_NONE) { - FtpWriteCustomReply(context, 502, "Server does not support PASV", reply); - return; - } - - FtpCloseDataConnection(context->getConn()); - - Comm::ConnectionPointer conn = new Comm::Connection; - ConnStateData * const connState = context->getConn(); - conn->flags = COMM_NONBLOCKING; - conn->local = connState->transparent() ? - connState->port->s : context->clientConnection->local; - conn->local.port(0); - const char *const note = connState->ftp.uri.termedBuf(); - comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); - if (!Comm::IsConnOpen(conn)) { - debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << - conn->local << " error: " << errno); - FtpWriteCustomReply(context, 451, "Internal error"); - return; - } - - typedef CommCbFunPtrCallT AcceptCall; - RefCount subCall = commCbCall(5, 5, "FtpAcceptDataConnection", - CommAcceptCbPtrFun(FtpAcceptDataConnection, connState)); - Subscription::Pointer sub = new CallSubscription(subCall); - connState->ftp.listener = subCall.getRaw(); - connState->ftp.dataListenConn = conn; - AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); - - char addr[MAX_IPSTRLEN]; - // remote server in interception setups and local address otherwise - const Ip::Address &server = connState->transparent() ? - context->clientConnection->local : conn->local; - server.toStr(addr, MAX_IPSTRLEN, AF_INET); - addr[MAX_IPSTRLEN - 1] = '\0'; - for (char *c = addr; *c != '\0'; ++c) { - if (*c == '.') - *c = ','; - } - - // conn->fd is the client data connection (and its local port) - const unsigned short port = comm_local_port(conn->fd); - conn->local.port(port); - - // In interception setups, we combine remote server address with a - // local port number and hope that traffic will be redirected to us. - MemBuf mb; - mb.init(); - - // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp - // versions block responses that use those alternative syntax rules! - mb.Printf("227 Entering Passive Mode (%s,%i,%i).\r\n", - addr, - static_cast(port / 256), - static_cast(port % 256)); - - debugs(11, 3, Raw("writing", mb.buf, mb.size)); - FtpWriteReply(context, mb); -} - -static void -FtpHandlePortReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - if (context->http->request->errType != ERR_NONE) { - FtpWriteCustomReply(context, 502, "Server does not support PASV (converted from PORT)", reply); - return; - } - - FtpWriteCustomReply(context, 200, "PORT successfully converted to PASV."); - - // and wait for RETR -} - -static void -FtpHandleErrorReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - ConnStateData *const connState = context->getConn(); - if (!connState->pinning.pinned) // we failed to connect to server - connState->ftp.uri.clean(); - // 421: we will close due to FTP_ERROR - FtpWriteErrorReply(context, reply, 421); -} - -static void -FtpHandleDataReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - ConnStateData *const conn = context->getConn(); - - if (reply != NULL && reply->sline.status() != Http::scOkay) { - FtpWriteForwardedReply(context, reply); - if (conn && Comm::IsConnOpen(conn->ftp.dataConn)) { - debugs(33, 3, "closing " << conn->ftp.dataConn << " on KO reply"); - FtpCloseDataConnection(conn); - } - return; - } - - if (!conn->ftp.dataConn) { - // We got STREAM_COMPLETE (or error) and closed the client data conn. - debugs(33, 3, "ignoring FTP srv data response after clt data closure"); - return; - } - - if (!FtpCheckDataConnPost(context)) { - FtpWriteCustomReply(context, 425, "Data connection is not established."); - FtpCloseDataConnection(conn); - return; - } - - debugs(33, 7, HERE << data.length); - - if (data.length <= 0) { - FtpWroteReplyData(conn->clientConnection, NULL, 0, Comm::OK, 0, context); - return; - } - - MemBuf mb; - mb.init(data.length + 1, data.length + 1); - mb.append(data.data, data.length); - - AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReplyData", - CommIoCbPtrFun(&FtpWroteReplyData, context)); - Comm::Write(conn->ftp.dataConn, &mb, call); - - context->noteSentBodyBytes(data.length); -} - -static void -FtpWroteReplyData(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) -{ - if (errflag == Comm::ERR_CLOSING) - return; - - ClientSocketContext *const context = static_cast(data); - ConnStateData *const connState = context->getConn(); - - if (errflag != Comm::OK) { - debugs(33, 3, HERE << "FTP reply data writing failed: " << - xstrerr(xerrno)); - FtpCloseDataConnection(connState); - FtpWriteCustomReply(context, 426, "Data connection error; transfer aborted"); - return; - } - - assert(context->http); - context->http->out.size += size; - - switch (context->socketState()) { - case STREAM_NONE: - debugs(33, 3, "Keep going"); - context->pullData(); - return; - case STREAM_COMPLETE: - debugs(33, 3, HERE << "FTP reply data transfer successfully complete"); - FtpWriteCustomReply(context, 226, "Transfer complete"); - break; - case STREAM_UNPLANNED_COMPLETE: - debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE"); - FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); - break; - case STREAM_FAILED: - debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_FAILED"); - FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); - break; - default: - fatal("unreachable code"); - } - - FtpCloseDataConnection(connState); -} - -static void -FtpHandleUploadReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - FtpWriteForwardedReply(context, reply); - // note that the client data connection may already be closed by now -} - -static void -FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply) -{ - const AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", - CommIoCbPtrFun(&FtpWroteReply, context)); - FtpWriteForwardedReply(context, reply, call); -} - -static void -FtpHandleEprtReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - if (context->http->request->errType != ERR_NONE) { - FtpWriteCustomReply(context, 502, "Server does not support PASV (converted from EPRT)", reply); - return; - } - - FtpWriteCustomReply(context, 200, "EPRT successfully converted to PASV."); - - // and wait for RETR -} - -static void -FtpHandleEpsvReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) -{ - if (context->http->request->errType != ERR_NONE) { - FtpWriteCustomReply(context, 502, "Cannot connect to server", reply); - return; - } - - FtpCloseDataConnection(context->getConn()); - - Comm::ConnectionPointer conn = new Comm::Connection; - ConnStateData * const connState = context->getConn(); - conn->flags = COMM_NONBLOCKING; - conn->local = connState->transparent() ? - connState->port->s : context->clientConnection->local; - conn->local.port(0); - const char *const note = connState->ftp.uri.termedBuf(); - comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); - if (!Comm::IsConnOpen(conn)) { - debugs(5, DBG_CRITICAL, "comm_open_listener failed: " << - conn->local << " error: " << errno); - FtpWriteCustomReply(context, 451, "Internal error"); - return; - } - - typedef CommCbFunPtrCallT AcceptCall; - RefCount subCall = commCbCall(5, 5, "FtpAcceptDataConnection", - CommAcceptCbPtrFun(FtpAcceptDataConnection, connState)); - Subscription::Pointer sub = new CallSubscription(subCall); - connState->ftp.listener = subCall.getRaw(); - connState->ftp.dataListenConn = conn; - AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); - - // conn->fd is the client data connection (and its local port) - const unsigned int port = comm_local_port(conn->fd); - conn->local.port(port); - - // In interception setups, we combine remote server address with a - // local port number and hope that traffic will be redirected to us. - MemBuf mb; - mb.init(); - mb.Printf("229 Entering Extended Passive Mode (|||%u|)\r\n", port); - - debugs(11, 3, Raw("writing", mb.buf, mb.size)); - FtpWriteReply(context, mb); -} - -/// writes FTP error response with given status and reply-derived error details -static void -FtpWriteErrorReply(ClientSocketContext *context, const HttpReply *reply, const int status) -{ - MemBuf mb; - mb.init(); - - assert(context->http); - const HttpRequest *request = context->http->request; - assert(request); - if (request->errType != ERR_NONE) - mb.Printf("%i-%s\r\n", status, errorPageName(request->errType)); - - if (request->errDetail > 0) { - // XXX: > 0 may not always mean that this is an errno - mb.Printf("%i-Error: (%d) %s\r\n", status, - request->errDetail, - strerror(request->errDetail)); - } - - // XXX: Remove hard coded names. Use an error page template instead. - const Adaptation::History::Pointer ah = request->adaptHistory(); - if (ah != NULL) { // XXX: add adapt::allMeta.getByName("X-Response-Info"); - const String desc = ah->allMeta.getByName("X-Response-Desc"); - if (info.size()) - mb.Printf("%i-Information: %s\r\n", status, info.termedBuf()); - if (desc.size()) - mb.Printf("%i-Description: %s\r\n", status, desc.termedBuf()); - } - - assert(reply != NULL); - const char *reason = reply->header.has(HDR_FTP_REASON) ? - reply->header.getStr(HDR_FTP_REASON): - reply->sline.reason(); - - mb.Printf("%i %s\r\n", status, reason); // error terminating line - - // TODO: errorpage.cc should detect FTP client and use - // configurable FTP-friendly error templates which we should - // write to the client "as is" instead of hiding most of the info - - FtpWriteReply(context, mb); -} - -/// writes FTP response based on HTTP reply that is not an FTP-response wrapper -static void -FtpWriteForwardedForeign(ClientSocketContext *context, const HttpReply *reply) -{ - ConnStateData *const connState = context->getConn(); - FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "foreign reply"); - //Close the data connection. - FtpCloseDataConnection(connState); - // 451: We intend to keep the control connection open. - FtpWriteErrorReply(context, reply, 451); -} - -static void -FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply, AsyncCall::Pointer call) -{ - assert(reply != NULL); - const HttpHeader &header = reply->header; - ConnStateData *const connState = context->getConn(); - - // adaptation and forwarding errors lack HDR_FTP_STATUS - if (!header.has(HDR_FTP_STATUS)) { - FtpWriteForwardedForeign(context, reply); - return; - } - - assert(header.has(HDR_FTP_REASON)); - - const int status = header.getInt(HDR_FTP_STATUS); - debugs(33, 7, HERE << "status: " << status); - - // Status 125 or 150 implies upload or data request, but we still check - // the state in case the server is buggy. - if ((status == 125 || status == 150) && - (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST || - connState->ftp.state == ConnStateData::FTP_HANDLE_DATA_REQUEST)) { - if (FtpCheckDataConnPost(context)) { - // If the data connection is ready, start reading data (here) - // and forward the response to client (further below). - debugs(33, 7, "data connection established, start data transfer"); - if (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) - connState->readSomeFtpData(); - } else { - // If we are waiting to accept the data connection, keep waiting. - if (Comm::IsConnOpen(connState->ftp.dataListenConn)) { - debugs(33, 7, "wait for the client to establish a data connection"); - connState->ftp.onDataAcceptCall = call; - // TODO: Add connect timeout for passive connections listener? - // TODO: Remember server response so that we can forward it? - } else { - // Either the connection was establised and closed after the - // data was transferred OR we failed to establish an active - // data connection and already sent the error to the client. - // In either case, there is nothing more to do. - debugs(33, 7, "done with data OR active connection failed"); - } - return; - } - } - - MemBuf mb; - mb.init(); - FtpPrintReply(mb, reply); - - debugs(11, 2, "FTP Client " << context->clientConnection); - debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << - "\n----------"); - - Comm::Write(context->clientConnection, &mb, call); -} - -static void -FtpPrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix) -{ - const HttpHeader &header = reply->header; - - HttpHeaderPos pos = HttpHeaderInitPos; - while (const HttpHeaderEntry *e = header.getEntry(&pos)) { - if (e->id == HDR_FTP_PRE) { - String raw; - if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw)) - mb.Printf("%s\r\n", raw.termedBuf()); - } - } - - if (header.has(HDR_FTP_STATUS)) { - const char *reason = header.getStr(HDR_FTP_REASON); - mb.Printf("%i %s\r\n", header.getInt(HDR_FTP_STATUS), - (reason ? reason : 0)); - } -} - -static void -FtpWroteEarlyReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) -{ - if (errflag == Comm::ERR_CLOSING) - return; - - if (errflag != Comm::OK) { - debugs(33, 3, HERE << "FTP reply writing failed: " << xstrerr(xerrno)); - conn->close(); - return; - } - - ConnStateData *const connState = static_cast(data); - ClientSocketContext::Pointer context = connState->getCurrentContext(); - if (context != NULL && context->http) { - context->http->out.size += size; - context->http->out.headers_sz += size; - } - - connState->flags.readMore = true; - connState->readSomeData(); -} - -static void -FtpWroteReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) -{ - if (errflag == Comm::ERR_CLOSING) - return; - - if (errflag != Comm::OK) { - debugs(33, 3, HERE << "FTP reply writing failed: " << - xstrerr(xerrno)); - conn->close(); - return; - } - - ClientSocketContext *const context = - static_cast(data); - ConnStateData *const connState = context->getConn(); - - assert(context->http); - context->http->out.size += size; - context->http->out.headers_sz += size; - - if (connState->ftp.state == ConnStateData::FTP_ERROR) { - debugs(33, 5, "closing on FTP server error"); - conn->close(); - return; - } - - const clientStream_status_t socketState = context->socketState(); - debugs(33, 5, "FTP client stream state " << socketState); - switch (socketState) { - case STREAM_UNPLANNED_COMPLETE: - case STREAM_FAILED: - conn->close(); - return; - - case STREAM_NONE: - case STREAM_COMPLETE: - connState->flags.readMore = true; - FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "FtpWroteReply"); - if (connState->in.bodyParser) - connState->finishDechunkingRequest(false); - context->keepaliveNextRequest(); - return; - } -} - -bool -FtpHandleRequest(ClientSocketContext *context, String &cmd, String ¶ms) { - if (HttpRequest *request = context->http->request) { - MemBuf *mb = new MemBuf; - Packer p; - mb->init(); - packerToMemInit(&p, mb); - request->pack(&p); - packerClean(&p); - - debugs(11, 2, "FTP Client " << context->clientConnection); - debugs(11, 2, "FTP Client REQUEST:\n---------\n" << mb->buf << - "\n----------"); - delete mb; - } - - static std::pair handlers[] = { - std::make_pair("LIST", FtpHandleDataRequest), - std::make_pair("NLST", FtpHandleDataRequest), - std::make_pair("MLSD", FtpHandleDataRequest), - std::make_pair("FEAT", FtpHandleFeatRequest), - std::make_pair("PASV", FtpHandlePasvRequest), - std::make_pair("PORT", FtpHandlePortRequest), - std::make_pair("RETR", FtpHandleDataRequest), - std::make_pair("EPRT", FtpHandleEprtRequest), - std::make_pair("EPSV", FtpHandleEpsvRequest), - std::make_pair("CWD", FtpHandleCwdRequest), - std::make_pair("PASS", FtpHandlePassRequest), - std::make_pair("CDUP", FtpHandleCdupRequest), - }; - - FtpRequestHandler *handler = NULL; - if (context->http->request->method == Http::METHOD_PUT) - handler = FtpHandleUploadRequest; - else { - for (size_t i = 0; i < sizeof(handlers) / sizeof(*handlers); ++i) { - if (cmd.caseCmp(handlers[i].first) == 0) { - handler = handlers[i].second; - break; - } - } - } - - return handler != NULL ? (*handler)(context, cmd, params) : true; -} - -/// Called to parse USER command, which is required to create an HTTP request -/// wrapper. Thus, errors are handled with FtpWriteEarlyReply() here. -bool -FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms) -{ - if (params.size() == 0) { - FtpWriteEarlyReply(connState, 501, "Missing username"); - return false; - } - - const String::size_type eou = params.rfind('@'); - if (eou == String::npos || eou + 1 >= params.size()) { - FtpWriteEarlyReply(connState, 501, "Missing host"); - return false; - } - - const String login = params.substr(0, eou); - String host = params.substr(eou + 1, params.size()); - // If we can parse it as raw IPv6 address, then surround with "[]". - // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is. - if (host.pos(":")) { - char ipBuf[MAX_IPSTRLEN]; - Ip::Address ipa; - ipa = host.termedBuf(); - if (!ipa.isAnyAddr()) { - ipa.toHostStr(ipBuf, MAX_IPSTRLEN); - host = ipBuf; - } - } - connState->ftp.host = host; - - String oldUri; - if (connState->ftp.readGreeting) - oldUri = connState->ftp.uri; - - connState->ftpSetWorkingDir(NULL); - connState->ftpBuildUri(); - - if (!connState->ftp.readGreeting) { - debugs(11, 3, "set URI to " << connState->ftp.uri); - } else if (oldUri.caseCmp(connState->ftp.uri) == 0) { - debugs(11, 5, "keep URI as " << oldUri); - } else { - debugs(11, 3, "reset URI from " << oldUri << " to " << connState->ftp.uri); - FtpCloseDataConnection(connState); - connState->ftp.readGreeting = false; - connState->unpinConnection(true); // close control connection to the server - FtpChangeState(connState, ConnStateData::FTP_BEGIN, "URI reset"); - } - - params.cut(eou); - - return true; -} - -bool -FtpHandleFeatRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_FEAT, "FtpHandleFeatRequest"); - - return true; -} - -bool -FtpHandlePasvRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - ConnStateData *const connState = context->getConn(); - assert(connState); - if (connState->ftp.gotEpsvAll) { - FtpSetReply(context, 500, "Bad PASV command"); - return false; - } - - if (params.size() > 0) { - FtpSetReply(context, 501, "Unexpected parameter"); - return false; - } - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASV, "FtpHandlePasvRequest"); - // no need to fake PASV request via FtpSetDataCommand() in true PASV case - return true; -} - -/// [Re]initializes dataConn for active data transfers. Does not connect. -static -bool FtpCreateDataConnection(ClientSocketContext *context, Ip::Address cltAddr) -{ - ConnStateData *const connState = context->getConn(); - assert(connState); - assert(connState->clientConnection != NULL); - assert(!connState->clientConnection->remote.isAnyAddr()); - - if (cltAddr != connState->clientConnection->remote) { - debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << connState->clientConnection->remote); - // Closing the control connection would not help with attacks because - // the client is evidently able to connect to us. Besides, closing - // makes retrials easier for the client and more damaging to us. - FtpSetReply(context, 501, "Prohibited parameter value"); - return false; - } - - FtpCloseDataConnection(context->getConn()); - - Comm::ConnectionPointer conn = new Comm::Connection(); - conn->remote = cltAddr; - - // Use local IP address of the control connection as the source address - // of the active data connection, or some clients will refuse to accept. - conn->flags |= COMM_DOBIND; - conn->local = connState->clientConnection->local; - // RFC 959 requires active FTP connections to originate from port 20 - // but that would preclude us from supporting concurrent transfers! (XXX?) - conn->local.port(0); - - debugs(11, 3, "will actively connect from " << conn->local << " to " << - conn->remote); - - context->getConn()->ftp.dataConn = conn; - context->getConn()->ftp.uploadAvailSize = 0; - return true; -} - -bool -FtpHandlePortRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - // TODO: Should PORT errors trigger FtpCloseDataConnection() cleanup? - - const ConnStateData *connState = context->getConn(); - if (connState->ftp.gotEpsvAll) { - FtpSetReply(context, 500, "Rejecting PORT after EPSV ALL"); - return false; - } - - if (!params.size()) { - FtpSetReply(context, 501, "Missing parameter"); - return false; - } - - Ip::Address cltAddr; - if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) { - FtpSetReply(context, 501, "Invalid parameter"); - return false; - } - - if (!FtpCreateDataConnection(context, cltAddr)) - return false; - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PORT, "FtpHandlePortRequest"); - FtpSetDataCommand(context); - return true; // forward our fake PASV request -} - -bool -FtpHandleDataRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - if (!FtpCheckDataConnPre(context)) - return false; - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_DATA_REQUEST, "FtpHandleDataRequest"); - - return true; -} - -bool -FtpHandleUploadRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - if (!FtpCheckDataConnPre(context)) - return false; - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_UPLOAD_REQUEST, "FtpHandleDataRequest"); - - return true; -} - -bool -FtpHandleEprtRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - debugs(11, 3, "Process an EPRT " << params); - - const ConnStateData *connState = context->getConn(); - if (connState->ftp.gotEpsvAll) { - FtpSetReply(context, 500, "Rejecting EPRT after EPSV ALL"); - return false; - } - - if (!params.size()) { - FtpSetReply(context, 501, "Missing parameter"); - return false; - } - - Ip::Address cltAddr; - if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) { - FtpSetReply(context, 501, "Invalid parameter"); - return false; - } - - if (!FtpCreateDataConnection(context, cltAddr)) - return false; - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_EPRT, "FtpHandleEprtRequest"); - FtpSetDataCommand(context); - return true; // forward our fake PASV request -} - -bool -FtpHandleEpsvRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - debugs(11, 3, "Process an EPSV command with params: " << params); - if (params.size() <= 0) { - // treat parameterless EPSV as "use the protocol of the ctrl conn" - } else if (params.caseCmp("ALL") == 0) { - ConnStateData *connState = context->getConn(); - FtpSetReply(context, 200, "EPSV ALL ok"); - connState->ftp.gotEpsvAll = true; - return false; - } else if (params.cmp("2") == 0) { - if (!Ip::EnableIpv6) { - FtpSetReply(context, 522, "Network protocol not supported, use (1)"); - return false; - } - } else if (params.cmp("1") != 0) { - FtpSetReply(context, 501, "Unsupported EPSV parameter"); - return false; - } - - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_EPSV, "FtpHandleEpsvRequest"); - FtpSetDataCommand(context); - return true; // forward our fake PASV request -} - -bool -FtpHandleCwdRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CWD, "FtpHandleCwdRequest"); - return true; -} - -bool -FtpHandlePassRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASS, "FtpHandlePassRequest"); - return true; -} - -bool -FtpHandleCdupRequest(ClientSocketContext *context, String &cmd, String ¶ms) -{ - FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CDUP, "FtpHandleCdupRequest"); - return true; -} - -// Convert client PORT, EPRT, PASV, or EPSV data command to Squid PASV command. -// Squid server-side decides what data command to use on that side. -void -FtpSetDataCommand(ClientSocketContext *context) -{ - ClientHttpRequest *const http = context->http; - assert(http != NULL); - HttpRequest *const request = http->request; - assert(request != NULL); - HttpHeader &header = request->header; - header.delById(HDR_FTP_COMMAND); - header.putStr(HDR_FTP_COMMAND, "PASV"); - header.delById(HDR_FTP_ARGUMENTS); - header.putStr(HDR_FTP_ARGUMENTS, ""); - debugs(11, 5, "client data command converted to fake PASV"); -} - -/// check that client data connection is ready for future I/O or at least -/// has a chance of becoming ready soon. -bool -FtpCheckDataConnPre(ClientSocketContext *context) -{ - ConnStateData *const connState = context->getConn(); - if (Comm::IsConnOpen(connState->ftp.dataConn)) - return true; - - if (Comm::IsConnOpen(connState->ftp.dataListenConn)) { - // We are still waiting for a client to connect to us after PASV. - // Perhaps client's data conn handshake has not reached us yet. - // After we talk to the server, FtpCheckDataConnPost() will recheck. - debugs(33, 3, "expecting clt data conn " << connState->ftp.dataListenConn); - return true; - } - - if (!connState->ftp.dataConn || connState->ftp.dataConn->remote.isAnyAddr()) { - debugs(33, 5, "missing " << connState->ftp.dataConn); - // TODO: use client address and default port instead. - FtpSetReply(context, 425, "Use PORT or PASV first"); - return false; - } - - // active transfer: open a connection from Squid to client - AsyncCall::Pointer connector = context->getConn()->ftp.connector = - commCbCall(17, 3, "FtpConnectDoneWrapper", - CommConnectCbPtrFun(FtpHandleConnectDone, context)); - - Comm::ConnOpener *cs = new Comm::ConnOpener(connState->ftp.dataConn, - connector, - Config.Timeout.connect); - AsyncJob::Start(cs); - return false; // ConnStateData::processFtpRequest waits FtpHandleConnectDone -} - -/// Check that client data connection is ready for immediate I/O. -static bool -FtpCheckDataConnPost(ClientSocketContext *context) -{ - ConnStateData *connState = context->getConn(); - assert(connState); - const Comm::ConnectionPointer &dataConn = connState->ftp.dataConn; - if (!Comm::IsConnOpen(dataConn)) { - debugs(33, 3, "missing client data conn: " << dataConn); - return false; - } - return true; -} - -void -FtpHandleConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data) -{ - ClientSocketContext *context = static_cast(data); - context->getConn()->ftp.connector = NULL; - - if (status != Comm::OK) { - conn->close(); - FtpSetReply(context, 425, "Cannot open data connection."); - assert(context->http && context->http->storeEntry() != NULL); - } else { - assert(context->getConn()->ftp.dataConn == conn); - assert(Comm::IsConnOpen(conn)); - fd_note(conn->fd, "active client ftp data"); - } - context->getConn()->resumeFtpRequest(context); -} - -void -FtpSetReply(ClientSocketContext *context, const int code, const char *msg) -{ - ClientHttpRequest *const http = context->http; - assert(http != NULL); - assert(http->storeEntry() == NULL); - - HttpReply *const reply = new HttpReply; - reply->sline.set(Http::ProtocolVersion(1, 1), Http::scNoContent); - HttpHeader &header = reply->header; - header.putTime(HDR_DATE, squid_curtime); - { - HttpHdrCc cc; - cc.Private(); - header.putCc(&cc); - } - header.putInt64(HDR_CONTENT_LENGTH, 0); - header.putInt(HDR_FTP_STATUS, code); - header.putStr(HDR_FTP_REASON, msg); - reply->hdrCacheInit(); - - setLogUri(http, urlCanonicalClean(http->request)); - - clientStreamNode *const node = context->getClientReplyContext(); - clientReplyContext *const repContext = - dynamic_cast(node->data.getRaw()); - assert(repContext != NULL); - - RequestFlags flags; - flags.cachable = false; // force releaseRequest() in storeCreateEntry() - flags.noCache = true; - repContext->createStoreEntry(http->request->method, flags); - http->storeEntry()->replaceHttpReply(reply); -} - -/// Whether Squid FTP gateway supports a given feature (e.g., a command). -static bool -FtpSupportedCommand(const String &name) -{ - static std::set BlackList; - if (BlackList.empty()) { - /* Add FTP commands that Squid cannot gateway correctly */ - - // we probably do not support AUTH TLS.* and AUTH SSL, - // but let's disclaim all AUTH support to KISS, for now - BlackList.insert("AUTH"); - } - - // we claim support for all commands that we do not know about - return BlackList.find(name.termedBuf()) == BlackList.end(); -} diff --git a/src/client_side.h b/src/client_side.h index b3f5d6c5fc..cadb4d0b0e 100644 --- a/src/client_side.h +++ b/src/client_side.h @@ -33,9 +33,11 @@ #ifndef SQUID_CLIENTSIDE_H #define SQUID_CLIENTSIDE_H +#include "clientStreamForward.h" #include "comm.h" #include "HttpControlMsg.h" #include "HttpParser.h" +#include "ipc/FdNotes.h" #include "SBuf.h" #if USE_AUTH #include "auth/UserRequest.h" @@ -194,10 +196,9 @@ class ConnStateData : public BodyProducer, public HttpControlMsgSink public: explicit ConnStateData(const MasterXaction::Pointer &xact); - ~ConnStateData(); + virtual ~ConnStateData(); void readSomeData(); - void readSomeFtpData(); bool areAllContextsForThisConnection() const; void freeAllContexts(); void notifyAllContexts(const int xerrno); ///< tell everybody about the err @@ -295,9 +296,10 @@ public: void expectNoForwarding(); ///< cleans up virgin request [body] forwarding state + /* BodyPipe API */ BodyPipe::Pointer expectRequestBody(int64_t size); - virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer); - virtual void noteBodyConsumerAborted(BodyPipe::Pointer); + virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer) = 0; + virtual void noteBodyConsumerAborted(BodyPipe::Pointer) = 0; bool handleReadData(); bool handleRequestBodyData(); @@ -323,8 +325,11 @@ public: CachePeer *pinnedPeer() const {return pinning.peer;} bool pinnedAuth() const {return pinning.auth;} + /// called just before a FwdState-dispatched job starts using connection + virtual void notePeerConnection(Comm::ConnectionPointer) {} + // pining related comm callbacks - void clientPinnedConnectionClosed(const CommCloseCbParams &io); + virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io); // comm callbacks void clientReadRequest(const CommIoCbParams &io); @@ -333,6 +338,7 @@ public: void requestTimeout(const CommTimeoutCbParams ¶ms); // AsyncJob API + virtual void start(); virtual bool doneAll() const { return BodyProducer::doneAll() && false;} virtual void swanSong(); @@ -343,43 +349,10 @@ public: /// The caller assumes responsibility for connection closure detection. void stopPinnedConnectionMonitoring(); - const bool isFtp; - enum FtpState { - FTP_BEGIN, - FTP_CONNECTED, - FTP_HANDLE_FEAT, - FTP_HANDLE_PASV, - FTP_HANDLE_PORT, - FTP_HANDLE_DATA_REQUEST, - FTP_HANDLE_UPLOAD_REQUEST, - FTP_HANDLE_EPRT, - FTP_HANDLE_EPSV, - FTP_HANDLE_CWD, - FTP_HANDLE_PASS, - FTP_HANDLE_CDUP, - FTP_ERROR - }; - struct { - String uri; - String host; - String workingDir; - FtpState state; - bool readGreeting; - bool gotEpsvAll; ///< restrict data conn setup commands to just EPSV - AsyncCall::Pointer onDataAcceptCall; ///< who to call upon data connection acceptance - Comm::ConnectionPointer dataListenConn; - Comm::ConnectionPointer dataConn; - Ip::Address serverDataAddr; - char uploadBuf[CLIENT_REQ_BUF_SZ]; - size_t uploadAvailSize; - AsyncCall::Pointer listener; ///< set when we are passively listening - AsyncCall::Pointer connector; ///< set when we are actively connecting - AsyncCall::Pointer reader; ///< set when we are reading FTP data - } ftp; - const char *ftpBuildUri(const char *file = NULL); - void ftpSetWorkingDir(const char *dir); - #if USE_OPENSSL + /// the second part of old httpsAccept, waiting for future HttpsServer home + void postHttpsAccept(); + /// called by FwdState when it is done bumping the server void httpsPeeked(Comm::ConnectionPointer serverConnection); @@ -422,12 +395,20 @@ public: void finishDechunkingRequest(bool withSuccess); - void resumeFtpRequest(ClientSocketContext *const context); - /* clt_conn_tag=tag annotation access */ const SBuf &connectionTag() const { return connectionTag_; } void connectionTag(const char *aTag) { connectionTag_ = aTag; } + /// handle a control message received by context from a peer and call back + virtual void writeControlMsgAndCall(ClientSocketContext *context, HttpReply *rep, AsyncCall::Pointer &call) = 0; + + /// ClientStream calls this to supply response header (once) and data + /// for the current ClientSocketContext. + virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData) = 0; + + /// remove no longer needed leading bytes from the input buffer + void consumeInput(const size_t byteCount); + protected: void startDechunkingRequest(); void abortChunkedRequestBody(const err_type error); @@ -436,11 +417,19 @@ protected: void startPinnedConnectionMonitoring(); void clientPinnedConnectionRead(const CommIoCbParams &io); -private: + // TODO: document + virtual ClientSocketContext *parseOneRequest(Http::ProtocolVersion &ver) = 0; + virtual void processParsedRequest(ClientSocketContext *context, const Http::ProtocolVersion &ver) = 0; + + /// returning N allows a pipeline of 1+N requests (see pipeline_prefetch) + virtual int pipelinePrefetchMax() const; + + /// timeout to use when waiting for the next request + virtual time_t idleTimeout() const = 0; + +protected: int connFinishedWithConn(int size); void clientAfterReadingRequests(); - void processFtpRequest(ClientSocketContext *const context); - void handleFtpRequestData(); bool concurrentRequestQueueFilled() const; @@ -451,10 +440,6 @@ private: Auth::UserRequest::Pointer auth_; #endif - HttpParser parser_; - - // XXX: CBDATA plays with public/private and leaves the following 'private' fields all public... :( - #if USE_OPENSSL bool switchedToHttps_; /// The SSL server host name appears in CONNECT request or the server ip address for the intercepted requests @@ -476,8 +461,6 @@ private: BodyPipe::Pointer bodyPipe; // set when we are reading request body SBuf connectionTag_; ///< clt_conn_tag=Tag annotation for client connection - - CBDATA_CLASS2(ConnStateData); }; void setLogUri(ClientHttpRequest * http, char const *uri, bool cleanUrl = false); @@ -486,8 +469,23 @@ const char *findTrailingHTTPVersion(const char *uriAndHTTPVersion, const char *e int varyEvaluateMatch(StoreEntry * entry, HttpRequest * req); +void clientStartListeningOn(AnyP::PortCfgPointer &port, const RefCount< CommCbFunPtrCallT > &subCall, const Ipc::FdNoteId noteId); + void clientOpenListenSockets(void); void clientConnectionsClose(void); void httpRequestFree(void *); +void clientSetKeepaliveFlag(ClientHttpRequest *http); + +/* misplaced declaratrions of Stream callbacks provided/used by client side */ +SQUIDCEXTERN CSR clientGetMoreData; +SQUIDCEXTERN CSS clientReplyStatus; +SQUIDCEXTERN CSD clientReplyDetach; +CSCB clientSocketRecipient; +CSD clientSocketDetach; + +/* TODO: Move to HttpServer. Warning: Move requires large code nonchanges! */ +ClientSocketContext *parseHttpRequest(ConnStateData *, HttpParser *, HttpRequestMethod *, Http::ProtocolVersion *); +void clientProcessRequest(ConnStateData *conn, HttpParser *hp, ClientSocketContext *context, const HttpRequestMethod& method, Http::ProtocolVersion http_ver); +void clientPostHttpsAccept(ConnStateData *connState); #endif /* SQUID_CLIENTSIDE_H */ diff --git a/src/client_side_reply.h b/src/client_side_reply.h index 9dc9f3a12e..e83ae20f26 100644 --- a/src/client_side_reply.h +++ b/src/client_side_reply.h @@ -31,6 +31,7 @@ #ifndef SQUID_CLIENTSIDEREPLY_H #define SQUID_CLIENTSIDEREPLY_H +#include "acl/forward.h" #include "client_side_request.h" #include "ip/forward.h" #include "RequestFlags.h" diff --git a/src/FtpServer.cc b/src/clients/FtpClient.cc similarity index 91% rename from src/FtpServer.cc rename to src/clients/FtpClient.cc index 449455bde9..0f15040e17 100644 --- a/src/FtpServer.cc +++ b/src/clients/FtpClient.cc @@ -6,7 +6,7 @@ #include "squid.h" #include "acl/FilledChecklist.h" -#include "FtpServer.h" +#include "clients/FtpClient.h" #include "Mem.h" #include "SquidConfig.h" #include "StatCounters.h" @@ -17,6 +17,7 @@ #include "comm/Write.h" #include "errorpage.h" #include "fd.h" +#include "ftp/Parsing.h" #include "ip/tools.h" #include "SquidString.h" #include "tools.h" @@ -1107,113 +1108,3 @@ ServerStateData::parseControlReply(size_t &bytesUsed) } }; // namespace Ftp - - -bool -Ftp::ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr) -{ - int h1, h2, h3, h4; - int p1, p2; - const int n = sscanf(buf, "%d,%d,%d,%d,%d,%d", - &h1, &h2, &h3, &h4, &p1, &p2); - - if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) - return false; - - if (forceIp) { - addr = forceIp; // but the above code still validates the IP we got - } else { - static char ipBuf[1024]; - snprintf(ipBuf, sizeof(ipBuf), "%d.%d.%d.%d", h1, h2, h3, h4); - addr = ipBuf; - - if (addr.isAnyAddr()) - return false; - } - - const int port = ((p1 << 8) + p2); - - if (port <= 0) - return false; - - if (Config.Ftp.sanitycheck && port < 1024) - return false; - - addr.port(port); - return true; -} - -bool -Ftp::ParseProtoIpPort(const char *buf, Ip::Address &addr) -{ - - const char delim = *buf; - const char *s = buf + 1; - const char *e = s; - const int proto = strtol(s, const_cast(&e), 10); - if ((proto != 1 && proto != 2) || *e != delim) - return false; - - s = e + 1; - e = strchr(s, delim); - char ip[MAX_IPSTRLEN]; - if (static_cast(e - s) >= sizeof(ip)) - return false; - strncpy(ip, s, e - s); - ip[e - s] = '\0'; - addr = ip; - - if (addr.isAnyAddr()) - return false; - - if ((proto == 2) != addr.isIPv6()) // proto ID mismatches address version - return false; - - s = e + 1; // skip port delimiter - const int port = strtol(s, const_cast(&e), 10); - if (port < 0 || *e != '|') - return false; - - if (Config.Ftp.sanitycheck && port < 1024) - return false; - - addr.port(port); - return true; -} - -const char * -Ftp::unescapeDoubleQuoted(const char *quotedPath) -{ - static MemBuf path; - path.reset(); - const char *s = quotedPath; - if (*s == '"') { - ++s; - bool parseDone = false; - while (!parseDone) { - if (const char *e = strchr(s, '"')) { - path.append(s, e - s); - s = e + 1; - if (*s == '"') { - path.append(s, 1); - ++s; - } else - parseDone = true; - } else { //parse error - parseDone = true; - path.reset(); - } - } - } - return path.content(); -} - -bool -Ftp::hasPathParameter(const String &cmd) -{ - static const char *pathCommandsStr[]= {"CWD","SMNT", "RETR", "STOR", "APPE", - "RNFR", "RNTO", "DELE", "RMD", "MKD", - "LIST", "NLST", "STAT", "MLSD", "MLST"}; - static const std::set pathCommands(pathCommandsStr, pathCommandsStr + sizeof(pathCommandsStr)/sizeof(pathCommandsStr[0])); - return pathCommands.find(cmd) != pathCommands.end(); -} diff --git a/src/FtpServer.h b/src/clients/FtpClient.h similarity index 97% rename from src/FtpServer.h rename to src/clients/FtpClient.h index 48c36c65fb..b2fc76971f 100644 --- a/src/FtpServer.h +++ b/src/clients/FtpClient.h @@ -162,8 +162,6 @@ bool ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr); bool ParseProtoIpPort(const char *buf, Ip::Address &addr); /// parses a ftp quoted quote-escaped path const char *unescapeDoubleQuoted(const char *quotedPath); -/// Return true if the FTP command takes as parameter a pathname -bool hasPathParameter(const String &cmd); -}; // namespace Ftp +} // namespace Ftp #endif /* SQUID_FTP_SERVER_H */ diff --git a/src/ftp.cc b/src/clients/FtpGateway.cc similarity index 99% rename from src/ftp.cc rename to src/clients/FtpGateway.cc index 0a5833a464..d32216e500 100644 --- a/src/ftp.cc +++ b/src/clients/FtpGateway.cc @@ -32,6 +32,7 @@ #include "squid.h" #include "acl/FilledChecklist.h" +#include "clients/FtpClient.h" #include "comm.h" #include "comm/ConnOpener.h" #include "comm/Read.h" @@ -41,7 +42,6 @@ #include "errorpage.h" #include "fd.h" #include "fde.h" -#include "FtpServer.h" #include "FwdState.h" #include "html_quote.h" #include "HttpHdrContRange.h" diff --git a/src/FtpGatewayServer.cc b/src/clients/FtpNative.cc similarity index 85% rename from src/FtpGatewayServer.cc rename to src/clients/FtpNative.cc index 979ce4c759..e4a29cdea1 100644 --- a/src/FtpGatewayServer.cc +++ b/src/clients/FtpNative.cc @@ -6,14 +6,15 @@ #include "squid.h" #include "anyp/PortCfg.h" -#include "FtpGatewayServer.h" -#include "FtpServer.h" +#include "client_side.h" +#include "clients/FtpClient.h" +#include "ftp/Parsing.h" #include "HttpHdrCc.h" #include "HttpRequest.h" +#include "servers/FtpServer.h" #include "Server.h" #include "SquidTime.h" #include "Store.h" -#include "client_side.h" #include "wordlist.h" namespace Ftp { @@ -31,8 +32,11 @@ public: protected: virtual void start(); - ConnStateData::FtpState clientState() const; - void clientState(ConnStateData::FtpState newState); + const Ftp::MasterState &master() const; + Ftp::MasterState &updateMaster(); + Ftp::ServerState clientState() const; + void clientState(Ftp::ServerState newState); + virtual void serverComplete(); virtual void failed(err_type error = ERR_NONE, int xerrno = 0); virtual void handleControlReply(); @@ -143,11 +147,11 @@ ServerStateData::~ServerStateData() void ServerStateData::start() { - if (!fwd->request->clientConnectionManager->ftp.readGreeting) + if (!master().clientReadGreeting) Ftp::ServerStateData::start(); else - if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || - clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + if (clientState() == fssHandleDataRequest || + clientState() == fssHandleUploadRequest) handleDataRequest(); else sendCommand(); @@ -178,17 +182,38 @@ ServerStateData::serverComplete() Ftp::ServerStateData::serverComplete(); } -ConnStateData::FtpState +Ftp::MasterState & +ServerStateData::updateMaster() +{ + CbcPointer &mgr = fwd->request->clientConnectionManager; + if (mgr.valid()) { + if (Ftp::Server *srv = dynamic_cast(mgr.get())) + return srv->master; + } + // this code will not be necessary once the master is inside MasterXaction + debugs(9, 3, "our server side is gone: " << mgr); + static Ftp::MasterState Master; + Master = Ftp::MasterState(); + return Master; +} + +const Ftp::MasterState & +ServerStateData::master() const +{ + return const_cast(this)->updateMaster(); +} + +Ftp::ServerState ServerStateData::clientState() const { - return fwd->request->clientConnectionManager->ftp.state; + return master().serverState; } void -ServerStateData::clientState(ConnStateData::FtpState newState) +ServerStateData::clientState(Ftp::ServerState newState) { - ConnStateData::FtpState &cltState = - fwd->request->clientConnectionManager->ftp.state; + // XXX: s/client/server/g + Ftp::ServerState &cltState = updateMaster().serverState; debugs(9, 3, "client state was " << cltState << " now: " << newState); cltState = newState; } @@ -215,7 +240,7 @@ void ServerStateData::failed(err_type error, int xerrno) { if (!doneWithServer()) - clientState(ConnStateData::FTP_ERROR); + clientState(fssError); // TODO: we need to customize ErrorState instead if (entry->isEmpty()) @@ -321,7 +346,8 @@ ServerStateData::forwardPreliminaryReply(const PreliminaryCb cb) { debugs(9, 5, HERE << "Forwarding preliminary reply to client"); - assert(thePreliminaryCb == NULL); + // we must prevent concurrent ConnStateData::sendControlMsg() calls + Must(thePreliminaryCb == NULL); thePreliminaryCb = cb; const HttpReply::Pointer reply = createHttpReply(Http::scContinue); @@ -340,7 +366,7 @@ ServerStateData::proceedAfterPreliminaryReply() { debugs(9, 5, HERE << "Proceeding after preliminary reply to client"); - assert(thePreliminaryCb != NULL); + Must(thePreliminaryCb != NULL); const PreliminaryCb cb = thePreliminaryCb; thePreliminaryCb = NULL; (this->*cb)(); @@ -384,7 +410,7 @@ ServerStateData::createHttpReply(const Http::StatusCode httpStatus, const int cl void ServerStateData::handleDataRequest() { - data.addr(fwd->request->clientConnectionManager->ftp.serverDataAddr); + data.addr(master().clientDataAddr); connectDataChannel(); } @@ -424,13 +450,13 @@ ServerStateData::startDataUpload() void ServerStateData::readGreeting() { - assert(!fwd->request->clientConnectionManager->ftp.readGreeting); + assert(!master().clientReadGreeting); switch (ctrl.replycode) { case 220: - fwd->request->clientConnectionManager->ftp.readGreeting = true; - if (clientState() == ConnStateData::FTP_BEGIN) - clientState(ConnStateData::FTP_CONNECTED); + updateMaster().clientReadGreeting = true; + if (clientState() == fssBegin) + clientState(fssConnected); // Do not forward server greeting to the client because our client // side code has greeted the client already. Also, a greeting may @@ -468,10 +494,10 @@ ServerStateData::sendCommand() else debugs(9, 5, HERE << "command: " << cmd << ", no parameters"); - if (clientState() == ConnStateData::FTP_HANDLE_PASV || - clientState() == ConnStateData::FTP_HANDLE_EPSV || - clientState() == ConnStateData::FTP_HANDLE_EPRT || - clientState() == ConnStateData::FTP_HANDLE_PORT) { + if (clientState() == fssHandlePasv || + clientState() == fssHandleEpsv || + clientState() == fssHandleEprt || + clientState() == fssHandlePort) { sendPassive(); return; } @@ -486,21 +512,21 @@ ServerStateData::sendCommand() writeCommand(mb.content()); state = - clientState() == ConnStateData::FTP_HANDLE_CDUP ? SENT_CDUP : - clientState() == ConnStateData::FTP_HANDLE_CWD ? SENT_CWD : - clientState() == ConnStateData::FTP_HANDLE_FEAT ? SENT_FEAT : - clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST ? SENT_DATA_REQUEST : - clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST ? SENT_DATA_REQUEST : - clientState() == ConnStateData::FTP_CONNECTED ? SENT_USER : - clientState() == ConnStateData::FTP_HANDLE_PASS ? SENT_PASS : + clientState() == fssHandleCdup ? SENT_CDUP : + clientState() == fssHandleCwd ? SENT_CWD : + clientState() == fssHandleFeat ? SENT_FEAT : + clientState() == fssHandleDataRequest ? SENT_DATA_REQUEST : + clientState() == fssHandleUploadRequest ? SENT_DATA_REQUEST : + clientState() == fssConnected ? SENT_USER : + clientState() == fssHandlePass ? SENT_PASS : SENT_COMMAND; } void ServerStateData::readReply() { - assert(clientState() == ConnStateData::FTP_CONNECTED || - clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + assert(clientState() == fssConnected || + clientState() == fssHandleUploadRequest); if (100 <= ctrl.replycode && ctrl.replycode < 200) forwardPreliminaryReply(&ServerStateData::scheduleReadControlReply); @@ -511,7 +537,7 @@ ServerStateData::readReply() void ServerStateData::readFeatReply() { - assert(clientState() == ConnStateData::FTP_HANDLE_FEAT); + assert(clientState() == fssHandleFeat); if (100 <= ctrl.replycode && ctrl.replycode < 200) return; // ignore preliminary replies @@ -522,12 +548,12 @@ ServerStateData::readFeatReply() void ServerStateData::readPasvReply() { - assert(clientState() == ConnStateData::FTP_HANDLE_PASV || clientState() == ConnStateData::FTP_HANDLE_EPSV || clientState() == ConnStateData::FTP_HANDLE_PORT || clientState() == ConnStateData::FTP_HANDLE_EPRT); + assert(clientState() == fssHandlePasv || clientState() == fssHandleEpsv || clientState() == fssHandlePort || clientState() == fssHandleEprt); if (100 <= ctrl.replycode && ctrl.replycode < 200) return; // ignore preliminary replies - if (handlePasvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) + if (handlePasvReply(updateMaster().clientDataAddr)) forwardReply(); else forwardError(); @@ -539,7 +565,7 @@ ServerStateData::readEpsvReply() if (100 <= ctrl.replycode && ctrl.replycode < 200) return; // ignore preliminary replies - if (handleEpsvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) { + if (handleEpsvReply(updateMaster().clientDataAddr)) { if (ctrl.message == NULL) return; // didn't get complete reply yet @@ -551,13 +577,13 @@ ServerStateData::readEpsvReply() void ServerStateData::readDataReply() { - assert(clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || - clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + assert(clientState() == fssHandleDataRequest || + clientState() == fssHandleUploadRequest); if (ctrl.replycode == 125 || ctrl.replycode == 150) { - if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST) + if (clientState() == fssHandleDataRequest) forwardPreliminaryReply(&ServerStateData::startDataDownload); - else // clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST + else // clientState() == fssHandleUploadRequest forwardPreliminaryReply(&ServerStateData::startDataUpload); } else forwardReply(); @@ -588,8 +614,8 @@ ServerStateData::stopDirTracking() { debugs(9, 5, "Got code from pwd: " << ctrl.replycode << ", msg: " << ctrl.last_reply); - if (ctrl.replycode == 257) - fwd->request->clientConnectionManager->ftpSetWorkingDir(Ftp::unescapeDoubleQuoted(ctrl.last_reply)); + if (ctrl.replycode == 257) + updateMaster().workingDir = Ftp::UnescapeDoubleQuoted(ctrl.last_reply); wordlistDestroy(&ctrl.message); safe_free(ctrl.last_command); @@ -608,7 +634,8 @@ ServerStateData::stopDirTracking() void ServerStateData::readCwdOrCdupReply() { - assert(clientState() == ConnStateData::FTP_HANDLE_CWD || clientState() == ConnStateData::FTP_HANDLE_CDUP); + assert(clientState() == fssHandleCwd || + clientState() == fssHandleCdup); debugs(9, 5, HERE << "Got code " << ctrl.replycode << ", msg: " << ctrl.last_reply); diff --git a/src/clients/Makefile.am b/src/clients/Makefile.am new file mode 100644 index 0000000000..bd71786313 --- /dev/null +++ b/src/clients/Makefile.am @@ -0,0 +1,11 @@ +include $(top_srcdir)/src/Common.am + +noinst_LTLIBRARIES = libclients.la + +libclients_la_SOURCES = \ + FtpClient.cc \ + FtpClient.h \ + FtpGateway.cc \ + FtpNative.cc \ + \ + forward.h diff --git a/src/clients/forward.h b/src/clients/forward.h new file mode 100644 index 0000000000..e3d6e439a1 --- /dev/null +++ b/src/clients/forward.h @@ -0,0 +1,19 @@ +#ifndef SQUID_CLIENTS_FORWARD_H +#define SQUID_CLIENTS_FORWARD_H + +class FwdState; +class HttpRequest; + +/** + * \defgroup ServerProtocolFTPAPI Server-Side FTP API + * \ingroup ServerProtocol + */ + +/// \ingroup ServerProtocolFTPAPI +void ftpStart(FwdState *); +/// \ingroup ServerProtocolFTPAPI +const char *ftpUrlWith2f(HttpRequest *); + +void ftpGatewayServerStart(FwdState *const); + +#endif /* SQUID_CLIENTS_FORWARD_H */ diff --git a/src/errorpage.cc b/src/errorpage.cc index da889df245..d5fde9e3ab 100644 --- a/src/errorpage.cc +++ b/src/errorpage.cc @@ -31,13 +31,13 @@ */ #include "squid.h" #include "cache_cf.h" +#include "clients/forward.h" #include "comm/Connection.h" #include "comm/Write.h" #include "disk.h" #include "err_detail_type.h" #include "errorpage.h" #include "fde.h" -#include "ftp.h" #include "html_quote.h" #include "HttpHeaderTools.h" #include "HttpReply.h" diff --git a/src/ftp.h b/src/ftp.h deleted file mode 100644 index ca4140200d..0000000000 --- a/src/ftp.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * DEBUG: section 09 File Transfer Protocol (FTP) - * AUTHOR: Harvest Derived - * - * SQUID Web Proxy Cache http://www.squid-cache.org/ - * ---------------------------------------------------------- - * - * Squid is the result of efforts by numerous individuals from - * the Internet community; see the CONTRIBUTORS file for full - * details. Many organizations have provided support for Squid's - * development; see the SPONSORS file for full details. Squid is - * Copyrighted (C) 2001 by the Regents of the University of - * California; see the COPYRIGHT file for full details. Squid - * incorporates software developed and/or copyrighted by other - * sources; see the CREDITS file for full details. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA. - * - */ - -#ifndef SQUID_FTP_H_ -#define SQUID_FTP_H_ - -class FwdState; - -/** - * \defgroup ServerProtocolFTPAPI Server-Side FTP API - * \ingroup ServerProtocol - */ - -/// \ingroup ServerProtocolFTPAPI -void ftpStart(FwdState *); -/// \ingroup ServerProtocolFTPAPI -const char *ftpUrlWith2f(HttpRequest *); - -#endif /* SQUID_FTP_H_ */ diff --git a/src/ftp/Makefile.am b/src/ftp/Makefile.am new file mode 100644 index 0000000000..4c4d083c0f --- /dev/null +++ b/src/ftp/Makefile.am @@ -0,0 +1,7 @@ +include $(top_srcdir)/src/Common.am + +noinst_LTLIBRARIES = libftp.la + +libftp_la_SOURCES = \ + Parsing.cc \ + Parsing.h diff --git a/src/ftp/Parsing.cc b/src/ftp/Parsing.cc new file mode 100644 index 0000000000..a3ad18c392 --- /dev/null +++ b/src/ftp/Parsing.cc @@ -0,0 +1,108 @@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + */ + +#include "squid.h" +#include "ftp/Parsing.h" +#include "ip/Address.h" +#include "MemBuf.h" +#include "SquidConfig.h" + +bool +Ftp::ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr) +{ + int h1, h2, h3, h4; + int p1, p2; + const int n = sscanf(buf, "%d,%d,%d,%d,%d,%d", + &h1, &h2, &h3, &h4, &p1, &p2); + + if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) + return false; + + if (forceIp) { + addr = forceIp; // but the above code still validates the IP we got + } else { + static char ipBuf[1024]; + snprintf(ipBuf, sizeof(ipBuf), "%d.%d.%d.%d", h1, h2, h3, h4); + addr = ipBuf; + + if (addr.isAnyAddr()) + return false; + } + + const int port = ((p1 << 8) + p2); + + if (port <= 0) + return false; + + if (Config.Ftp.sanitycheck && port < 1024) + return false; + + addr.port(port); + return true; +} + +bool +Ftp::ParseProtoIpPort(const char *buf, Ip::Address &addr) +{ + + const char delim = *buf; + const char *s = buf + 1; + const char *e = s; + const int proto = strtol(s, const_cast(&e), 10); + if ((proto != 1 && proto != 2) || *e != delim) + return false; + + s = e + 1; + e = strchr(s, delim); + char ip[MAX_IPSTRLEN]; + if (static_cast(e - s) >= sizeof(ip)) + return false; + strncpy(ip, s, e - s); + ip[e - s] = '\0'; + addr = ip; + + if (addr.isAnyAddr()) + return false; + + if ((proto == 2) != addr.isIPv6()) // proto ID mismatches address version + return false; + + s = e + 1; // skip port delimiter + const int port = strtol(s, const_cast(&e), 10); + if (port < 0 || *e != '|') + return false; + + if (Config.Ftp.sanitycheck && port < 1024) + return false; + + addr.port(port); + return true; +} + +const char * +Ftp::UnescapeDoubleQuoted(const char *quotedPath) +{ + static MemBuf path; + path.reset(); + const char *s = quotedPath; + if (*s == '"') { + ++s; + bool parseDone = false; + while (!parseDone) { + if (const char *e = strchr(s, '"')) { + path.append(s, e - s); + s = e + 1; + if (*s == '"') { + path.append(s, 1); + ++s; + } else + parseDone = true; + } else { //parse error + parseDone = true; + path.reset(); + } + } + } + return path.content(); +} diff --git a/src/ftp/Parsing.h b/src/ftp/Parsing.h new file mode 100644 index 0000000000..1363a88fb9 --- /dev/null +++ b/src/ftp/Parsing.h @@ -0,0 +1,15 @@ +#ifndef SQUID_FTP_PARSING_H +#define SQUID_FTP_PARSING_H + +#include "ip/forward.h" + +namespace Ftp { + +// TODO: Document +bool ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr); +bool ParseProtoIpPort(const char *buf, Ip::Address &addr); +const char *UnescapeDoubleQuoted(const char *quotedPath); + +} // namespace Ftp + +#endif /* SQUID_FTP_PARSING_H */ diff --git a/src/servers/FtpServer.cc b/src/servers/FtpServer.cc new file mode 100644 index 0000000000..5c38fe2e3d --- /dev/null +++ b/src/servers/FtpServer.cc @@ -0,0 +1,1594 @@ +/* + * DEBUG: section 33 Transfer protocol servers + */ + +#include "squid.h" +#include "base/Subscription.h" +#include "clientStream.h" +#include "comm/ConnOpener.h" +#include "comm/Read.h" +#include "comm/TcpAcceptor.h" +#include "comm/Write.h" +#include "client_side_reply.h" +#include "client_side_request.h" +#include "errorpage.h" +#include "fd.h" +#include "ftp/Parsing.h" +#include "globals.h" +#include "HttpHdrCc.h" +#include "ip/tools.h" +#include "ipc/FdNotes.h" +#include "servers/forward.h" +#include "servers/FtpServer.h" +#include "SquidConfig.h" +#include "StatCounters.h" +#include "tools.h" + +CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server); + +namespace Ftp { +static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = ""); +static bool SupportedCommand(const String &name); +}; + +Ftp::Server::Server(const MasterXaction::Pointer &xact): + AsyncJob("Ftp::Server"), + ConnStateData(xact), + uri(), + host(), + gotEpsvAll(false), + onDataAcceptCall(), + dataListenConn(), + dataConn(), + uploadAvailSize(0), + listener(), + connector(), + reader() +{ + assert(xact->squidPort->transport.protocol == AnyP::PROTO_FTP); + flags.readMore = false; // we need to announce ourselves first +} + +Ftp::Server::~Server() +{ + closeDataConnection(); +} + +int +Ftp::Server::pipelinePrefetchMax() const +{ + return 0; // no support for concurrent FTP requests +} + +time_t +Ftp::Server::idleTimeout() const +{ + return Config.Timeout.ftpClientIdle; +} + +void +Ftp::Server::start() +{ + ConnStateData::start(); + + if (transparent()) { + char buf[MAX_IPSTRLEN]; + clientConnection->local.toUrl(buf, MAX_IPSTRLEN); + host = buf; + calcUri(); + debugs(33, 5, HERE << "FTP transparent URL: " << uri); + } + + writeEarlyReply(220, "Service ready"); +} + +/// schedules another data connection read if needed +void +Ftp::Server::maybeReadUploadData() +{ + if (reader != NULL) + return; + + const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize; + if (availSpace <= 0) + return; + + debugs(33, 4, HERE << dataConn << ": reading FTP data..."); + + typedef CommCbMemFunT Dialer; + reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData); + comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace, + reader); +} + +/// react to the freshly parsed request +void +Ftp::Server::doProcessRequest() +{ + // zero pipelinePrefetchMax() ensures that there is only parsed request + ClientSocketContext::Pointer context = getCurrentContext(); + Must(context != NULL); + Must(getConcurrentRequestCount() == 1); + + ClientHttpRequest *const http = context->http; + assert(http != NULL); + HttpRequest *const request = http->request; + assert(request != NULL); + debugs(33, 9, request); + + HttpHeader &header = request->header; + assert(header.has(HDR_FTP_COMMAND)); + String &cmd = header.findEntry(HDR_FTP_COMMAND)->value; + assert(header.has(HDR_FTP_ARGUMENTS)); + String ¶ms = header.findEntry(HDR_FTP_ARGUMENTS)->value; + + const bool fwd = !http->storeEntry() && handleRequest(cmd, params); + + if (http->storeEntry() != NULL) { + debugs(33, 4, "got an immediate response"); + assert(http->storeEntry() != NULL); + clientSetKeepaliveFlag(http); + context->pullData(); + } else if (fwd) { + debugs(33, 4, "forwarding request to server side"); + assert(http->storeEntry() == NULL); + clientProcessRequest(this, NULL /*parser*/, context.getRaw(), + request->method, request->http_ver); + } else { + debugs(33, 4, "will resume processing later"); + } +} + +void +Ftp::Server::processParsedRequest(ClientSocketContext *context, const Http::ProtocolVersion &) +{ + // Process FTP request asynchronously to make sure FTP + // data connection accept callback is fired first. + CallJobHere(33, 4, CbcPointer(this), + Ftp::Server, doProcessRequest); +} + +/// imports more upload data from the data connection +void +Ftp::Server::readUploadData(const CommIoCbParams &io) +{ + debugs(33,5,HERE << io.conn << " size " << io.size); + Must(reader != NULL); + reader = NULL; + + assert(Comm::IsConnOpen(dataConn)); + assert(io.conn->fd == dataConn->fd); + + if (io.flag == Comm::OK && bodyPipe != NULL) { + if (io.size > 0) { + kb_incr(&(statCounter.client_http.kbytes_in), io.size); + + char *const current_buf = uploadBuf + uploadAvailSize; + if (io.buf != current_buf) + memmove(current_buf, io.buf, io.size); + uploadAvailSize += io.size; + shovelUploadData(); + } else if (io.size == 0) { + debugs(33, 5, HERE << io.conn << " closed"); + closeDataConnection(); + if (uploadAvailSize <= 0) + finishDechunkingRequest(true); + } + } else { // not Comm::Flags::OK or unexpected read + debugs(33, 5, HERE << io.conn << " closed"); + closeDataConnection(); + finishDechunkingRequest(false); + } + +} + +/// shovel upload data from the internal buffer to the body pipe if possible +void +Ftp::Server::shovelUploadData() +{ + assert(bodyPipe != NULL); + + debugs(33,5, HERE << "handling FTP request data for " << clientConnection); + const size_t putSize = bodyPipe->putMoreData(uploadBuf, + uploadAvailSize); + if (putSize > 0) { + uploadAvailSize -= putSize; + if (uploadAvailSize > 0) + memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize); + } + + if (Comm::IsConnOpen(dataConn)) + maybeReadUploadData(); + else if (uploadAvailSize <= 0) + finishDechunkingRequest(true); +} + +void +Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer) +{ + shovelUploadData(); +} + +void +Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr) +{ + ConnStateData::noteBodyConsumerAborted(ptr); + closeDataConnection(); +} + +/// accept a new FTP control connection and hand it to a dedicated Server +void +Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams ¶ms) +{ + MasterXaction::Pointer xact = params.xaction; + AnyP::PortCfgPointer s = xact->squidPort; + + // NP: it is possible the port was reconfigured when the call or accept() was queued. + + if (params.flag != Comm::OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, "ftpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, HERE << params.conn << ": accepted"); + fd_note(params.conn->fd, "client ftp connect"); + + if (s->tcp_keepalive.enabled) + commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout); + + ++incoming_sockets_accepted; + + AsyncJob::Start(new Server(xact)); +} + +void +Ftp::StartListening() +{ + for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) { + if (MAXTCPLISTENPORTS == NHttpSockets) { + debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" << + " limit of " << MAXTCPLISTENPORTS << " ports."); + break; + } + + // direct new connections accepted by listenConn to Accept() + typedef CommCbFunPtrCallT AcceptCall; + RefCount subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection", + CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection, + CommAcceptCbParams(NULL))); + clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket); + } +} + +void +Ftp::StopListening() +{ + for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) { + if (s->listenConn != NULL) { + debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local); + s->listenConn->close(); + s->listenConn = NULL; + } + } +} + +void +Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn) +{ + // find request + ClientSocketContext::Pointer context = getCurrentContext(); + Must(context != NULL); + ClientHttpRequest *const http = context->http; + Must(http != NULL); + HttpRequest *const request = http->request; + Must(request != NULL); + + // this is not an idle connection, so we do not want I/O monitoring + const bool monitor = false; + + // make FTP peer connection exclusive to our request + pinConnection(conn, request, conn->getPeer(), false, monitor); +} + +void +Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io) +{ + ConnStateData::clientPinnedConnectionClosed(io); + + // if the server control connection is gone, reset state to login again + // TODO: merge with similar code in ftpHandleUserRequest() + debugs(33, 5, "will need to re-login due to FTP server closure"); + master.clientReadGreeting = false; + changeState(fssBegin, "server closure"); + // XXX: Not enough. Gateway::ServerStateData::sendCommand() will not + // re-login because clientState() is not ConnStateData::FTP_CONNECTED. +} + +/// computes uri member from host and, if tracked, working dir with file name +void +Ftp::Server::calcUri(const char *file) +{ + uri = "ftp://"; + uri.append(host); + if (port->ftp_track_dirs && master.workingDir.size()) { + if (master.workingDir[0] != '/') + uri.append("/"); + uri.append(master.workingDir); + } + + if (uri[uri.size() - 1] != '/') + uri.append("/"); + + if (port->ftp_track_dirs && file) { + // remove any '/' from the beginning of path + while (*file == '/') + ++file; + uri.append(file); + } +} + +/// Starts waiting for a data connection. Returns listening port. +/// On errors, responds with an error and returns zero. +unsigned int +Ftp::Server::listenForDataConnection() +{ + closeDataConnection(); + + Comm::ConnectionPointer conn = new Comm::Connection; + conn->flags = COMM_NONBLOCKING; + conn->local = transparent() ? port->s : clientConnection->local; + conn->local.port(0); + const char *const note = uri.termedBuf(); + comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); + if (!Comm::IsConnOpen(conn)) { + debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " << + conn->local << " error: " << errno); + writeCustomReply(451, "Internal error"); + return 0; + } + + typedef CommCbMemFunT AcceptDialer; + typedef AsyncCallT AcceptCall; + RefCount call = static_cast(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection)); + Subscription::Pointer sub = new CallSubscription(call); + listener = call.getRaw(); + dataListenConn = conn; + AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); + + const unsigned int listeningPort = comm_local_port(conn->fd); + conn->local.port(listeningPort); + return listeningPort; +} + +void +Ftp::Server::acceptDataConnection(const CommAcceptCbParams ¶ms) +{ + if (params.flag != Comm::OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, dataListenConn << ": accept " + "failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, "accepted " << params.conn); + fd_note(params.conn->fd, "passive client ftp data"); + ++incoming_sockets_accepted; + + if (!clientConnection) { + debugs(33, 5, "late data connection?"); + closeDataConnection(); // in case we are still listening + params.conn->close(); + } else + if (params.conn->remote != clientConnection->remote) { + debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote); + params.conn->close(); + // Some FTP servers close control connection here, but it may make + // things worse from DoS p.o.v. and no better from data stealing p.o.v. + } else { + closeDataConnection(); + dataConn = params.conn; + uploadAvailSize = 0; + debugs(33, 7, "ready for data"); + if (onDataAcceptCall != NULL) { + AsyncCall::Pointer call = onDataAcceptCall; + onDataAcceptCall = NULL; + // If we got an upload request, start reading data from the client. + if (master.serverState == fssHandleUploadRequest) + maybeReadUploadData(); + else + Must(master.serverState == fssHandleDataRequest); + MemBuf mb; + mb.init(); + mb.Printf("150 Data connection opened.\r\n"); + Comm::Write(clientConnection, &mb, call); + } + } +} + +void +Ftp::Server::closeDataConnection() +{ + if (listener != NULL) { + listener->cancel("no longer needed"); + listener = NULL; + } + + if (Comm::IsConnOpen(dataListenConn)) { + debugs(33, 5, HERE << "FTP closing client data listen socket: " << + *dataListenConn); + dataListenConn->close(); + } + dataListenConn = NULL; + + if (reader != NULL) { + // Comm::ReadCancel can deal with negative FDs + Comm::ReadCancel(dataConn->fd, reader); + reader = NULL; + } + + if (Comm::IsConnOpen(dataConn)) { + debugs(33, 5, HERE << "FTP closing client data connection: " << + *dataConn); + dataConn->close(); + } + dataConn = NULL; +} + +/// Writes FTP [error] response before we fully parsed the FTP request and +/// created the corresponding HTTP request wrapper for that FTP request. +void +Ftp::Server::writeEarlyReply(const int code, const char *msg) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + MemBuf mb; + mb.init(); + mb.Printf("%i %s\r\n", code, msg); + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply); + Comm::Write(clientConnection, &mb, call); + + flags.readMore = false; + + // TODO: Create master transaction. Log it in wroteEarlyReply(). +} + +void +Ftp::Server::writeReply(MemBuf &mb) +{ + debugs(11, 2, "FTP Client " << clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply); + Comm::Write(clientConnection, &mb, call); +} + +void +Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + const bool sendDetails = reply != NULL && + reply->header.has(HDR_FTP_STATUS) && reply->header.has(HDR_FTP_REASON); + + MemBuf mb; + mb.init(); + if (sendDetails) { + mb.Printf("%i-%s\r\n", code, msg); + mb.Printf(" Server reply:\r\n"); + Ftp::PrintReply(mb, reply, " "); + mb.Printf("%i \r\n", code); + } else + mb.Printf("%i %s\r\n", code, msg); + + writeReply(mb); +} + +void +Ftp::Server::changeState(const ServerState newState, const char *reason) +{ + if (master.serverState == newState) { + debugs(33, 3, "client state unchanged at " << master.serverState << + " because " << reason); + master.serverState = newState; + } else { + debugs(33, 3, "client state was " << master.serverState << + ", now " << newState << " because " << reason); + master.serverState = newState; + } +} + +/// whether the given FTP command has a pathname parameter +static bool +ftpHasPathParameter(const String &cmd) +{ + static const char *pathCommandsStr[]= {"CWD","SMNT", "RETR", "STOR", "APPE", + "RNFR", "RNTO", "DELE", "RMD", "MKD", + "LIST", "NLST", "STAT", "MLSD", "MLST"}; + static const std::set pathCommands(pathCommandsStr, pathCommandsStr + sizeof(pathCommandsStr)/sizeof(pathCommandsStr[0])); + return pathCommands.find(cmd) != pathCommands.end(); +} + +/// Parses a single FTP request on the control connection. +/// Returns NULL on errors and incomplete requests. +ClientSocketContext * +Ftp::Server::parseOneRequest(Http::ProtocolVersion &ver) +{ + ver = Http::ProtocolVersion(1, 1); + + // TODO: Use tokenizer for parsing instead of raw pointer manipulation. + const char *inBuf = in.buf.rawContent(); + + const char *const eor = + static_cast(memchr(inBuf, '\n', + min(static_cast(in.buf.length()), Config.maxRequestHeaderSize))); + + if (eor == NULL && in.buf.length() >= Config.maxRequestHeaderSize) { + changeState(fssError, "huge req"); + writeEarlyReply(421, "Too large request"); + return NULL; + } + + if (eor == NULL) { + debugs(33, 5, HERE << "Incomplete request, waiting for end of request"); + return NULL; + } + + const size_t req_sz = eor + 1 - inBuf; + + // skip leading whitespaces + const char *boc = inBuf; // beginning of command + while (boc < eor && isspace(*boc)) ++boc; + if (boc >= eor) { + debugs(33, 5, HERE << "Empty request, ignoring"); + consumeInput(req_sz); + return NULL; + } + + const char *eoc = boc; // end of command + while (eoc < eor && !isspace(*eoc)) ++eoc; + in.buf.setAt(eoc - inBuf, '\0'); + + const char *bop = eoc + 1; // beginning of parameter + while (bop < eor && isspace(*bop)) ++bop; + if (bop < eor) { + const char *eop = eor - 1; + while (isspace(*eop)) --eop; + assert(eop >= bop); + in.buf.setAt(eop + 1 - inBuf, '\0'); + } else + bop = NULL; + + debugs(33, 7, HERE << "Parsed FTP command " << boc << " with " << + (bop == NULL ? "no " : "") << "parameters" << + (bop != NULL ? ": " : "") << bop); + + // TODO: Use SBuf instead of String + const String cmd = boc; + String params = bop; + + consumeInput(req_sz); + + if (!master.clientReadGreeting) { + // the first command must be USER + if (!pinning.pinned && cmd.caseCmp("USER") != 0) { + writeEarlyReply(530, "Must login first"); + return NULL; + } + } + + // We need to process USER request now because it sets ftp server Hostname. + if (cmd.caseCmp("USER") == 0 && !handleUserRequest(cmd, params)) + return NULL; + + if (!Ftp::SupportedCommand(cmd)) { + writeEarlyReply(502, "Unknown or unsupported command"); + return NULL; + } + + const HttpRequestMethod method = + !cmd.caseCmp("APPE") || !cmd.caseCmp("STOR") || !cmd.caseCmp("STOU") ? + Http::METHOD_PUT : Http::METHOD_GET; + + const char *aPath = params.size() > 0 && ftpHasPathParameter(cmd) ? + params.termedBuf() : NULL; + calcUri(aPath); + char *newUri = xstrdup(uri.termedBuf()); + HttpRequest *const request = HttpRequest::CreateFromUrlAndMethod(newUri, method); + if (!request) { + debugs(33, 5, HERE << "Invalid FTP URL: " << uri); + writeEarlyReply(501, "Invalid host"); + uri.clean(); + safe_free(newUri); + return NULL; + } + + request->flags.ftpNative = true; + request->http_ver = ver; + + // Our fake Request-URIs are not distinctive enough for caching to work + request->flags.cachable = false; // XXX: reset later by maybeCacheable() + request->flags.noCache = true; + + request->header.putStr(HDR_FTP_COMMAND, cmd.termedBuf()); + request->header.putStr(HDR_FTP_ARGUMENTS, params.termedBuf() != NULL ? + params.termedBuf() : ""); + if (method == Http::METHOD_PUT) { + request->header.putStr(HDR_EXPECT, "100-continue"); + request->header.putStr(HDR_TRANSFER_ENCODING, "chunked"); + } + + ClientHttpRequest *const http = new ClientHttpRequest(this); + http->request = request; + HTTPMSGLOCK(http->request); + http->req_sz = req_sz; + http->uri = newUri; + + ClientSocketContext *const result = + new ClientSocketContext(clientConnection, http); + + StoreIOBuffer tempBuffer; + tempBuffer.data = result->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; + + ClientStreamData newServer = new clientReplyContext(http); + ClientStreamData newClient = result; + clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, + clientReplyStatus, newServer, clientSocketRecipient, + clientSocketDetach, newClient, tempBuffer); + + Must(!getConcurrentRequestCount()); + result->registerWithConn(); + result->flags.parsed_ok = 1; + flags.readMore = false; + return result; +} + +void +Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data) +{ + // the caller guarantees that we are dealing with the current context only + ClientSocketContext::Pointer context = getCurrentContext(); + assert(context != NULL); + + if (context->http && context->http->al != NULL && + !context->http->al->reply && reply) { + context->http->al->reply = reply; + HTTPMSGLOCK(context->http->al->reply); + } + + static ReplyHandler handlers[] = { + NULL, // fssBegin + NULL, // fssConnected + &Ftp::Server::handleFeatReply, // fssHandleFeat + &Ftp::Server::handlePasvReply, // fssHandlePasv + &Ftp::Server::handlePortReply, // fssHandlePort + &Ftp::Server::handleDataReply, // fssHandleDataRequest + &Ftp::Server::handleUploadReply, // fssHandleUploadRequest + &Ftp::Server::handleEprtReply,// fssHandleEprt + &Ftp::Server::handleEpsvReply,// fssHandleEpsv + NULL, // fssHandleCwd + NULL, //fssHandlePass + NULL, // fssHandleCdup + &Ftp::Server::handleErrorReply // fssError + }; + const Server &server = dynamic_cast(*context->getConn()); + if (const ReplyHandler handler = handlers[server.master.serverState]) + (this->*handler)(reply, data); + else + writeForwardedReply(reply); +} + +void +Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (getCurrentContext()->http->request->errType != ERR_NONE) { + writeCustomReply(502, "Server does not support FEAT", reply); + return; + } + + HttpReply *filteredReply = reply->clone(); + HttpHeader &filteredHeader = filteredReply->header; + + // Remove all unsupported commands from the response wrapper. + int deletedCount = 0; + HttpHeaderPos pos = HttpHeaderInitPos; + bool hasEPRT = false; + bool hasEPSV = false; + int prependSpaces = 1; + while (const HttpHeaderEntry *e = filteredHeader.getEntry(&pos)) { + if (e->id == HDR_FTP_PRE) { + // assume RFC 2389 FEAT response format, quoted by Squid: + // <"> SP NAME [SP PARAMS] <"> + // but accommodate MS servers sending four SPs before NAME + if (e->value.size() < 4) + continue; + const char *raw = e->value.termedBuf(); + if (raw[0] != '"' || raw[1] != ' ') + continue; + const char *beg = raw + 1 + strspn(raw + 1, " "); // after quote and spaces + // command name ends with (SP parameter) or quote + const char *end = beg + strcspn(beg, " \""); + + if (end <= beg) + continue; + + // compute the number of spaces before the command + prependSpaces = beg - raw - 1; + + const String cmd = e->value.substr(beg-raw, end-raw); + + if (!Ftp::SupportedCommand(cmd)) + filteredHeader.delAt(pos, deletedCount); + + if (cmd == "EPRT") + hasEPRT = true; + else if (cmd == "EPSV") + hasEPSV = true; + } + } + + char buf[256]; + int insertedCount = 0; + if (!hasEPRT) { + snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT"); + filteredHeader.putStr(HDR_FTP_PRE, buf); + ++insertedCount; + } + if (!hasEPSV) { + snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV"); + filteredHeader.putStr(HDR_FTP_PRE, buf); + ++insertedCount; + } + + if (deletedCount || insertedCount) { + filteredHeader.refreshMask(); + debugs(33, 5, "deleted " << deletedCount << " inserted " << insertedCount); + } + + writeForwardedReply(filteredReply); +} + +void +Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer data) +{ + ClientSocketContext::Pointer context = getCurrentContext(); + assert(context != NULL); + + if (context->http->request->errType != ERR_NONE) { + writeCustomReply(502, "Server does not support PASV", reply); + return; + } + + const unsigned short localPort = listenForDataConnection(); + if (!localPort) + return; + + char addr[MAX_IPSTRLEN]; + // remote server in interception setups and local address otherwise + const Ip::Address &server = transparent() ? + clientConnection->local : dataListenConn->local; + server.toStr(addr, MAX_IPSTRLEN, AF_INET); + addr[MAX_IPSTRLEN - 1] = '\0'; + for (char *c = addr; *c != '\0'; ++c) { + if (*c == '.') + *c = ','; + } + + // In interception setups, we combine remote server address with a + // local port number and hope that traffic will be redirected to us. + // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp + // versions block responses that use those alternative syntax rules! + MemBuf mb; + mb.init(); + mb.Printf("227 Entering Passive Mode (%s,%i,%i).\r\n", + addr, + static_cast(localPort / 256), + static_cast(localPort % 256)); + debugs(11, 3, Raw("writing", mb.buf, mb.size)); + writeReply(mb); +} + +void +Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (getCurrentContext()->http->request->errType != ERR_NONE) { + writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply); + return; + } + + writeCustomReply(200, "PORT successfully converted to PASV."); + + // and wait for RETR +} + +void +Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (!pinning.pinned) // we failed to connect to server + uri.clean(); + // 421: we will close due to fssError + writeErrorReply(reply, 421); +} + +void +Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (reply != NULL && reply->sline.status() != Http::scOkay) { + writeForwardedReply(reply); + if (Comm::IsConnOpen(dataConn)) { + debugs(33, 3, "closing " << dataConn << " on KO reply"); + closeDataConnection(); + } + return; + } + + if (!dataConn) { + // We got STREAM_COMPLETE (or error) and closed the client data conn. + debugs(33, 3, "ignoring FTP srv data response after clt data closure"); + return; + } + + if (!checkDataConnPost()) { + writeCustomReply(425, "Data connection is not established."); + closeDataConnection(); + return; + } + + debugs(33, 7, HERE << data.length); + + if (data.length <= 0) { + replyDataWritingCheckpoint(); // skip the actual write call + return; + } + + MemBuf mb; + mb.init(data.length + 1, data.length + 1); + mb.append(data.data, data.length); + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData); + Comm::Write(dataConn, &mb, call); + + getCurrentContext()->noteSentBodyBytes(data.length); +} + +/// called when we are done writing a chunk of the response data +void +Ftp::Server::wroteReplyData(const CommIoCbParams &io) +{ + if (io.flag == Comm::ERR_CLOSING) + return; + + if (io.flag != Comm::OK) { + debugs(33, 3, HERE << "FTP reply data writing failed: " << + xstrerr(io.xerrno)); + closeDataConnection(); + writeCustomReply(426, "Data connection error; transfer aborted"); + return; + } + + assert(getCurrentContext()->http); + getCurrentContext()->http->out.size += io.size; + replyDataWritingCheckpoint(); +} + +/// ClientStream checks after (actual or skipped) reply data writing +void +Ftp::Server::replyDataWritingCheckpoint() { + switch (getCurrentContext()->socketState()) { + case STREAM_NONE: + debugs(33, 3, "Keep going"); + getCurrentContext()->pullData(); + return; + case STREAM_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer successfully complete"); + writeCustomReply(226, "Transfer complete"); + break; + case STREAM_UNPLANNED_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE"); + writeCustomReply(451, "Server error; transfer aborted"); + break; + case STREAM_FAILED: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_FAILED"); + writeCustomReply(451, "Server error; transfer aborted"); + break; + default: + fatal("unreachable code"); + } + + closeDataConnection(); +} + +void +Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer data) +{ + writeForwardedReply(reply); + // note that the client data connection may already be closed by now +} + +void +Ftp::Server::writeForwardedReply(const HttpReply *reply) +{ + assert(reply != NULL); + const HttpHeader &header = reply->header; + // adaptation and forwarding errors lack HDR_FTP_STATUS + if (!header.has(HDR_FTP_STATUS)) { + writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply + return; + } + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply); + writeForwardedReplyAndCall(reply, call); +} + +void +Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (getCurrentContext()->http->request->errType != ERR_NONE) { + writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply); + return; + } + + writeCustomReply(200, "EPRT successfully converted to PASV."); + + // and wait for RETR +} + +void +Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer data) +{ + if (getCurrentContext()->http->request->errType != ERR_NONE) { + writeCustomReply(502, "Cannot connect to server", reply); + return; + } + + const unsigned short localPort = listenForDataConnection(); + if (!localPort) + return; + + // In interception setups, we combine remote server address with a + // local port number and hope that traffic will be redirected to us. + MemBuf mb; + mb.init(); + mb.Printf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort); + + debugs(11, 3, Raw("writing", mb.buf, mb.size)); + writeReply(mb); +} + +/// writes FTP error response with given status and reply-derived error details +void +Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode) +{ + const HttpRequest *request = getCurrentContext()->http->request; + assert(request); + + MemBuf mb; + mb.init(); + + if (request->errType != ERR_NONE) + mb.Printf("%i-%s\r\n", scode, errorPageName(request->errType)); + + if (request->errDetail > 0) { + // XXX: > 0 may not always mean that this is an errno + mb.Printf("%i-Error: (%d) %s\r\n", scode, + request->errDetail, + strerror(request->errDetail)); + } + + // XXX: Remove hard coded names. Use an error page template instead. + const Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { // XXX: add adapt::allMeta.getByName("X-Response-Info"); + const String desc = ah->allMeta.getByName("X-Response-Desc"); + if (info.size()) + mb.Printf("%i-Information: %s\r\n", scode, info.termedBuf()); + if (desc.size()) + mb.Printf("%i-Description: %s\r\n", scode, desc.termedBuf()); + } + + assert(reply != NULL); + const char *reason = reply->header.has(HDR_FTP_REASON) ? + reply->header.getStr(HDR_FTP_REASON): + reply->sline.reason(); + + mb.Printf("%i %s\r\n", scode, reason); // error terminating line + + // TODO: errorpage.cc should detect FTP client and use + // configurable FTP-friendly error templates which we should + // write to the client "as is" instead of hiding most of the info + + writeReply(mb); +} + +/// writes FTP response based on HTTP reply that is not an FTP-response wrapper +void +Ftp::Server::writeForwardedForeign(const HttpReply *reply) +{ + changeState(fssConnected, "foreign reply"); + closeDataConnection(); + // 451: We intend to keep the control connection open. + writeErrorReply(reply, 451); +} + +void +Ftp::Server::writeControlMsgAndCall(ClientSocketContext *context, HttpReply *reply, AsyncCall::Pointer &call) +{ + // the caller guarantees that we are dealing with the current context only + // the caller should also make sure reply->header.has(HDR_FTP_STATUS) + writeForwardedReplyAndCall(reply, call); +} + +void +Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call) +{ + assert(reply != NULL); + const HttpHeader &header = reply->header; + + // without status, the caller must use the writeForwardedForeign() path + Must(header.has(HDR_FTP_STATUS)); + Must(header.has(HDR_FTP_REASON)); + const int scode = header.getInt(HDR_FTP_STATUS); + debugs(33, 7, HERE << "scode: " << scode); + + // Status 125 or 150 implies upload or data request, but we still check + // the state in case the server is buggy. + if ((scode == 125 || scode == 150) && + (master.serverState == fssHandleUploadRequest || + master.serverState == fssHandleDataRequest)) { + if (checkDataConnPost()) { + // If the data connection is ready, start reading data (here) + // and forward the response to client (further below). + debugs(33, 7, "data connection established, start data transfer"); + if (master.serverState == fssHandleUploadRequest) + maybeReadUploadData(); + } else { + // If we are waiting to accept the data connection, keep waiting. + if (Comm::IsConnOpen(dataListenConn)) { + debugs(33, 7, "wait for the client to establish a data connection"); + onDataAcceptCall = call; + // TODO: Add connect timeout for passive connections listener? + // TODO: Remember server response so that we can forward it? + } else { + // Either the connection was establised and closed after the + // data was transferred OR we failed to establish an active + // data connection and already sent the error to the client. + // In either case, there is nothing more to do. + debugs(33, 7, "done with data OR active connection failed"); + } + return; + } + } + + MemBuf mb; + mb.init(); + Ftp::PrintReply(mb, reply); + + debugs(11, 2, "FTP Client " << clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + Comm::Write(clientConnection, &mb, call); +} + +static void +Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix) +{ + const HttpHeader &header = reply->header; + + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = header.getEntry(&pos)) { + if (e->id == HDR_FTP_PRE) { + String raw; + if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw)) + mb.Printf("%s\r\n", raw.termedBuf()); + } + } + + if (header.has(HDR_FTP_STATUS)) { + const char *reason = header.getStr(HDR_FTP_REASON); + mb.Printf("%i %s\r\n", header.getInt(HDR_FTP_STATUS), + (reason ? reason : 0)); + } +} + +void +Ftp::Server::wroteEarlyReply(const CommIoCbParams &io) +{ + if (io.flag == Comm::ERR_CLOSING) + return; + + if (io.flag != Comm::OK) { + debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno)); + io.conn->close(); + return; + } + + ClientSocketContext::Pointer context = getCurrentContext(); + if (context != NULL && context->http) { + context->http->out.size += io.size; + context->http->out.headers_sz += io.size; + } + + flags.readMore = true; + readSomeData(); +} + +void +Ftp::Server::wroteReply(const CommIoCbParams &io) +{ + if (io.flag == Comm::ERR_CLOSING) + return; + + if (io.flag != Comm::OK) { + debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno)); + io.conn->close(); + return; + } + + ClientSocketContext::Pointer context = getCurrentContext(); + assert(context->http); + context->http->out.size += io.size; + context->http->out.headers_sz += io.size; + + if (master.serverState == fssError) { + debugs(33, 5, "closing on FTP server error"); + io.conn->close(); + return; + } + + const clientStream_status_t socketState = context->socketState(); + debugs(33, 5, "FTP client stream state " << socketState); + switch (socketState) { + case STREAM_UNPLANNED_COMPLETE: + case STREAM_FAILED: + io.conn->close(); + return; + + case STREAM_NONE: + case STREAM_COMPLETE: + flags.readMore = true; + changeState(fssConnected, "Ftp::Server::wroteReply"); + if (in.bodyParser) + finishDechunkingRequest(false); + context->keepaliveNextRequest(); + return; + } +} + +bool +Ftp::Server::handleRequest(String &cmd, String ¶ms) { + HttpRequest *request = getCurrentContext()->http->request; + Must(request); + + if (do_debug(11, 2)) { + MemBuf mb; + Packer p; + mb.init(); + packerToMemInit(&p, &mb); + request->pack(&p); + packerClean(&p); + + debugs(11, 2, "FTP Client " << clientConnection); + debugs(11, 2, "FTP Client REQUEST:\n---------\n" << mb.buf << + "\n----------"); + } + + // TODO: optimize using a static map with case-insensitive lookup + static std::pair handlers[] = { + std::make_pair("LIST", &Ftp::Server::handleDataRequest), + std::make_pair("NLST", &Ftp::Server::handleDataRequest), + std::make_pair("MLSD", &Ftp::Server::handleDataRequest), + std::make_pair("FEAT", &Ftp::Server::handleFeatRequest), + std::make_pair("PASV", &Ftp::Server::handlePasvRequest), + std::make_pair("PORT", &Ftp::Server::handlePortRequest), + std::make_pair("RETR", &Ftp::Server::handleDataRequest), + std::make_pair("EPRT", &Ftp::Server::handleEprtRequest), + std::make_pair("EPSV", &Ftp::Server::handleEpsvRequest), + std::make_pair("CWD", &Ftp::Server::handleCwdRequest), + std::make_pair("PASS", &Ftp::Server::handlePassRequest), + std::make_pair("CDUP", &Ftp::Server::handleCdupRequest) + }; + + RequestHandler handler = NULL; + if (request->method == Http::METHOD_PUT) + handler = &Ftp::Server::handleUploadRequest; + else { + for (size_t i = 0; i < sizeof(handlers) / sizeof(*handlers); ++i) { + if (cmd.caseCmp(handlers[i].first) == 0) { + handler = handlers[i].second; + break; + } + } + } + + // TODO: complain about unknown commands + return handler != NULL ? (this->*handler)(cmd, params) : true; +} + +/// Called to parse USER command, which is required to create an HTTP request +/// wrapper. Thus, errors are handled with writeEarlyReply() here. +bool +Ftp::Server::handleUserRequest(const String &cmd, String ¶ms) +{ + if (params.size() == 0) { + writeEarlyReply(501, "Missing username"); + return false; + } + + const String::size_type eou = params.rfind('@'); + if (eou == String::npos || eou + 1 >= params.size()) { + writeEarlyReply(501, "Missing host"); + return false; + } + + const String login = params.substr(0, eou); + host = params.substr(eou + 1, params.size()); + // If we can parse it as raw IPv6 address, then surround with "[]". + // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is. + if (host.pos(":")) { + char ipBuf[MAX_IPSTRLEN]; + Ip::Address ipa; + ipa = host.termedBuf(); + if (!ipa.isAnyAddr()) { + ipa.toHostStr(ipBuf, MAX_IPSTRLEN); + host = ipBuf; + } + } + + String oldUri; + if (master.clientReadGreeting) + oldUri = uri; + + master.workingDir = NULL; + calcUri(); + + if (!master.clientReadGreeting) { + debugs(11, 3, "set URI to " << uri); + } else if (oldUri.caseCmp(uri) == 0) { + debugs(11, 5, "keep URI as " << oldUri); + } else { + debugs(11, 3, "reset URI from " << oldUri << " to " << uri); + closeDataConnection(); + master.clientReadGreeting = false; + unpinConnection(true); // close control connection to peer + changeState(fssBegin, "URI reset"); + } + + params.cut(eou); + + return true; +} + +bool +Ftp::Server::handleFeatRequest(String &cmd, String ¶ms) +{ + changeState(fssHandleFeat, "ftpHandleFeatRequest"); + return true; +} + +bool +Ftp::Server::handlePasvRequest(String &cmd, String ¶ms) +{ + if (gotEpsvAll) { + setReply(500, "Bad PASV command"); + return false; + } + + if (params.size() > 0) { + setReply(501, "Unexpected parameter"); + return false; + } + + changeState(fssHandlePasv, "ftpHandlePasvRequest"); + // no need to fake PASV request via setDataCommand() in true PASV case + return true; +} + +/// [Re]initializes dataConn for active data transfers. Does not connect. +bool +Ftp::Server::createDataConnection(Ip::Address cltAddr) +{ + assert(clientConnection != NULL); + assert(!clientConnection->remote.isAnyAddr()); + + if (cltAddr != clientConnection->remote) { + debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote); + // Closing the control connection would not help with attacks because + // the client is evidently able to connect to us. Besides, closing + // makes retrials easier for the client and more damaging to us. + setReply(501, "Prohibited parameter value"); + return false; + } + + closeDataConnection(); + + Comm::ConnectionPointer conn = new Comm::Connection(); + conn->remote = cltAddr; + + // Use local IP address of the control connection as the source address + // of the active data connection, or some clients will refuse to accept. + conn->flags |= COMM_DOBIND; + conn->local = clientConnection->local; + // RFC 959 requires active FTP connections to originate from port 20 + // but that would preclude us from supporting concurrent transfers! (XXX?) + conn->local.port(0); + + debugs(11, 3, "will actively connect from " << conn->local << " to " << + conn->remote); + + dataConn = conn; + uploadAvailSize = 0; + return true; +} + +bool +Ftp::Server::handlePortRequest(String &cmd, String ¶ms) +{ + // TODO: Should PORT errors trigger closeDataConnection() cleanup? + + if (gotEpsvAll) { + setReply(500, "Rejecting PORT after EPSV ALL"); + return false; + } + + if (!params.size()) { + setReply(501, "Missing parameter"); + return false; + } + + Ip::Address cltAddr; + if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) { + setReply(501, "Invalid parameter"); + return false; + } + + if (!createDataConnection(cltAddr)) + return false; + + changeState(fssHandlePort, "ftpHandlePortRequest"); + setDataCommand(); + return true; // forward our fake PASV request +} + +bool +Ftp::Server::handleDataRequest(String &cmd, String ¶ms) +{ + if (!checkDataConnPre()) + return false; + + changeState(fssHandleDataRequest, "ftpHandleDataRequest"); + + return true; +} + +bool +Ftp::Server::handleUploadRequest(String &cmd, String ¶ms) +{ + if (!checkDataConnPre()) + return false; + + changeState(fssHandleUploadRequest, "ftpHandleDataRequest"); + + return true; +} + +bool +Ftp::Server::handleEprtRequest(String &cmd, String ¶ms) +{ + debugs(11, 3, "Process an EPRT " << params); + + if (gotEpsvAll) { + setReply(500, "Rejecting EPRT after EPSV ALL"); + return false; + } + + if (!params.size()) { + setReply(501, "Missing parameter"); + return false; + } + + Ip::Address cltAddr; + if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) { + setReply(501, "Invalid parameter"); + return false; + } + + if (!createDataConnection(cltAddr)) + return false; + + changeState(fssHandleEprt, "ftpHandleEprtRequest"); + setDataCommand(); + return true; // forward our fake PASV request +} + +bool +Ftp::Server::handleEpsvRequest(String &cmd, String ¶ms) +{ + debugs(11, 3, "Process an EPSV command with params: " << params); + if (params.size() <= 0) { + // treat parameterless EPSV as "use the protocol of the ctrl conn" + } else if (params.caseCmp("ALL") == 0) { + setReply(200, "EPSV ALL ok"); + gotEpsvAll = true; + return false; + } else if (params.cmp("2") == 0) { + if (!Ip::EnableIpv6) { + setReply(522, "Network protocol not supported, use (1)"); + return false; + } + } else if (params.cmp("1") != 0) { + setReply(501, "Unsupported EPSV parameter"); + return false; + } + + changeState(fssHandleEpsv, "ftpHandleEpsvRequest"); + setDataCommand(); + return true; // forward our fake PASV request +} + +bool +Ftp::Server::handleCwdRequest(String &cmd, String ¶ms) +{ + changeState(fssHandleCwd, "ftpHandleCwdRequest"); + return true; +} + +bool +Ftp::Server::handlePassRequest(String &cmd, String ¶ms) +{ + changeState(fssHandlePass, "ftpHandlePassRequest"); + return true; +} + +bool +Ftp::Server::handleCdupRequest(String &cmd, String ¶ms) +{ + changeState(fssHandleCdup, "ftpHandleCdupRequest"); + return true; +} + +// Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command. +// Squid FTP client decides what data command to use with peers. +void +Ftp::Server::setDataCommand() +{ + ClientHttpRequest *const http = getCurrentContext()->http; + assert(http != NULL); + HttpRequest *const request = http->request; + assert(request != NULL); + HttpHeader &header = request->header; + header.delById(HDR_FTP_COMMAND); + header.putStr(HDR_FTP_COMMAND, "PASV"); + header.delById(HDR_FTP_ARGUMENTS); + header.putStr(HDR_FTP_ARGUMENTS, ""); + debugs(11, 5, "client data command converted to fake PASV"); +} + +/// check that client data connection is ready for future I/O or at least +/// has a chance of becoming ready soon. +bool +Ftp::Server::checkDataConnPre() +{ + if (Comm::IsConnOpen(dataConn)) + return true; + + if (Comm::IsConnOpen(dataListenConn)) { + // We are still waiting for a client to connect to us after PASV. + // Perhaps client's data conn handshake has not reached us yet. + // After we talk to the server, checkDataConnPost() will recheck. + debugs(33, 3, "expecting clt data conn " << dataListenConn); + return true; + } + + if (!dataConn || dataConn->remote.isAnyAddr()) { + debugs(33, 5, "missing " << dataConn); + // TODO: use client address and default port instead. + setReply(425, "Use PORT or PASV first"); + return false; + } + + // active transfer: open a data connection from Squid to client + typedef CommCbMemFunT Dialer; + connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData); + Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector, + Config.Timeout.connect); + AsyncJob::Start(cs); + return false; // ConnStateData::processFtpRequest waits handleConnectDone +} + +/// Check that client data connection is ready for immediate I/O. +bool +Ftp::Server::checkDataConnPost() const +{ + if (!Comm::IsConnOpen(dataConn)) { + debugs(33, 3, "missing client data conn: " << dataConn); + return false; + } + return true; +} + +/// Done establishing a data connection to the user. +void +Ftp::Server::connectedForData(const CommConnectCbParams ¶ms) +{ + connector = NULL; + + if (params.flag != Comm::OK) { + /* it might have been a timeout with a partially open link */ + if (params.conn != NULL) + params.conn->close(); + setReply(425, "Cannot open data connection."); + ClientSocketContext::Pointer context = getCurrentContext(); + Must(context->http); + Must(context->http->storeEntry() != NULL); + } else { + Must(dataConn == params.conn); + Must(Comm::IsConnOpen(params.conn)); + fd_note(params.conn->fd, "active client ftp data"); + } + + doProcessRequest(); +} + +void +Ftp::Server::setReply(const int code, const char *msg) +{ + ClientSocketContext::Pointer context = getCurrentContext(); + ClientHttpRequest *const http = context->http; + assert(http != NULL); + assert(http->storeEntry() == NULL); + + HttpReply *const reply = new HttpReply; + reply->sline.set(Http::ProtocolVersion(1, 1), Http::scNoContent); + HttpHeader &header = reply->header; + header.putTime(HDR_DATE, squid_curtime); + { + HttpHdrCc cc; + cc.Private(); + header.putCc(&cc); + } + header.putInt64(HDR_CONTENT_LENGTH, 0); + header.putInt(HDR_FTP_STATUS, code); + header.putStr(HDR_FTP_REASON, msg); + reply->hdrCacheInit(); + + setLogUri(http, urlCanonicalClean(http->request)); + + clientStreamNode *const node = context->getClientReplyContext(); + clientReplyContext *const repContext = + dynamic_cast(node->data.getRaw()); + assert(repContext != NULL); + + RequestFlags reqFlags; + reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry() + reqFlags.noCache = true; + repContext->createStoreEntry(http->request->method, reqFlags); + http->storeEntry()->replaceHttpReply(reply); +} + +/// Whether Squid FTP gateway supports a given feature (e.g., a command). +static bool +Ftp::SupportedCommand(const String &name) +{ + static std::set BlackList; + if (BlackList.empty()) { + /* Add FTP commands that Squid cannot gateway correctly */ + + // we probably do not support AUTH TLS.* and AUTH SSL, + // but let's disclaim all AUTH support to KISS, for now + BlackList.insert("AUTH"); + } + + // we claim support for all commands that we do not know about + return BlackList.find(name.termedBuf()) == BlackList.end(); +} + diff --git a/src/servers/FtpServer.h b/src/servers/FtpServer.h new file mode 100644 index 0000000000..f659895db2 --- /dev/null +++ b/src/servers/FtpServer.h @@ -0,0 +1,150 @@ +/* + * DEBUG: section 33 Client-side Routines + */ + +#ifndef SQUID_SERVERS_FTP_SERVER_H +#define SQUID_SERVERS_FTP_SERVER_H + +#include "client_side.h" + +namespace Ftp { + +typedef enum { + fssBegin, + fssConnected, + fssHandleFeat, + fssHandlePasv, + fssHandlePort, + fssHandleDataRequest, + fssHandleUploadRequest, + fssHandleEprt, + fssHandleEpsv, + fssHandleCwd, + fssHandlePass, + fssHandleCdup, + fssError +} ServerState; + +// TODO: This should become a part of MasterXaction when we start sending +// master transactions to the clients/ code. +/// Transaction information shared among our FTP client and server jobs. +class MasterState +{ +public: + Ip::Address clientDataAddr; ///< address of our FTP client data connection + String workingDir; + ServerState serverState; ///< what our FTP server is doing + bool clientReadGreeting; ///< whether our FTP client read their FTP server greeting + + MasterState(): serverState(fssBegin), clientReadGreeting(false) {} +}; + +/// Manages a control connection from an FTP client. +class Server: public ConnStateData +{ +public: + explicit Server(const MasterXaction::Pointer &xact); + virtual ~Server(); + + MasterState master; ///< info shared among our FTP client and server jobs + +protected: + friend void StartListening(); + + /* ConnStateData API */ + virtual ClientSocketContext *parseOneRequest(Http::ProtocolVersion &ver); + virtual void processParsedRequest(ClientSocketContext *context, const Http::ProtocolVersion &ver); + virtual void notePeerConnection(Comm::ConnectionPointer conn); + virtual void clientPinnedConnectionClosed(const CommCloseCbParams &io); + virtual void handleReply(HttpReply *header, StoreIOBuffer receivedData); + virtual int pipelinePrefetchMax() const; + virtual void writeControlMsgAndCall(ClientSocketContext *context, HttpReply *rep, AsyncCall::Pointer &call); + virtual time_t idleTimeout() const; + + /* BodyPipe API */ + virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer); + virtual void noteBodyConsumerAborted(BodyPipe::Pointer ptr); + + /* AsyncJob API */ + virtual void start(); + + /* Comm callbacks */ + static void AcceptCtrlConnection(const CommAcceptCbParams ¶ms); + void acceptDataConnection(const CommAcceptCbParams ¶ms); + void readUploadData(const CommIoCbParams &io); + void wroteEarlyReply(const CommIoCbParams &io); + void wroteReply(const CommIoCbParams &io); + void wroteReplyData(const CommIoCbParams &io); + void connectedForData(const CommConnectCbParams ¶ms); + + unsigned int listenForDataConnection(); + bool createDataConnection(Ip::Address cltAddr); + void closeDataConnection(); + + void calcUri(const char *file = NULL); + void changeState(const Ftp::ServerState newState, const char *reason); + bool handleUserRequest(const String &cmd, String ¶ms); + bool checkDataConnPost() const; + void replyDataWritingCheckpoint(); + void maybeReadUploadData(); + + void setReply(const int code, const char *msg); + void writeCustomReply(const int code, const char *msg, const HttpReply *reply = NULL); + void writeEarlyReply(const int code, const char *msg); + void writeErrorReply(const HttpReply *reply, const int status); + void writeForwardedForeign(const HttpReply *reply); + void writeForwardedReply(const HttpReply *reply); + void writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call); + void writeReply(MemBuf &mb); + + bool handleRequest(String &cmd, String ¶ms); + void setDataCommand(); + bool checkDataConnPre(); + + /// a method handling an FTP command; selected by handleRequest() + typedef bool (Ftp::Server::*RequestHandler)(String &cmd, String ¶ms); + bool handleFeatRequest(String &cmd, String ¶ms); + bool handlePasvRequest(String &cmd, String ¶ms); + bool handlePortRequest(String &cmd, String ¶ms); + bool handleDataRequest(String &cmd, String ¶ms); + bool handleUploadRequest(String &cmd, String ¶ms); + bool handleEprtRequest(String &cmd, String ¶ms); + bool handleEpsvRequest(String &cmd, String ¶ms); + bool handleCwdRequest(String &cmd, String ¶ms); + bool handlePassRequest(String &cmd, String ¶ms); + bool handleCdupRequest(String &cmd, String ¶ms); + + /// a method handling an FTP response; selected by handleReply() + typedef void (Ftp::Server::*ReplyHandler)(const HttpReply *reply, StoreIOBuffer data); + void handleFeatReply(const HttpReply *header, StoreIOBuffer receivedData); + void handlePasvReply(const HttpReply *header, StoreIOBuffer receivedData); + void handlePortReply(const HttpReply *header, StoreIOBuffer receivedData); + void handleErrorReply(const HttpReply *header, StoreIOBuffer receivedData); + void handleDataReply(const HttpReply *header, StoreIOBuffer receivedData); + void handleUploadReply(const HttpReply *header, StoreIOBuffer receivedData); + void handleEprtReply(const HttpReply *header, StoreIOBuffer receivedData); + void handleEpsvReply(const HttpReply *header, StoreIOBuffer receivedData); + +private: + void doProcessRequest(); + void shovelUploadData(); + + String uri; ///< a URI reconstructed from various FTP message details + String host; ///< intended dest. of a transparently intercepted FTP conn + bool gotEpsvAll; ///< restrict data conn setup commands to just EPSV + AsyncCall::Pointer onDataAcceptCall; ///< who to call upon data conn acceptance + Comm::ConnectionPointer dataListenConn; ///< data connection listening socket + Comm::ConnectionPointer dataConn; ///< data connection + char uploadBuf[CLIENT_REQ_BUF_SZ]; ///< data connection input buffer + size_t uploadAvailSize; ///< number of yet unused uploadBuf bytes + + AsyncCall::Pointer listener; ///< set when we are passively listening + AsyncCall::Pointer connector; ///< set when we are actively connecting + AsyncCall::Pointer reader; ///< set when we are reading FTP data + + CBDATA_CLASS2(Server); +}; + +} // namespace Ftp + +#endif /* SQUID_SERVERS_FTP_SERVER_H */ diff --git a/src/servers/HttpServer.cc b/src/servers/HttpServer.cc new file mode 100644 index 0000000000..58646d1b4a --- /dev/null +++ b/src/servers/HttpServer.cc @@ -0,0 +1,193 @@ +/* + * DEBUG: section 33 Client-side Routines + */ + +#include "squid.h" +#include "client_side.h" +#include "client_side_request.h" +#include "comm/Write.h" +#include "HttpHeaderTools.h" +#include "profiler/Profiler.h" +#include "servers/forward.h" +#include "SquidConfig.h" + +namespace Http { + +/// Manages a connection from an HTTP client. +class Server: public ConnStateData +{ +public: + Server(const MasterXaction::Pointer &xact, const bool beHttpsServer); + virtual ~Server() {} + + void readSomeHttpData(); + +protected: + /* ConnStateData API */ + virtual ClientSocketContext *parseOneRequest(Http::ProtocolVersion &ver); + virtual void processParsedRequest(ClientSocketContext *context, const Http::ProtocolVersion &ver); + virtual void handleReply(HttpReply *rep, StoreIOBuffer receivedData); + virtual void writeControlMsgAndCall(ClientSocketContext *context, HttpReply *rep, AsyncCall::Pointer &call); + virtual time_t idleTimeout() const; + + /* BodyPipe API */ + virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer); + virtual void noteBodyConsumerAborted(BodyPipe::Pointer); + + /* AsyncJob API */ + virtual void start(); + +private: + void processHttpRequest(ClientSocketContext *const context); + void handleHttpRequestData(); + + HttpParser parser_; + HttpRequestMethod method_; ///< parsed HTTP method + + /// temporary hack to avoid creating a true HttpsServer class + const bool isHttpsServer; + + CBDATA_CLASS2(Server); +}; + +} // namespace Http + +CBDATA_NAMESPACED_CLASS_INIT(Http, Server); + +Http::Server::Server(const MasterXaction::Pointer &xact, bool beHttpsServer): + AsyncJob("Http::Server"), + ConnStateData(xact), + isHttpsServer(beHttpsServer) +{ +} + +time_t +Http::Server::idleTimeout() const +{ + return Config.Timeout.clientIdlePconn; +} + +void +Http::Server::start() +{ + ConnStateData::start(); + +#if USE_OPENSSL + // XXX: Until we create an HttpsServer class, use this hack to allow old + // client_side.cc code to manipulate ConnStateData object directly + if (isHttpsServer) { + postHttpsAccept(); + return; + } +#endif + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(33, 5, + TimeoutDialer, this, Http::Server::requestTimeout); + commSetConnTimeout(clientConnection, Config.Timeout.request, timeoutCall); + readSomeData(); +} + +void +Http::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer) +{ + if (!handleRequestBodyData()) + return; + + // too late to read more body + if (!isOpen() || stoppedReceiving()) + return; + + readSomeData(); +} + +ClientSocketContext * +Http::Server::parseOneRequest(Http::ProtocolVersion &ver) +{ + ClientSocketContext *context = NULL; + PROF_start(HttpServer_parseOneRequest); + HttpParserInit(&parser_, in.buf.c_str(), in.buf.length()); + context = parseHttpRequest(this, &parser_, &method_, &ver); + PROF_stop(HttpServer_parseOneRequest); + return context; +} + +void +Http::Server::processParsedRequest(ClientSocketContext *context, const Http::ProtocolVersion &ver) +{ + /* We have an initial client stream in place should it be needed */ + /* setup our private context */ + context->registerWithConn(); + clientProcessRequest(this, &parser_, context, method_, ver); +} + +void +Http::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr) +{ + ConnStateData::noteBodyConsumerAborted(ptr); + stopReceiving("virgin request body consumer aborted"); // closes ASAP +} + +void +Http::Server::handleReply(HttpReply *rep, StoreIOBuffer receivedData) +{ + // the caller guarantees that we are dealing with the current context only + ClientSocketContext::Pointer context = getCurrentContext(); + Must(context != NULL); + const ClientHttpRequest *http = context->http; + Must(http != NULL); + + // After sending Transfer-Encoding: chunked (at least), always send + // the last-chunk if there was no error, ignoring responseFinishedOrFailed. + const bool mustSendLastChunk = http->request->flags.chunkedReply && + !http->request->flags.streamError && + !context->startOfOutput(); + const bool responseFinishedOrFailed = !rep && + !receivedData.data && + !receivedData.length; + if (responseFinishedOrFailed && !mustSendLastChunk) { + context->writeComplete(context->clientConnection, NULL, 0, Comm::OK); + return; + } + + if (!context->startOfOutput()) { + context->sendBody(rep, receivedData); + return; + } + + assert(rep); + http->al->reply = rep; + HTTPMSGLOCK(http->al->reply); + context->sendStartOfMessage(rep, receivedData); +} + +void +Http::Server::writeControlMsgAndCall(ClientSocketContext *context, HttpReply *rep, AsyncCall::Pointer &call) +{ + // apply selected clientReplyContext::buildReplyHeader() mods + // it is not clear what headers are required for control messages + rep->header.removeHopByHopEntries(); + rep->header.putStr(HDR_CONNECTION, "keep-alive"); + httpHdrMangleList(&rep->header, getCurrentContext()->http->request, ROR_REPLY); + + MemBuf *mb = rep->pack(); + + debugs(11, 2, "HTTP Client " << clientConnection); + debugs(11, 2, "HTTP Client CONTROL MSG:\n---------\n" << mb->buf << "\n----------"); + + Comm::Write(context->clientConnection, mb, call); + + delete mb; +} + +ConnStateData * +Http::NewServer(MasterXactionPointer &xact) +{ + return new Server(xact, false); +} + +ConnStateData * +Https::NewServer(MasterXactionPointer &xact) +{ + return new Http::Server(xact, true); +} diff --git a/src/servers/Makefile.am b/src/servers/Makefile.am new file mode 100644 index 0000000000..ae0fc2d05e --- /dev/null +++ b/src/servers/Makefile.am @@ -0,0 +1,10 @@ +include $(top_srcdir)/src/Common.am + +noinst_LTLIBRARIES = libservers.la + +libservers_la_SOURCES = \ + FtpServer.cc \ + FtpServer.h \ + HttpServer.cc \ + \ + forward.h diff --git a/src/servers/forward.h b/src/servers/forward.h new file mode 100644 index 0000000000..50e20e5dda --- /dev/null +++ b/src/servers/forward.h @@ -0,0 +1,31 @@ +#ifndef SQUID_SERVERS_FORWARD_H +#define SQUID_SERVERS_FORWARD_H + +class MasterXaction; +template class RefCount; +typedef RefCount MasterXactionPointer; + +namespace Http { + +/// create a new HTTP connection handler; never returns NULL +ConnStateData *NewServer(MasterXactionPointer &xact); + +} // namespace Http + +namespace Https { + +/// create a new HTTPS connection handler; never returns NULL +ConnStateData *NewServer(MasterXactionPointer &xact); + +} // namespace Https + +namespace Ftp { + +/// accept connections on all configured ftp_ports +void StartListening(); +/// reject new connections to any configured ftp_port +void StopListening(); + +} // namespace Ftp + +#endif /* SQUID_SERVERS_FORWARD_H */