From: Alex Rousskov Date: Wed, 30 Jul 2014 17:33:14 +0000 (-0600) Subject: Merged from trunk (r13515). X-Git-Tag: SQUID_3_5_0_1~117^2~13 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8ea0d84754ce24036c43f6df893e667ae6d52b6d;p=thirdparty%2Fsquid.git Merged from trunk (r13515). Needs more work to handle FTP adaptation failures better. --- 8ea0d84754ce24036c43f6df893e667ae6d52b6d diff --cc src/FtpGatewayServer.cc index dbb4a6471e,0000000000..979ce4c759 mode 100644,000000..100644 --- a/src/FtpGatewayServer.cc +++ b/src/FtpGatewayServer.cc @@@ -1,688 -1,0 +1,688 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#include "squid.h" + +#include "anyp/PortCfg.h" +#include "FtpGatewayServer.h" +#include "FtpServer.h" +#include "HttpHdrCc.h" +#include "HttpRequest.h" +#include "Server.h" +#include "SquidTime.h" +#include "Store.h" +#include "client_side.h" +#include "wordlist.h" + +namespace Ftp { + +namespace Gateway { + +class ServerStateData: public Ftp::ServerStateData +{ +public: + ServerStateData(FwdState *const fwdState); + ~ServerStateData(); + + virtual void processReplyBody(); + +protected: + virtual void start(); + + ConnStateData::FtpState clientState() const; + void clientState(ConnStateData::FtpState newState); + virtual void serverComplete(); + virtual void failed(err_type error = ERR_NONE, int xerrno = 0); + virtual void handleControlReply(); + virtual void handleRequestBodyProducerAborted(); + virtual bool mayReadVirginReplyBody() const; + virtual void completeForwarding(); + void forwardReply(); + void forwardError(err_type error = ERR_NONE, int xerrno = 0); + void failedErrorMessage(err_type error, int xerrno); + HttpReply *createHttpReply(const Http::StatusCode httpStatus, const int clen = 0); + void handleDataRequest(); + void startDataDownload(); + void startDataUpload(); + bool startDirTracking(); + void stopDirTracking(); + bool weAreTrackingDir() const {return savedReply.message != NULL;} + + typedef void (ServerStateData::*PreliminaryCb)(); + void forwardPreliminaryReply(const PreliminaryCb cb); + void proceedAfterPreliminaryReply(); + PreliminaryCb thePreliminaryCb; + + typedef void (ServerStateData::*SM_FUNC)(); + static const SM_FUNC SM_FUNCS[]; + void readGreeting(); + void sendCommand(); + void readReply(); + void readFeatReply(); + void readPasvReply(); + void readDataReply(); + void readTransferDoneReply(); + void readEpsvReply(); + void readCwdOrCdupReply(); + void readUserOrPassReply(); + - virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno); ++ virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag err, int xerrno); + void scheduleReadControlReply(); + + bool forwardingCompleted; ///< completeForwarding() has been called + + struct { + wordlist *message; ///< reply message, one wordlist entry per message line + char *lastCommand; ///< the command caused the reply + char *lastReply; ///< last line of reply: reply status plus message + int replyCode; ///< the reply status + } savedReply; ///< set and delayed while we are tracking using PWD + + CBDATA_CLASS2(ServerStateData); +}; + +CBDATA_CLASS_INIT(ServerStateData); + +const ServerStateData::SM_FUNC ServerStateData::SM_FUNCS[] = { + &ServerStateData::readGreeting, // BEGIN + &ServerStateData::readUserOrPassReply, // SENT_USER + &ServerStateData::readUserOrPassReply, // SENT_PASS + NULL,/*&ServerStateData::readReply*/ // SENT_TYPE + NULL,/*&ServerStateData::readReply*/ // SENT_MDTM + NULL,/*&ServerStateData::readReply*/ // SENT_SIZE + NULL, // SENT_EPRT + NULL, // SENT_PORT + &ServerStateData::readEpsvReply, // SENT_EPSV_ALL + &ServerStateData::readEpsvReply, // SENT_EPSV_1 + &ServerStateData::readEpsvReply, // SENT_EPSV_2 + &ServerStateData::readPasvReply, // SENT_PASV + &ServerStateData::readCwdOrCdupReply, // SENT_CWD + NULL,/*&ServerStateData::readDataReply,*/ // SENT_LIST + NULL,/*&ServerStateData::readDataReply,*/ // SENT_NLST + NULL,/*&ServerStateData::readReply*/ // SENT_REST + NULL,/*&ServerStateData::readDataReply*/ // SENT_RETR + NULL,/*&ServerStateData::readReply*/ // SENT_STOR + NULL,/*&ServerStateData::readReply*/ // SENT_QUIT + &ServerStateData::readTransferDoneReply, // READING_DATA + &ServerStateData::readReply, // WRITING_DATA + NULL,/*&ServerStateData::readReply*/ // SENT_MKDIR + &ServerStateData::readFeatReply, // SENT_FEAT + NULL,/*&ServerStateData::readPwdReply*/ // SENT_PWD + &ServerStateData::readCwdOrCdupReply, // SENT_CDUP + &ServerStateData::readDataReply,// SENT_DATA_REQUEST + &ServerStateData::readReply, // SENT_COMMAND + NULL +}; + +ServerStateData::ServerStateData(FwdState *const fwdState): + AsyncJob("Ftp::Gateway::ServerStateData"), Ftp::ServerStateData(fwdState), + forwardingCompleted(false) +{ + savedReply.message = NULL; + savedReply.lastCommand = NULL; + savedReply.lastReply = NULL; + savedReply.replyCode = 0; + + // Nothing we can do at request creation time can mark the response as + // uncachable, unfortunately. This prevents "found KEY_PRIVATE" WARNINGs. + entry->releaseRequest(); +} + +ServerStateData::~ServerStateData() +{ + closeServer(); // TODO: move to Server.cc? + if (savedReply.message) + wordlistDestroy(&savedReply.message); + + xfree(savedReply.lastCommand); + xfree(savedReply.lastReply); +} + +void +ServerStateData::start() +{ + if (!fwd->request->clientConnectionManager->ftp.readGreeting) + Ftp::ServerStateData::start(); + else + if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + handleDataRequest(); + else + sendCommand(); +} + +/// Keep control connection for future requests, after we are done with it. +/// Similar to COMPLETE_PERSISTENT_MSG handling in http.cc. +void +ServerStateData::serverComplete() +{ + CbcPointer &mgr = fwd->request->clientConnectionManager; + if (mgr.valid()) { + if (Comm::IsConnOpen(ctrl.conn)) { + debugs(9, 7, "completing FTP server " << ctrl.conn << + " after " << ctrl.replycode); + fwd->unregister(ctrl.conn); + if (ctrl.replycode == 221) { // Server sends FTP 221 before closing + mgr->unpinConnection(false); + ctrl.close(); + } else { + mgr->pinConnection(ctrl.conn, fwd->request, + ctrl.conn->getPeer(), + fwd->request->flags.connectionAuth); + ctrl.forget(); + } + } + } + Ftp::ServerStateData::serverComplete(); +} + +ConnStateData::FtpState +ServerStateData::clientState() const +{ + return fwd->request->clientConnectionManager->ftp.state; +} + +void +ServerStateData::clientState(ConnStateData::FtpState newState) +{ + ConnStateData::FtpState &cltState = + fwd->request->clientConnectionManager->ftp.state; + debugs(9, 3, "client state was " << cltState << " now: " << newState); + cltState = newState; +} + +/** + * Ensure we do not double-complete on the forward entry. + * We complete forwarding when the response adaptation is over + * (but we may still be waiting for 226 from the FTP server) and + * also when we get that 226 from the server (and adaptation is done). + * + \todo Rewrite FwdState to ignore double completion? + */ +void +ServerStateData::completeForwarding() +{ + debugs(9, 5, forwardingCompleted); + if (forwardingCompleted) + return; + forwardingCompleted = true; + Ftp::ServerStateData::completeForwarding(); +} + +void +ServerStateData::failed(err_type error, int xerrno) +{ + if (!doneWithServer()) + clientState(ConnStateData::FTP_ERROR); + + // TODO: we need to customize ErrorState instead + if (entry->isEmpty()) + failedErrorMessage(error, xerrno); // as a reply + + Ftp::ServerStateData::failed(error, xerrno); +} + +void +ServerStateData::failedErrorMessage(err_type error, int xerrno) +{ + const Http::StatusCode httpStatus = failedHttpStatus(error); + HttpReply *const reply = createHttpReply(httpStatus); + entry->replaceHttpReply(reply); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + fwd->request->detailError(error, xerrno); +} + +void +ServerStateData::processReplyBody() +{ + debugs(9, 3, HERE << "starting"); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * probably was aborted because content length exceeds one + * of the maximum size limits. + */ + abortTransaction("entry aborted after calling appendSuccessHeader()"); + return; + } + +#if USE_ADAPTATION + + if (adaptationAccessCheckPending) { + debugs(9,3, HERE << "returning due to adaptationAccessCheckPending"); + return; + } + +#endif + + if (data.readBuf != NULL && data.readBuf->hasContent()) { + const mb_size_t csize = data.readBuf->contentSize(); + debugs(9, 5, HERE << "writing " << csize << " bytes to the reply"); + addVirginReplyBody(data.readBuf->content(), csize); + data.readBuf->consume(csize); + } + + entry->flush(); + + maybeReadVirginBody(); +} + +void +ServerStateData::handleControlReply() +{ + if (!request->clientConnectionManager.valid()) { + debugs(9, 5, "client connection gone"); + closeServer(); + return; + } + + Ftp::ServerStateData::handleControlReply(); + if (ctrl.message == NULL) + return; // didn't get complete reply yet + + assert(state < END); + assert(this->SM_FUNCS[state] != NULL); + (this->*SM_FUNCS[state])(); +} + +void +ServerStateData::handleRequestBodyProducerAborted() +{ + ::ServerStateData::handleRequestBodyProducerAborted(); + + failed(ERR_READ_ERROR); +} + +bool +ServerStateData::mayReadVirginReplyBody() const +{ + // TODO: move this method to the regular FTP server? + return Comm::IsConnOpen(data.conn); +} + +void +ServerStateData::forwardReply() +{ + assert(entry->isEmpty()); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + + HttpReply *const reply = createHttpReply(Http::scNoContent); + + setVirginReply(reply); + adaptOrFinalizeReply(); + + serverComplete(); +} + +void +ServerStateData::forwardPreliminaryReply(const PreliminaryCb cb) +{ + debugs(9, 5, HERE << "Forwarding preliminary reply to client"); + + assert(thePreliminaryCb == NULL); + thePreliminaryCb = cb; + + const HttpReply::Pointer reply = createHttpReply(Http::scContinue); + + // the Sink will use this to call us back after writing 1xx to the client + typedef NullaryMemFunT CbDialer; + const AsyncCall::Pointer call = JobCallback(11, 3, CbDialer, this, + ServerStateData::proceedAfterPreliminaryReply); + + CallJobHere1(9, 4, request->clientConnectionManager, ConnStateData, + ConnStateData::sendControlMsg, HttpControlMsg(reply, call)); +} + +void +ServerStateData::proceedAfterPreliminaryReply() +{ + debugs(9, 5, HERE << "Proceeding after preliminary reply to client"); + + assert(thePreliminaryCb != NULL); + const PreliminaryCb cb = thePreliminaryCb; + thePreliminaryCb = NULL; + (this->*cb)(); +} + +void +ServerStateData::forwardError(err_type error, int xerrno) +{ + failed(error, xerrno); +} + +HttpReply * +ServerStateData::createHttpReply(const Http::StatusCode httpStatus, const int clen) +{ + HttpReply *const reply = new HttpReply; + reply->sline.set(Http::ProtocolVersion(1, 1), httpStatus); + HttpHeader &header = reply->header; + header.putTime(HDR_DATE, squid_curtime); + { + HttpHdrCc cc; + cc.Private(); + header.putCc(&cc); + } + if (clen >= 0) + header.putInt64(HDR_CONTENT_LENGTH, clen); + + if (ctrl.message) { + for (wordlist *W = ctrl.message; W && W->next; W = W->next) + header.putStr(HDR_FTP_PRE, httpHeaderQuoteString(W->key).termedBuf()); + } + if (ctrl.replycode > 0) + header.putInt(HDR_FTP_STATUS, ctrl.replycode); + if (ctrl.last_reply) + header.putStr(HDR_FTP_REASON, ctrl.last_reply); + + reply->hdrCacheInit(); + + return reply; +} + +void +ServerStateData::handleDataRequest() +{ + data.addr(fwd->request->clientConnectionManager->ftp.serverDataAddr); + connectDataChannel(); +} + +void +ServerStateData::startDataDownload() +{ + assert(Comm::IsConnOpen(data.conn)); + + debugs(9, 3, HERE << "begin data transfer from " << data.conn->remote << + " (" << data.conn->local << ")"); + + HttpReply *const reply = createHttpReply(Http::scOkay, -1); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + setVirginReply(reply); + adaptOrFinalizeReply(); + + maybeReadVirginBody(); + state = READING_DATA; +} + +void +ServerStateData::startDataUpload() +{ + assert(Comm::IsConnOpen(data.conn)); + + debugs(9, 3, HERE << "begin data transfer to " << data.conn->remote << + " (" << data.conn->local << ")"); + + if (!startRequestBodyFlow()) { // register to receive body data + failed(); + return; + } + + state = WRITING_DATA; +} + +void +ServerStateData::readGreeting() +{ + assert(!fwd->request->clientConnectionManager->ftp.readGreeting); + + switch (ctrl.replycode) { + case 220: + fwd->request->clientConnectionManager->ftp.readGreeting = true; + if (clientState() == ConnStateData::FTP_BEGIN) + clientState(ConnStateData::FTP_CONNECTED); + + // Do not forward server greeting to the client because our client + // side code has greeted the client already. Also, a greeting may + // confuse a client that has changed the gateway destination mid-air. + + start(); + break; + case 120: + if (NULL != ctrl.message) + debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ctrl.message->key); + forwardPreliminaryReply(&ServerStateData::scheduleReadControlReply); + break; + default: + failed(); + break; + } +} + +void +ServerStateData::sendCommand() +{ + if (!fwd->request->header.has(HDR_FTP_COMMAND)) { + abortTransaction("Internal error: FTP gateway request with no command"); + return; + } + + HttpHeader &header = fwd->request->header; + assert(header.has(HDR_FTP_COMMAND)); + const String &cmd = header.findEntry(HDR_FTP_COMMAND)->value; + assert(header.has(HDR_FTP_ARGUMENTS)); + const String ¶ms = header.findEntry(HDR_FTP_ARGUMENTS)->value; + + if (params.size() > 0) + debugs(9, 5, HERE << "command: " << cmd << ", parameters: " << params); + else + debugs(9, 5, HERE << "command: " << cmd << ", no parameters"); + + if (clientState() == ConnStateData::FTP_HANDLE_PASV || + clientState() == ConnStateData::FTP_HANDLE_EPSV || + clientState() == ConnStateData::FTP_HANDLE_EPRT || + clientState() == ConnStateData::FTP_HANDLE_PORT) { + sendPassive(); + return; + } + + static MemBuf mb; + mb.reset(); + if (params.size() > 0) + mb.Printf("%s %s%s", cmd.termedBuf(), params.termedBuf(), Ftp::crlf); + else + mb.Printf("%s%s", cmd.termedBuf(), Ftp::crlf); + + writeCommand(mb.content()); + + state = + clientState() == ConnStateData::FTP_HANDLE_CDUP ? SENT_CDUP : + clientState() == ConnStateData::FTP_HANDLE_CWD ? SENT_CWD : + clientState() == ConnStateData::FTP_HANDLE_FEAT ? SENT_FEAT : + clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST ? SENT_DATA_REQUEST : + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST ? SENT_DATA_REQUEST : + clientState() == ConnStateData::FTP_CONNECTED ? SENT_USER : + clientState() == ConnStateData::FTP_HANDLE_PASS ? SENT_PASS : + SENT_COMMAND; +} + +void +ServerStateData::readReply() +{ + assert(clientState() == ConnStateData::FTP_CONNECTED || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + forwardPreliminaryReply(&ServerStateData::scheduleReadControlReply); + else + forwardReply(); +} + +void +ServerStateData::readFeatReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_FEAT); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; // ignore preliminary replies + + forwardReply(); +} + +void +ServerStateData::readPasvReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_PASV || clientState() == ConnStateData::FTP_HANDLE_EPSV || clientState() == ConnStateData::FTP_HANDLE_PORT || clientState() == ConnStateData::FTP_HANDLE_EPRT); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; // ignore preliminary replies + + if (handlePasvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) + forwardReply(); + else + forwardError(); +} + +void +ServerStateData::readEpsvReply() +{ + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; // ignore preliminary replies + + if (handleEpsvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) { + if (ctrl.message == NULL) + return; // didn't get complete reply yet + + forwardReply(); + } else + forwardError(); +} + +void +ServerStateData::readDataReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + + if (ctrl.replycode == 125 || ctrl.replycode == 150) { + if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST) + forwardPreliminaryReply(&ServerStateData::startDataDownload); + else // clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST + forwardPreliminaryReply(&ServerStateData::startDataUpload); + } else + forwardReply(); +} + +bool +ServerStateData::startDirTracking() +{ + if (!fwd->request->clientConnectionManager->port->ftp_track_dirs) + return false; + + debugs(9, 5, "Start directory tracking"); + savedReply.message = ctrl.message; + savedReply.lastCommand = ctrl.last_command; + savedReply.lastReply = ctrl.last_reply; + savedReply.replyCode = ctrl.replycode; + + ctrl.last_command = NULL; + ctrl.last_reply = NULL; + ctrl.message = NULL; + ctrl.offset = 0; + writeCommand("PWD\r\n"); + return true; +} + +void +ServerStateData::stopDirTracking() +{ + debugs(9, 5, "Got code from pwd: " << ctrl.replycode << ", msg: " << ctrl.last_reply); + + if (ctrl.replycode == 257) + fwd->request->clientConnectionManager->ftpSetWorkingDir(Ftp::unescapeDoubleQuoted(ctrl.last_reply)); + + wordlistDestroy(&ctrl.message); + safe_free(ctrl.last_command); + safe_free(ctrl.last_reply); + + ctrl.message = savedReply.message; + ctrl.last_command = savedReply.lastCommand; + ctrl.last_reply = savedReply.lastReply; + ctrl.replycode = savedReply.replyCode; + + savedReply.message = NULL; + savedReply.lastReply = NULL; + savedReply.lastCommand = NULL; +} + +void +ServerStateData::readCwdOrCdupReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_CWD || clientState() == ConnStateData::FTP_HANDLE_CDUP); + + debugs(9, 5, HERE << "Got code " << ctrl.replycode << ", msg: " << ctrl.last_reply); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; + + if (weAreTrackingDir()) { // we are tracking + stopDirTracking(); // and forward the delayed response below + } else if (startDirTracking()) + return; + + forwardReply(); +} + +void +ServerStateData::readUserOrPassReply() +{ + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; //Just ignore + + if (weAreTrackingDir()) { // we are tracking + stopDirTracking(); // and forward the delayed response below + } else if (ctrl.replycode == 230) { // successful login + if (startDirTracking()) + return; + } + + forwardReply(); +} + +void +ServerStateData::readTransferDoneReply() +{ + debugs(9, 3, HERE); + + if (ctrl.replycode != 226 && ctrl.replycode != 250) { + debugs(9, DBG_IMPORTANT, HERE << "Got code " << ctrl.replycode << + " after reading data"); + } + + serverComplete(); +} + +void - ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno) ++ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag err, int xerrno) +{ + debugs(9, 3, HERE); + data.opener = NULL; + - if (err != COMM_OK) { ++ if (err != Comm::OK) { + debugs(9, 2, HERE << "Failed to connect FTP server data channel."); + forwardError(ERR_CONNECT_FAIL, xerrno); + return; + } + + debugs(9, 2, HERE << "Connected FTP server data channel: " << conn); + + data.opened(conn, dataCloser()); + + sendCommand(); +} + +void +ServerStateData::scheduleReadControlReply() +{ + Ftp::ServerStateData::scheduleReadControlReply(0); +} + +}; // namespace Gateway + +}; // namespace Ftp + +void +ftpGatewayServerStart(FwdState *const fwdState) +{ + AsyncJob::Start(new Ftp::Gateway::ServerStateData(fwdState)); +} diff --cc src/FtpServer.cc index 4456445bc2,0000000000..449455bde9 mode 100644,000000..100644 --- a/src/FtpServer.cc +++ b/src/FtpServer.cc @@@ -1,1217 -1,0 +1,1219 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#include "squid.h" + +#include "acl/FilledChecklist.h" +#include "FtpServer.h" +#include "Mem.h" +#include "SquidConfig.h" +#include "StatCounters.h" +#include "client_side.h" +#include "comm/ConnOpener.h" ++#include "comm/Read.h" +#include "comm/TcpAcceptor.h" +#include "comm/Write.h" +#include "errorpage.h" +#include "fd.h" +#include "ip/tools.h" +#include "SquidString.h" +#include "tools.h" +#include "wordlist.h" +#include + +namespace Ftp { + +const char *const crlf = "\r\n"; + +/// \ingroup ServerProtocolFTPInternal +static char * +escapeIAC(const char *buf) +{ + int n; + char *ret; + unsigned const char *p; + unsigned char *r; + + for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p) + if (*p == 255) + ++n; + + ret = (char *)xmalloc(n); + + for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) { + *r = *p; + ++r; + + if (*p == 255) { + *r = 255; + ++r; + } + } + + *r = '\0'; + ++r; + assert((r - (unsigned char *)ret) == n ); + return ret; +} + +/// configures the channel with a descriptor and registers a close handler +void +FtpChannel::opened(const Comm::ConnectionPointer &newConn, + const AsyncCall::Pointer &aCloser) +{ + assert(!Comm::IsConnOpen(conn)); + assert(closer == NULL); + + assert(Comm::IsConnOpen(newConn)); + assert(aCloser != NULL); + + conn = newConn; + closer = aCloser; + comm_add_close_handler(conn->fd, closer); +} + +/// planned close: removes the close handler and calls comm_close +void +FtpChannel::close() +{ + // channels with active listeners will be closed when the listener handler dies. + if (Comm::IsConnOpen(conn)) { + comm_remove_close_handler(conn->fd, closer); + conn->close(); // we do not expect to be called back + } + clear(); +} + +void +FtpChannel::forget() +{ + if (Comm::IsConnOpen(conn)) { + commUnsetConnTimeout(conn); + comm_remove_close_handler(conn->fd, closer); + } + clear(); +} + +void +FtpChannel::clear() +{ + conn = NULL; + closer = NULL; +} + +ServerStateData::ServerStateData(FwdState *fwdState): + AsyncJob("Ftp::ServerStateData"), ::ServerStateData(fwdState) +{ + ++statCounter.server.all.requests; + ++statCounter.server.ftp.requests; + + ctrl.last_command = xstrdup("Connect to server"); + ctrl.buf = static_cast(memAllocBuf(4096, &ctrl.size)); + ctrl.offset = 0; + + typedef CommCbMemFunT Dialer; + const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this, + ServerStateData::ctrlClosed); + ctrl.opened(fwdState->serverConnection(), closer); +} + +void +ServerStateData::DataChannel::addr(const Ip::Address &import) +{ + static char addrBuf[MAX_IPSTRLEN]; + import.toStr(addrBuf, sizeof(addrBuf)); + xfree(host); + host = xstrdup(addrBuf); + port = import.port(); +} + +ServerStateData::~ServerStateData() +{ + if (data.opener != NULL) { + data.opener->cancel("Ftp::ServerStateData destructed"); + data.opener = NULL; + } + data.close(); + + if (ctrl.buf) { + memFreeBuf(ctrl.size, ctrl.buf); + ctrl.buf = NULL; + } + if (ctrl.message) + wordlistDestroy(&ctrl.message); + safe_free(ctrl.last_command); + safe_free(ctrl.last_reply); + + if (data.readBuf) { + if (!data.readBuf->isNull()) + data.readBuf->clean(); + + delete data.readBuf; + } + + safe_free(old_request); + + safe_free(old_reply); + + fwd = NULL; // refcounted +} + +void +ServerStateData::start() +{ + scheduleReadControlReply(0); +} + +void +ServerStateData::initReadBuf() +{ + if (data.readBuf == NULL) { + data.readBuf = new MemBuf; + data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF); + } +} + +/** + * Close the FTP server connection(s). Used by serverComplete(). + */ +void +ServerStateData::closeServer() +{ + if (Comm::IsConnOpen(ctrl.conn)) { + debugs(9,3, HERE << "closing FTP server FD " << ctrl.conn->fd << ", this " << this); + fwd->unregister(ctrl.conn); + ctrl.close(); + } + + if (Comm::IsConnOpen(data.conn)) { + debugs(9,3, HERE << "closing FTP data FD " << data.conn->fd << ", this " << this); + data.close(); + } + + debugs(9,3, HERE << "FTP ctrl and data connections closed. this " << this); +} + +/** + * Did we close all FTP server connection(s)? + * + \retval true Both server control and data channels are closed. And not waiting for a new data connection to open. + \retval false Either control channel or data is still active. + */ +bool +ServerStateData::doneWithServer() const +{ + return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn); +} + +void +ServerStateData::failed(err_type error, int xerrno) +{ + debugs(9,3,HERE << "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry); + + const char *command, *reply; + const Http::StatusCode httpStatus = failedHttpStatus(error); + ErrorState *const ftperr = new ErrorState(error, httpStatus, fwd->request); + ftperr->xerrno = xerrno; + + ftperr->ftp.server_msg = ctrl.message; + ctrl.message = NULL; + + if (old_request) + command = old_request; + else + command = ctrl.last_command; + + if (command && strncmp(command, "PASS", 4) == 0) + command = "PASS "; + + if (old_reply) + reply = old_reply; + else + reply = ctrl.last_reply; + + if (command) + ftperr->ftp.request = xstrdup(command); + + if (reply) + ftperr->ftp.reply = xstrdup(reply); + + fwd->request->detailError(error, xerrno); + fwd->fail(ftperr); + + closeServer(); // we failed, so no serverComplete() +} + +Http::StatusCode +ServerStateData::failedHttpStatus(err_type &error) +{ + if (error == ERR_NONE) + error = ERR_FTP_FAILURE; + return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout : + Http::scBadGateway; +} + +/** + * DPW 2007-04-23 + * Looks like there are no longer anymore callers that set + * buffered_ok=1. Perhaps it can be removed at some point. + */ +void +ServerStateData::scheduleReadControlReply(int buffered_ok) +{ + debugs(9, 3, HERE << ctrl.conn); + + if (buffered_ok && ctrl.offset > 0) { + /* We've already read some reply data */ + handleControlReply(); + } else { + /* + * Cancel the timeout on the Data socket (if any) and + * establish one on the control socket. + */ + if (Comm::IsConnOpen(data.conn)) { + commUnsetConnTimeout(data.conn); + } + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, ServerStateData::timeout); + commSetConnTimeout(ctrl.conn, Config.Timeout.read, timeoutCall); + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, ServerStateData::readControlReply); + comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader); + } +} + +void +ServerStateData::readControlReply(const CommIoCbParams &io) +{ + debugs(9, 3, HERE << "FD " << io.fd << ", Read " << io.size << " bytes"); + + if (io.size > 0) { + kb_incr(&(statCounter.server.all.kbytes_in), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_in), io.size); + } + - if (io.flag == COMM_ERR_CLOSING) ++ if (io.flag == Comm::ERR_CLOSING) + return; + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + abortTransaction("entry aborted during control reply read"); + return; + } + + assert(ctrl.offset < ctrl.size); + - if (io.flag == COMM_OK && io.size > 0) { ++ if (io.flag == Comm::OK && io.size > 0) { + fd_bytes(io.fd, io.size, FD_READ); + } + - if (io.flag != COMM_OK) { ++ if (io.flag != Comm::OK) { + debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT, + "ftpReadControlReply: read error: " << xstrerr(io.xerrno)); + + if (ignoreErrno(io.xerrno)) { + scheduleReadControlReply(0); + } else { + failed(ERR_READ_ERROR, io.xerrno); + /* failed closes ctrl.conn and frees ftpState */ + } + return; + } + + if (io.size == 0) { + if (entry->store_status == STORE_PENDING) { + failed(ERR_FTP_FAILURE, 0); + /* failed closes ctrl.conn and frees ftpState */ + return; + } + + /* XXX this may end up having to be serverComplete() .. */ + abortTransaction("zero control reply read"); + return; + } + + unsigned int len =io.size + ctrl.offset; + ctrl.offset = len; + assert(len <= ctrl.size); + if (Comm::IsConnOpen(ctrl.conn)) + commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply + handleControlReply(); +} + +void +ServerStateData::handleControlReply() +{ + debugs(9, 3, HERE); + + size_t bytes_used = 0; + wordlistDestroy(&ctrl.message); + + if (!parseControlReply(bytes_used)) { + /* didn't get complete reply yet */ + + if (ctrl.offset == ctrl.size) { + ctrl.buf = (char *)memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size); + } + + scheduleReadControlReply(0); + return; + } + + assert(ctrl.message); // the entire FTP server response, line by line + assert(ctrl.replycode >= 0); // FTP status code (from the last line) + assert(ctrl.last_reply); // FTP reason (from the last line) + + if (ctrl.offset == bytes_used) { + /* used it all up */ + ctrl.offset = 0; + } else { + /* Got some data past the complete reply */ + assert(bytes_used < ctrl.offset); + ctrl.offset -= bytes_used; + memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset); + } + + debugs(9, 3, HERE << "state=" << state << ", code=" << ctrl.replycode); +} + +bool +ServerStateData::handlePasvReply(Ip::Address &srvAddr) +{ + int code = ctrl.replycode; + char *buf; + debugs(9, 3, HERE); + + if (code != 227) { + debugs(9, 2, "PASV not supported by remote end"); + return false; + } + + /* 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). */ + /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ + debugs(9, 5, HERE << "scanning: " << ctrl.last_reply); + + buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789"); + + const char *forceIp = Config.Ftp.sanitycheck ? + fd_table[ctrl.conn->fd].ipaddr : NULL; + if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) { + debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " << + ctrl.conn->remote << ": " << ctrl.last_reply); + return false; + } + + data.addr(srvAddr); + + return true; +} + +bool +ServerStateData::handleEpsvReply(Ip::Address &remoteAddr) +{ + int code = ctrl.replycode; + char *buf; + debugs(9, 3, HERE); + + if (code != 229 && code != 522) { + if (code == 200) { + /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */ + /* vsftpd for one send '200 EPSV ALL ok.' without even port info. + * Its okay to re-send EPSV 1/2 but nothing else. */ + debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV"); + } else { + debugs(9, 2, "EPSV not supported by remote end"); + } + return sendPassive(); + } + + if (code == 522) { + /* server response with list of supported methods */ + /* 522 Network protocol not supported, use (1) */ + /* 522 Network protocol not supported, use (1,2) */ + /* 522 Network protocol not supported, use (2) */ + /* TODO: handle the (1,2) case. We might get it back after EPSV ALL + * which means close data + control without self-destructing and re-open from scratch. */ + debugs(9, 5, HERE << "scanning: " << ctrl.last_reply); + buf = ctrl.last_reply; + while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(') + ++buf; + if (buf != NULL && *buf == '\n') + ++buf; + + if (buf == NULL || *buf == '\0') { + /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */ + debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints"); + return sendPassive(); + } else if (strcmp(buf, "(1)") == 0) { + state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */ + return sendPassive(); + } else if (strcmp(buf, "(2)") == 0) { + if (Ip::EnableIpv6) { + /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */ + if (state == SENT_EPSV_2) { + return sendEprt(); + } else { + /* or try the next Passive mode down the chain. */ + return sendPassive(); + } + } else { + /* Server only accept EPSV in IPv6 traffic. */ + state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */ + return sendPassive(); + } + } else { + /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */ + debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf); + return sendPassive(); + } + failed(ERR_FTP_FAILURE, 0); + return false; + } + + /* 229 Entering Extended Passive Mode (|||port|) */ + /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ + debugs(9, 5, "scanning: " << ctrl.last_reply); + + buf = ctrl.last_reply + strcspn(ctrl.last_reply, "("); + + char h1, h2, h3, h4; + unsigned short port; + int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4); + + if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) { + debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " << + ctrl.conn->remote << ": " << + ctrl.last_reply); + + return sendPassive(); + } + + if (0 == port) { + debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " << + ctrl.conn->remote << ": " << + ctrl.last_reply); + + return sendPassive(); + } + + if (Config.Ftp.sanitycheck) { + if (port < 1024) { + debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " << + ctrl.conn->remote << ": " << + ctrl.last_reply); + + return sendPassive(); + } + } + + remoteAddr = ctrl.conn->remote; + remoteAddr.port(port); + data.addr(remoteAddr); + return true; +} + +// The server-side EPRT and PORT commands are not yet implemented. +// The ServerStateData::sendEprt() will fail because of the unimplemented +// openListenSocket() or sendPort() methods +bool +ServerStateData::sendEprt() +{ + if (!Config.Ftp.eprt) { + /* Disabled. Switch immediately to attempting old PORT command. */ + debugs(9, 3, "EPRT disabled by local administrator"); + return sendPort(); + } + + debugs(9, 3, HERE); + + if (!openListenSocket()) { + failed(ERR_FTP_FAILURE, 0); + return false; + } + + debugs(9, 3, "Listening for FTP data connection with FD " << data.conn); + if (!Comm::IsConnOpen(data.conn)) { + /* XXX Need to set error message */ + failed(ERR_FTP_FAILURE, 0); + return false; + } + + static MemBuf mb; + mb.reset(); + char buf[MAX_IPSTRLEN]; + /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */ + /* Which can be used by EITHER protocol. */ + debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port()); + mb.Printf("EPRT |%d|%s|%d|%s", + ( data.conn->local.isIPv6() ? 2 : 1 ), + data.conn->local.toStr(buf,MAX_IPSTRLEN), + comm_local_port(data.conn->fd), Ftp::crlf ); + + state = SENT_EPRT; + writeCommand(mb.content()); + return true; +} + +bool +ServerStateData::sendPort() +{ + failed(ERR_FTP_FAILURE, 0); + return false; +} + +bool +ServerStateData::sendPassive() +{ + debugs(9, 3, HERE); + + /** \par + * Checks for EPSV ALL special conditions: + * If enabled to be sent, squid MUST NOT request any other connect methods. + * If 'ALL' is sent and fails the entire FTP Session fails. + * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */ + if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) { + // We are here because the last "EPSV 1" failed, but because of epsv_all + // no other method allowed. + debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent."); + failed(ERR_FTP_FAILURE, 0); + return false; + } + + + /// Closes any old FTP-Data connection which may exist. */ + data.close(); + + /** \par + * Checks for previous EPSV/PASV failures on this server/session. + * Diverts to EPRT immediately if they are not working. */ + if (!Config.Ftp.passive || state == SENT_PASV) { + sendEprt(); + return true; + } + + static MemBuf mb; + mb.reset(); + /** \par + * Send EPSV (ALL,2,1) or PASV on the control channel. + * + * - EPSV ALL is used if enabled. + * - EPSV 2 is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6. + * - EPSV 1 is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4. + * - PASV is used if EPSV 1 fails. + */ + switch (state) { + case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */ + if (ctrl.conn->local.isIPv6()) { + debugs(9, 5, HERE << "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed."); + mb.Printf("EPSV 2%s", Ftp::crlf); + state = SENT_EPSV_2; + break; + } + // else fall through to skip EPSV 2 + + case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */ + if (ctrl.conn->local.isIPv4()) { + debugs(9, 5, HERE << "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed."); + mb.Printf("EPSV 1%s", Ftp::crlf); + state = SENT_EPSV_1; + break; + } else if (Config.Ftp.epsv_all) { + debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent."); + failed(ERR_FTP_FAILURE, 0); + return false; + } + // else fall through to skip EPSV 1 + + case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */ + debugs(9, 5, HERE << "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead."); + mb.Printf("PASV%s", Ftp::crlf); + state = SENT_PASV; + break; + + default: { + bool doEpsv = true; + if (Config.accessList.ftp_epsv) { + ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL); + doEpsv = (checklist.fastCheck() == ACCESS_ALLOWED); + } + if (!doEpsv) { + debugs(9, 5, HERE << "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")"); + mb.Printf("PASV%s", Ftp::crlf); + state = SENT_PASV; + } else if (Config.Ftp.epsv_all) { + debugs(9, 5, HERE << "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")"); + mb.Printf("EPSV ALL%s", Ftp::crlf); + state = SENT_EPSV_ALL; + } else { + if (ctrl.conn->local.isIPv6()) { + debugs(9, 5, HERE << "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2"); + mb.Printf("EPSV 2%s", Ftp::crlf); + state = SENT_EPSV_2; + } + if (ctrl.conn->local.isIPv4()) { + debugs(9, 5, HERE << "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1"); + mb.Printf("EPSV 1%s", Ftp::crlf); + state = SENT_EPSV_1; + } + } + break; + } + } + + if (ctrl.message) + wordlistDestroy(&ctrl.message); + ctrl.message = NULL; //No message to return to client. + ctrl.offset = 0; //reset readed response, to make room read the next response + + writeCommand(mb.content()); + + /* + * ugly hack for ftp servers like ftp.netscape.com that sometimes + * dont acknowledge PASV commands. Use connect timeout to be faster then read timeout (minutes). + */ + /* XXX: resurrect or remove + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, + TimeoutDialer, this, FtpStateData::timeout); + commSetConnTimeout(ctrl.conn, Config.Timeout.connect, timeoutCall); + */ + + return true; +} + + +void +ServerStateData::connectDataChannel() +{ + safe_free(ctrl.last_command); + + safe_free(ctrl.last_reply); + + ctrl.last_command = xstrdup("Connect to server data port"); + + // Generate a new data channel descriptor to be opened. + Comm::ConnectionPointer conn = new Comm::Connection; - conn->local = ctrl.conn->local; ++ conn->setAddrs(ctrl.conn->local, data.host); + conn->local.port(0); - conn->remote = data.host; + conn->remote.port(data.port); ++ conn->tos = ctrl.conn->tos; ++ conn->nfmark = ctrl.conn->nfmark; + + debugs(9, 3, HERE << "connecting to " << conn->remote); + + data.opener = commCbCall(9,3, "Ftp::ServerStateData::dataChannelConnected", + CommConnectCbPtrFun(ServerStateData::dataChannelConnected, this)); + Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect); + cs->setHost(data.host); + AsyncJob::Start(cs); +} + +void - ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data) ++ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data) +{ + ServerStateData *ftpState = static_cast(data); + ftpState->dataChannelConnected(conn, status, xerrno); +} + +bool +ServerStateData::openListenSocket() +{ + return false; +} + +/// creates a data channel Comm close callback +AsyncCall::Pointer +ServerStateData::dataCloser() +{ + typedef CommCbMemFunT Dialer; + return JobCallback(9, 5, Dialer, this, ServerStateData::dataClosed); +} + +/// handler called by Comm when FTP data channel is closed unexpectedly +void +ServerStateData::dataClosed(const CommCloseCbParams &io) +{ + debugs(9, 4, HERE); + if (data.listenConn != NULL) { + data.listenConn->close(); + data.listenConn = NULL; + // NP clear() does the: data.fd = -1; + } + data.clear(); +} + +void +ServerStateData::writeCommand(const char *buf) +{ + char *ebuf; + /* trace FTP protocol communications at level 2 */ + debugs(9, 2, "ftp<< " << buf); + + if (Config.Ftp.telnet) + ebuf = escapeIAC(buf); + else + ebuf = xstrdup(buf); + + safe_free(ctrl.last_command); + + safe_free(ctrl.last_reply); + + ctrl.last_command = ebuf; + + if (!Comm::IsConnOpen(ctrl.conn)) { + debugs(9, 2, HERE << "cannot send to closing ctrl " << ctrl.conn); + // TODO: assert(ctrl.closer != NULL); + return; + } + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this, + ServerStateData::writeCommandCallback); + Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL); + + scheduleReadControlReply(0); +} + +void +ServerStateData::writeCommandCallback(const CommIoCbParams &io) +{ + + debugs(9, 5, HERE << "wrote " << io.size << " bytes"); + + if (io.size > 0) { + fd_bytes(io.fd, io.size, FD_WRITE); + kb_incr(&(statCounter.server.all.kbytes_out), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_out), io.size); + } + - if (io.flag == COMM_ERR_CLOSING) ++ if (io.flag == Comm::ERR_CLOSING) + return; + + if (io.flag) { + debugs(9, DBG_IMPORTANT, "ftpWriteCommandCallback: " << io.conn << ": " << xstrerr(io.xerrno)); + failed(ERR_WRITE_ERROR, io.xerrno); + /* failed closes ctrl.conn and frees ftpState */ + return; + } +} + +/// handler called by Comm when FTP control channel is closed unexpectedly +void +ServerStateData::ctrlClosed(const CommCloseCbParams &io) +{ + debugs(9, 4, HERE); + ctrl.clear(); + mustStop("Ftp::ServerStateData::ctrlClosed"); +} + +void +ServerStateData::timeout(const CommTimeoutCbParams &io) +{ + debugs(9, 4, HERE << io.conn << ": '" << entry->url() << "'" ); + + if (abortOnBadEntry("entry went bad while waiting for a timeout")) + return; + + failed(ERR_READ_TIMEOUT, 0); + /* failed() closes ctrl.conn and frees ftpState */ +} + +const Comm::ConnectionPointer & +ServerStateData::dataConnection() const +{ + return data.conn; +} + +void +ServerStateData::maybeReadVirginBody() +{ + // too late to read + if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing()) + return; + + if (data.read_pending) + return; + + initReadBuf(); + + const int read_sz = replyBodySpace(*data.readBuf, 0); + + debugs(11,9, HERE << "FTP may read up to " << read_sz << " bytes"); + + if (read_sz < 2) // see http.cc + return; + + data.read_pending = true; + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, + TimeoutDialer, this, ServerStateData::timeout); + commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall); + + debugs(9,5,HERE << "queueing read on FD " << data.conn->fd); + + typedef CommCbMemFunT Dialer; + entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz, + JobCallback(9, 5, Dialer, this, ServerStateData::dataRead)); +} + +void +ServerStateData::dataRead(const CommIoCbParams &io) +{ + int j; + int bin; + + data.read_pending = false; + + debugs(9, 3, HERE << "FD " << io.fd << " Read " << io.size << " bytes"); + + if (io.size > 0) { + kb_incr(&(statCounter.server.all.kbytes_in), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_in), io.size); + } + - if (io.flag == COMM_ERR_CLOSING) ++ if (io.flag == Comm::ERR_CLOSING) + return; + + assert(io.fd == data.conn->fd); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + abortTransaction("entry aborted during dataRead"); + return; + } + - if (io.flag == COMM_OK && io.size > 0) { - debugs(9, 5, HERE << "appended " << io.size << " bytes to readBuf"); ++ if (io.flag == Comm::OK && io.size > 0) { ++ debugs(9, 5, "appended " << io.size << " bytes to readBuf"); + data.readBuf->appended(io.size); +#if USE_DELAY_POOLS + DelayId delayId = entry->mem_obj->mostBytesAllowed(); + delayId.bytesIn(io.size); +#endif + ++ IOStats.Ftp.reads; + + for (j = io.size - 1, bin = 0; j; ++bin) + j >>= 1; + + ++ IOStats.Ftp.read_hist[bin]; + } + - if (io.flag != COMM_OK) { ++ if (io.flag != Comm::OK) { + debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT, + HERE << "read error: " << xstrerr(io.xerrno)); + + if (ignoreErrno(io.xerrno)) { + maybeReadVirginBody(); + } else { + failed(ERR_READ_ERROR, 0); + /* failed closes ctrl.conn and frees ftpState */ + return; + } + } else if (io.size == 0) { + debugs(9,3, HERE << "Calling dataComplete() because io.size == 0"); + /* + * DPW 2007-04-23 + * Dangerous curves ahead. This call to dataComplete was + * calling scheduleReadControlReply, handleControlReply, + * and then ftpReadTransferDone. If ftpReadTransferDone + * gets unexpected status code, it closes down the control + * socket and our FtpStateData object gets destroyed. As + * a workaround we no longer set the 'buffered_ok' flag in + * the scheduleReadControlReply call. + */ + dataComplete(); + } + + processReplyBody(); +} + +void +ServerStateData::dataComplete() +{ + debugs(9, 3,HERE); + + /* Connection closed; transfer done. */ + + /// Close data channel, if any, to conserve resources while we wait. + data.close(); + + /* expect the "transfer complete" message on the control socket */ + /* + * DPW 2007-04-23 + * Previously, this was the only place where we set the + * 'buffered_ok' flag when calling scheduleReadControlReply(). + * It caused some problems if the FTP server returns an unexpected + * status code after the data command. FtpStateData was being + * deleted in the middle of dataRead(). + */ + /* AYJ: 2011-01-13: Bug 2581. + * 226 status is possibly waiting in the ctrl buffer. + * The connection will hang if we DONT send buffered_ok. + * This happens on all transfers which can be completly sent by the + * server before the 150 started status message is read in by Squid. + * ie all transfers of about one packet hang. + */ + scheduleReadControlReply(1); +} + +/** + * Quickly abort the transaction + * + \todo destruction should be sufficient as the destructor should cleanup, + * including canceling close handlers + */ +void +ServerStateData::abortTransaction(const char *reason) +{ + debugs(9, 3, HERE << "aborting transaction for " << reason << + "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this); + if (Comm::IsConnOpen(ctrl.conn)) { + ctrl.conn->close(); + return; + } + + fwd->handleUnregisteredServerEnd(); + mustStop("ServerStateData::abortTransaction"); +} + +/** + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ +void +ServerStateData::switchTimeoutToDataChannel() +{ + commUnsetConnTimeout(ctrl.conn); + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, + ServerStateData::timeout); + commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall); +} + +void +ServerStateData::sentRequestBody(const CommIoCbParams &io) +{ + if (io.size > 0) + kb_incr(&(statCounter.server.ftp.kbytes_out), io.size); + ::ServerStateData::sentRequestBody(io); +} + +/** + * called after we wrote the last byte of the request body + */ +void +ServerStateData::doneSendingRequestBody() +{ + ::ServerStateData::doneSendingRequestBody(); + debugs(9,3, HERE); + dataComplete(); + /* NP: RFC 959 3.3. DATA CONNECTION MANAGEMENT + * if transfer type is 'stream' call dataComplete() + * otherwise leave open. (reschedule control channel read?) + */ +} + +/// Parses FTP server control response into ctrl structure fields, +/// setting bytesUsed and returning true on success. +bool +ServerStateData::parseControlReply(size_t &bytesUsed) +{ + char *s; + char *sbuf; + char *end; + int usable; + int complete = 0; + wordlist *head = NULL; + wordlist *list; + wordlist **tail = &head; + size_t linelen; + debugs(9, 3, HERE); + /* + * We need a NULL-terminated buffer for scanning, ick + */ + const size_t len = ctrl.offset; + sbuf = (char *)xmalloc(len + 1); + xstrncpy(sbuf, ctrl.buf, len + 1); + end = sbuf + len - 1; + + while (*end != '\r' && *end != '\n' && end > sbuf) + --end; + + usable = end - sbuf; + + debugs(9, 3, HERE << "usable = " << usable); + + if (usable == 0) { + debugs(9, 3, HERE << "didn't find end of line"); + safe_free(sbuf); + return false; + } + + debugs(9, 3, HERE << len << " bytes to play with"); + ++end; + s = sbuf; + s += strspn(s, crlf); + + for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { + if (complete) + break; + + debugs(9, 5, HERE << "s = {" << s << "}"); + + linelen = strcspn(s, crlf) + 1; + + if (linelen < 2) + break; + + if (linelen > 3) + complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' '); + + list = new wordlist(); + + list->key = (char *)xmalloc(linelen); + + xstrncpy(list->key, s, linelen); + + /* trace the FTP communication chat at level 2 */ + debugs(9, 2, "ftp>> " << list->key); + + if (complete) { + // use list->key for last_reply because s contains the new line + ctrl.last_reply = xstrdup(list->key + 4); + ctrl.replycode = atoi(list->key); + } + + *tail = list; + + tail = &list->next; + } + + bytesUsed = static_cast(s - sbuf); + safe_free(sbuf); + + if (!complete) { + wordlistDestroy(&head); + return false; + } + + ctrl.message = head; + assert(ctrl.replycode >= 0); + assert(ctrl.last_reply); + assert(ctrl.message); + return true; +} + +}; // namespace Ftp + + +bool +Ftp::ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr) +{ + int h1, h2, h3, h4; + int p1, p2; + const int n = sscanf(buf, "%d,%d,%d,%d,%d,%d", + &h1, &h2, &h3, &h4, &p1, &p2); + + if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) + return false; + + if (forceIp) { + addr = forceIp; // but the above code still validates the IP we got + } else { + static char ipBuf[1024]; + snprintf(ipBuf, sizeof(ipBuf), "%d.%d.%d.%d", h1, h2, h3, h4); + addr = ipBuf; + + if (addr.isAnyAddr()) + return false; + } + + const int port = ((p1 << 8) + p2); + + if (port <= 0) + return false; + + if (Config.Ftp.sanitycheck && port < 1024) + return false; + + addr.port(port); + return true; +} + +bool +Ftp::ParseProtoIpPort(const char *buf, Ip::Address &addr) +{ + + const char delim = *buf; + const char *s = buf + 1; + const char *e = s; + const int proto = strtol(s, const_cast(&e), 10); + if ((proto != 1 && proto != 2) || *e != delim) + return false; + + s = e + 1; + e = strchr(s, delim); + char ip[MAX_IPSTRLEN]; + if (static_cast(e - s) >= sizeof(ip)) + return false; + strncpy(ip, s, e - s); + ip[e - s] = '\0'; + addr = ip; + + if (addr.isAnyAddr()) + return false; + + if ((proto == 2) != addr.isIPv6()) // proto ID mismatches address version + return false; + + s = e + 1; // skip port delimiter + const int port = strtol(s, const_cast(&e), 10); + if (port < 0 || *e != '|') + return false; + + if (Config.Ftp.sanitycheck && port < 1024) + return false; + + addr.port(port); + return true; +} + +const char * +Ftp::unescapeDoubleQuoted(const char *quotedPath) +{ + static MemBuf path; + path.reset(); + const char *s = quotedPath; + if (*s == '"') { + ++s; + bool parseDone = false; + while (!parseDone) { + if (const char *e = strchr(s, '"')) { + path.append(s, e - s); + s = e + 1; + if (*s == '"') { + path.append(s, 1); + ++s; + } else + parseDone = true; + } else { //parse error + parseDone = true; + path.reset(); + } + } + } + return path.content(); +} + +bool +Ftp::hasPathParameter(const String &cmd) +{ + static const char *pathCommandsStr[]= {"CWD","SMNT", "RETR", "STOR", "APPE", + "RNFR", "RNTO", "DELE", "RMD", "MKD", + "LIST", "NLST", "STAT", "MLSD", "MLST"}; + static const std::set pathCommands(pathCommandsStr, pathCommandsStr + sizeof(pathCommandsStr)/sizeof(pathCommandsStr[0])); + return pathCommands.find(cmd) != pathCommands.end(); +} diff --cc src/FtpServer.h index 88a962c6e5,0000000000..48c36c65fb mode 100644,000000..100644 --- a/src/FtpServer.h +++ b/src/FtpServer.h @@@ -1,169 -1,0 +1,169 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#ifndef SQUID_FTP_SERVER_H +#define SQUID_FTP_SERVER_H + +#include "Server.h" + +class String; +namespace Ftp { + +extern const char *const crlf; + +/// common code for FTP server control and data channels +/// does not own the channel descriptor, which is managed by FtpStateData +class FtpChannel +{ +public: + /// called after the socket is opened, sets up close handler + void opened(const Comm::ConnectionPointer &conn, const AsyncCall::Pointer &aCloser); + + /** Handles all operations needed to properly close the active channel FD. + * clearing the close handler, clearing the listen socket properly, and calling comm_close + */ + void close(); + + void forget(); /// remove the close handler, leave connection open + + void clear(); ///< just drops conn and close handler. does not close active connections. + + Comm::ConnectionPointer conn; ///< channel descriptor + + /** A temporary handle to the connection being listened on. + * Closing this will also close the waiting Data channel acceptor. + * If a data connection has already been accepted but is still waiting in the event queue + * the callback will still happen and needs to be handled (usually dropped). + */ + Comm::ConnectionPointer listenConn; + + AsyncCall::Pointer opener; ///< Comm opener handler callback. +private: + AsyncCall::Pointer closer; ///< Comm close handler callback +}; + +/// Base class for FTP over HTTP and FTP Gateway server state. +class ServerStateData: public ::ServerStateData +{ +public: + ServerStateData(FwdState *fwdState); + virtual ~ServerStateData(); + + virtual void failed(err_type error = ERR_NONE, int xerrno = 0); + virtual void timeout(const CommTimeoutCbParams &io); + virtual const Comm::ConnectionPointer & dataConnection() const; + virtual void abortTransaction(const char *reason); + void writeCommand(const char *buf); + + /// extracts remoteAddr from PASV response, validates it, + /// sets data address details, and returns true on success + bool handlePasvReply(Ip::Address &remoteAddr); + bool handleEpsvReply(Ip::Address &remoteAddr); + + bool sendEprt(); + bool sendPort(); + bool sendPassive(); + void connectDataChannel(); + bool openListenSocket(); + virtual void maybeReadVirginBody(); + void switchTimeoutToDataChannel(); + + // \todo: optimize ctrl and data structs member order, to minimize size + /// FTP control channel info; the channel is opened once per transaction + struct CtrlChannel: public FtpChannel { + char *buf; + size_t size; + size_t offset; + wordlist *message; + char *last_command; + char *last_reply; + int replycode; + } ctrl; + + /// FTP data channel info; the channel may be opened/closed a few times + struct DataChannel: public FtpChannel { + MemBuf *readBuf; + char *host; + unsigned short port; + bool read_pending; + + void addr(const Ip::Address &addr); ///< import host and port + } data; + + enum { + BEGIN, + SENT_USER, + SENT_PASS, + SENT_TYPE, + SENT_MDTM, + SENT_SIZE, + SENT_EPRT, + SENT_PORT, + SENT_EPSV_ALL, + SENT_EPSV_1, + SENT_EPSV_2, + SENT_PASV, + SENT_CWD, + SENT_LIST, + SENT_NLST, + SENT_REST, + SENT_RETR, + SENT_STOR, + SENT_QUIT, + READING_DATA, + WRITING_DATA, + SENT_MKDIR, + SENT_FEAT, + SENT_PWD, + SENT_CDUP, + SENT_DATA_REQUEST, // LIST, NLST or RETR requests.. + SENT_COMMAND, // General command + END + } ftp_state_t; + + int state; + char *old_request; + char *old_reply; + +protected: + virtual void start(); + + void initReadBuf(); + virtual void closeServer(); + virtual bool doneWithServer() const; + virtual Http::StatusCode failedHttpStatus(err_type &error); + void ctrlClosed(const CommCloseCbParams &io); + void scheduleReadControlReply(int buffered_ok); + void readControlReply(const CommIoCbParams &io); + virtual void handleControlReply(); + void writeCommandCallback(const CommIoCbParams &io); + static CNCB dataChannelConnected; - virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno) = 0; ++ virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno) = 0; + void dataRead(const CommIoCbParams &io); + void dataComplete(); + AsyncCall::Pointer dataCloser(); + virtual void dataClosed(const CommCloseCbParams &io); + + // sending of the request body to the server + virtual void sentRequestBody(const CommIoCbParams &io); + virtual void doneSendingRequestBody(); + +private: + bool parseControlReply(size_t &bytesUsed); + + CBDATA_CLASS2(ServerStateData); +}; + +/// parses and validates "A1,A2,A3,A4,P1,P2" IP,port sequence +bool ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr); +/// parses and validates EPRT "" proto,ip,port sequence +bool ParseProtoIpPort(const char *buf, Ip::Address &addr); +/// parses a ftp quoted quote-escaped path +const char *unescapeDoubleQuoted(const char *quotedPath); +/// Return true if the FTP command takes as parameter a pathname +bool hasPathParameter(const String &cmd); +}; // namespace Ftp + +#endif /* SQUID_FTP_SERVER_H */ diff --cc src/anyp/PortCfg.cc index 6392d03d4e,c1d357ef8e..da137358a4 --- a/src/anyp/PortCfg.cc +++ b/src/anyp/PortCfg.cc @@@ -9,7 -9,10 +9,11 @@@ #include #include - CBDATA_NAMESPACED_CLASS_INIT(AnyP, PortCfg); + AnyP::PortCfgPointer HttpPortList; + #if USE_OPENSSL + AnyP::PortCfgPointer HttpsPortList; + #endif ++AnyP::PortCfgPointer FtpPortList; int NHttpSockets = 0; int HttpSockets[MAXTCPLISTENPORTS]; diff --cc src/anyp/PortCfg.h index 6a592a6a55,a27f08f265..9e4c160ac6 --- a/src/anyp/PortCfg.h +++ b/src/anyp/PortCfg.h @@@ -94,16 -94,22 +94,27 @@@ public long sslContextFlags; ///< flags modifying the use of SSL long sslOptions; ///< SSL engine options #endif + - bool ftp_track_dirs; ///< Whether to track FTP directories - - CBDATA_CLASS2(PortCfg); // namespaced ++ bool ftp_track_dirs; ///< whether ftp_port should track FTP directories }; } // namespace AnyP + /// list of Squid http_port configured + extern AnyP::PortCfgPointer HttpPortList; + + #if USE_OPENSSL + /// list of Squid https_port configured + extern AnyP::PortCfgPointer HttpsPortList; + #endif + ++/// list of Squid ftp_port configured ++extern AnyP::PortCfgPointer FtpPortList; ++ + #if !defined(MAXTCPLISTENPORTS) // Max number of TCP listening ports #define MAXTCPLISTENPORTS 128 + #endif // TODO: kill this global array. Need to check performance of array vs list though. extern int NHttpSockets; diff --cc src/cf.data.pre index e544e65b62,54fd5b8a98..1474d1e882 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@@ -1979,20 -1866,6 +1866,20 @@@ DOC_STAR See http_port for a list of available options. DOC_END +NAME: ftp_port +TYPE: PortCfg +DEFAULT: none - LOC: Config.Sockaddr.ftp ++LOC: FtpPortList +DOC_START + Usage: [ip:]port [options] + + Ftp options: + ftp-track-dirs=on|off + Enables tracking of FTP directories by injecting extra + PWD commands and adjusting Request-URI (in wrapping HTTP + requests) to reflect the current FTP server directory. + Disabled by default. + NAME: tcp_outgoing_tos tcp_outgoing_ds tcp_outgoing_dscp TYPE: acl_tos DEFAULT: none diff --cc src/client_side.cc index b3af191854,65344d0a72..5ceebc57d9 --- a/src/client_side.cc +++ b/src/client_side.cc @@@ -93,8 -93,8 +93,9 @@@ #include "clientStream.h" #include "comm.h" #include "comm/Connection.h" +#include "comm/ConnOpener.h" #include "comm/Loops.h" + #include "comm/Read.h" #include "comm/TcpAcceptor.h" #include "comm/Write.h" #include "CommCalls.h" @@@ -292,15 -237,9 +293,15 @@@ ClientSocketContext::getClientReplyCont return (clientStreamNode *)http->client_stream.tail->prev->data; } +ConnStateData * +ClientSocketContext::getConn() const +{ + return http->getConn(); +} + /** - * This routine should be called to grow the inbuf and then - * call comm_read(). + * This routine should be called to grow the in.buf and then + * call Comm::Read(). */ void ConnStateData::readSomeData() @@@ -315,28 -254,9 +316,28 @@@ typedef CommCbMemFunT Dialer; reader = JobCallback(33, 5, Dialer, this, ConnStateData::clientReadRequest); - comm_read(clientConnection, in.buf, reader); + Comm::Read(clientConnection, reader); } +void +ConnStateData::readSomeFtpData() +{ + if (ftp.reader != NULL) + return; + + const size_t availSpace = sizeof(ftp.uploadBuf) - ftp.uploadAvailSize; + if (availSpace <= 0) + return; + + debugs(33, 4, HERE << ftp.dataConn << ": reading FTP data..."); + + typedef CommCbMemFunT Dialer; + ftp.reader = JobCallback(33, 5, Dialer, this, + ConnStateData::clientReadFtpData); + comm_read(ftp.dataConn, ftp.uploadBuf + ftp.uploadAvailSize, availSpace, + ftp.reader); +} + void ClientSocketContext::removeFromConnectionList(ConnStateData * conn) { @@@ -3233,59 -3064,6 +3227,59 @@@ ConnStateData::clientReadRequest(const clientAfterReadingRequests(); } +void +ConnStateData::clientReadFtpData(const CommIoCbParams &io) +{ + debugs(33,5,HERE << io.conn << " size " << io.size); + Must(ftp.reader != NULL); + ftp.reader = NULL; + + assert(Comm::IsConnOpen(ftp.dataConn)); + assert(io.conn->fd == ftp.dataConn->fd); + - if (io.flag == COMM_OK && bodyPipe != NULL) { ++ if (io.flag == Comm::OK && bodyPipe != NULL) { + if (io.size > 0) { + kb_incr(&(statCounter.client_http.kbytes_in), io.size); + + char *const current_buf = ftp.uploadBuf + ftp.uploadAvailSize; + if (io.buf != current_buf) + memmove(current_buf, io.buf, io.size); + ftp.uploadAvailSize += io.size; + handleFtpRequestData(); + } else if (io.size == 0) { + debugs(33, 5, HERE << io.conn << " closed"); + FtpCloseDataConnection(this); + if (ftp.uploadAvailSize <= 0) + finishDechunkingRequest(true); + } - } else { //not COMM_OK or unexpected read ++ } else { // not Comm::Flags::OK or unexpected read + debugs(33, 5, HERE << io.conn << " closed"); + FtpCloseDataConnection(this); + finishDechunkingRequest(false); + } + +} + +void +ConnStateData::handleFtpRequestData() +{ + assert(bodyPipe != NULL); + + debugs(33,5, HERE << "handling FTP request data for " << clientConnection); + const size_t putSize = bodyPipe->putMoreData(ftp.uploadBuf, + ftp.uploadAvailSize); + if (putSize > 0) { + ftp.uploadAvailSize -= putSize; + if (ftp.uploadAvailSize > 0) + memmove(ftp.uploadBuf, ftp.uploadBuf + putSize, ftp.uploadAvailSize); + } + + if (Comm::IsConnOpen(ftp.dataConn)) + readSomeFtpData(); + else if (ftp.uploadAvailSize <= 0) + finishDechunkingRequest(true); +} + /** * called when new request data has been read from the socket * @@@ -3936,50 -3690,6 +3924,46 @@@ httpsAccept(const CommAcceptCbParams &p } } +/** handle a new FTP connection */ +static void +ftpAccept(const CommAcceptCbParams ¶ms) +{ + MasterXaction::Pointer xact = params.xaction; + AnyP::PortCfgPointer s = xact->squidPort; + - if (!s.valid()) { - // it is possible the call or accept() was still queued when the port was reconfigured - debugs(33, 2, "FTP accept failure: port reconfigured."); - return; - } ++ // NP: it is possible the port was reconfigured when the call or accept() was queued. + - if (params.flag != COMM_OK) { ++ if (params.flag != Comm::OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, "ftpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, HERE << params.conn << ": accepted"); + fd_note(params.conn->fd, "client ftp connect"); + + if (s->tcp_keepalive.enabled) { + commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout); + } + + ++incoming_sockets_accepted; + + // Socket is ready, setup the connection manager to start using it + ConnStateData *connState = new ConnStateData(xact); + + if (connState->transparent()) { + char buf[MAX_IPSTRLEN]; + connState->clientConnection->local.toUrl(buf, MAX_IPSTRLEN); + connState->ftp.host = buf; + const char *uri = connState->ftpBuildUri(); + debugs(33, 5, HERE << "FTP transparent URL: " << uri); + } + + FtpWriteEarlyReply(connState, 220, "Service ready"); + + // TODO: Merge common httpAccept() parts, applying USE_DELAY_POOLS to FTP. +} + void ConnStateData::sslCrtdHandleReplyWrapper(void *data, const HelperReply &reply) { @@@ -4415,42 -4127,12 +4401,42 @@@ clientHttpsConnectionsOpen(void } #endif +static void +clientFtpConnectionsOpen(void) +{ - AnyP::PortCfg *s; - - for (s = Config.Sockaddr.ftp; s; s = s->next) { ++ for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) { + if (MAXTCPLISTENPORTS == NHttpSockets) { + debugs(1, DBG_IMPORTANT, "Ignoring 'ftp_port' lines exceeding the limit."); + debugs(1, DBG_IMPORTANT, "The limit is " << MAXTCPLISTENPORTS << " FTP ports."); + continue; + } + + // Fill out a Comm::Connection which IPC will open as a listener for us + s->listenConn = new Comm::Connection; + s->listenConn->local = s->s; + s->listenConn->flags = COMM_NONBLOCKING | (s->flags.tproxyIntercept ? COMM_TRANSPARENT : 0) | + (s->flags.natIntercept ? COMM_INTERCEPTION : 0); + + // setup the subscriptions such that new connections accepted by listenConn are handled by FTP + typedef CommCbFunPtrCallT AcceptCall; - RefCount subCall = commCbCall(5, 5, "ftpAccept", CommAcceptCbPtrFun(ftpAccept, s)); ++ RefCount subCall = commCbCall(5, 5, "ftpAccept", CommAcceptCbPtrFun(ftpAccept, CommAcceptCbParams(NULL))); + Subscription::Pointer sub = new CallSubscription(subCall); + + AsyncCall::Pointer listenCall = asyncCall(33, 2, "clientListenerConnectionOpened", + ListeningStartedDialer(&clientListenerConnectionOpened, + s, Ipc::fdnFtpSocket, sub)); + Ipc::StartListening(SOCK_STREAM, IPPROTO_TCP, s->listenConn, Ipc::fdnFtpSocket, listenCall); + HttpSockets[NHttpSockets] = -1; + ++NHttpSockets; + } +} + /// process clientHttpConnectionsOpen result static void - clientListenerConnectionOpened(AnyP::PortCfg *s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub) + clientListenerConnectionOpened(AnyP::PortCfgPointer &s, const Ipc::FdNoteId portTypeNote, const Subscription::Pointer &sub) { + Must(s != NULL); + if (!OpenedHttpSocket(s->listenConn, portTypeNote)) return; @@@ -4485,9 -4165,9 +4470,9 @@@ clientOpenListenSockets(void } void -clientHttpConnectionsClose(void) +clientConnectionsClose(void) { - for (AnyP::PortCfg *s = Config.Sockaddr.http; s; s = s->next) { + for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) { if (s->listenConn != NULL) { debugs(1, DBG_IMPORTANT, "Closing HTTP port " << s->listenConn->local); s->listenConn->close(); @@@ -4505,14 -4185,6 +4490,14 @@@ } #endif - for (AnyP::PortCfg *s = Config.Sockaddr.ftp; s; s = s->next) { ++ for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) { + if (s->listenConn != NULL) { + debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local); + s->listenConn->close(); + s->listenConn = NULL; + } + } + // TODO see if we can drop HttpSockets array entirely */ for (int i = 0; i < NHttpSockets; ++i) { HttpSockets[i] = -1; @@@ -4936,1314 -4577,3 +4920,1314 @@@ ConnStateData::unpinConnection(const bo /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host * connection has gone away */ } + +const char * +ConnStateData::ftpBuildUri(const char *file) +{ + ftp.uri = "ftp://"; + ftp.uri.append(ftp.host); + if (port->ftp_track_dirs && ftp.workingDir.size()) { + if (ftp.workingDir[0] != '/') + ftp.uri.append("/"); + ftp.uri.append(ftp.workingDir); + } + + if (ftp.uri[ftp.uri.size() - 1] != '/') + ftp.uri.append("/"); + + if (port->ftp_track_dirs && file) { + //remove any '/' from the beginning of path + while (*file == '/') + ++file; + ftp.uri.append(file); + } + + return ftp.uri.termedBuf(); +} + +void +ConnStateData::ftpSetWorkingDir(const char *dir) +{ + ftp.workingDir = dir; +} + +static void +FtpAcceptDataConnection(const CommAcceptCbParams ¶ms) +{ + ConnStateData *connState = static_cast(params.data); + - if (params.flag != COMM_OK) { ++ if (params.flag != Comm::OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, HERE << connState->ftp.dataListenConn << ": accept " + "failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, "accepted " << params.conn); + fd_note(params.conn->fd, "passive client ftp data"); + ++incoming_sockets_accepted; + + if (!connState->clientConnection) { + debugs(33, 5, "late data connection?"); + FtpCloseDataConnection(connState); // in case we are still listening + params.conn->close(); + } else + if (params.conn->remote != connState->clientConnection->remote) { + debugs(33, 2, "rogue data conn? ctrl: " << connState->clientConnection->remote); + params.conn->close(); + // Some FTP servers close control connection here, but it may make + // things worse from DoS p.o.v. and no better from data stealing p.o.v. + } else { + FtpCloseDataConnection(connState); + connState->ftp.dataConn = params.conn; + connState->ftp.uploadAvailSize = 0; + debugs(33, 7, "ready for data"); + if (connState->ftp.onDataAcceptCall != NULL) { + AsyncCall::Pointer call = connState->ftp.onDataAcceptCall; + connState->ftp.onDataAcceptCall = NULL; + // If we got an upload request, start reading data from the client. + if (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + connState->readSomeFtpData(); + else + Must(connState->ftp.state == ConnStateData::FTP_HANDLE_DATA_REQUEST); + MemBuf mb; + mb.init(); + mb.Printf("150 Data connection opened.\r\n"); + Comm::Write(connState->clientConnection, &mb, call); + } + } +} + +static void +FtpCloseDataConnection(ConnStateData *conn) +{ + if (conn->ftp.listener != NULL) { + conn->ftp.listener->cancel("no longer needed"); + conn->ftp.listener = NULL; + } + + if (Comm::IsConnOpen(conn->ftp.dataListenConn)) { + debugs(33, 5, HERE << "FTP closing client data listen socket: " << + *conn->ftp.dataListenConn); + conn->ftp.dataListenConn->close(); + } + conn->ftp.dataListenConn = NULL; + + if (conn->ftp.reader != NULL) { - // comm_read_cancel can deal with negative FDs - comm_read_cancel(conn->ftp.dataConn->fd, conn->ftp.reader); ++ // Comm::ReadCancel can deal with negative FDs ++ Comm::ReadCancel(conn->ftp.dataConn->fd, conn->ftp.reader); + conn->ftp.reader = NULL; + } + + if (Comm::IsConnOpen(conn->ftp.dataConn)) { + debugs(33, 5, HERE << "FTP closing client data connection: " << + *conn->ftp.dataConn); + conn->ftp.dataConn->close(); + } + conn->ftp.dataConn = NULL; +} + +/// Writes FTP [error] response before we fully parsed the FTP request and +/// created the corresponding HTTP request wrapper for that FTP request. +static void +FtpWriteEarlyReply(ConnStateData *connState, const int code, const char *msg) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + MemBuf mb; + mb.init(); + mb.Printf("%i %s\r\n", code, msg); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteEarlyReply", + CommIoCbPtrFun(&FtpWroteEarlyReply, connState)); + Comm::Write(connState->clientConnection, &mb, call); + + connState->flags.readMore = false; + + // TODO: Create master transaction. Log it in FtpWroteEarlyReply. +} + +static void +FtpWriteReply(ClientSocketContext *context, MemBuf &mb) +{ + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", + CommIoCbPtrFun(&FtpWroteReply, context)); + Comm::Write(context->clientConnection, &mb, call); +} + +static void +FtpWriteCustomReply(ClientSocketContext *context, const int code, const char *msg, const HttpReply *reply) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + const bool sendDetails = reply != NULL && + reply->header.has(HDR_FTP_STATUS) && reply->header.has(HDR_FTP_REASON); + + MemBuf mb; + mb.init(); + if (sendDetails) { + mb.Printf("%i-%s\r\n", code, msg); + mb.Printf(" Server reply:\r\n"); + FtpPrintReply(mb, reply, " "); + mb.Printf("%i \r\n", code); + } else + mb.Printf("%i %s\r\n", code, msg); + + FtpWriteReply(context, mb); +} + +static void +FtpChangeState(ConnStateData *connState, const ConnStateData::FtpState newState, const char *reason) +{ + assert(connState); + if (connState->ftp.state == newState) { + debugs(33, 3, "client state unchanged at " << connState->ftp.state << + " because " << reason); + connState->ftp.state = newState; + } else { + debugs(33, 3, "client state was " << connState->ftp.state << + ", now " << newState << " because " << reason); + connState->ftp.state = newState; + } +} + +/** Parse an FTP request + * + * \note Sets result->flags.parsed_ok to 0 if failed to parse the request, + * to 1 if the request was correctly parsed. + * \param[in] connState a ConnStateData. The caller must make sure it is not null + * \param[out] mehtod_p will be set as a side-effect of the parsing. + * Pointed-to value will be set to Http::METHOD_NONE in case of + * parsing failure + * \param[out] http_ver will be set as a side-effect of the parsing + * \return NULL on incomplete requests, + * a ClientSocketContext structure on success or failure. + */ +static ClientSocketContext * +FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::ProtocolVersion *http_ver) +{ + *http_ver = Http::ProtocolVersion(1, 1); + + // TODO: Use tokenizer for parsing instead of raw pointer manipulation. + const char *inBuf = connState->in.buf.rawContent(); + + const char *const eor = + static_cast(memchr(inBuf, '\n', + min(static_cast(connState->in.buf.length()), Config.maxRequestHeaderSize))); + + if (eor == NULL && connState->in.buf.length() >= Config.maxRequestHeaderSize) { + FtpChangeState(connState, ConnStateData::FTP_ERROR, "huge req"); + FtpWriteEarlyReply(connState, 421, "Too large request"); + return NULL; + } + + if (eor == NULL) { + debugs(33, 5, HERE << "Incomplete request, waiting for end of request"); + return NULL; + } + + const size_t req_sz = eor + 1 - inBuf; + + // skip leading whitespaces + const char *boc = inBuf; // beginning of command + while (boc < eor && isspace(*boc)) ++boc; + if (boc >= eor) { + debugs(33, 5, HERE << "Empty request, ignoring"); + connNoteUseOfBuffer(connState, req_sz); + return NULL; + } + + const char *eoc = boc; // end of command + while (eoc < eor && !isspace(*eoc)) ++eoc; + connState->in.buf.setAt(eoc - inBuf, '\0'); + + const char *bop = eoc + 1; // beginning of parameter + while (bop < eor && isspace(*bop)) ++bop; + if (bop < eor) { + const char *eop = eor - 1; + while (isspace(*eop)) --eop; + assert(eop >= bop); + connState->in.buf.setAt(eop + 1 - inBuf, '\0'); + } else + bop = NULL; + + debugs(33, 7, HERE << "Parsed FTP command " << boc << " with " << + (bop == NULL ? "no " : "") << "parameters" << + (bop != NULL ? ": " : "") << bop); + + // TODO: Use SBuf instead of String + const String cmd = boc; + String params = bop; + + connNoteUseOfBuffer(connState, req_sz); + + if (!connState->ftp.readGreeting) { + // the first command must be USER + if (!connState->pinning.pinned && cmd.caseCmp("USER") != 0) { + FtpWriteEarlyReply(connState, 530, "Must login first"); + return NULL; + } + } + + // We need to process USER request now because it sets ftp server Hostname. + if (cmd.caseCmp("USER") == 0 && + !FtpHandleUserRequest(connState, cmd, params)) + return NULL; + + if (!FtpSupportedCommand(cmd)) { + FtpWriteEarlyReply(connState, 502, "Unknown or unsupported command"); + return NULL; + } + + *method_p = !cmd.caseCmp("APPE") || !cmd.caseCmp("STOR") || + !cmd.caseCmp("STOU") ? Http::METHOD_PUT : Http::METHOD_GET; + + char *uri; + const char *aPath = params.size() > 0 && Ftp::hasPathParameter(cmd)? + params.termedBuf() : NULL; + uri = xstrdup(connState->ftpBuildUri(aPath)); + HttpRequest *const request = + HttpRequest::CreateFromUrlAndMethod(uri, *method_p); + if (request == NULL) { + debugs(33, 5, HERE << "Invalid FTP URL: " << connState->ftp.uri); + FtpWriteEarlyReply(connState, 501, "Invalid host"); + connState->ftp.uri.clean(); + safe_free(uri); + return NULL; + } + + request->http_ver = *http_ver; + + // Our fake Request-URIs are not distinctive enough for caching to work + request->flags.cachable = false; // XXX: reset later by maybeCacheable() + request->flags.noCache = true; + + request->header.putStr(HDR_FTP_COMMAND, cmd.termedBuf()); + request->header.putStr(HDR_FTP_ARGUMENTS, params.termedBuf() != NULL ? + params.termedBuf() : ""); + if (*method_p == Http::METHOD_PUT) { + request->header.putStr(HDR_EXPECT, "100-continue"); + request->header.putStr(HDR_TRANSFER_ENCODING, "chunked"); + } + + ClientHttpRequest *const http = new ClientHttpRequest(connState); + http->request = request; + HTTPMSGLOCK(http->request); + http->req_sz = req_sz; + http->uri = uri; + + ClientSocketContext *const result = + new ClientSocketContext(connState->clientConnection, http); + + StoreIOBuffer tempBuffer; + tempBuffer.data = result->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; + + ClientStreamData newServer = new clientReplyContext(http); + ClientStreamData newClient = result; + clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, + clientReplyStatus, newServer, clientSocketRecipient, + clientSocketDetach, newClient, tempBuffer); + + result->registerWithConn(); + result->flags.parsed_ok = 1; + connState->flags.readMore = false; + return result; +} + +static void +FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer data) +{ + if (context->http && context->http->al != NULL && + !context->http->al->reply && reply) { + context->http->al->reply = reply; + HTTPMSGLOCK(context->http->al->reply); + } + + static FtpReplyHandler *handlers[] = { + NULL, // FTP_BEGIN + NULL, // FTP_CONNECTED + FtpHandleFeatReply, // FTP_HANDLE_FEAT + FtpHandlePasvReply, // FTP_HANDLE_PASV + FtpHandlePortReply, // FTP_HANDLE_PORT + FtpHandleDataReply, // FTP_HANDLE_DATA_REQUEST + FtpHandleUploadReply, // FTP_HANDLE_UPLOAD_REQUEST + FtpHandleEprtReply,// FTP_HANDLE_EPRT + FtpHandleEpsvReply,// FTP_HANDLE_EPSV + NULL, // FTP_HANDLE_CWD + NULL, //FTP_HANDLE_PASS + NULL, // FTP_HANDLE_CDUP + FtpHandleErrorReply // FTP_ERROR + }; + const ConnStateData::FtpState state = context->getConn()->ftp.state; + FtpReplyHandler *const handler = handlers[state]; + if (handler) + (*handler)(context, reply, data); + else + FtpWriteForwardedReply(context, reply); +} + +static void +FtpHandleFeatReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support FEAT", reply); + return; + } + + HttpReply *filteredReply = reply->clone(); + HttpHeader &filteredHeader = filteredReply->header; + + // Remove all unsupported commands from the response wrapper. + int deletedCount = 0; + HttpHeaderPos pos = HttpHeaderInitPos; + bool hasEPRT = false; + bool hasEPSV = false; + int prependSpaces = 1; + while (const HttpHeaderEntry *e = filteredHeader.getEntry(&pos)) { + if (e->id == HDR_FTP_PRE) { + // assume RFC 2389 FEAT response format, quoted by Squid: + // <"> SP NAME [SP PARAMS] <"> + // but accommodate MS servers sending four SPs before NAME + if (e->value.size() < 4) + continue; + const char *raw = e->value.termedBuf(); + if (raw[0] != '"' || raw[1] != ' ') + continue; + const char *beg = raw + 1 + strspn(raw + 1, " "); // after quote and spaces + // command name ends with (SP parameter) or quote + const char *end = beg + strcspn(beg, " \""); + + if (end <= beg) + continue; + + // compute the number of spaces before the command + prependSpaces = beg - raw - 1; + + const String cmd = e->value.substr(beg-raw, end-raw); + + if (!FtpSupportedCommand(cmd)) + filteredHeader.delAt(pos, deletedCount); + + if (cmd == "EPRT") + hasEPRT = true; + else if (cmd == "EPSV") + hasEPSV = true; + } + } + + char buf[256]; + int insertedCount = 0; + if (!hasEPRT) { + snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT"); + filteredHeader.putStr(HDR_FTP_PRE, buf); + ++insertedCount; + } + if (!hasEPSV) { + snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV"); + filteredHeader.putStr(HDR_FTP_PRE, buf); + ++insertedCount; + } + + if (deletedCount || insertedCount) { + filteredHeader.refreshMask(); + debugs(33, 5, "deleted " << deletedCount << " inserted " << insertedCount); + } + + FtpWriteForwardedReply(context, filteredReply); +} + +static void +FtpHandlePasvReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support PASV", reply); + return; + } + + FtpCloseDataConnection(context->getConn()); + + Comm::ConnectionPointer conn = new Comm::Connection; + ConnStateData * const connState = context->getConn(); + conn->flags = COMM_NONBLOCKING; + conn->local = connState->transparent() ? + connState->port->s : context->clientConnection->local; + conn->local.port(0); + const char *const note = connState->ftp.uri.termedBuf(); + comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); + if (!Comm::IsConnOpen(conn)) { + debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << + conn->local << " error: " << errno); + FtpWriteCustomReply(context, 451, "Internal error"); + return; + } + + typedef CommCbFunPtrCallT AcceptCall; + RefCount subCall = commCbCall(5, 5, "FtpAcceptDataConnection", + CommAcceptCbPtrFun(FtpAcceptDataConnection, connState)); + Subscription::Pointer sub = new CallSubscription(subCall); + connState->ftp.listener = subCall.getRaw(); + connState->ftp.dataListenConn = conn; + AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); + + char addr[MAX_IPSTRLEN]; + // remote server in interception setups and local address otherwise + const Ip::Address &server = connState->transparent() ? + context->clientConnection->local : conn->local; + server.toStr(addr, MAX_IPSTRLEN, AF_INET); + addr[MAX_IPSTRLEN - 1] = '\0'; + for (char *c = addr; *c != '\0'; ++c) { + if (*c == '.') + *c = ','; + } + + // conn->fd is the client data connection (and its local port) + const unsigned short port = comm_local_port(conn->fd); + conn->local.port(port); + + // In interception setups, we combine remote server address with a + // local port number and hope that traffic will be redirected to us. + MemBuf mb; + mb.init(); + + // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp + // versions block responses that use those alternative syntax rules! + mb.Printf("227 Entering Passive Mode (%s,%i,%i).\r\n", + addr, + static_cast(port / 256), + static_cast(port % 256)); + + debugs(11, 3, Raw("writing", mb.buf, mb.size)); + FtpWriteReply(context, mb); +} + +static void +FtpHandlePortReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support PASV (converted from PORT)", reply); + return; + } + + FtpWriteCustomReply(context, 200, "PORT successfully converted to PASV."); + + // and wait for RETR +} + +static void +FtpHandleErrorReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + ConnStateData *const connState = context->getConn(); + if (!connState->pinning.pinned) // we failed to connect to server + connState->ftp.uri.clean(); + // 421: we will close due to FTP_ERROR + FtpWriteErrorReply(context, reply, 421); +} + +static void +FtpHandleDataReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + ConnStateData *const conn = context->getConn(); + + if (reply != NULL && reply->sline.status() != Http::scOkay) { + FtpWriteForwardedReply(context, reply); + if (conn && Comm::IsConnOpen(conn->ftp.dataConn)) { + debugs(33, 3, "closing " << conn->ftp.dataConn << " on KO reply"); + FtpCloseDataConnection(conn); + } + return; + } + + if (!conn->ftp.dataConn) { + // We got STREAM_COMPLETE (or error) and closed the client data conn. + debugs(33, 3, "ignoring FTP srv data response after clt data closure"); + return; + } + + if (!FtpCheckDataConnPost(context)) { + FtpWriteCustomReply(context, 425, "Data connection is not established."); + FtpCloseDataConnection(conn); + return; + } + + debugs(33, 7, HERE << data.length); + + if (data.length <= 0) { - FtpWroteReplyData(conn->clientConnection, NULL, 0, COMM_OK, 0, context); ++ FtpWroteReplyData(conn->clientConnection, NULL, 0, Comm::OK, 0, context); + return; + } + + MemBuf mb; + mb.init(data.length + 1, data.length + 1); + mb.append(data.data, data.length); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReplyData", + CommIoCbPtrFun(&FtpWroteReplyData, context)); + Comm::Write(conn->ftp.dataConn, &mb, call); + + context->noteSentBodyBytes(data.length); +} + +static void - FtpWroteReplyData(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) ++FtpWroteReplyData(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) +{ - if (errflag == COMM_ERR_CLOSING) ++ if (errflag == Comm::ERR_CLOSING) + return; + + ClientSocketContext *const context = static_cast(data); + ConnStateData *const connState = context->getConn(); + - if (errflag != COMM_OK) { ++ if (errflag != Comm::OK) { + debugs(33, 3, HERE << "FTP reply data writing failed: " << + xstrerr(xerrno)); + FtpCloseDataConnection(connState); + FtpWriteCustomReply(context, 426, "Data connection error; transfer aborted"); + return; + } + + assert(context->http); + context->http->out.size += size; + + switch (context->socketState()) { + case STREAM_NONE: + debugs(33, 3, "Keep going"); + context->pullData(); + return; + case STREAM_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer successfully complete"); + FtpWriteCustomReply(context, 226, "Transfer complete"); + break; + case STREAM_UNPLANNED_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE"); + FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); + break; + case STREAM_FAILED: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_FAILED"); + FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); + break; + default: + fatal("unreachable code"); + } + + FtpCloseDataConnection(connState); +} + +static void +FtpHandleUploadReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + FtpWriteForwardedReply(context, reply); + // note that the client data connection may already be closed by now +} + +static void +FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply) +{ + const AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", + CommIoCbPtrFun(&FtpWroteReply, context)); + FtpWriteForwardedReply(context, reply, call); +} + +static void +FtpHandleEprtReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support PASV (converted from EPRT)", reply); + return; + } + + FtpWriteCustomReply(context, 200, "EPRT successfully converted to PASV."); + + // and wait for RETR +} + +static void +FtpHandleEpsvReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Cannot connect to server", reply); + return; + } + + FtpCloseDataConnection(context->getConn()); + + Comm::ConnectionPointer conn = new Comm::Connection; + ConnStateData * const connState = context->getConn(); + conn->flags = COMM_NONBLOCKING; + conn->local = connState->transparent() ? + connState->port->s : context->clientConnection->local; + conn->local.port(0); + const char *const note = connState->ftp.uri.termedBuf(); + comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); + if (!Comm::IsConnOpen(conn)) { + debugs(5, DBG_CRITICAL, "comm_open_listener failed: " << + conn->local << " error: " << errno); + FtpWriteCustomReply(context, 451, "Internal error"); + return; + } + + typedef CommCbFunPtrCallT AcceptCall; + RefCount subCall = commCbCall(5, 5, "FtpAcceptDataConnection", + CommAcceptCbPtrFun(FtpAcceptDataConnection, connState)); + Subscription::Pointer sub = new CallSubscription(subCall); + connState->ftp.listener = subCall.getRaw(); + connState->ftp.dataListenConn = conn; + AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); + + // conn->fd is the client data connection (and its local port) + const unsigned int port = comm_local_port(conn->fd); + conn->local.port(port); + + // In interception setups, we combine remote server address with a + // local port number and hope that traffic will be redirected to us. + MemBuf mb; + mb.init(); + mb.Printf("229 Entering Extended Passive Mode (|||%u|)\r\n", port); + + debugs(11, 3, Raw("writing", mb.buf, mb.size)); + FtpWriteReply(context, mb); +} + +/// writes FTP error response with given status and reply-derived error details +static void +FtpWriteErrorReply(ClientSocketContext *context, const HttpReply *reply, const int status) +{ + MemBuf mb; + mb.init(); + + assert(context->http); + const HttpRequest *request = context->http->request; + assert(request); + if (request->errType != ERR_NONE) + mb.Printf("%i-%s\r\n", status, errorPageName(request->errType)); + + if (request->errDetail > 0) { + // XXX: > 0 may not always mean that this is an errno + mb.Printf("%i-Error: (%d) %s\r\n", status, + request->errDetail, + strerror(request->errDetail)); + } + + // XXX: Remove hard coded names. Use an error page template instead. + const Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { // XXX: add adapt::allMeta.getByName("X-Response-Info"); + const String desc = ah->allMeta.getByName("X-Response-Desc"); + if (info.size()) + mb.Printf("%i-Information: %s\r\n", status, info.termedBuf()); + if (desc.size()) + mb.Printf("%i-Description: %s\r\n", status, desc.termedBuf()); + } + + assert(reply != NULL); + const char *reason = reply->header.has(HDR_FTP_REASON) ? + reply->header.getStr(HDR_FTP_REASON): + reply->sline.reason(); + + mb.Printf("%i %s\r\n", status, reason); // error terminating line + + // TODO: errorpage.cc should detect FTP client and use + // configurable FTP-friendly error templates which we should + // write to the client "as is" instead of hiding most of the info + + FtpWriteReply(context, mb); +} + +/// writes FTP response based on HTTP reply that is not an FTP-response wrapper +static void +FtpWriteForwardedForeign(ClientSocketContext *context, const HttpReply *reply) +{ + ConnStateData *const connState = context->getConn(); + FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "foreign reply"); + //Close the data connection. + FtpCloseDataConnection(connState); + // 451: We intend to keep the control connection open. + FtpWriteErrorReply(context, reply, 451); +} + +static void +FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply, AsyncCall::Pointer call) +{ + assert(reply != NULL); + const HttpHeader &header = reply->header; + ConnStateData *const connState = context->getConn(); + + // adaptation and forwarding errors lack HDR_FTP_STATUS + if (!header.has(HDR_FTP_STATUS)) { + FtpWriteForwardedForeign(context, reply); + return; + } + + assert(header.has(HDR_FTP_REASON)); + + const int status = header.getInt(HDR_FTP_STATUS); + debugs(33, 7, HERE << "status: " << status); + + // Status 125 or 150 implies upload or data request, but we still check + // the state in case the server is buggy. + if ((status == 125 || status == 150) && + (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST || + connState->ftp.state == ConnStateData::FTP_HANDLE_DATA_REQUEST)) { + if (FtpCheckDataConnPost(context)) { + // If the data connection is ready, start reading data (here) + // and forward the response to client (further below). + debugs(33, 7, "data connection established, start data transfer"); + if (connState->ftp.state == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + connState->readSomeFtpData(); + } else { + // If we are waiting to accept the data connection, keep waiting. + if (Comm::IsConnOpen(connState->ftp.dataListenConn)) { + debugs(33, 7, "wait for the client to establish a data connection"); + connState->ftp.onDataAcceptCall = call; + // TODO: Add connect timeout for passive connections listener? + // TODO: Remember server response so that we can forward it? + } else { + // Either the connection was establised and closed after the + // data was transferred OR we failed to establish an active + // data connection and already sent the error to the client. + // In either case, there is nothing more to do. + debugs(33, 7, "done with data OR active connection failed"); + } + return; + } + } + + MemBuf mb; + mb.init(); + FtpPrintReply(mb, reply); + + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + Comm::Write(context->clientConnection, &mb, call); +} + +static void +FtpPrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix) +{ + const HttpHeader &header = reply->header; + + HttpHeaderPos pos = HttpHeaderInitPos; + while (const HttpHeaderEntry *e = header.getEntry(&pos)) { + if (e->id == HDR_FTP_PRE) { + String raw; + if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw)) + mb.Printf("%s\r\n", raw.termedBuf()); + } + } + + if (header.has(HDR_FTP_STATUS)) { + const char *reason = header.getStr(HDR_FTP_REASON); + mb.Printf("%i %s\r\n", header.getInt(HDR_FTP_STATUS), + (reason ? reason : 0)); + } +} + +static void - FtpWroteEarlyReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) ++FtpWroteEarlyReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) +{ - if (errflag == COMM_ERR_CLOSING) ++ if (errflag == Comm::ERR_CLOSING) + return; + - if (errflag != COMM_OK) { ++ if (errflag != Comm::OK) { + debugs(33, 3, HERE << "FTP reply writing failed: " << xstrerr(xerrno)); + conn->close(); + return; + } + + ConnStateData *const connState = static_cast(data); + ClientSocketContext::Pointer context = connState->getCurrentContext(); + if (context != NULL && context->http) { + context->http->out.size += size; + context->http->out.headers_sz += size; + } + + connState->flags.readMore = true; + connState->readSomeData(); +} + +static void - FtpWroteReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) ++FtpWroteReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, Comm::Flag errflag, int xerrno, void *data) +{ - if (errflag == COMM_ERR_CLOSING) ++ if (errflag == Comm::ERR_CLOSING) + return; + - if (errflag != COMM_OK) { ++ if (errflag != Comm::OK) { + debugs(33, 3, HERE << "FTP reply writing failed: " << + xstrerr(xerrno)); + conn->close(); + return; + } + + ClientSocketContext *const context = + static_cast(data); + ConnStateData *const connState = context->getConn(); + + assert(context->http); + context->http->out.size += size; + context->http->out.headers_sz += size; + + if (connState->ftp.state == ConnStateData::FTP_ERROR) { + debugs(33, 5, "closing on FTP server error"); + conn->close(); + return; + } + + const clientStream_status_t socketState = context->socketState(); + debugs(33, 5, "FTP client stream state " << socketState); + switch (socketState) { + case STREAM_UNPLANNED_COMPLETE: + case STREAM_FAILED: + conn->close(); + return; + + case STREAM_NONE: + case STREAM_COMPLETE: + connState->flags.readMore = true; + FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "FtpWroteReply"); + if (connState->in.bodyParser) + connState->finishDechunkingRequest(false); + context->keepaliveNextRequest(); + return; + } +} + +bool +FtpHandleRequest(ClientSocketContext *context, String &cmd, String ¶ms) { + if (HttpRequest *request = context->http->request) { + MemBuf *mb = new MemBuf; + Packer p; + mb->init(); + packerToMemInit(&p, mb); + request->pack(&p); + packerClean(&p); + + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REQUEST:\n---------\n" << mb->buf << + "\n----------"); + delete mb; + } + + static std::pair handlers[] = { + std::make_pair("LIST", FtpHandleDataRequest), + std::make_pair("NLST", FtpHandleDataRequest), + std::make_pair("MLSD", FtpHandleDataRequest), + std::make_pair("FEAT", FtpHandleFeatRequest), + std::make_pair("PASV", FtpHandlePasvRequest), + std::make_pair("PORT", FtpHandlePortRequest), + std::make_pair("RETR", FtpHandleDataRequest), + std::make_pair("EPRT", FtpHandleEprtRequest), + std::make_pair("EPSV", FtpHandleEpsvRequest), + std::make_pair("CWD", FtpHandleCwdRequest), + std::make_pair("PASS", FtpHandlePassRequest), + std::make_pair("CDUP", FtpHandleCdupRequest), + }; + + FtpRequestHandler *handler = NULL; + if (context->http->request->method == Http::METHOD_PUT) + handler = FtpHandleUploadRequest; + else { + for (size_t i = 0; i < sizeof(handlers) / sizeof(*handlers); ++i) { + if (cmd.caseCmp(handlers[i].first) == 0) { + handler = handlers[i].second; + break; + } + } + } + + return handler != NULL ? (*handler)(context, cmd, params) : true; +} + +/// Called to parse USER command, which is required to create an HTTP request +/// wrapper. Thus, errors are handled with FtpWriteEarlyReply() here. +bool +FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms) +{ + if (params.size() == 0) { + FtpWriteEarlyReply(connState, 501, "Missing username"); + return false; + } + + const String::size_type eou = params.rfind('@'); + if (eou == String::npos || eou + 1 >= params.size()) { + FtpWriteEarlyReply(connState, 501, "Missing host"); + return false; + } + + const String login = params.substr(0, eou); + String host = params.substr(eou + 1, params.size()); + // If we can parse it as raw IPv6 address, then surround with "[]". + // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is. + if (host.pos(":")) { + char ipBuf[MAX_IPSTRLEN]; + Ip::Address ipa; + ipa = host.termedBuf(); + if (!ipa.isAnyAddr()) { + ipa.toHostStr(ipBuf, MAX_IPSTRLEN); + host = ipBuf; + } + } + connState->ftp.host = host; + + String oldUri; + if (connState->ftp.readGreeting) + oldUri = connState->ftp.uri; + + connState->ftpSetWorkingDir(NULL); + connState->ftpBuildUri(); + + if (!connState->ftp.readGreeting) { + debugs(11, 3, "set URI to " << connState->ftp.uri); + } else if (oldUri.caseCmp(connState->ftp.uri) == 0) { + debugs(11, 5, "keep URI as " << oldUri); + } else { + debugs(11, 3, "reset URI from " << oldUri << " to " << connState->ftp.uri); + FtpCloseDataConnection(connState); + connState->ftp.readGreeting = false; + connState->unpinConnection(true); // close control connection to the server + FtpChangeState(connState, ConnStateData::FTP_BEGIN, "URI reset"); + } + + params.cut(eou); + + return true; +} + +bool +FtpHandleFeatRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_FEAT, "FtpHandleFeatRequest"); + + return true; +} + +bool +FtpHandlePasvRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + ConnStateData *const connState = context->getConn(); + assert(connState); + if (connState->ftp.gotEpsvAll) { + FtpSetReply(context, 500, "Bad PASV command"); + return false; + } + + if (params.size() > 0) { + FtpSetReply(context, 501, "Unexpected parameter"); + return false; + } + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASV, "FtpHandlePasvRequest"); + // no need to fake PASV request via FtpSetDataCommand() in true PASV case + return true; +} + +/// [Re]initializes dataConn for active data transfers. Does not connect. +static +bool FtpCreateDataConnection(ClientSocketContext *context, Ip::Address cltAddr) +{ + ConnStateData *const connState = context->getConn(); + assert(connState); + assert(connState->clientConnection != NULL); + assert(!connState->clientConnection->remote.isAnyAddr()); + + if (cltAddr != connState->clientConnection->remote) { + debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << connState->clientConnection->remote); + // Closing the control connection would not help with attacks because + // the client is evidently able to connect to us. Besides, closing + // makes retrials easier for the client and more damaging to us. + FtpSetReply(context, 501, "Prohibited parameter value"); + return false; + } + + FtpCloseDataConnection(context->getConn()); + + Comm::ConnectionPointer conn = new Comm::Connection(); + conn->remote = cltAddr; + + // Use local IP address of the control connection as the source address + // of the active data connection, or some clients will refuse to accept. + conn->flags |= COMM_DOBIND; + conn->local = connState->clientConnection->local; + // RFC 959 requires active FTP connections to originate from port 20 + // but that would preclude us from supporting concurrent transfers! (XXX?) + conn->local.port(0); + + debugs(11, 3, "will actively connect from " << conn->local << " to " << + conn->remote); + + context->getConn()->ftp.dataConn = conn; + context->getConn()->ftp.uploadAvailSize = 0; + return true; +} + +bool +FtpHandlePortRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + // TODO: Should PORT errors trigger FtpCloseDataConnection() cleanup? + + const ConnStateData *connState = context->getConn(); + if (connState->ftp.gotEpsvAll) { + FtpSetReply(context, 500, "Rejecting PORT after EPSV ALL"); + return false; + } + + if (!params.size()) { + FtpSetReply(context, 501, "Missing parameter"); + return false; + } + + Ip::Address cltAddr; + if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) { + FtpSetReply(context, 501, "Invalid parameter"); + return false; + } + + if (!FtpCreateDataConnection(context, cltAddr)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PORT, "FtpHandlePortRequest"); + FtpSetDataCommand(context); + return true; // forward our fake PASV request +} + +bool +FtpHandleDataRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (!FtpCheckDataConnPre(context)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_DATA_REQUEST, "FtpHandleDataRequest"); + + return true; +} + +bool +FtpHandleUploadRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (!FtpCheckDataConnPre(context)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_UPLOAD_REQUEST, "FtpHandleDataRequest"); + + return true; +} + +bool +FtpHandleEprtRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + debugs(11, 3, "Process an EPRT " << params); + + const ConnStateData *connState = context->getConn(); + if (connState->ftp.gotEpsvAll) { + FtpSetReply(context, 500, "Rejecting EPRT after EPSV ALL"); + return false; + } + + if (!params.size()) { + FtpSetReply(context, 501, "Missing parameter"); + return false; + } + + Ip::Address cltAddr; + if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) { + FtpSetReply(context, 501, "Invalid parameter"); + return false; + } + + if (!FtpCreateDataConnection(context, cltAddr)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_EPRT, "FtpHandleEprtRequest"); + FtpSetDataCommand(context); + return true; // forward our fake PASV request +} + +bool +FtpHandleEpsvRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + debugs(11, 3, "Process an EPSV command with params: " << params); + if (params.size() <= 0) { + // treat parameterless EPSV as "use the protocol of the ctrl conn" + } else if (params.caseCmp("ALL") == 0) { + ConnStateData *connState = context->getConn(); + FtpSetReply(context, 200, "EPSV ALL ok"); + connState->ftp.gotEpsvAll = true; + return false; + } else if (params.cmp("2") == 0) { + if (!Ip::EnableIpv6) { + FtpSetReply(context, 522, "Network protocol not supported, use (1)"); + return false; + } + } else if (params.cmp("1") != 0) { + FtpSetReply(context, 501, "Unsupported EPSV parameter"); + return false; + } + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_EPSV, "FtpHandleEpsvRequest"); + FtpSetDataCommand(context); + return true; // forward our fake PASV request +} + +bool +FtpHandleCwdRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CWD, "FtpHandleCwdRequest"); + return true; +} + +bool +FtpHandlePassRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASS, "FtpHandlePassRequest"); + return true; +} + +bool +FtpHandleCdupRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CDUP, "FtpHandleCdupRequest"); + return true; +} + +// Convert client PORT, EPRT, PASV, or EPSV data command to Squid PASV command. +// Squid server-side decides what data command to use on that side. +void +FtpSetDataCommand(ClientSocketContext *context) +{ + ClientHttpRequest *const http = context->http; + assert(http != NULL); + HttpRequest *const request = http->request; + assert(request != NULL); + HttpHeader &header = request->header; + header.delById(HDR_FTP_COMMAND); + header.putStr(HDR_FTP_COMMAND, "PASV"); + header.delById(HDR_FTP_ARGUMENTS); + header.putStr(HDR_FTP_ARGUMENTS, ""); + debugs(11, 5, "client data command converted to fake PASV"); +} + +/// check that client data connection is ready for future I/O or at least +/// has a chance of becoming ready soon. +bool +FtpCheckDataConnPre(ClientSocketContext *context) +{ + ConnStateData *const connState = context->getConn(); + if (Comm::IsConnOpen(connState->ftp.dataConn)) + return true; + + if (Comm::IsConnOpen(connState->ftp.dataListenConn)) { + // We are still waiting for a client to connect to us after PASV. + // Perhaps client's data conn handshake has not reached us yet. + // After we talk to the server, FtpCheckDataConnPost() will recheck. + debugs(33, 3, "expecting clt data conn " << connState->ftp.dataListenConn); + return true; + } + + if (!connState->ftp.dataConn || connState->ftp.dataConn->remote.isAnyAddr()) { + debugs(33, 5, "missing " << connState->ftp.dataConn); + // TODO: use client address and default port instead. + FtpSetReply(context, 425, "Use PORT or PASV first"); + return false; + } + + // active transfer: open a connection from Squid to client + AsyncCall::Pointer connector = context->getConn()->ftp.connector = + commCbCall(17, 3, "FtpConnectDoneWrapper", + CommConnectCbPtrFun(FtpHandleConnectDone, context)); + + Comm::ConnOpener *cs = new Comm::ConnOpener(connState->ftp.dataConn, + connector, + Config.Timeout.connect); + AsyncJob::Start(cs); + return false; // ConnStateData::processFtpRequest waits FtpHandleConnectDone +} + +/// Check that client data connection is ready for immediate I/O. +static bool +FtpCheckDataConnPost(ClientSocketContext *context) +{ + ConnStateData *connState = context->getConn(); + assert(connState); + const Comm::ConnectionPointer &dataConn = connState->ftp.dataConn; + if (!Comm::IsConnOpen(dataConn)) { + debugs(33, 3, "missing client data conn: " << dataConn); + return false; + } + return true; +} + +void - FtpHandleConnectDone(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data) ++FtpHandleConnectDone(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data) +{ + ClientSocketContext *context = static_cast(data); + context->getConn()->ftp.connector = NULL; + - if (status != COMM_OK) { ++ if (status != Comm::OK) { + conn->close(); + FtpSetReply(context, 425, "Cannot open data connection."); + assert(context->http && context->http->storeEntry() != NULL); + } else { + assert(context->getConn()->ftp.dataConn == conn); + assert(Comm::IsConnOpen(conn)); + fd_note(conn->fd, "active client ftp data"); + } + context->getConn()->resumeFtpRequest(context); +} + +void +FtpSetReply(ClientSocketContext *context, const int code, const char *msg) +{ + ClientHttpRequest *const http = context->http; + assert(http != NULL); + assert(http->storeEntry() == NULL); + + HttpReply *const reply = new HttpReply; + reply->sline.set(Http::ProtocolVersion(1, 1), Http::scNoContent); + HttpHeader &header = reply->header; + header.putTime(HDR_DATE, squid_curtime); + { + HttpHdrCc cc; + cc.Private(); + header.putCc(&cc); + } + header.putInt64(HDR_CONTENT_LENGTH, 0); + header.putInt(HDR_FTP_STATUS, code); + header.putStr(HDR_FTP_REASON, msg); + reply->hdrCacheInit(); + + setLogUri(http, urlCanonicalClean(http->request)); + + clientStreamNode *const node = context->getClientReplyContext(); + clientReplyContext *const repContext = + dynamic_cast(node->data.getRaw()); + assert(repContext != NULL); + + RequestFlags flags; + flags.cachable = false; // force releaseRequest() in storeCreateEntry() + flags.noCache = true; + repContext->createStoreEntry(http->request->method, flags); + http->storeEntry()->replaceHttpReply(reply); +} + +/// Whether Squid FTP gateway supports a given feature (e.g., a command). +static bool +FtpSupportedCommand(const String &name) +{ + static std::set BlackList; + if (BlackList.empty()) { + /* Add FTP commands that Squid cannot gateway correctly */ + + // we probably do not support AUTH TLS.* and AUTH SSL, + // but let's disclaim all AUTH support to KISS, for now + BlackList.insert("AUTH"); + } + + // we claim support for all commands that we do not know about + return BlackList.find(name.termedBuf()) == BlackList.end(); +} diff --cc src/client_side.h index edf6103d0f,5d8359372a..b3f5d6c5fc --- a/src/client_side.h +++ b/src/client_side.h @@@ -293,16 -296,17 +299,16 @@@ public virtual void noteMoreBodySpaceAvailable(BodyPipe::Pointer); virtual void noteBodyConsumerAborted(BodyPipe::Pointer); - bool handleReadData(SBuf *buf); + bool handleReadData(); bool handleRequestBodyData(); - /** - * Correlate the current ConnStateData object with the pinning_fd socket descriptor. - */ - void pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth); - /** - * Decorrelate the ConnStateData object from its pinned CachePeer - */ - void unpinConnection(); + /// forward future client requests using the given server connection + /// optionally, monitor pinned server connection for server-side closures + void pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth, bool monitor = true); + /// undo pinConnection() and, optionally, close the pinned connection + void unpinConnection(const bool andClose); + /// returns validated pinnned server connection (and stops its monitoring) + Comm::ConnectionPointer borrowPinnedConnection(HttpRequest *request, const CachePeer *aPeer); /** * Checks if there is pinning info if it is valid. It can close the server side connection * if pinned info is not valid. @@@ -414,12 -381,13 +420,16 @@@ bool switchedToHttps() const { return false; } #endif + void finishDechunkingRequest(bool withSuccess); + + void resumeFtpRequest(ClientSocketContext *const context); + + /* clt_conn_tag=tag annotation access */ + const SBuf &connectionTag() const { return connectionTag_; } + void connectionTag(const char *aTag) { connectionTag_ = aTag; } + protected: void startDechunkingRequest(); - void finishDechunkingRequest(bool withSuccess); void abortChunkedRequestBody(const err_type error); err_type handleChunkedRequestBody(size_t &putSize); @@@ -427,16 -395,10 +437,15 @@@ void clientPinnedConnectionRead(const CommIoCbParams &io); private: - int connReadWasError(comm_err_t flag, int size, int xerrno); int connFinishedWithConn(int size); void clientAfterReadingRequests(); + void processFtpRequest(ClientSocketContext *const context); + void handleFtpRequestData(); + bool concurrentRequestQueueFilled() const; + void pinNewConnection(const Comm::ConnectionPointer &pinServer, HttpRequest *request, CachePeer *aPeer, bool auth); + #if USE_AUTH /// some user details that can be used to perform authentication on this connection Auth::UserRequest::Pointer auth_; diff --cc src/ftp.cc index 73b37b4034,0735e2f80a..0a5833a464 --- a/src/ftp.cc +++ b/src/ftp.cc @@@ -33,7 -33,10 +33,9 @@@ #include "squid.h" #include "acl/FilledChecklist.h" #include "comm.h" + #include "comm/ConnOpener.h" + #include "comm/Read.h" #include "comm/TcpAcceptor.h" -#include "comm/Write.h" #include "CommCalls.h" #include "compat/strtoll.h" #include "errorpage.h" @@@ -177,9 -271,14 +177,9 @@@ public void setCurrentOffset(int64_t offset) { currentOffset = offset; } int64_t getCurrentOffset() const { return currentOffset; } - virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno); - static CNCB ftpPasvCallback; ++ virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag err, int xerrno); static PF ftpDataWrite; - void ftpTimeout(const CommTimeoutCbParams &io); - void ctrlClosed(const CommCloseCbParams &io); - void dataClosed(const CommCloseCbParams &io); - void ftpReadControlReply(const CommIoCbParams &io); - void ftpWriteCommandCallback(const CommIoCbParams &io); + virtual void timeout(const CommTimeoutCbParams &io); void ftpAcceptDataConnection(const CommAcceptCbParams &io); static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm); @@@ -1826,20 -2688,69 +1829,20 @@@ ftpReadPasv(FtpStateData * ftpState } void - FtpStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno) -FtpStateData::ftpPasvCallback(const Comm::ConnectionPointer &conn, Comm::Flag status, int xerrno, void *data) ++FtpStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, Comm::Flag err, int xerrno) { - FtpStateData *ftpState = (FtpStateData *)data; debugs(9, 3, HERE); - ftpState->data.opener = NULL; + data.opener = NULL; - if (err != COMM_OK) { - if (status != Comm::OK) { ++ if (err != Comm::OK) { debugs(9, 2, HERE << "Failed to connect. Retrying via another method."); // ABORT on timeouts. server may be waiting on a broken TCP link. - if (err == COMM_TIMEOUT) - if (status == Comm::TIMEOUT) - ftpState->writeCommand("ABOR"); ++ if (err == Comm::TIMEOUT) + writeCommand("ABOR"); // try another connection attempt with some other method - ftpSendPassive(ftpState); + ftpSendPassive(this); return; } @@@ -2066,10 -2977,12 +2069,10 @@@ FtpStateData::ftpAcceptDataConnection(c } } - /** On COMM_OK start using the accepted data socket and discard the temporary listen socket. */ + /** On Comm::OK start using the accepted data socket and discard the temporary listen socket. */ data.close(); data.opened(io.conn, dataCloser()); - static char ntoapeer[MAX_IPSTRLEN]; - io.conn->remote.toStr(ntoapeer,sizeof(ntoapeer)); - data.host = xstrdup(ntoapeer); + data.addr(io.conn->remote); debugs(9, 3, HERE << "Connected data socket on " << io.conn << ". FD table says: " << diff --cc src/tests/stub_client_side.cc index ce479480c9,5aff3910f4..ec8f1dc48c --- a/src/tests/stub_client_side.cc +++ b/src/tests/stub_client_side.cc @@@ -51,10 -51,10 +51,10 @@@ void ConnStateData::stopSending(const c void ConnStateData::expectNoForwarding() STUB void ConnStateData::noteMoreBodySpaceAvailable(BodyPipe::Pointer) STUB void ConnStateData::noteBodyConsumerAborted(BodyPipe::Pointer) STUB - bool ConnStateData::handleReadData(SBuf *buf) STUB_RETVAL(false) + bool ConnStateData::handleReadData() STUB_RETVAL(false) bool ConnStateData::handleRequestBodyData() STUB_RETVAL(false) - void ConnStateData::pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth, bool monitor = true) STUB -void ConnStateData::pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth) STUB -void ConnStateData::unpinConnection() STUB ++void ConnStateData::pinConnection(const Comm::ConnectionPointer &pinServerConn, HttpRequest *request, CachePeer *peer, bool auth, bool monitor) STUB +void ConnStateData::unpinConnection(const bool andClose) STUB const Comm::ConnectionPointer ConnStateData::validatePinnedConnection(HttpRequest *request, const CachePeer *peer) STUB_RETVAL(NULL) void ConnStateData::clientPinnedConnectionClosed(const CommCloseCbParams &io) STUB void ConnStateData::clientReadRequest(const CommIoCbParams &io) STUB diff --cc src/tools.cc index e924ed1724,324bdaab0d..91f4b2d963 --- a/src/tools.cc +++ b/src/tools.cc @@@ -1152,14 -1151,6 +1151,14 @@@ getMyPort(void } #endif - if ((p = Config.Sockaddr.ftp)) { ++ if ((p = FtpPortList) != NULL) { + // skip any special interception ports - while (p && p->flags.isIntercepted()) ++ while (p != NULL && p->flags.isIntercepted()) + p = p->next; - if (p) ++ if (p != NULL) + return p->s.port(); + } + debugs(21, DBG_CRITICAL, "ERROR: No forward-proxy ports configured."); return 0; // Invalid port. This will result in invalid URLs on bad configurations. }