From: Alex Rousskov Date: Tue, 27 Aug 2013 15:45:11 +0000 (-0600) Subject: Merged from trunk r12948. X-Git-Tag: SQUID_3_5_0_1~117^2~48 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=73950ceb1794fada8d593195d05155345bd7afca;p=thirdparty%2Fsquid.git Merged from trunk r12948. --- 73950ceb1794fada8d593195d05155345bd7afca diff --cc src/FtpGatewayServer.cc index 94e134e88f,0000000000..7c3243d317 mode 100644,000000..100644 --- a/src/FtpGatewayServer.cc +++ b/src/FtpGatewayServer.cc @@@ -1,546 -1,0 +1,536 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#include "squid.h" + +#include "FtpGatewayServer.h" +#include "FtpServer.h" +#include "HttpHdrCc.h" +#include "HttpRequest.h" +#include "Server.h" +#include "SquidTime.h" +#include "Store.h" +#include "client_side.h" +#include "wordlist.h" + +namespace Ftp { + +namespace Gateway { + +class ServerStateData: public Ftp::ServerStateData +{ +public: + ServerStateData(FwdState *const fwdState); + ~ServerStateData(); + + virtual void processReplyBody(); + +protected: + virtual void start(); + + ConnStateData::FtpState clientState() const; + void clientState(ConnStateData::FtpState newState); + virtual void serverComplete(); + virtual void failed(err_type error = ERR_NONE, int xerrno = 0); + virtual void handleControlReply(); + virtual void handleRequestBodyProducerAborted(); + virtual bool mayReadVirginReplyBody() const; + virtual void completeForwarding(); + void forwardReply(); + void forwardError(err_type error = ERR_NONE, int xerrno = 0); + void failedErrorMessage(err_type error, int xerrno); + HttpReply *createHttpReply(const Http::StatusCode httpStatus, const int clen = 0); + void handleDataRequest(); + void startDataDownload(); + void startDataUpload(); + + typedef void (ServerStateData::*PreliminaryCb)(); + void forwardPreliminaryReply(const PreliminaryCb cb); + void proceedAfterPreliminaryReply(); + PreliminaryCb thePreliminaryCb; + + enum { + BEGIN, + SENT_COMMAND, + SENT_PASV, + SENT_PORT, + SENT_DATA_REQUEST, + READING_DATA, + UPLOADING_DATA, + END + }; + typedef void (ServerStateData::*SM_FUNC)(); + static const SM_FUNC SM_FUNCS[]; + void readGreeting(); + void sendCommand(); + void readReply(); + void readPasvReply(); + void readPortReply(); + void readDataReply(); + void readTransferDoneReply(); + + virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno); + void scheduleReadControlReply(); + + bool forwardingCompleted; ///< completeForwarding() has been called + + CBDATA_CLASS2(ServerStateData); +}; + +CBDATA_CLASS_INIT(ServerStateData); + +const ServerStateData::SM_FUNC ServerStateData::SM_FUNCS[] = { + &ServerStateData::readGreeting, // BEGIN + &ServerStateData::readReply, // SENT_COMMAND + &ServerStateData::readPasvReply, // SENT_PASV + &ServerStateData::readPortReply, // SENT_PORT + &ServerStateData::readDataReply, // SENT_DATA_REQUEST + &ServerStateData::readTransferDoneReply, // READING_DATA + &ServerStateData::readReply, // UPLOADING_DATA + NULL // END +}; + +ServerStateData::ServerStateData(FwdState *const fwdState): + AsyncJob("Ftp::Gateway::ServerStateData"), Ftp::ServerStateData(fwdState), + forwardingCompleted(false) +{ +} + +ServerStateData::~ServerStateData() +{ + closeServer(); // TODO: move to Server.cc? +} + +void +ServerStateData::start() +{ + if (!fwd->request->clientConnectionManager->ftp.readGreeting) + Ftp::ServerStateData::start(); + else + if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + handleDataRequest(); + else + sendCommand(); +} + +/// Keep control connection for future requests, after we are done with it. +/// Similar to COMPLETE_PERSISTENT_MSG handling in http.cc. +void +ServerStateData::serverComplete() +{ + if (Comm::IsConnOpen(ctrl.conn)) { + debugs(9, 5, "preserve FTP server FD " << ctrl.conn->fd); + fwd->unregister(ctrl.conn); + ctrl.forget(); + // fwd->request->clientConnectionManager has this connection pinned + } + Ftp::ServerStateData::serverComplete(); +} + +ConnStateData::FtpState +ServerStateData::clientState() const +{ + return fwd->request->clientConnectionManager->ftp.state; +} + +void +ServerStateData::clientState(ConnStateData::FtpState newState) +{ + ConnStateData::FtpState &cltState = + fwd->request->clientConnectionManager->ftp.state; + debugs(9, 3, "client state was " << cltState << " now: " << newState); + cltState = newState; +} + +/** + * Ensure we do not double-complete on the forward entry. + * We complete forwarding when the response adaptation is over + * (but we may still be waiting for 226 from the FTP server) and + * also when we get that 226 from the server (and adaptation is done). + * + \todo Rewrite FwdState to ignore double completion? + */ +void +ServerStateData::completeForwarding() +{ + debugs(9, 5, forwardingCompleted); + if (forwardingCompleted) + return; + forwardingCompleted = true; + Ftp::ServerStateData::completeForwarding(); +} + +void +ServerStateData::failed(err_type error, int xerrno) +{ + if (!doneWithServer()) + clientState(ConnStateData::FTP_ERROR); + + // TODO: we need to customize ErrorState instead + if (entry->isEmpty()) + failedErrorMessage(error, xerrno); // as a reply + + Ftp::ServerStateData::failed(error, xerrno); +} + +void +ServerStateData::failedErrorMessage(err_type error, int xerrno) +{ + const Http::StatusCode httpStatus = failedHttpStatus(error); + HttpReply *const reply = createHttpReply(httpStatus); + entry->replaceHttpReply(reply); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + fwd->request->detailError(error, xerrno); +} + +void +ServerStateData::processReplyBody() +{ + debugs(9, 3, HERE << "starting"); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + /* + * probably was aborted because content length exceeds one + * of the maximum size limits. + */ + abortTransaction("entry aborted after calling appendSuccessHeader()"); + return; + } + +#if USE_ADAPTATION + + if (adaptationAccessCheckPending) { + debugs(9,3, HERE << "returning due to adaptationAccessCheckPending"); + return; + } + +#endif + + if (data.readBuf != NULL && data.readBuf->hasContent()) { + const mb_size_t csize = data.readBuf->contentSize(); + debugs(9, 5, HERE << "writing " << csize << " bytes to the reply"); + addVirginReplyBody(data.readBuf->content(), csize); + data.readBuf->consume(csize); + } + + entry->flush(); + + maybeReadVirginBody(); +} + +void +ServerStateData::handleControlReply() +{ + Ftp::ServerStateData::handleControlReply(); + if (ctrl.message == NULL) + return; // didn't get complete reply yet + + assert(state < END); + (this->*SM_FUNCS[state])(); +} + +void +ServerStateData::handleRequestBodyProducerAborted() +{ + ::ServerStateData::handleRequestBodyProducerAborted(); + + failed(ERR_READ_ERROR); +} + +bool +ServerStateData::mayReadVirginReplyBody() const +{ + // TODO: move this method to the regular FTP server? + return Comm::IsConnOpen(data.conn); +} + +void +ServerStateData::forwardReply() +{ + assert(entry->isEmpty()); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + + HttpReply *const reply = createHttpReply(Http::scNoContent); + + setVirginReply(reply); + adaptOrFinalizeReply(); + + serverComplete(); +} + +void +ServerStateData::forwardPreliminaryReply(const PreliminaryCb cb) +{ + debugs(9, 5, HERE << "Forwarding preliminary reply to client"); + + assert(thePreliminaryCb == NULL); + thePreliminaryCb = cb; + + const HttpReply::Pointer reply = createHttpReply(Http::scContinue); + + // the Sink will use this to call us back after writing 1xx to the client + typedef NullaryMemFunT CbDialer; + const AsyncCall::Pointer call = JobCallback(11, 3, CbDialer, this, + ServerStateData::proceedAfterPreliminaryReply); + + CallJobHere1(9, 4, request->clientConnectionManager, ConnStateData, + ConnStateData::sendControlMsg, HttpControlMsg(reply, call)); +} + +void +ServerStateData::proceedAfterPreliminaryReply() +{ + debugs(9, 5, HERE << "Proceeding after preliminary reply to client"); + + assert(thePreliminaryCb != NULL); + const PreliminaryCb cb = thePreliminaryCb; + thePreliminaryCb = NULL; + (this->*cb)(); +} + +void +ServerStateData::forwardError(err_type error, int xerrno) +{ + failed(error, xerrno); +} + +HttpReply * +ServerStateData::createHttpReply(const Http::StatusCode httpStatus, const int clen) +{ + HttpReply *const reply = new HttpReply; + reply->sline.set(Http::ProtocolVersion(1, 1), httpStatus); + HttpHeader &header = reply->header; + header.putTime(HDR_DATE, squid_curtime); + { + HttpHdrCc cc; + cc.Private(); + header.putCc(&cc); + } + if (clen >= 0) + header.putInt64(HDR_CONTENT_LENGTH, clen); + if (ctrl.replycode > 0) + header.putInt(HDR_FTP_STATUS, ctrl.replycode); + if (ctrl.message) { + for (wordlist *W = ctrl.message; W; W = W->next) + header.putStr(HDR_FTP_REASON, W->key); + } else if (ctrl.last_command) + header.putStr(HDR_FTP_REASON, ctrl.last_command); + + reply->hdrCacheInit(); + + return reply; +} + +void +ServerStateData::handleDataRequest() +{ - data.addr = fwd->request->clientConnectionManager->ftp.serverDataAddr; - - if (!data.addr.IsSockAddr()) { // should never happen - debugs(9, DBG_IMPORTANT, HERE << "Inconsistent FTP server state: " - "data.addr=" << data.addr); - failed(); - return; - } - ++ data.addr(fwd->request->clientConnectionManager->ftp.serverDataAddr); + connectDataChannel(); +} + +void +ServerStateData::startDataDownload() +{ + assert(Comm::IsConnOpen(data.conn)); + + debugs(9, 3, HERE << "begin data transfer from " << data.conn->remote << + " (" << data.conn->local << ")"); + + HttpReply *const reply = createHttpReply(Http::scOkay, -1); + EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT); + setVirginReply(reply); + adaptOrFinalizeReply(); + + switchTimeoutToDataChannel(); + maybeReadVirginBody(); + state = READING_DATA; +} + +void +ServerStateData::startDataUpload() +{ + assert(Comm::IsConnOpen(data.conn)); + + debugs(9, 3, HERE << "begin data transfer to " << data.conn->remote << + " (" << data.conn->local << ")"); + + if (!startRequestBodyFlow()) { // register to receive body data + failed(); + return; + } + + state = UPLOADING_DATA; +} + +void +ServerStateData::readGreeting() +{ + assert(!fwd->request->clientConnectionManager->ftp.readGreeting); + + switch (ctrl.replycode) { + case 220: + fwd->request->clientConnectionManager->ftp.readGreeting = true; + if (clientState() == ConnStateData::FTP_BEGIN) + clientState(ConnStateData::FTP_CONNECTED); + + // Do not forward server greeting to client as a preliminary + // reply because it may confuse web browsers. Should we + // forward greeting as part of the final reply? + //ctrl.replycode = 120; // change status for forwarded server greeting + //forwardPreliminaryReply(&ServerStateData::start); + + start(); + break; + case 120: + if (NULL != ctrl.message) + debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ctrl.message->key); + forwardPreliminaryReply(&ServerStateData::scheduleReadControlReply); + break; + default: + failed(); + break; + } +} + +void +ServerStateData::sendCommand() +{ + if (!fwd->request->header.has(HDR_FTP_COMMAND)) { + abortTransaction("Internal error: FTP gateway request with no command"); + return; + } + + HttpHeader &header = fwd->request->header; + assert(header.has(HDR_FTP_COMMAND)); + const String &cmd = header.findEntry(HDR_FTP_COMMAND)->value; + assert(header.has(HDR_FTP_ARGUMENTS)); + const String ¶ms = header.findEntry(HDR_FTP_ARGUMENTS)->value; + + if (params.size() > 0) + debugs(9, 5, HERE << "command: " << cmd << ", parameters: " << params); + else + debugs(9, 5, HERE << "command: " << cmd << ", no parameters"); + + static MemBuf mb; + mb.reset(); + if (params.size() > 0) + mb.Printf("%s %s%s", cmd.termedBuf(), params.termedBuf(), Ftp::crlf); + else + mb.Printf("%s%s", cmd.termedBuf(), Ftp::crlf); + + writeCommand(mb.content()); + + state = + clientState() == ConnStateData::FTP_HANDLE_PASV ? SENT_PASV : + clientState() == ConnStateData::FTP_HANDLE_PORT ? SENT_PORT : + clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST ? SENT_DATA_REQUEST : + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST ? SENT_DATA_REQUEST : + SENT_COMMAND; +} + +void +ServerStateData::readReply() +{ + assert(clientState() == ConnStateData::FTP_CONNECTED || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + forwardPreliminaryReply(&ServerStateData::scheduleReadControlReply); + else + forwardReply(); +} + +void +ServerStateData::readPasvReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_PASV); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; // ignore preliminary replies + - if (handlePasvReply()) { - fwd->request->clientConnectionManager->ftp.serverDataAddr = data.addr; ++ if (handlePasvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) + forwardReply(); - } else ++ else + forwardError(); +} + +/// In fact, we are handling a PASV reply here (XXX: remove duplication) +void +ServerStateData::readPortReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_PORT); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; // ignore preliminary replies + - if (handlePasvReply()) { - fwd->request->clientConnectionManager->ftp.serverDataAddr = data.addr; ++ if (handlePasvReply(fwd->request->clientConnectionManager->ftp.serverDataAddr)) + forwardReply(); - } else ++ else + forwardError(); +} + +void +ServerStateData::readDataReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST || + clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST); + + if (ctrl.replycode == 150) { + if (clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST) + forwardPreliminaryReply(&ServerStateData::startDataDownload); + else // clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST + forwardPreliminaryReply(&ServerStateData::startDataUpload); + } else + forwardReply(); +} + +void +ServerStateData::readTransferDoneReply() +{ + debugs(9, 3, HERE); + + if (ctrl.replycode != 226 && ctrl.replycode != 250) { + debugs(9, DBG_IMPORTANT, HERE << "Got code " << ctrl.replycode << + " after reading data"); + } + + serverComplete(); +} + +void +ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno) +{ + debugs(9, 3, HERE); + data.opener = NULL; + + if (err != COMM_OK) { + debugs(9, 2, HERE << "Failed to connect FTP server data channel."); + forwardError(ERR_CONNECT_FAIL, xerrno); + return; + } + + debugs(9, 2, HERE << "Connected FTP server data channel: " << conn); + + data.opened(conn, dataCloser()); + + sendCommand(); +} + +void +ServerStateData::scheduleReadControlReply() +{ + Ftp::ServerStateData::scheduleReadControlReply(0); +} + +}; // namespace Gateway + +}; // namespace Ftp + +void +ftpGatewayServerStart(FwdState *const fwdState) +{ + AsyncJob::Start(new Ftp::Gateway::ServerStateData(fwdState)); +} diff --cc src/FtpServer.cc index 995d221ea7,0000000000..5b03dc0517 mode 100644,000000..100644 --- a/src/FtpServer.cc +++ b/src/FtpServer.cc @@@ -1,848 -1,0 +1,860 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#include "squid.h" + +#include "FtpServer.h" +#include "Mem.h" +#include "SquidConfig.h" +#include "StatCounters.h" +#include "client_side.h" +#include "comm/ConnOpener.h" +#include "comm/Write.h" +#include "errorpage.h" +#include "fd.h" +#include "tools.h" +#include "wordlist.h" + +namespace Ftp { + +const char *const crlf = "\r\n"; + +/// \ingroup ServerProtocolFTPInternal +static char * +escapeIAC(const char *buf) +{ + int n; + char *ret; + unsigned const char *p; + unsigned char *r; + + for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p) + if (*p == 255) + ++n; + + ret = (char *)xmalloc(n); + + for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) { + *r = *p; + ++r; + + if (*p == 255) { + *r = 255; + ++r; + } + } + + *r = '\0'; + ++r; + assert((r - (unsigned char *)ret) == n ); + return ret; +} + +/// configures the channel with a descriptor and registers a close handler +void - ServerChannel::opened(const Comm::ConnectionPointer &newConn, ++FtpChannel::opened(const Comm::ConnectionPointer &newConn, + const AsyncCall::Pointer &aCloser) +{ + assert(!Comm::IsConnOpen(conn)); + assert(closer == NULL); + + assert(Comm::IsConnOpen(newConn)); + assert(aCloser != NULL); + + conn = newConn; + closer = aCloser; + comm_add_close_handler(conn->fd, closer); +} + +/// planned close: removes the close handler and calls comm_close +void - ServerChannel::close() ++FtpChannel::close() +{ + // channels with active listeners will be closed when the listener handler dies. + if (Comm::IsConnOpen(conn)) { + comm_remove_close_handler(conn->fd, closer); + conn->close(); // we do not expect to be called back + } + clear(); +} + +void - ServerChannel::forget() ++FtpChannel::forget() +{ + if (Comm::IsConnOpen(conn)) + comm_remove_close_handler(conn->fd, closer); + clear(); +} + +void - ServerChannel::clear() ++FtpChannel::clear() +{ + conn = NULL; + closer = NULL; +} + +ServerStateData::ServerStateData(FwdState *fwdState): + AsyncJob("Ftp::ServerStateData"), ::ServerStateData(fwdState) +{ + ++statCounter.server.all.requests; + ++statCounter.server.ftp.requests; + + ctrl.last_command = xstrdup("Connect to server"); + ctrl.buf = static_cast(memAllocBuf(4096, &ctrl.size)); + ctrl.offset = 0; + + typedef CommCbMemFunT Dialer; + const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this, + ServerStateData::ctrlClosed); + ctrl.opened(fwdState->serverConnection(), closer); +} + ++void ++ServerStateData::DataChannel::addr(const Ip::Address &import) ++{ ++ static char addrBuf[MAX_IPSTRLEN]; ++ import.toStr(addrBuf, sizeof(addrBuf)); ++ xfree(host); ++ host = xstrdup(addrBuf); ++ port = import.port(); ++} ++ +ServerStateData::~ServerStateData() +{ + if (data.opener != NULL) { + data.opener->cancel("Ftp::ServerStateData destructed"); + data.opener = NULL; + } + data.close(); + + if (ctrl.buf) { + memFreeBuf(ctrl.size, ctrl.buf); + ctrl.buf = NULL; + } + if (ctrl.message) + wordlistDestroy(&ctrl.message); + safe_free(ctrl.last_command); + safe_free(ctrl.last_reply); + + if (data.readBuf) { + if (!data.readBuf->isNull()) + data.readBuf->clean(); + + delete data.readBuf; + } + + safe_free(old_request); + + safe_free(old_reply); + + fwd = NULL; // refcounted +} + +void +ServerStateData::start() +{ + scheduleReadControlReply(0); +} + +void +ServerStateData::initReadBuf() +{ + if (data.readBuf == NULL) { + data.readBuf = new MemBuf; + data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF); + } +} + +/** + * Close the FTP server connection(s). Used by serverComplete(). + */ +void +ServerStateData::closeServer() +{ + if (Comm::IsConnOpen(ctrl.conn)) { + debugs(9,3, HERE << "closing FTP server FD " << ctrl.conn->fd << ", this " << this); + fwd->unregister(ctrl.conn); + ctrl.close(); + } + + if (Comm::IsConnOpen(data.conn)) { + debugs(9,3, HERE << "closing FTP data FD " << data.conn->fd << ", this " << this); + data.close(); + } + + debugs(9,3, HERE << "FTP ctrl and data connections closed. this " << this); +} + +/** + * Did we close all FTP server connection(s)? + * + \retval true Both server control and data channels are closed. And not waiting for a new data connection to open. + \retval false Either control channel or data is still active. + */ +bool +ServerStateData::doneWithServer() const +{ + return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn); +} + +void +ServerStateData::failed(err_type error, int xerrno) +{ + debugs(9,3,HERE << "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry); + + const char *command, *reply; + const Http::StatusCode httpStatus = failedHttpStatus(error); + ErrorState *const ftperr = new ErrorState(error, httpStatus, fwd->request); + ftperr->xerrno = xerrno; + + ftperr->ftp.server_msg = ctrl.message; + ctrl.message = NULL; + + if (old_request) + command = old_request; + else + command = ctrl.last_command; + + if (command && strncmp(command, "PASS", 4) == 0) + command = "PASS "; + + if (old_reply) + reply = old_reply; + else + reply = ctrl.last_reply; + + if (command) + ftperr->ftp.request = xstrdup(command); + + if (reply) + ftperr->ftp.reply = xstrdup(reply); + + fwd->fail(ftperr); + + closeServer(); // we failed, so no serverComplete() +} + +Http::StatusCode +ServerStateData::failedHttpStatus(err_type &error) +{ + if (error == ERR_NONE) + error = ERR_FTP_FAILURE; + return error == ERR_READ_TIMEOUT ? Http::scGateway_Timeout : + Http::scBadGateway; +} + +/** + * DPW 2007-04-23 + * Looks like there are no longer anymore callers that set + * buffered_ok=1. Perhaps it can be removed at some point. + */ +void +ServerStateData::scheduleReadControlReply(int buffered_ok) +{ + debugs(9, 3, HERE << ctrl.conn); + + if (buffered_ok && ctrl.offset > 0) { + /* We've already read some reply data */ + handleControlReply(); + } else { + /* + * Cancel the timeout on the Data socket (if any) and + * establish one on the control socket. + */ + if (Comm::IsConnOpen(data.conn)) { + commUnsetConnTimeout(data.conn); + } + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, ServerStateData::timeout); + commSetConnTimeout(ctrl.conn, Config.Timeout.read, timeoutCall); + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, ServerStateData::readControlReply); + comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader); + } +} + +void +ServerStateData::readControlReply(const CommIoCbParams &io) +{ + debugs(9, 3, HERE << "FD " << io.fd << ", Read " << io.size << " bytes"); + + if (io.size > 0) { + kb_incr(&(statCounter.server.all.kbytes_in), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_in), io.size); + } + + if (io.flag == COMM_ERR_CLOSING) + return; + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + abortTransaction("entry aborted during control reply read"); + return; + } + + assert(ctrl.offset < ctrl.size); + + if (io.flag == COMM_OK && io.size > 0) { + fd_bytes(io.fd, io.size, FD_READ); + } + + if (io.flag != COMM_OK) { + debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT, + "ftpReadControlReply: read error: " << xstrerr(io.xerrno)); + + if (ignoreErrno(io.xerrno)) { + scheduleReadControlReply(0); + } else { + failed(ERR_READ_ERROR, io.xerrno); + /* failed closes ctrl.conn and frees ftpState */ + } + return; + } + + if (io.size == 0) { + if (entry->store_status == STORE_PENDING) { + failed(ERR_FTP_FAILURE, 0); + /* failed closes ctrl.conn and frees ftpState */ + return; + } + + /* XXX this may end up having to be serverComplete() .. */ + abortTransaction("zero control reply read"); + return; + } + + unsigned int len =io.size + ctrl.offset; + ctrl.offset = len; + assert(len <= ctrl.size); + handleControlReply(); +} + +void +ServerStateData::handleControlReply() +{ + debugs(9, 3, HERE); + + size_t bytes_used = 0; + wordlistDestroy(&ctrl.message); + ctrl.message = parseControlReply(ctrl.buf, ctrl.offset, &ctrl.replycode, + &bytes_used); + + if (ctrl.message == NULL) { + /* didn't get complete reply yet */ + + if (ctrl.offset == ctrl.size) { + ctrl.buf = (char *)memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size); + } + + scheduleReadControlReply(0); + return; + } else if (ctrl.offset == bytes_used) { + /* used it all up */ + ctrl.offset = 0; + } else { + /* Got some data past the complete reply */ + assert(bytes_used < ctrl.offset); + ctrl.offset -= bytes_used; + memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset); + } + + /* Move the last line of the reply message to ctrl.last_reply */ + const wordlist *W; + for (W = ctrl.message; W && W->next; W = W->next); + if (W) { + safe_free(ctrl.last_reply); + ctrl.last_reply = xstrdup(W->key); + } + + debugs(9, 3, HERE << "state=" << state << ", code=" << ctrl.replycode); +} + +bool - ServerStateData::handlePasvReply() ++ServerStateData::handlePasvReply(Ip::Address &srvAddr) +{ + int code = ctrl.replycode; + char *buf; + debugs(9, 3, HERE); + + if (code != 227) { + debugs(9, 2, "PASV not supported by remote end"); + return false; + } + + /* 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). */ + /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ + debugs(9, 5, HERE << "scanning: " << ctrl.last_reply); + + buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789"); + + const char *forceIp = Config.Ftp.sanitycheck ? + fd_table[ctrl.conn->fd].ipaddr : NULL; - if (!Ftp::ParseIpPort(buf, forceIp, data.addr)) { ++ if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) { + debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " << + ctrl.conn->remote << ": " << ctrl.last_reply); + return false; + } + ++ data.addr(srvAddr); ++ + return true; +} + +void +ServerStateData::connectDataChannel() +{ + safe_free(ctrl.last_command); + + safe_free(ctrl.last_reply); + + ctrl.last_command = xstrdup("Connect to server data port"); + ++ // Generate a new data channel descriptor to be opened. + Comm::ConnectionPointer conn = new Comm::Connection; + conn->local = ctrl.conn->local; - conn->local.SetPort(0); - conn->remote = data.addr; ++ conn->local.port(0); ++ conn->remote = data.host; ++ conn->remote.port(data.port); + + debugs(9, 3, HERE << "connecting to " << conn->remote); + + data.opener = commCbCall(9,3, "Ftp::ServerStateData::dataChannelConnected", + CommConnectCbPtrFun(ServerStateData::dataChannelConnected, this)); + Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect); - char buf[MAX_IPSTRLEN]; - data.addr.ToHostname(buf, MAX_IPSTRLEN); - cs->setHost(buf); ++ cs->setHost(data.host); + AsyncJob::Start(cs); +} + +void +ServerStateData::dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data) +{ + ServerStateData *ftpState = static_cast(data); + ftpState->dataChannelConnected(conn, status, xerrno); +} + +/// creates a data channel Comm close callback +AsyncCall::Pointer +ServerStateData::dataCloser() +{ + typedef CommCbMemFunT Dialer; + return JobCallback(9, 5, Dialer, this, ServerStateData::dataClosed); +} + +/// handler called by Comm when FTP data channel is closed unexpectedly +void +ServerStateData::dataClosed(const CommCloseCbParams &io) +{ + debugs(9, 4, HERE); + if (data.listenConn != NULL) { + data.listenConn->close(); + data.listenConn = NULL; + // NP clear() does the: data.fd = -1; + } + data.clear(); +} + +void +ServerStateData::writeCommand(const char *buf) +{ + char *ebuf; + /* trace FTP protocol communications at level 2 */ + debugs(9, 2, "ftp<< " << buf); + + if (Config.Ftp.telnet) + ebuf = escapeIAC(buf); + else + ebuf = xstrdup(buf); + + safe_free(ctrl.last_command); + + safe_free(ctrl.last_reply); + + ctrl.last_command = ebuf; + + if (!Comm::IsConnOpen(ctrl.conn)) { + debugs(9, 2, HERE << "cannot send to closing ctrl " << ctrl.conn); + // TODO: assert(ctrl.closer != NULL); + return; + } + + typedef CommCbMemFunT Dialer; + AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this, + ServerStateData::writeCommandCallback); + Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL); + + scheduleReadControlReply(0); +} + +void +ServerStateData::writeCommandCallback(const CommIoCbParams &io) +{ + + debugs(9, 5, HERE << "wrote " << io.size << " bytes"); + + if (io.size > 0) { + fd_bytes(io.fd, io.size, FD_WRITE); + kb_incr(&(statCounter.server.all.kbytes_out), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_out), io.size); + } + + if (io.flag == COMM_ERR_CLOSING) + return; + + if (io.flag) { + debugs(9, DBG_IMPORTANT, "ftpWriteCommandCallback: " << io.conn << ": " << xstrerr(io.xerrno)); + failed(ERR_WRITE_ERROR, io.xerrno); + /* failed closes ctrl.conn and frees ftpState */ + return; + } +} + +/// handler called by Comm when FTP control channel is closed unexpectedly +void +ServerStateData::ctrlClosed(const CommCloseCbParams &io) +{ + debugs(9, 4, HERE); + ctrl.clear(); + mustStop("Ftp::ServerStateData::ctrlClosed"); +} + +void +ServerStateData::timeout(const CommTimeoutCbParams &io) +{ + debugs(9, 4, HERE << io.conn << ": '" << entry->url() << "'" ); + + if (abortOnBadEntry("entry went bad while waiting for a timeout")) + return; + + failed(ERR_READ_TIMEOUT, 0); + /* failed() closes ctrl.conn and frees ftpState */ +} + +const Comm::ConnectionPointer & +ServerStateData::dataConnection() const +{ + return data.conn; +} + +void +ServerStateData::maybeReadVirginBody() +{ + // too late to read + if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing()) + return; + + if (data.read_pending) + return; + + initReadBuf(); + + const int read_sz = replyBodySpace(*data.readBuf, 0); + + debugs(11,9, HERE << "FTP may read up to " << read_sz << " bytes"); + + if (read_sz < 2) // see http.cc + return; + + data.read_pending = true; + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, + TimeoutDialer, this, ServerStateData::timeout); + commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall); + + debugs(9,5,HERE << "queueing read on FD " << data.conn->fd); + + typedef CommCbMemFunT Dialer; + entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz, + JobCallback(9, 5, Dialer, this, ServerStateData::dataRead)); +} + +void +ServerStateData::dataRead(const CommIoCbParams &io) +{ + int j; + int bin; + + data.read_pending = false; + + debugs(9, 3, HERE << "FD " << io.fd << " Read " << io.size << " bytes"); + + if (io.size > 0) { + kb_incr(&(statCounter.server.all.kbytes_in), io.size); + kb_incr(&(statCounter.server.ftp.kbytes_in), io.size); + } + + if (io.flag == COMM_ERR_CLOSING) + return; + + assert(io.fd == data.conn->fd); + + if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) { + abortTransaction("entry aborted during dataRead"); + return; + } + + if (io.flag == COMM_OK && io.size > 0) { + debugs(9, 5, HERE << "appended " << io.size << " bytes to readBuf"); + data.readBuf->appended(io.size); +#if USE_DELAY_POOLS + DelayId delayId = entry->mem_obj->mostBytesAllowed(); + delayId.bytesIn(io.size); +#endif + ++ IOStats.Ftp.reads; + + for (j = io.size - 1, bin = 0; j; ++bin) + j >>= 1; + + ++ IOStats.Ftp.read_hist[bin]; + } + + if (io.flag != COMM_OK) { + debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT, + HERE << "read error: " << xstrerr(io.xerrno)); + + if (ignoreErrno(io.xerrno)) { + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = + JobCallback(9, 5, TimeoutDialer, this, + ServerStateData::timeout); + commSetConnTimeout(io.conn, Config.Timeout.read, timeoutCall); + + maybeReadVirginBody(); + } else { + failed(ERR_READ_ERROR, 0); + /* failed closes ctrl.conn and frees ftpState */ + return; + } + } else if (io.size == 0) { + debugs(9,3, HERE << "Calling dataComplete() because io.size == 0"); + /* + * DPW 2007-04-23 + * Dangerous curves ahead. This call to dataComplete was + * calling scheduleReadControlReply, handleControlReply, + * and then ftpReadTransferDone. If ftpReadTransferDone + * gets unexpected status code, it closes down the control + * socket and our FtpStateData object gets destroyed. As + * a workaround we no longer set the 'buffered_ok' flag in + * the scheduleReadControlReply call. + */ + dataComplete(); + } + + processReplyBody(); +} + +void +ServerStateData::dataComplete() +{ + debugs(9, 3,HERE); + + /* Connection closed; transfer done. */ + + /// Close data channel, if any, to conserve resources while we wait. + data.close(); + + /* expect the "transfer complete" message on the control socket */ + /* + * DPW 2007-04-23 + * Previously, this was the only place where we set the + * 'buffered_ok' flag when calling scheduleReadControlReply(). + * It caused some problems if the FTP server returns an unexpected + * status code after the data command. FtpStateData was being + * deleted in the middle of dataRead(). + */ + /* AYJ: 2011-01-13: Bug 2581. + * 226 status is possibly waiting in the ctrl buffer. + * The connection will hang if we DONT send buffered_ok. + * This happens on all transfers which can be completly sent by the + * server before the 150 started status message is read in by Squid. + * ie all transfers of about one packet hang. + */ + scheduleReadControlReply(1); +} + +/** + * Quickly abort the transaction + * + \todo destruction should be sufficient as the destructor should cleanup, + * including canceling close handlers + */ +void +ServerStateData::abortTransaction(const char *reason) +{ + debugs(9, 3, HERE << "aborting transaction for " << reason << + "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this); + if (Comm::IsConnOpen(ctrl.conn)) { + ctrl.conn->close(); + return; + } + + fwd->handleUnregisteredServerEnd(); + mustStop("ServerStateData::abortTransaction"); +} + +/** + * Cancel the timeout on the Control socket and establish one + * on the data socket + */ +void +ServerStateData::switchTimeoutToDataChannel() +{ + commUnsetConnTimeout(ctrl.conn); + + typedef CommCbMemFunT TimeoutDialer; + AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, + ServerStateData::timeout); + commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall); +} + +void +ServerStateData::sentRequestBody(const CommIoCbParams &io) +{ + if (io.size > 0) + kb_incr(&(statCounter.server.ftp.kbytes_out), io.size); + ::ServerStateData::sentRequestBody(io); +} + +/** + * called after we wrote the last byte of the request body + */ +void +ServerStateData::doneSendingRequestBody() +{ + ::ServerStateData::doneSendingRequestBody(); + debugs(9,3, HERE); + dataComplete(); + /* NP: RFC 959 3.3. DATA CONNECTION MANAGEMENT + * if transfer type is 'stream' call dataComplete() + * otherwise leave open. (reschedule control channel read?) + */ +} + +wordlist * +ServerStateData::parseControlReply(char *buf, size_t len, int *codep, size_t *used) +{ + char *s; + char *sbuf; + char *end; + int usable; + int complete = 0; + wordlist *head = NULL; + wordlist *list; + wordlist **tail = &head; + size_t offset; + size_t linelen; + int code = -1; + debugs(9, 3, HERE); + /* + * We need a NULL-terminated buffer for scanning, ick + */ + sbuf = (char *)xmalloc(len + 1); + xstrncpy(sbuf, buf, len + 1); + end = sbuf + len - 1; + + while (*end != '\r' && *end != '\n' && end > sbuf) + --end; + + usable = end - sbuf; + + debugs(9, 3, HERE << "usable = " << usable); + + if (usable == 0) { + debugs(9, 3, HERE << "didn't find end of line"); + safe_free(sbuf); + return NULL; + } + + debugs(9, 3, HERE << len << " bytes to play with"); + ++end; + s = sbuf; + s += strspn(s, crlf); + + for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) { + if (complete) + break; + + debugs(9, 5, HERE << "s = {" << s << "}"); + + linelen = strcspn(s, crlf) + 1; + + if (linelen < 2) + break; + + if (linelen > 3) + complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' '); + + if (complete) + code = atoi(s); + + offset = 0; + + if (linelen > 3) + if (*s >= '0' && *s <= '9' && (*(s + 3) == '-' || *(s + 3) == ' ')) + offset = 4; + + list = new wordlist(); + + list->key = (char *)xmalloc(linelen - offset); + + xstrncpy(list->key, s + offset, linelen - offset); + + /* trace the FTP communication chat at level 2 */ + debugs(9, 2, "ftp>> " << code << " " << list->key); + + *tail = list; + + tail = &list->next; + } + + *used = (size_t) (s - sbuf); + safe_free(sbuf); + + if (!complete) + wordlistDestroy(&head); + + if (codep) + *codep = code; + + return head; +} + +}; // namespace Ftp + + +bool +Ftp::ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr) +{ + int h1, h2, h3, h4; + int p1, p2; + const int n = sscanf(buf, "%d,%d,%d,%d,%d,%d", + &h1, &h2, &h3, &h4, &p1, &p2); + + if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) + return false; + + if (forceIp) { + addr = forceIp; // but the above code still validates the IP we got + } else { + static char ipBuf[1024]; + snprintf(ipBuf, sizeof(ipBuf), "%d.%d.%d.%d", h1, h2, h3, h4); + addr = ipBuf; + - if (addr.IsAnyAddr()) ++ if (addr.isAnyAddr()) + return false; + } + + const int port = ((p1 << 8) + p2); + + if (port <= 0) + return false; + + if (Config.Ftp.sanitycheck && port < 1024) + return false; + - addr.SetPort(port); ++ addr.port(port); + return true; +} diff --cc src/FtpServer.h index cc7f94b10a,0000000000..7dfc7854eb mode 100644,000000..100644 --- a/src/FtpServer.h +++ b/src/FtpServer.h @@@ -1,120 -1,0 +1,126 @@@ +/* + * DEBUG: section 09 File Transfer Protocol (FTP) + * + */ + +#ifndef SQUID_FTP_SERVER_H +#define SQUID_FTP_SERVER_H + +#include "Server.h" + +namespace Ftp { + +extern const char *const crlf; + +/// common code for FTP server control and data channels +/// does not own the channel descriptor, which is managed by FtpStateData - class ServerChannel ++class FtpChannel +{ +public: + /// called after the socket is opened, sets up close handler + void opened(const Comm::ConnectionPointer &conn, const AsyncCall::Pointer &aCloser); + + /** Handles all operations needed to properly close the active channel FD. + * clearing the close handler, clearing the listen socket properly, and calling comm_close + */ + void close(); + + void forget(); /// remove the close handler, leave connection open + + void clear(); ///< just drops conn and close handler. does not close active connections. + + Comm::ConnectionPointer conn; ///< channel descriptor + + /** A temporary handle to the connection being listened on. + * Closing this will also close the waiting Data channel acceptor. + * If a data connection has already been accepted but is still waiting in the event queue + * the callback will still happen and needs to be handled (usually dropped). + */ + Comm::ConnectionPointer listenConn; + + AsyncCall::Pointer opener; ///< Comm opener handler callback. +private: + AsyncCall::Pointer closer; ///< Comm close handler callback +}; + +/// Base class for FTP over HTTP and FTP Gateway server state. +class ServerStateData: public ::ServerStateData +{ +public: + ServerStateData(FwdState *fwdState); + virtual ~ServerStateData(); + + virtual void failed(err_type error = ERR_NONE, int xerrno = 0); + virtual void timeout(const CommTimeoutCbParams &io); + virtual const Comm::ConnectionPointer & dataConnection() const; + virtual void abortTransaction(const char *reason); + void writeCommand(const char *buf); - bool handlePasvReply(); ++ ++ /// extracts remoteAddr from PASV response, validates it, ++ /// sets data address details, and returns true on success ++ bool handlePasvReply(Ip::Address &remoteAddr); + void connectDataChannel(); + virtual void maybeReadVirginBody(); + void switchTimeoutToDataChannel(); + + // \todo: optimize ctrl and data structs member order, to minimize size + /// FTP control channel info; the channel is opened once per transaction - struct CtrlChannel: public ServerChannel { ++ struct CtrlChannel: public FtpChannel { + char *buf; + size_t size; + size_t offset; + wordlist *message; + char *last_command; + char *last_reply; + int replycode; + } ctrl; + + /// FTP data channel info; the channel may be opened/closed a few times - struct DataChannel: public ServerChannel { ++ struct DataChannel: public FtpChannel { + MemBuf *readBuf; - Ip::Address addr; ++ char *host; ++ unsigned short port; + bool read_pending; ++ ++ void addr(const Ip::Address &addr); ///< import host and port + } data; + + int state; + char *old_request; + char *old_reply; + +protected: + virtual void start(); + + void initReadBuf(); + virtual void closeServer(); + virtual bool doneWithServer() const; + virtual Http::StatusCode failedHttpStatus(err_type &error); + void ctrlClosed(const CommCloseCbParams &io); + void scheduleReadControlReply(int buffered_ok); + void readControlReply(const CommIoCbParams &io); + virtual void handleControlReply(); + void writeCommandCallback(const CommIoCbParams &io); + static CNCB dataChannelConnected; + virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno) = 0; + void dataRead(const CommIoCbParams &io); + void dataComplete(); + AsyncCall::Pointer dataCloser(); + virtual void dataClosed(const CommCloseCbParams &io); + + // sending of the request body to the server + virtual void sentRequestBody(const CommIoCbParams &io); + virtual void doneSendingRequestBody(); + +private: + static wordlist *parseControlReply(char *buf, size_t len, int *codep, size_t *used); + + CBDATA_CLASS2(ServerStateData); +}; + +/// parses and validates "A1,A2,A3,A4,P1,P2" IP,port sequence +bool ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr); + +}; // namespace Ftp + +#endif /* SQUID_FTP_SERVER_H */ diff --cc src/FwdState.cc index a9de6934dd,cbb5c3a808..a8f7644580 --- a/src/FwdState.cc +++ b/src/FwdState.cc @@@ -47,9 -47,8 +47,9 @@@ #include "event.h" #include "fd.h" #include "fde.h" - #include "forward.h" #include "ftp.h" +#include "FtpGatewayServer.h" + #include "FwdState.h" #include "globals.h" #include "gopher.h" #include "hier_code.h" diff --cc src/Makefile.am index 24c59f762f,7be4f7aae3..52abf5faa3 --- a/src/Makefile.am +++ b/src/Makefile.am @@@ -355,10 -353,8 +353,12 @@@ squid_SOURCES = fqdncache.cc \ ftp.h \ ftp.cc \ + FtpServer.h \ + FtpServer.cc \ + FtpGatewayServer.h \ + FtpGatewayServer.cc \ + FwdState.cc \ + FwdState.h \ Generic.h \ globals.h \ gopher.h \ diff --cc src/client_side.cc index 0349c08046,a513e59488..b2681c3216 --- a/src/client_side.cc +++ b/src/client_side.cc @@@ -99,14 -98,12 +99,13 @@@ #include "comm/Write.h" #include "CommCalls.h" #include "errorpage.h" - #include "eui/Config.h" #include "fd.h" #include "fde.h" - #include "forward.h" #include "fqdncache.h" + #include "FwdState.h" #include "globals.h" #include "http.h" +#include "HttpHdrCc.h" #include "HttpHdrContRange.h" #include "HttpHeaderTools.h" #include "HttpReply.h" @@@ -249,42 -230,6 +233,40 @@@ static void clientUpdateSocketStats(Log char *skipLeadingSpace(char *aString); static void connNoteUseOfBuffer(ConnStateData* conn, size_t byteCount); - static ConnStateData *connStateCreate(const Comm::ConnectionPointer &client, AnyP::PortCfg *port); - +static IOACB FtpAcceptDataConnection; +static void FtpCloseDataConnection(ConnStateData *conn); +static ClientSocketContext *FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::ProtocolVersion *http_ver); +static bool FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms); +static CNCB FtpHandleConnectDone; + +static void FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer data); +typedef void FtpReplyHandler(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data); +static FtpReplyHandler FtpHandlePasvReply; +static FtpReplyHandler FtpHandlePortReply; +static FtpReplyHandler FtpHandleErrorReply; +static FtpReplyHandler FtpHandleDataReply; +static FtpReplyHandler FtpHandleUploadReply; + +static void FtpWriteEarlyReply(ConnStateData *conn, const int code, const char *msg); +static void FtpWriteReply(ClientSocketContext *context, MemBuf &mb); +static void FtpWriteCustomReply(ClientSocketContext *context, const int code, const char *msg, const HttpReply *reply = NULL); +static void FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply); +static void FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply, AsyncCall::Pointer call); +static void FtpPrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = ""); +static IOCB FtpWroteEarlyReply; +static IOCB FtpWroteReply; +static IOCB FtpWroteReplyData; + +typedef bool FtpRequestHandler(ClientSocketContext *context, String &cmd, String ¶ms); +static FtpRequestHandler FtpHandleRequest; +static FtpRequestHandler FtpHandlePasvRequest; +static FtpRequestHandler FtpHandlePortRequest; +static FtpRequestHandler FtpHandleDataRequest; +static FtpRequestHandler FtpHandleUploadRequest; + +static bool FtpCheckDataConnection(ClientSocketContext *context); +static void FtpSetReply(ClientSocketContext *context, const int code, const char *msg); + clientStreamNode * ClientSocketContext::getTail() const { @@@ -726,10 -630,10 +698,10 @@@ ClientHttpRequest::logRequest( debugs(33, 9, "clientLogRequest: http.code='" << al->http.code << "'"); - if (loggingEntry() && loggingEntry()->mem_obj) + if (loggingEntry() && loggingEntry()->mem_obj && loggingEntry()->objectLen() >= 0) al->cache.objectSize = loggingEntry()->contentLen(); - al->cache.caddr.SetNoAddr(); + al->cache.caddr.setNoAddr(); if (getConn() != NULL) { al->cache.caddr = getConn()->log_addr; @@@ -2997,18 -2935,28 +3068,28 @@@ connStripBufferWhitespace (ConnStateDat } } - static int - connOkToAddRequest(ConnStateData * conn) + /** + * Limit the number of concurrent requests. + * \return true when there are available position(s) in the pipeline queue for another request. + * \return false when the pipeline queue is full or disabled. + */ + bool + ConnStateData::concurrentRequestQueueFilled() const { - const int limit = !conn->isFtp && Config.onoff.pipeline_prefetch ? 2 : 1; - const int result = conn->getConcurrentRequestCount() < limit; + const int existingRequestCount = getConcurrentRequestCount(); + + // default to the configured pipeline size. + // add 1 because the head of pipeline is counted in concurrent requests and not prefetch queue - const int concurrentRequestLimit = Config.pipeline_max_prefetch + 1; ++ const int concurrentRequestLimit = (isFtp ? 0 : Config.pipeline_max_prefetch) + 1; - if (!result) { - debugs(33, 3, HERE << conn->clientConnection << " max concurrent requests reached"); - debugs(33, 5, HERE << conn->clientConnection << " defering new request until one is done"); + // when queue filled already we cant add more. + if (existingRequestCount >= concurrentRequestLimit) { + debugs(33, 3, clientConnection << " max concurrent requests reached (" << concurrentRequestLimit << ")"); + debugs(33, 5, clientConnection << " deferring new request until one is done"); + return true; } - return result; + return false; } /** @@@ -3501,23 -3319,37 +3526,38 @@@ clientLifetimeTimeout(const CommTimeout io.conn->close(); } - ConnStateData * - connStateCreate(const Comm::ConnectionPointer &client, AnyP::PortCfg *port) -ConnStateData::ConnStateData(const MasterXaction::Pointer &xact) : ++ConnStateData::ConnStateData(const MasterXaction::Pointer &xact): + AsyncJob("ConnStateData"), ++ isFtp(strcmp(xact->squidPort->protocol, "ftp") == 0), // TODO: convert into a method? + #if USE_SSL + sslBumpMode(Ssl::bumpEnd), + switchedToHttps_(false), + sslServerBump(NULL), + #endif + stoppedSending_(NULL), + stoppedReceiving_(NULL) { - ConnStateData *result = new ConnStateData(port->protocol); + pinning.host = NULL; + pinning.port = -1; + pinning.pinned = false; + pinning.auth = false; + pinning.zeroReply = false; + pinning.peer = NULL; + + // store the details required for creating more MasterXaction objects as new requests come in + clientConnection = xact->tcpClient; + port = cbdataReference(xact->squidPort.get()); + log_addr = xact->tcpClient->remote; + log_addr.applyMask(Config.Addrs.client_netmask); - result->clientConnection = client; - result->log_addr = client->remote; - result->log_addr.ApplyMask(Config.Addrs.client_netmask); - result->in.buf = (char *)memAllocBuf(CLIENT_REQ_BUF_SZ, &result->in.allocatedSize); - result->port = cbdataReference(port); + in.buf = (char *)memAllocBuf(CLIENT_REQ_BUF_SZ, &in.allocatedSize); if (port->disable_pmtu_discovery != DISABLE_PMTU_OFF && - (result->transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) { + (transparent() || port->disable_pmtu_discovery == DISABLE_PMTU_ALWAYS)) { #if defined(IP_MTU_DISCOVER) && defined(IP_PMTUDISC_DONT) int i = IP_PMTUDISC_DONT; - if (setsockopt(client->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) - debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << client << " : " << xstrerror()); + if (setsockopt(clientConnection->fd, SOL_IP, IP_MTU_DISCOVER, &i, sizeof(i)) < 0) + debugs(33, 2, "WARNING: Path MTU discovery disabling failed on " << clientConnection << " : " << xstrerror()); #else static bool reported = false; @@@ -3545,22 -3377,9 +3585,9 @@@ } #endif - #if USE_SQUID_EUI - if (Eui::TheConfig.euiLookup) { - if (client->remote.IsIPv4()) { - result->clientConnection->remoteEui48.lookup(client->remote); - } else if (client->remote.IsIPv6()) { - result->clientConnection->remoteEui64.lookup(client->remote); - } - } - #endif - - clientdbEstablished(client->remote, 1); - - if (!result->isFtp) - result->flags.readMore = true; + clientdbEstablished(clientConnection->remote, 1); - return result; - flags.readMore = true; ++ flags.readMore = !isFtp; } /** Handle a new connection on HTTP socket. */ @@@ -3915,42 -3757,6 +3965,51 @@@ httpsAccept(const CommAcceptCbParams &p } } +/** handle a new FTP connection */ +static void +ftpAccept(const CommAcceptCbParams ¶ms) +{ - AnyP::PortCfg *s = static_cast(params.data); ++ MasterXaction::Pointer xact = params.xaction; ++ AnyP::PortCfgPointer s = xact->squidPort; ++ ++ if (!s.valid()) { ++ // it is possible the call or accept() was still queued when the port was reconfigured ++ debugs(33, 2, "FTP accept failure: port reconfigured."); ++ return; ++ } + + if (params.flag != COMM_OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, "ftpAccept: " << s->listenConn << ": accept failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, HERE << params.conn << ": accepted"); + fd_note(params.conn->fd, "client ftp connect"); + + if (s->tcp_keepalive.enabled) { + commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout); + } + + ++incoming_sockets_accepted; + + // Socket is ready, setup the connection manager to start using it - ConnStateData *connState = connStateCreate(params.conn, s); ++ ConnStateData *connState = new ConnStateData(xact); + + if (connState->transparent()) { + char buf[MAX_IPSTRLEN]; - connState->clientConnection->local.ToURL(buf,MAX_IPSTRLEN); ++ connState->clientConnection->local.toUrl(buf, MAX_IPSTRLEN); + connState->ftp.uri = "ftp://"; + connState->ftp.uri.append(buf); + connState->ftp.uri.append("/"); + debugs(33, 5, HERE << "FTP transparent URL: " << connState->ftp.uri); + } + + FtpWriteEarlyReply(connState, 220, "Service ready"); ++ ++ // TODO: Merge common httpAccept() parts, applying USE_DELAY_POOLS to FTP. +} + void ConnStateData::sslCrtdHandleReplyWrapper(void *data, const HelperReply &reply) { @@@ -4851,866 -4585,3 +4887,866 @@@ ConnStateData::unpinConnection( /* NOTE: pinning.pinned should be kept. This combined with fd == -1 at the end of a request indicates that the host * connection has gone away */ } + + +static void +FtpAcceptDataConnection(const CommAcceptCbParams ¶ms) +{ + ConnStateData *connState = static_cast(params.data); + + if (params.flag != COMM_OK) { + // Its possible the call was still queued when the client disconnected + debugs(33, 2, HERE << connState->ftp.dataListenConn << ": accept " + "failure: " << xstrerr(params.xerrno)); + return; + } + + debugs(33, 4, HERE << params.conn << ": accepted"); + fd_note(params.conn->fd, "client ftp data connect"); + ++incoming_sockets_accepted; + + FtpCloseDataConnection(connState); + connState->ftp.dataConn = params.conn; + connState->ftp.uploadAvailSize = 0; +} + +static void +FtpCloseDataConnection(ConnStateData *conn) +{ + if (Comm::IsConnOpen(conn->ftp.dataListenConn)) { + debugs(33, 5, HERE << "FTP closing client data listen socket: " << + *conn->ftp.dataListenConn); + conn->ftp.dataListenConn->close(); + } + conn->ftp.dataListenConn = NULL; + + if (Comm::IsConnOpen(conn->ftp.dataConn)) { + debugs(33, 5, HERE << "FTP closing client data connection: " << + *conn->ftp.dataConn); + conn->ftp.dataConn->close(); + } + conn->ftp.dataConn = NULL; + conn->ftp.reader = NULL; +} + +/// Writes FTP [error] response before we fully parsed the FTP request and +/// created the corresponding HTTP request wrapper for that FTP request. +static void +FtpWriteEarlyReply(ConnStateData *connState, const int code, const char *msg) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + MemBuf mb; + mb.init(); + mb.Printf("%i %s\r\n", code, msg); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteEarlyReply", + CommIoCbPtrFun(&FtpWroteEarlyReply, connState)); + Comm::Write(connState->clientConnection, &mb, call); + + connState->flags.readMore = false; + + // TODO: Create master transaction. Log it in FtpWroteEarlyReply. +} + +static void +FtpWriteReply(ClientSocketContext *context, MemBuf &mb) +{ + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", + CommIoCbPtrFun(&FtpWroteReply, context)); + Comm::Write(context->clientConnection, &mb, call); +} + +static void +FtpWriteCustomReply(ClientSocketContext *context, const int code, const char *msg, const HttpReply *reply) +{ + debugs(33, 7, HERE << code << ' ' << msg); + assert(99 < code && code < 1000); + + const bool sendDetails = reply != NULL && + reply->header.has(HDR_FTP_STATUS) && reply->header.has(HDR_FTP_REASON); + + MemBuf mb; + mb.init(); + if (sendDetails) { + mb.Printf("%i-%s\r\n", code, msg); + mb.Printf(" Server reply:\r\n"); + FtpPrintReply(mb, reply, " "); + mb.Printf("%i \r\n", code); + } else + mb.Printf("%i %s\r\n", code, msg); + + FtpWriteReply(context, mb); +} + +static void +FtpChangeState(ConnStateData *connState, const ConnStateData::FtpState newState, const char *reason) +{ + assert(connState); + if (connState->ftp.state == newState) { + debugs(33, 3, "client state unchanged at " << connState->ftp.state << + " because " << reason); + connState->ftp.state = newState; + } else { + debugs(33, 3, "client state was " << connState->ftp.state << + ", now " << newState << " because " << reason); + connState->ftp.state = newState; + } +} + +/** Parse an FTP request + * + * \note Sets result->flags.parsed_ok to 0 if failed to parse the request, + * to 1 if the request was correctly parsed. + * \param[in] connState a ConnStateData. The caller must make sure it is not null + * \param[out] mehtod_p will be set as a side-effect of the parsing. + * Pointed-to value will be set to Http::METHOD_NONE in case of + * parsing failure + * \param[out] http_ver will be set as a side-effect of the parsing + * \return NULL on incomplete requests, + * a ClientSocketContext structure on success or failure. + */ +static ClientSocketContext * +FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::ProtocolVersion *http_ver) +{ + *http_ver = Http::ProtocolVersion(1, 1); + + const char *const eor = + static_cast(memchr(connState->in.buf, '\n', + min(connState->in.notYetUsed, Config.maxRequestHeaderSize))); + const size_t req_sz = eor + 1 - connState->in.buf; + + if (eor == NULL && connState->in.notYetUsed >= Config.maxRequestHeaderSize) { + FtpChangeState(connState, ConnStateData::FTP_ERROR, "huge req"); + FtpWriteEarlyReply(connState, 421, "Too large request"); + return NULL; + } + + if (eor == NULL) { + debugs(33, 5, HERE << "Incomplete request, waiting for end of request"); + return NULL; + } + + connNoteUseOfBuffer(connState, req_sz); + + // skip leading whitespaces + const char *boc = connState->in.buf; + while (boc < eor && isspace(*boc)) ++boc; + if (boc >= eor) { + debugs(33, 5, HERE << "Empty request, ignoring"); + return NULL; + } + + const char *eoc = boc; + while (eoc < eor && !isspace(*eoc)) ++eoc; + connState->in.buf[eoc - connState->in.buf] = '\0'; + + const char *bop = eoc + 1; + while (bop < eor && isspace(*bop)) ++bop; + if (bop < eor) { + const char *eop = eor - 1; + while (isspace(*eop)) --eop; + assert(eop >= bop); + connState->in.buf[eop + 1 - connState->in.buf] = '\0'; + } else + bop = NULL; + + debugs(33, 7, HERE << "Parsed FTP command " << boc << " with " << + (bop == NULL ? "no " : "") << "parameters" << + (bop != NULL ? ": " : "") << bop); + + const String cmd = boc; + String params = bop; + + *method_p = !cmd.caseCmp("APPE") || !cmd.caseCmp("STOR") || + !cmd.caseCmp("STOU") ? Http::METHOD_PUT : Http::METHOD_GET; + + if (connState->ftp.uri.size() == 0) { + // the first command must be USER + if (cmd.caseCmp("USER") != 0) { + FtpWriteEarlyReply(connState, 530, "Must login first"); + return NULL; + } + + if (params.size() == 0) { + FtpWriteEarlyReply(connState, 501, "Missing username"); + return NULL; + } + } + + // We need to process USER request now because it sets request URI. + if (cmd.caseCmp("USER") == 0 && + !FtpHandleUserRequest(connState, cmd, params)) + return NULL; + + assert(connState->ftp.uri.size() > 0); + char *uri = xstrdup(connState->ftp.uri.termedBuf()); + HttpRequest *const request = + HttpRequest::CreateFromUrlAndMethod(uri, *method_p); + if (request == NULL) { + debugs(33, 5, HERE << "Invalid FTP URL: " << connState->ftp.uri); + FtpWriteEarlyReply(connState, 501, "Invalid host"); + connState->ftp.uri.clean(); + safe_free(uri); + return NULL; + } + + request->http_ver = *http_ver; + request->header.putStr(HDR_FTP_COMMAND, cmd.termedBuf()); + request->header.putStr(HDR_FTP_ARGUMENTS, params.termedBuf() != NULL ? + params.termedBuf() : ""); + if (*method_p == Http::METHOD_PUT) + request->header.putStr(HDR_TRANSFER_ENCODING, "chunked"); + + ClientHttpRequest *const http = new ClientHttpRequest(connState); + http->request = request; + HTTPMSGLOCK(http->request); + http->req_sz = req_sz; + http->uri = uri; + + ClientSocketContext *const result = + ClientSocketContextNew(connState->clientConnection, http); + + StoreIOBuffer tempBuffer; + tempBuffer.data = result->reqbuf; + tempBuffer.length = HTTP_REQBUF_SZ; + + ClientStreamData newServer = new clientReplyContext(http); + ClientStreamData newClient = result; + clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach, + clientReplyStatus, newServer, clientSocketRecipient, + clientSocketDetach, newClient, tempBuffer); + + result->registerWithConn(); + result->flags.parsed_ok = 1; + connState->flags.readMore = false; + return result; +} + +static void +FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer data) +{ + if (context->http && context->http->al != NULL && + !context->http->al->reply && reply) { + context->http->al->reply = reply; + HTTPMSGLOCK(context->http->al->reply); + } + + static FtpReplyHandler *handlers[] = { + NULL, // FTP_BEGIN + NULL, // FTP_CONNECTED + FtpHandlePasvReply, // FTP_HANDLE_PASV + FtpHandlePortReply, // FTP_HANDLE_PORT + FtpHandleDataReply, // FTP_HANDLE_DATA_REQUEST + FtpHandleUploadReply, // FTP_HANDLE_DATA_REQUEST + FtpHandleErrorReply // FTP_ERROR + }; + const ConnStateData::FtpState state = context->getConn()->ftp.state; + FtpReplyHandler *const handler = handlers[state]; + if (handler) + (*handler)(context, reply, data); + else + FtpWriteForwardedReply(context, reply); +} + +static void +FtpHandlePasvReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support PASV", reply); + return; + } + + FtpCloseDataConnection(context->getConn()); + + Comm::ConnectionPointer conn = new Comm::Connection; + ConnStateData * const connState = context->getConn(); + conn->flags = COMM_NONBLOCKING; + conn->local = connState->transparent() ? + connState->port->s : context->clientConnection->local; - conn->local.SetPort(0); ++ conn->local.port(0); + const char *const note = connState->ftp.uri.termedBuf(); + comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note); + if (!Comm::IsConnOpen(conn)) { + debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << + conn->local << " error: " << errno); + FtpWriteCustomReply(context, 451, "Internal error"); + return; + } + + typedef CommCbFunPtrCallT AcceptCall; + RefCount subCall = commCbCall(5, 5, "FtpAcceptDataConnection", + CommAcceptCbPtrFun(FtpAcceptDataConnection, connState)); + Subscription::Pointer sub = new CallSubscription(subCall); + AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub)); + + connState->ftp.dataListenConn = conn; + + char addr[MAX_IPSTRLEN]; + // remote server in interception setups and local address otherwise + const Ip::Address &server = connState->transparent() ? + context->clientConnection->local : conn->local; - server.NtoA(addr, MAX_IPSTRLEN, AF_INET); ++ server.toStr(addr, MAX_IPSTRLEN, AF_INET); + addr[MAX_IPSTRLEN - 1] = '\0'; + for (char *c = addr; *c != '\0'; ++c) { + if (*c == '.') + *c = ','; + } + + // conn->fd is the client data connection (and its local port) + const unsigned short port = comm_local_port(conn->fd); - conn->local.SetPort(port); ++ conn->local.port(port); + + // In interception setups, we combine remote server address with a + // local port number and hope that traffic will be redirected to us. + MemBuf mb; + mb.init(); + mb.Printf("227 =%s,%i,%i\r\n", addr, static_cast(port >> 8), + static_cast(port % 256)); + + debugs(11, 3, Raw("writing", mb.buf, mb.size)); + FtpWriteReply(context, mb); +} + +static void +FtpHandlePortReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + if (context->http->request->errType != ERR_NONE) { + FtpWriteCustomReply(context, 502, "Server does not support PASV (converted from PORT)", reply); + return; + } + + FtpWriteCustomReply(context, 200, "PORT successfully converted to PASV."); + + // and wait for RETR +} + +static void +FtpHandleErrorReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + int code; + ConnStateData *const connState = context->getConn(); + if (!connState->pinning.pinned) { // we failed to connect to server + connState->ftp.uri.clean(); + code = 530; + } else + code = 421; + const char *const msg = err_type_str[context->http->request->errType]; + FtpWriteCustomReply(context, code, msg, reply); +} + +static void +FtpHandleDataReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + ConnStateData *const conn = context->getConn(); + + if (reply != NULL && reply->sline.status() != Http::scOkay) { + FtpWriteForwardedReply(context, reply); + if (conn && Comm::IsConnOpen(conn->ftp.dataConn)) { + debugs(33, 3, "closing " << conn->ftp.dataConn << " on KO reply"); + conn->ftp.dataConn->close(); + } + return; + } + + debugs(33, 7, HERE << data.length); + + if (data.length <= 0) { + FtpWroteReplyData(conn->clientConnection, NULL, 0, COMM_OK, 0, context); + return; + } + + if (!Comm::IsConnOpen(conn->ftp.dataConn)) { + debugs(33, 3, HERE << "got FTP reply data when client data connection " + "is closed, ignoring"); + return; + } + + MemBuf mb; + mb.init(data.length + 1, data.length + 1); + mb.append(data.data, data.length); + + AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReplyData", + CommIoCbPtrFun(&FtpWroteReplyData, context)); + Comm::Write(conn->ftp.dataConn, &mb, call); + + context->noteSentBodyBytes(data.length); +} + +static void +FtpWroteReplyData(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) +{ + if (errflag == COMM_ERR_CLOSING) + return; + + ClientSocketContext *const context = static_cast(data); + ConnStateData *const connState = context->getConn(); + + if (errflag != COMM_OK) { + debugs(33, 3, HERE << "FTP reply data writing failed: " << + xstrerr(xerrno)); + FtpCloseDataConnection(connState); + FtpWriteCustomReply(context, 426, "Data connection error; transfer aborted"); + return; + } + + assert(context->http); + context->http->out.size += size; + + switch (context->socketState()) { + case STREAM_NONE: + debugs(33, 3, "Keep going"); + context->pullData(); + return; + case STREAM_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer successfully complete"); + FtpWriteCustomReply(context, 226, "Transfer complete"); + break; + case STREAM_UNPLANNED_COMPLETE: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE"); + FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); + break; + case STREAM_FAILED: + debugs(33, 3, HERE << "FTP reply data transfer failed: STREAM_FAILED"); + FtpWriteCustomReply(context, 451, "Server error; transfer aborted"); + break; + default: + fatal("unreachable code"); + } + + FtpCloseDataConnection(connState); +} + +static void +FtpHandleUploadReply(ClientSocketContext *context, const HttpReply *reply, StoreIOBuffer data) +{ + FtpWriteForwardedReply(context, reply); +} + +static void +FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply) +{ + const AsyncCall::Pointer call = commCbCall(33, 5, "FtpWroteReply", + CommIoCbPtrFun(&FtpWroteReply, context)); + FtpWriteForwardedReply(context, reply, call); +} + +static void +FtpWriteForwardedForeign(ClientSocketContext *context, const HttpReply *reply) +{ + ConnStateData *const connState = context->getConn(); + FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "foreign reply"); + + assert(context->http); + const HttpRequest *request = context->http->request; + assert(request); + + assert(reply != NULL); + const int status = 421; + const char *reason = reply->sline.reason(); + MemBuf mb; + mb.init(); + + if (request->errType != ERR_NONE) + mb.Printf("%i-%s\r\n", status, errorPageName(request->errType)); + + if (request->errDetail > 0) { + // XXX: > 0 may not always mean that this is an errno + mb.Printf("%i-Error: (%d) %s\r\n", status, + request->errDetail, + strerror(request->errDetail)); + } + + // XXX: Remove hard coded names. Use an error page template instead. + const Adaptation::History::Pointer ah = request->adaptHistory(); + if (ah != NULL) { // XXX: add adapt::allMeta.getByName("X-Response-Info"); + const String desc = ah->allMeta.getByName("X-Response-Desc"); + if (info.size()) + mb.Printf("%i-Information: %s\r\n", status, info.termedBuf()); + if (desc.size()) + mb.Printf("%i-Description: %s\r\n", status, desc.termedBuf()); + } + + mb.Printf("%i %s\r\n", status, reason); // error terminating line + + // TODO: errorpage.cc should detect FTP client and use + // configurable FTP-friendly error templates which we should + // write to the client "as is" instead of hiding most of the info + + FtpWriteReply(context, mb); + return; +} + +static void +FtpWriteForwardedReply(ClientSocketContext *context, const HttpReply *reply, AsyncCall::Pointer call) +{ + assert(reply != NULL); + const HttpHeader &header = reply->header; + ConnStateData *const connState = context->getConn(); + + // adaptation and forwarding errors lack HDR_FTP_STATUS + if (!header.has(HDR_FTP_STATUS)) { + FtpWriteForwardedForeign(context, reply); + return; + } + + assert(header.has(HDR_FTP_REASON)); + + const int status = header.getInt(HDR_FTP_STATUS); + debugs(33, 7, HERE << "status: " << status); + + if (status == 150 && connState->ftp.state == + ConnStateData::FTP_HANDLE_UPLOAD_REQUEST) + connState->readSomeFtpData(); + + MemBuf mb; + mb.init(); + FtpPrintReply(mb, reply); + + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REPLY:\n---------\n" << mb.buf << + "\n----------"); + + Comm::Write(context->clientConnection, &mb, call); +} + +static void +FtpPrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix) +{ + const HttpHeader &header = reply->header; + + char status[4]; + if (header.has(HDR_FTP_STATUS)) + snprintf(status, sizeof(status), "%i", header.getInt(HDR_FTP_STATUS)); + else + status[0] = '\0'; + + HttpHeaderPos pos = HttpHeaderInitPos; + const HttpHeaderEntry *e = header.getEntry(&pos); + while (e) { + const HttpHeaderEntry *const next = header.getEntry(&pos); + if (e->id == HDR_FTP_REASON) { + const bool isLastLine = next == NULL || next->id != HDR_FTP_REASON; + const int separator = status[0] == '\0' || isLastLine ? ' ' : '-'; + mb.Printf("%s%s%c%s\r\n", prefix, status, separator, + e->value.termedBuf()); + } + e = next; + } +} + +static void +FtpWroteEarlyReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) +{ + if (errflag == COMM_ERR_CLOSING) + return; + + if (errflag != COMM_OK) { + debugs(33, 3, HERE << "FTP reply writing failed: " << xstrerr(xerrno)); + conn->close(); + return; + } + + ConnStateData *const connState = static_cast(data); + ClientSocketContext::Pointer context = connState->getCurrentContext(); + if (context != NULL && context->http) { + context->http->out.size += size; + context->http->out.headers_sz += size; + } + + connState->flags.readMore = true; + connState->readSomeData(); +} + +static void +FtpWroteReply(const Comm::ConnectionPointer &conn, char *bufnotused, size_t size, comm_err_t errflag, int xerrno, void *data) +{ + if (errflag == COMM_ERR_CLOSING) + return; + + if (errflag != COMM_OK) { + debugs(33, 3, HERE << "FTP reply writing failed: " << + xstrerr(xerrno)); + conn->close(); + return; + } + + ClientSocketContext *const context = + static_cast(data); + ConnStateData *const connState = context->getConn(); + + assert(context->http); + context->http->out.size += size; + context->http->out.headers_sz += size; + + if (connState->ftp.state == ConnStateData::FTP_ERROR) { + debugs(33, 5, "closing on FTP server error"); + conn->close(); + return; + } + + const clientStream_status_t socketState = context->socketState(); + debugs(33, 5, "FTP client stream state " << socketState); + switch (socketState) { + case STREAM_UNPLANNED_COMPLETE: + case STREAM_FAILED: + conn->close(); + return; + + case STREAM_NONE: + case STREAM_COMPLETE: + connState->flags.readMore = true; + FtpChangeState(connState, ConnStateData::FTP_CONNECTED, "FtpWroteReply"); + if (connState->in.bodyParser) + connState->finishDechunkingRequest(false); + context->keepaliveNextRequest(); + return; + } +} + +bool +FtpHandleRequest(ClientSocketContext *context, String &cmd, String ¶ms) { + if (HttpRequest *request = context->http->request) { + MemBuf *mb = new MemBuf; + Packer p; + mb->init(); + packerToMemInit(&p, mb); + request->pack(&p); + packerClean(&p); + + debugs(11, 2, "FTP Client " << context->clientConnection); + debugs(11, 2, "FTP Client REQUEST:\n---------\n" << mb->buf << + "\n----------"); + delete mb; + } + + static std::pair handlers[] = { + std::make_pair("LIST", FtpHandleDataRequest), + std::make_pair("NLST", FtpHandleDataRequest), + std::make_pair("PASV", FtpHandlePasvRequest), + std::make_pair("PORT", FtpHandlePortRequest), + std::make_pair("RETR", FtpHandleDataRequest) + }; + + FtpRequestHandler *handler = NULL; + if (context->http->request->method == Http::METHOD_PUT) + handler = FtpHandleUploadRequest; + else { + for (size_t i = 0; i < sizeof(handlers) / sizeof(*handlers); ++i) { + if (cmd.caseCmp(handlers[i].first) == 0) { + handler = handlers[i].second; + break; + } + } + } + + return handler != NULL ? (*handler)(context, cmd, params) : true; +} + +/// Called to parse USER command, which is required to create an HTTP request +/// wrapper. Thus, errors are handled with FtpWriteEarlyReply() here. +bool +FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms) +{ + if (params.size() == 0) { + FtpWriteEarlyReply(connState, 501, "Missing username"); + return false; + } + + const String::size_type eou = params.rfind('@'); + if (eou == String::npos || eou + 1 >= params.size()) { + if (connState->ftp.uri.size() > 0) + return true; + FtpWriteEarlyReply(connState, 501, "Missing host"); + return false; + } + + static const String scheme = "ftp://"; + const String host = params.substr(eou + 1, params.size()); + String uri = scheme; + uri.append(host); + uri.append("/"); + + if (connState->ftp.uri.size() == 0) + connState->ftp.uri = uri; + else if (uri.caseCmp(connState->ftp.uri) != 0) { + debugs(11, 3, "expected " << connState->ftp.uri); + debugs(11, 3, " but got " << uri); + FtpWriteEarlyReply(connState, 501, "Cannot change host"); + return false; + } + + params.cut(eou); + + return true; +} + +bool +FtpHandlePasvRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (params.size() > 0) { + FtpSetReply(context, 501, "Unexpected parameter"); + return false; + } + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASV, "FtpHandlePasvRequest"); + + return true; +} + +#include "FtpServer.h" /* XXX: For Ftp::ParseIpPort() */ + +bool +FtpHandlePortRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (!params.size()) { + FtpSetReply(context, 501, "Missing parameter"); + return false; + } + + Ip::Address cltAddr; + if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) { + FtpSetReply(context, 501, "Invalid parameter"); + return false; + } + + FtpCloseDataConnection(context->getConn()); + debugs(11, 3, "will actively connect to " << cltAddr); + + Comm::ConnectionPointer conn = new Comm::Connection(); + conn->remote = cltAddr; + + // TODO: should we use getOutgoingAddress() here instead? - if (conn->remote.IsIPv4()) - conn->local.SetIPv4(); ++ if (conn->remote.isIPv4()) ++ conn->local.setIPv4(); + + // RFC 959 requires active FTP connections to originate from port 20 + // but that would preclude us from supporting concurrent transfers! (XXX?) + // conn->flags |= COMM_DOBIND; - // conn->local.SetPort(20); ++ // conn->local.port(20); + + context->getConn()->ftp.dataConn = conn; + context->getConn()->ftp.uploadAvailSize = 0; // XXX: FtpCloseDataConnection should do that + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PORT, "FtpHandlePortRequest"); + + // convert client PORT command to Squid PASV command because Squid + // does not support active FTP transfers on the server side (yet?) + ClientHttpRequest *const http = context->http; + assert(http != NULL); + HttpRequest *const request = http->request; + assert(request != NULL); + HttpHeader &header = request->header; + header.delById(HDR_FTP_COMMAND); + header.putStr(HDR_FTP_COMMAND, "PASV"); + header.delById(HDR_FTP_ARGUMENTS); + header.putStr(HDR_FTP_ARGUMENTS, ""); + return true; // forward our fake PASV request +} + +bool +FtpHandleDataRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (!FtpCheckDataConnection(context)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_DATA_REQUEST, "FtpHandleDataRequest"); + + return true; +} + +bool +FtpHandleUploadRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + if (!FtpCheckDataConnection(context)) + return false; + + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_UPLOAD_REQUEST, "FtpHandleDataRequest"); + + return true; +} + +bool +FtpCheckDataConnection(ClientSocketContext *context) +{ + ConnStateData *const connState = context->getConn(); + if (Comm::IsConnOpen(connState->ftp.dataConn)) + return true; + + if (Comm::IsConnOpen(connState->ftp.dataListenConn)) { + FtpSetReply(context, 425, "Data connection is not established"); + return false; + } + - if (!connState->ftp.dataConn || connState->ftp.dataConn->remote.IsAnyAddr()) { ++ if (!connState->ftp.dataConn || connState->ftp.dataConn->remote.isAnyAddr()) { + // XXX: use client address and default port instead. + FtpSetReply(context, 425, "Use PORT or PASV first"); + return false; + } + + // active transfer: open a connection from Squid to client + AsyncCall::Pointer connector = context->getConn()->ftp.connector = + commCbCall(17, 3, "FtpConnectDoneWrapper", + CommConnectCbPtrFun(FtpHandleConnectDone, context)); + + Comm::ConnOpener *cs = new Comm::ConnOpener(connState->ftp.dataConn, + connector, + Config.Timeout.connect); + AsyncJob::Start(cs); + return false; // ConnStateData::processFtpRequest waits FtpHandleConnectDone +} + +void +FtpHandleConnectDone(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data) +{ + ClientSocketContext *context = static_cast(data); + context->getConn()->ftp.connector = NULL; + + if (status != COMM_OK) { + conn->close(); + FtpSetReply(context, 425, "Cannot open data connection."); + assert(context->http && context->http->storeEntry() != NULL); + } else { + assert(context->getConn()->ftp.dataConn == conn); + context->getConn()->ftp.uploadAvailSize = 0; + assert(Comm::IsConnOpen(context->getConn()->ftp.dataConn)); + } + context->getConn()->resumeFtpRequest(context); +} + +void +FtpSetReply(ClientSocketContext *context, const int code, const char *msg) +{ + ClientHttpRequest *const http = context->http; + assert(http != NULL); + assert(http->storeEntry() == NULL); + + HttpReply *const reply = new HttpReply; + reply->sline.set(Http::ProtocolVersion(1, 1), Http::scNoContent); + HttpHeader &header = reply->header; + header.putTime(HDR_DATE, squid_curtime); + { + HttpHdrCc cc; + cc.Private(); + header.putCc(&cc); + } + header.putInt64(HDR_CONTENT_LENGTH, 0); + header.putInt(HDR_FTP_STATUS, code); + header.putStr(HDR_FTP_REASON, msg); + reply->hdrCacheInit(); + + setLogUri(http, http->uri, true); + + clientStreamNode *const node = context->getClientReplyContext(); + clientReplyContext *const repContext = + dynamic_cast(node->data.getRaw()); + assert(repContext != NULL); + repContext->createStoreEntry(http->request->method, RequestFlags()); + http->storeEntry()->replaceHttpReply(reply); +} diff --cc src/client_side.h index d1ac64d613,89ad23fddb..88a0524007 --- a/src/client_side.h +++ b/src/client_side.h @@@ -411,10 -384,13 +413,16 @@@ private int connReadWasError(comm_err_t flag, int size, int xerrno); int connFinishedWithConn(int size); void clientAfterReadingRequests(); + void processFtpRequest(ClientSocketContext *const context); + void handleFtpRequestData(); + - private: + bool concurrentRequestQueueFilled() const; + + #if USE_AUTH + /// some user details that can be used to perform authentication on this connection + Auth::UserRequest::Pointer auth_; + #endif + HttpParser parser_; // XXX: CBDATA plays with public/private and leaves the following 'private' fields all public... :( diff --cc src/ftp.cc index c512bd8a3a,fb320b0664..c61f35a088 --- a/src/ftp.cc +++ b/src/ftp.cc @@@ -38,8 -40,7 +38,8 @@@ #include "errorpage.h" #include "fd.h" #include "fde.h" - #include "forward.h" +#include "FtpServer.h" + #include "FwdState.h" #include "html_quote.h" #include "HttpHdrContRange.h" #include "HttpHeader.h" @@@ -144,17 -148,43 +144,12 @@@ class FtpStateData /// \ingroup ServerProtocolFTPInternal typedef void (FTPSM) (FtpStateData *); -/// common code for FTP control and data channels -/// does not own the channel descriptor, which is managed by FtpStateData -class FtpChannel -{ -public: - FtpChannel() {}; - - /// called after the socket is opened, sets up close handler - void opened(const Comm::ConnectionPointer &conn, const AsyncCall::Pointer &aCloser); - - /** Handles all operations needed to properly close the active channel FD. - * clearing the close handler, clearing the listen socket properly, and calling comm_close - */ - void close(); - - void clear(); ///< just drops conn and close handler. does not close active connections. - - Comm::ConnectionPointer conn; ///< channel descriptor - - /** A temporary handle to the connection being listened on. - * Closing this will also close the waiting Data channel acceptor. - * If a data connection has already been accepted but is still waiting in the event queue - * the callback will still happen and needs to be handled (usually dropped). - */ - Comm::ConnectionPointer listenConn; - - AsyncCall::Pointer opener; ///< Comm opener handler callback. -private: - AsyncCall::Pointer closer; ///< Comm close handler callback -}; - /// \ingroup ServerProtocolFTPInternal -class FtpStateData : public ServerStateData +class FtpStateData : public Ftp::ServerStateData { - public: - void *operator new (size_t); - void operator delete (void *); - void *toCbdata() { return this; } - - FtpStateData(FwdState *fwdState); - FtpStateData(FwdState *, const Comm::ConnectionPointer &conn); - ~FtpStateData(); ++ FtpStateData(FwdState *); + virtual ~FtpStateData(); char user[MAX_URL]; char password[MAX_URL]; int password_url; @@@ -178,11 -211,28 +173,8 @@@ char typecode; MemBuf listing; ///< FTP directory listing in HTML format. - // \todo: optimize ctrl and data structs member order, to minimize size - /// FTP control channel info; the channel is opened once per transaction - struct CtrlChannel: public FtpChannel { - char *buf; - size_t size; - size_t offset; - wordlist *message; - char *last_command; - char *last_reply; - int replycode; - } ctrl; - - /// FTP data channel info; the channel may be opened/closed a few times - struct DataChannel: public FtpChannel { - MemBuf *readBuf; - char *host; - unsigned short port; - bool read_pending; - } data; - struct _ftp_flags flags; - private: - CBDATA_CLASS(FtpStateData); - public: // these should all be private virtual void start(); @@@ -1889,10 -2453,30 +1868,12 @@@ ftpReadEPSV(FtpStateData* ftpState } } - ftpState->data.addr = fd_table[ftpState->ctrl.conn->fd].ipaddr; - ftpState->data.addr.SetPort(port); + ftpState->data.port = port; + + safe_free(ftpState->data.host); + ftpState->data.host = xstrdup(fd_table[ftpState->ctrl.conn->fd].ipaddr); - safe_free(ftpState->ctrl.last_command); - - safe_free(ftpState->ctrl.last_reply); - - ftpState->ctrl.last_command = xstrdup("Connect to server data port"); - - // Generate a new data channel descriptor to be opened. - Comm::ConnectionPointer conn = new Comm::Connection; - conn->local = ftpState->ctrl.conn->local; - conn->local.port(0); - conn->remote = ftpState->ctrl.conn->remote; - conn->remote.port(port); - - debugs(9, 3, HERE << "connecting to " << conn->remote); - - ftpState->data.opener = commCbCall(9,3, "FtpStateData::ftpPasvCallback", CommConnectCbPtrFun(FtpStateData::ftpPasvCallback, ftpState)); - Comm::ConnOpener *cs = new Comm::ConnOpener(conn, ftpState->data.opener, Config.Timeout.connect); - cs->setHost(ftpState->data.host); - AsyncJob::Start(cs); + ftpState->connectDataChannel(); } /** \ingroup ServerProtocolFTPInternal @@@ -2047,9 -2631,48 +2028,10 @@@ FtpStateData::processHeadResponse( static void ftpReadPasv(FtpStateData * ftpState) { - if (ftpState->handlePasvReply()) - int code = ftpState->ctrl.replycode; - int h1, h2, h3, h4; - int p1, p2; - int n; - unsigned short port; - Ip::Address ipa_remote; - char *buf; - LOCAL_ARRAY(char, ipaddr, 1024); - debugs(9, 3, HERE); - - if (code != 227) { - debugs(9, 2, "PASV not supported by remote end"); - ftpSendEPRT(ftpState); - return; - } - - /* 227 Entering Passive Mode (h1,h2,h3,h4,p1,p2). */ - /* ANSI sez [^0-9] is undefined, it breaks on Watcom cc */ - debugs(9, 5, HERE << "scanning: " << ftpState->ctrl.last_reply); - - buf = ftpState->ctrl.last_reply + strcspn(ftpState->ctrl.last_reply, "0123456789"); - - n = sscanf(buf, "%d,%d,%d,%d,%d,%d", &h1, &h2, &h3, &h4, &p1, &p2); - - if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) { - debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " << - ftpState->ctrl.conn->remote << ": " << - ftpState->ctrl.last_reply); - - ftpSendEPRT(ftpState); - return; - } - - snprintf(ipaddr, 1024, "%d.%d.%d.%d", h1, h2, h3, h4); - - ipa_remote = ipaddr; - - if ( ipa_remote.isAnyAddr() ) { - debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " << - ftpState->ctrl.conn->remote << ": " << - ftpState->ctrl.last_reply); - ++ Ip::Address srvAddr; // unused ++ if (ftpState->handlePasvReply(srvAddr)) + ftpState->connectDataChannel(); + else { ftpSendEPRT(ftpState); return; } @@@ -2109,10 -2782,10 +2091,10 @@@ ftpOpenListenSocket(FtpStateData * ftpS temp->flags |= COMM_REUSEADDR; } else { /* if not running in fallback mode a new port needs to be retrieved */ - temp->local.SetPort(0); + temp->local.port(0); } - ftpState->listenForDataChannel(temp, ftpState->entry->url()); + ftpState->listenForDataChannel(temp); } /// \ingroup ServerProtocolFTPInternal @@@ -2299,7 -2972,9 +2281,7 @@@ FtpStateData::ftpAcceptDataConnection(c /** On COMM_OK start using the accepted data socket and discard the temporary listen socket. */ data.close(); data.opened(io.conn, dataCloser()); - data.addr = io.conn->remote; - static char ntoapeer[MAX_IPSTRLEN]; - io.conn->remote.toStr(ntoapeer,sizeof(ntoapeer)); - data.host = xstrdup(ntoapeer); ++ data.addr(io.conn->remote); debugs(9, 3, HERE << "Connected data socket on " << io.conn << ". FD table says: " <<