From: Christos Tsantilas Date: Wed, 11 Dec 2013 22:13:41 +0000 (+0200) Subject: FTP gw tracking of server directory X-Git-Tag: SQUID_3_5_0_1~117^2~26 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=54a6c0cd3f3bae55203788e0c0d048adfed04a5a;p=thirdparty%2Fsquid.git FTP gw tracking of server directory This feature works as follows: - Squid probes the remote directory by inserting PWD commands after the initial login and after every CWD command sent by the client. - Squid remembers the current directory - client-side generates the URL in pseudo HTTP requests from the domain name and the current directory (e.g. GET ftp://ftp.example.com/pub). - For FTP commands with directories or file names as arguments (e.g. file donwload/upload, directory listing), these arguments are appended to the current directory (e.g. “RETR project/file.txt” becomes “GET ftp://ftp.example.com/pub/project/file.txt”;). The feature can be controlled via a new ftp-track-dirs=[on|off] option for the ftp_port squid.conf parameter. Ftp server directory tracking is disabled by default. --- diff --git a/src/FtpGatewayServer.cc b/src/FtpGatewayServer.cc index 1d10abbc0a..3c687f14c5 100644 --- a/src/FtpGatewayServer.cc +++ b/src/FtpGatewayServer.cc @@ -5,6 +5,7 @@ #include "squid.h" +#include "anyp/PortCfg.h" #include "FtpGatewayServer.h" #include "FtpServer.h" #include "HttpHdrCc.h" @@ -45,6 +46,9 @@ protected: void handleDataRequest(); void startDataDownload(); void startDataUpload(); + bool startDirTracking(); + void stopDirTracking(); + bool weAreTrackingDir() const {return savedReply.message != NULL;} typedef void (ServerStateData::*PreliminaryCb)(); void forwardPreliminaryReply(const PreliminaryCb cb); @@ -61,12 +65,21 @@ protected: void readDataReply(); void readTransferDoneReply(); void readEpsvReply(); + void readCwdOrCdupReply(); + void readUserOrPassReply(); virtual void dataChannelConnected(const Comm::ConnectionPointer &conn, comm_err_t err, int xerrno); void scheduleReadControlReply(); bool forwardingCompleted; ///< completeForwarding() has been called + struct { + wordlist *message; ///< reply message, one wordlist entry per message line + char *lastCommand; ///< the command caused the reply + char *lastReply; ///< last line of reply: reply status plus message + int replyCode; ///< the reply status + } savedReply; ///< set and delayed while we are tracking using PWD + CBDATA_CLASS2(ServerStateData); }; @@ -74,8 +87,8 @@ CBDATA_CLASS_INIT(ServerStateData); const ServerStateData::SM_FUNC ServerStateData::SM_FUNCS[] = { &ServerStateData::readGreeting, // BEGIN - NULL,/*&ServerStateData::readReply*/ // SENT_USER - NULL,/*&ServerStateData::readReply*/ // SENT_PASS + &ServerStateData::readUserOrPassReply, // SENT_USER + &ServerStateData::readUserOrPassReply, // SENT_PASS NULL,/*&ServerStateData::readReply*/ // SENT_TYPE NULL,/*&ServerStateData::readReply*/ // SENT_MDTM NULL,/*&ServerStateData::readReply*/ // SENT_SIZE @@ -85,7 +98,7 @@ const ServerStateData::SM_FUNC ServerStateData::SM_FUNCS[] = { &ServerStateData::readEpsvReply, // SENT_EPSV_1 &ServerStateData::readEpsvReply, // SENT_EPSV_2 &ServerStateData::readPasvReply, // SENT_PASV - NULL,/*&ServerStateData::readReply*/ // SENT_CWD + &ServerStateData::readCwdOrCdupReply, // SENT_CWD NULL,/*&ServerStateData::readDataReply,*/ // SENT_LIST NULL,/*&ServerStateData::readDataReply,*/ // SENT_NLST NULL,/*&ServerStateData::readReply*/ // SENT_REST @@ -96,6 +109,8 @@ const ServerStateData::SM_FUNC ServerStateData::SM_FUNCS[] = { &ServerStateData::readReply, // WRITING_DATA NULL,/*&ServerStateData::readReply*/ // SENT_MKDIR &ServerStateData::readFeatReply, // SENT_FEAT + NULL,/*&ServerStateData::readPwdReply*/ // SENT_PWD + &ServerStateData::readCwdOrCdupReply, // SENT_CDUP &ServerStateData::readDataReply,// SENT_DATA_REQUEST &ServerStateData::readReply, // SENT_COMMAND NULL @@ -105,6 +120,11 @@ ServerStateData::ServerStateData(FwdState *const fwdState): AsyncJob("Ftp::Gateway::ServerStateData"), Ftp::ServerStateData(fwdState), forwardingCompleted(false) { + savedReply.message = false; + savedReply.lastCommand = NULL; + savedReply.lastReply = NULL; + savedReply.replyCode = 0; + // Nothing we can do at request creation time can mark the response as // uncachable, unfortunately. This prevents "found KEY_PRIVATE" WARNINGs. entry->releaseRequest(); @@ -113,6 +133,11 @@ ServerStateData::ServerStateData(FwdState *const fwdState): ServerStateData::~ServerStateData() { closeServer(); // TODO: move to Server.cc? + if (savedReply.message) + wordlistDestroy(&savedReply.message); + + xfree(savedReply.lastCommand); + xfree(savedReply.lastReply); } void @@ -247,6 +272,12 @@ ServerStateData::processReplyBody() void ServerStateData::handleControlReply() { + if (!request->clientConnectionManager.valid()) { + debugs(9, 5, "client connection gone"); + closeServer(); + return; + } + Ftp::ServerStateData::handleControlReply(); if (ctrl.message == NULL) return; // didn't get complete reply yet @@ -456,9 +487,13 @@ ServerStateData::sendCommand() writeCommand(mb.content()); state = + clientState() == ConnStateData::FTP_HANDLE_CDUP ? SENT_CDUP : + clientState() == ConnStateData::FTP_HANDLE_CWD ? SENT_CWD : clientState() == ConnStateData::FTP_HANDLE_FEAT ? SENT_FEAT : clientState() == ConnStateData::FTP_HANDLE_DATA_REQUEST ? SENT_DATA_REQUEST : clientState() == ConnStateData::FTP_HANDLE_UPLOAD_REQUEST ? SENT_DATA_REQUEST : + clientState() == ConnStateData::FTP_CONNECTED ? SENT_USER : + clientState() == ConnStateData::FTP_HANDLE_PASS ? SENT_PASS : SENT_COMMAND; } @@ -529,6 +564,82 @@ ServerStateData::readDataReply() forwardReply(); } +bool +ServerStateData::startDirTracking() +{ + if (!fwd->request->clientConnectionManager->port->ftp_track_dirs) + return false; + + debugs(9, 5, "Start directory tracking"); + savedReply.message = ctrl.message; + savedReply.lastCommand = ctrl.last_command; + savedReply.lastReply = ctrl.last_reply; + savedReply.replyCode = ctrl.replycode; + + ctrl.last_command = NULL; + ctrl.last_reply = NULL; + ctrl.message = NULL; + ctrl.offset = 0; + writeCommand("PWD\r\n"); + return true; +} + +void +ServerStateData::stopDirTracking() +{ + debugs(9, 5, "Got code from pwd: " << ctrl.replycode << ", msg: " << ctrl.last_reply); + + if (ctrl.replycode == 257) + fwd->request->clientConnectionManager->ftpSetWorkingDir(Ftp::unescapeDoubleQuoted(ctrl.last_reply)); + + wordlistDestroy(&ctrl.message); + safe_free(ctrl.last_command); + safe_free(ctrl.last_reply); + + ctrl.message = savedReply.message; + ctrl.last_command = savedReply.lastCommand; + ctrl.last_reply = savedReply.lastReply; + ctrl.replycode = savedReply.replyCode; + + savedReply.message = NULL; + savedReply.lastReply = NULL; + savedReply.lastCommand = NULL; +} + +void +ServerStateData::readCwdOrCdupReply() +{ + assert(clientState() == ConnStateData::FTP_HANDLE_CWD || clientState() == ConnStateData::FTP_HANDLE_CDUP); + + debugs(9, 5, HERE << "Got code " << ctrl.replycode << ", msg: " << ctrl.last_reply); + + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; + + if (weAreTrackingDir()) { // we are tracking + stopDirTracking(); // and forward the delayed response below + } else if (startDirTracking()) + return; + + forwardReply(); +} + +void +ServerStateData::readUserOrPassReply() +{ + if (100 <= ctrl.replycode && ctrl.replycode < 200) + return; //Just ignore + + if (weAreTrackingDir()) { // we are tracking + stopDirTracking(); // and forward the delayed response below + } else if (ctrl.replycode == 230) { // successful login + if (startDirTracking()) + return; + } + + forwardReply(); +} + void ServerStateData::readTransferDoneReply() { diff --git a/src/FtpServer.cc b/src/FtpServer.cc index 54797bbe76..5b3a84c009 100644 --- a/src/FtpServer.cc +++ b/src/FtpServer.cc @@ -16,8 +16,10 @@ #include "errorpage.h" #include "fd.h" #include "ip/tools.h" +#include "SquidString.h" #include "tools.h" #include "wordlist.h" +#include namespace Ftp { @@ -1173,3 +1175,40 @@ Ftp::ParseProtoIpPort(const char *buf, Ip::Address &addr) addr.port(port); return true; } + +const char * +Ftp::unescapeDoubleQuoted(const char *quotedPath) +{ + static MemBuf path; + path.reset(); + const char *s = quotedPath; + if (*s == '"') { + ++s; + bool parseDone = false; + while (!parseDone) { + if (const char *e = strchr(s, '"')) { + path.append(s, e - s); + s = e + 1; + if (*s == '"') { + path.append(s, 1); + ++s; + } else + parseDone = true; + } else { //parse error + parseDone = true; + path.reset(); + } + } + } + return path.content(); +} + +bool +Ftp::hasPathParameter(const String &cmd) +{ + static const char *pathCommandsStr[]= {"CWD","SMNT", "RETR", "STOR", "APPE", + "RNFR", "RNTO", "DELE", "RMD", "MKD", + "LIST", "NLST", "STAT"}; + static const std::set pathCommands(pathCommandsStr, pathCommandsStr + sizeof(pathCommandsStr)/sizeof(pathCommandsStr[0])); + return pathCommands.find(cmd) != pathCommands.end(); +} diff --git a/src/FtpServer.h b/src/FtpServer.h index 34dff80170..88a962c6e5 100644 --- a/src/FtpServer.h +++ b/src/FtpServer.h @@ -8,6 +8,7 @@ #include "Server.h" +class String; namespace Ftp { extern const char *const crlf; @@ -115,6 +116,8 @@ public: WRITING_DATA, SENT_MKDIR, SENT_FEAT, + SENT_PWD, + SENT_CDUP, SENT_DATA_REQUEST, // LIST, NLST or RETR requests.. SENT_COMMAND, // General command END @@ -157,7 +160,10 @@ private: bool ParseIpPort(const char *buf, const char *forceIp, Ip::Address &addr); /// parses and validates EPRT "" proto,ip,port sequence bool ParseProtoIpPort(const char *buf, Ip::Address &addr); - +/// parses a ftp quoted quote-escaped path +const char *unescapeDoubleQuoted(const char *quotedPath); +/// Return true if the FTP command takes as parameter a pathname +bool hasPathParameter(const String &cmd); }; // namespace Ftp #endif /* SQUID_FTP_SERVER_H */ diff --git a/src/anyp/PortCfg.cc b/src/anyp/PortCfg.cc index 41a6b9ee1c..61981f67d7 100644 --- a/src/anyp/PortCfg.cc +++ b/src/anyp/PortCfg.cc @@ -17,10 +17,11 @@ AnyP::PortCfg::PortCfg(const char *aProtocol) : next(NULL), protocol(xstrdup(aProtocol)), name(NULL), - defaultsite(NULL) + defaultsite(NULL), #if USE_SSL - ,dynamicCertMemCacheSize(std::numeric_limits::max()) + dynamicCertMemCacheSize(std::numeric_limits::max()), #endif + ftp_track_dirs(false) {} AnyP::PortCfg::~PortCfg() @@ -65,6 +66,7 @@ AnyP::PortCfg::clone() const b->connection_auth_disabled = connection_auth_disabled; b->disable_pmtu_discovery = disable_pmtu_discovery; b->tcp_keepalive = tcp_keepalive; + b->ftp_track_dirs = ftp_track_dirs; #if 0 // TODO: AYJ: 2009-07-18: for now SSL does not clone. Configure separate ports with IPs and SSL settings diff --git a/src/anyp/PortCfg.h b/src/anyp/PortCfg.h index 9553caeeb2..bddc060707 100644 --- a/src/anyp/PortCfg.h +++ b/src/anyp/PortCfg.h @@ -87,6 +87,8 @@ public: long sslOptions; ///< SSL engine options #endif + bool ftp_track_dirs; ///< Whether to track FTP directories + CBDATA_CLASS2(PortCfg); // namespaced }; diff --git a/src/cache_cf.cc b/src/cache_cf.cc index 14c1c8eced..289ab50fd7 100644 --- a/src/cache_cf.cc +++ b/src/cache_cf.cc @@ -3780,6 +3780,10 @@ parse_port_option(AnyP::PortCfg * s, char *token) } else if (strncmp(token, "dynamic_cert_mem_cache_size=", 28) == 0) { parseBytesOptionValue(&s->dynamicCertMemCacheSize, B_BYTES_STR, token + 28); #endif + } else if (strcmp(token, "ftp-track-dirs=on") == 0) { + s->ftp_track_dirs = true; + } else if (strcmp(token, "ftp-track-dirs=off") == 0) { + s->ftp_track_dirs = false; } else { debugs(3, DBG_CRITICAL, "FATAL: Unknown http(s)_port option '" << token << "'."); self_destruct(); diff --git a/src/cf.data.pre b/src/cf.data.pre index 70d336ba15..361d601def 100644 --- a/src/cf.data.pre +++ b/src/cf.data.pre @@ -1924,7 +1924,14 @@ TYPE: PortCfg DEFAULT: none LOC: Config.Sockaddr.ftp DOC_START - Usage: [ip:]port + Usage: [ip:]port [options] + + Ftp options: + ftp-track-dirs=on|off + Enables tracking of FTP directories by injecting extra + PWD commands and adjusting Request-URI (in wrapping HTTP + requests) to reflect the current FTP server directory. + Disabled by default. NAME: tcp_outgoing_tos tcp_outgoing_ds tcp_outgoing_dscp TYPE: acl_tos diff --git a/src/client_side.cc b/src/client_side.cc index 627cb847e2..a6c8f35c57 100644 --- a/src/client_side.cc +++ b/src/client_side.cc @@ -101,6 +101,7 @@ #include "errorpage.h" #include "fd.h" #include "fde.h" +#include "FtpServer.h" #include "fqdncache.h" #include "FwdState.h" #include "globals.h" @@ -273,6 +274,9 @@ static FtpRequestHandler FtpHandleDataRequest; static FtpRequestHandler FtpHandleUploadRequest; static FtpRequestHandler FtpHandleEprtRequest; static FtpRequestHandler FtpHandleEpsvRequest; +static FtpRequestHandler FtpHandleCwdRequest; +static FtpRequestHandler FtpHandlePassRequest; +static FtpRequestHandler FtpHandleCdupRequest; static bool FtpCheckDataConnPre(ClientSocketContext *context); static bool FtpCheckDataConnPost(ClientSocketContext *context); @@ -4020,10 +4024,9 @@ ftpAccept(const CommAcceptCbParams ¶ms) if (connState->transparent()) { char buf[MAX_IPSTRLEN]; connState->clientConnection->local.toUrl(buf, MAX_IPSTRLEN); - connState->ftp.uri = "ftp://"; - connState->ftp.uri.append(buf); - connState->ftp.uri.append("/"); - debugs(33, 5, HERE << "FTP transparent URL: " << connState->ftp.uri); + connState->ftp.host = buf; + const char *uri = connState->ftpBuildUri(); + debugs(33, 5, HERE << "FTP transparent URL: " << uri); } FtpWriteEarlyReply(connState, 220, "Service ready"); @@ -4974,6 +4977,35 @@ ConnStateData::unpinConnection(const bool andClose) * connection has gone away */ } +const char * +ConnStateData::ftpBuildUri(const char *file) +{ + ftp.uri = "ftp://"; + ftp.uri.append(ftp.host); + if (port->ftp_track_dirs && ftp.workingDir.size()) { + if (ftp.workingDir[0] != '/') + ftp.uri.append("/"); + ftp.uri.append(ftp.workingDir); + } + + if (ftp.uri[ftp.uri.size() - 1] != '/') + ftp.uri.append("/"); + + if (port->ftp_track_dirs && file) { + //remove any '/' from the beginning of path + while (*file == '/') + ++file; + ftp.uri.append(file); + } + + return ftp.uri.termedBuf(); +} + +void +ConnStateData::ftpSetWorkingDir(const char *dir) +{ + ftp.workingDir = dir; +} static void FtpAcceptDataConnection(const CommAcceptCbParams ¶ms) @@ -5172,15 +5204,15 @@ FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::Pro const String cmd = boc; String params = bop; - if (connState->ftp.uri.size() == 0) { + if (!connState->ftp.readGreeting) { // the first command must be USER - if (cmd.caseCmp("USER") != 0) { + if (!connState->pinning.pinned && cmd.caseCmp("USER") != 0) { FtpWriteEarlyReply(connState, 530, "Must login first"); return NULL; } } - // We need to process USER request now because it sets request URI. + // We need to process USER request now because it sets ftp server Hostname. if (cmd.caseCmp("USER") == 0 && !FtpHandleUserRequest(connState, cmd, params)) return NULL; @@ -5193,8 +5225,10 @@ FtpParseRequest(ConnStateData *connState, HttpRequestMethod *method_p, Http::Pro *method_p = !cmd.caseCmp("APPE") || !cmd.caseCmp("STOR") || !cmd.caseCmp("STOU") ? Http::METHOD_PUT : Http::METHOD_GET; - assert(connState->ftp.uri.size() > 0); - char *uri = xstrdup(connState->ftp.uri.termedBuf()); + char *uri; + const char *aPath = params.size() > 0 && Ftp::hasPathParameter(cmd)? + params.termedBuf() : NULL; + uri = xstrdup(connState->ftpBuildUri(aPath)); HttpRequest *const request = HttpRequest::CreateFromUrlAndMethod(uri, *method_p); if (request == NULL) { @@ -5263,6 +5297,9 @@ FtpHandleReply(ClientSocketContext *context, HttpReply *reply, StoreIOBuffer dat FtpHandleUploadReply, // FTP_HANDLE_UPLOAD_REQUEST FtpHandleEprtReply,// FTP_HANDLE_EPRT FtpHandleEpsvReply,// FTP_HANDLE_EPSV + NULL, // FTP_HANDLE_CWD + NULL, //FTP_HANDLE_PASS + NULL, // FTP_HANDLE_CDUP FtpHandleErrorReply // FTP_ERROR }; const ConnStateData::FtpState state = context->getConn()->ftp.state; @@ -5783,6 +5820,9 @@ FtpHandleRequest(ClientSocketContext *context, String &cmd, String ¶ms) { std::make_pair("RETR", FtpHandleDataRequest), std::make_pair("EPRT", FtpHandleEprtRequest), std::make_pair("EPSV", FtpHandleEpsvRequest), + std::make_pair("CWD", FtpHandleCwdRequest), + std::make_pair("PASS", FtpHandlePassRequest), + std::make_pair("CDUP", FtpHandleCdupRequest), }; FtpRequestHandler *handler = NULL; @@ -5816,21 +5856,35 @@ FtpHandleUserRequest(ConnStateData *connState, const String &cmd, String ¶ms return false; } - static const String scheme = "ftp://"; const String login = params.substr(0, eou); const String host = params.substr(eou + 1, params.size()); + // If we can parse it as raw IPv6 address, then surround with "[]". + // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is. + if (host.pos(":")) { + char ipBuf[MAX_IPSTRLEN]; + Ip::Address ipa; + ipa = host.termedBuf(); + if (!ipa.isAnyAddr()) { + ipa.toHostStr(ipBuf, MAX_IPSTRLEN); + connState->ftp.host = ipBuf; + } + } + if (connState->ftp.host.size() == 0) + connState->ftp.host = host; + + String oldUri; + if (connState->ftp.readGreeting) + oldUri = connState->ftp.uri; - String uri = scheme; - uri.append(host); - uri.append("/"); + connState->ftpSetWorkingDir(NULL); + connState->ftpBuildUri(); - if (!connState->ftp.uri.size()) { - connState->ftp.uri = uri; + if (!connState->ftp.readGreeting) { debugs(11, 3, "set URI to " << connState->ftp.uri); - } else if (uri.caseCmp(connState->ftp.uri) == 0) { - debugs(11, 5, "keep URI as " << uri); + } else if (oldUri.caseCmp(connState->ftp.uri) == 0) { + debugs(11, 5, "keep URI as " << oldUri); } else { - debugs(11, 3, "reset URI from " << connState->ftp.uri << " to " << uri); + debugs(11, 3, "reset URI from " << oldUri << " to " << connState->ftp.uri); FtpCloseDataConnection(connState); connState->ftp.readGreeting = false; connState->unpinConnection(true); // close control connection to the server @@ -5909,8 +5963,6 @@ bool FtpCreateDataConnection(ClientSocketContext *context, Ip::Address cltAddr) return true; } -#include "FtpServer.h" /* XXX: For Ftp::ParseIpPort() */ - bool FtpHandlePortRequest(ClientSocketContext *context, String &cmd, String ¶ms) { @@ -6019,6 +6071,26 @@ FtpHandleEpsvRequest(ClientSocketContext *context, String &cmd, String ¶ms) return true; // forward our fake PASV request } +bool +FtpHandleCwdRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CWD, "FtpHandleCwdRequest"); + return true; +} + +bool +FtpHandlePassRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_PASS, "FtpHandlePassRequest"); + return true; +} + +bool +FtpHandleCdupRequest(ClientSocketContext *context, String &cmd, String ¶ms) +{ + FtpChangeState(context->getConn(), ConnStateData::FTP_HANDLE_CDUP, "FtpHandleCdupRequest"); + return true; +} // Convert client PORT, EPRT, PASV, or EPSV data command to Squid PASV command. // Squid server-side decides what data command to use on that side. diff --git a/src/client_side.h b/src/client_side.h index f29edfc160..8d523328ea 100644 --- a/src/client_side.h +++ b/src/client_side.h @@ -347,10 +347,15 @@ public: FTP_HANDLE_UPLOAD_REQUEST, FTP_HANDLE_EPRT, FTP_HANDLE_EPSV, + FTP_HANDLE_CWD, + FTP_HANDLE_PASS, + FTP_HANDLE_CDUP, FTP_ERROR }; struct { String uri; + String host; + String workingDir; FtpState state; bool readGreeting; bool gotEpsvAll; ///< restrict data conn setup commands to just EPSV @@ -363,6 +368,8 @@ public: AsyncCall::Pointer connector; ///< set when we are actively connecting AsyncCall::Pointer reader; ///< set when we are reading FTP data } ftp; + const char *ftpBuildUri(const char *file = NULL); + void ftpSetWorkingDir(const char *dir); #if USE_SSL /// called by FwdState when it is done bumping the server diff --git a/src/ftp.cc b/src/ftp.cc index 27cce1991c..8941c98037 100644 --- a/src/ftp.cc +++ b/src/ftp.cc @@ -334,6 +334,9 @@ FTPSM *FTP_SM_FUNCS[] = { ftpWriteTransferDone, /* WRITING_DATA (STOR) */ ftpReadMkdir, /* SENT_MKDIR */ NULL, /* SENT_FEAT */ + NULL, /* SENT_PWD */ + NULL, /* SENT_CDUP*/ + NULL, /* SENT_DATA_REQUEST */ NULL /* SENT_COMMAND */ };