src/servers/FtpServer.cc

   1 /*
   2  * DEBUG: section 33    Transfer protocol servers
   3  */
   4
   5 #include "squid.h"
   6 #include "base/CharacterSet.h"
   7 #include "base/RefCount.h"
   8 #include "base/Subscription.h"
   9 #include "client_side_reply.h"
  10 #include "client_side_request.h"
  11 #include "clientStream.h"
  12 #include "comm/ConnOpener.h"
  13 #include "comm/Read.h"
  14 #include "comm/TcpAcceptor.h"
  15 #include "comm/Write.h"
  16 #include "errorpage.h"
  17 #include "fd.h"
  18 #include "ftp/Elements.h"
  19 #include "ftp/Parsing.h"
  20 #include "globals.h"
  21 #include "http/one/RequestParser.h"
  22 #include "HttpHdrCc.h"
  23 #include "ip/tools.h"
  24 #include "ipc/FdNotes.h"
  25 #include "parser/Tokenizer.h"
  26 #include "servers/forward.h"
  27 #include "servers/FtpServer.h"
  28 #include "SquidConfig.h"
  29 #include "StatCounters.h"
  30 #include "tools.h"
  31
  32 #include <set>
  33 #include <map>
  34
  35 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  36
  37 namespace Ftp
  38 {
  39 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  40 static bool SupportedCommand(const SBuf &name);
  41 static bool CommandHasPathParameter(const SBuf &cmd);
  42 };
  43
  44 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  45         AsyncJob("Ftp::Server"),
  46         ConnStateData(xact),
  47         master(new MasterState),
  48         uri(),
  49         host(),
  50         gotEpsvAll(false),
  51         onDataAcceptCall(),
  52         dataListenConn(),
  53         dataConn(),
  54         uploadAvailSize(0),
  55         listener(),
  56         connector(),
  57         reader()
  58 {
  59     flags.readMore = false; // we need to announce ourselves first
  60 }
  61
  62 Ftp::Server::~Server()
  63 {
  64     closeDataConnection();
  65 }
  66
  67 int
  68 Ftp::Server::pipelinePrefetchMax() const
  69 {
  70     return 0; // no support for concurrent FTP requests
  71 }
  72
  73 time_t
  74 Ftp::Server::idleTimeout() const
  75 {
  76     return Config.Timeout.ftpClientIdle;
  77 }
  78
  79 void
  80 Ftp::Server::start()
  81 {
  82     ConnStateData::start();
  83
  84     if (transparent()) {
  85         char buf[MAX_IPSTRLEN];
  86         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  87         host = buf;
  88         calcUri(NULL);
  89         debugs(33, 5, "FTP transparent URL: " << uri);
  90     }
  91
  92     writeEarlyReply(220, "Service ready");
  93 }
  94
  95 /// schedules another data connection read if needed
  96 void
  97 Ftp::Server::maybeReadUploadData()
  98 {
  99     if (reader != NULL)
 100         return;
 101
 102     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 103     if (availSpace <= 0)
 104         return;
 105
 106     debugs(33, 4, dataConn << ": reading FTP data...");
 107
 108     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 109     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 110     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 111               reader);
 112 }
 113
 114 /// react to the freshly parsed request
 115 void
 116 Ftp::Server::doProcessRequest()
 117 {
 118     // zero pipelinePrefetchMax() ensures that there is only parsed request
 119     ClientSocketContext::Pointer context = getCurrentContext();
 120     Must(context != NULL);
 121     Must(getConcurrentRequestCount() == 1);
 122
 123     ClientHttpRequest *const http = context->http;
 124     assert(http != NULL);
 125
 126     HttpRequest *const request = http->request;
 127     Must(http->storeEntry() || request);
 128     const bool mayForward = !http->storeEntry() && handleRequest(request);
 129
 130     if (http->storeEntry() != NULL) {
 131         debugs(33, 4, "got an immediate response");
 132         clientSetKeepaliveFlag(http);
 133         context->pullData();
 134     } else if (mayForward) {
 135         debugs(33, 4, "forwarding request to server side");
 136         assert(http->storeEntry() == NULL);
 137         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 138     } else {
 139         debugs(33, 4, "will resume processing later");
 140     }
 141 }
 142
 143 void
 144 Ftp::Server::processParsedRequest(ClientSocketContext *context)
 145 {
 146     Must(getConcurrentRequestCount() == 1);
 147
 148     // Process FTP request asynchronously to make sure FTP
 149     // data connection accept callback is fired first.
 150     CallJobHere(33, 4, CbcPointer<Server>(this),
 151                 Ftp::Server, doProcessRequest);
 152 }
 153
 154 /// imports more upload data from the data connection
 155 void
 156 Ftp::Server::readUploadData(const CommIoCbParams &io)
 157 {
 158     debugs(33, 5, io.conn << " size " << io.size);
 159     Must(reader != NULL);
 160     reader = NULL;
 161
 162     assert(Comm::IsConnOpen(dataConn));
 163     assert(io.conn->fd == dataConn->fd);
 164
 165     if (io.flag == Comm::OK && bodyPipe != NULL) {
 166         if (io.size > 0) {
 167             kb_incr(&(statCounter.client_http.kbytes_in), io.size);
 168
 169             char *const current_buf = uploadBuf + uploadAvailSize;
 170             if (io.buf != current_buf)
 171                 memmove(current_buf, io.buf, io.size);
 172             uploadAvailSize += io.size;
 173             shovelUploadData();
 174         } else if (io.size == 0) {
 175             debugs(33, 5, io.conn << " closed");
 176             closeDataConnection();
 177             if (uploadAvailSize <= 0)
 178                 finishDechunkingRequest(true);
 179         }
 180     } else { // not Comm::Flags::OK or unexpected read
 181         debugs(33, 5, io.conn << " closed");
 182         closeDataConnection();
 183         finishDechunkingRequest(false);
 184     }
 185
 186 }
 187
 188 /// shovel upload data from the internal buffer to the body pipe if possible
 189 void
 190 Ftp::Server::shovelUploadData()
 191 {
 192     assert(bodyPipe != NULL);
 193
 194     debugs(33, 5, "handling FTP request data for " << clientConnection);
 195     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 196                            uploadAvailSize);
 197     if (putSize > 0) {
 198         uploadAvailSize -= putSize;
 199         if (uploadAvailSize > 0)
 200             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 201     }
 202
 203     if (Comm::IsConnOpen(dataConn))
 204         maybeReadUploadData();
 205     else if (uploadAvailSize <= 0)
 206         finishDechunkingRequest(true);
 207 }
 208
 209 void
 210 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 211 {
 212     shovelUploadData();
 213 }
 214
 215 void
 216 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 217 {
 218     ConnStateData::noteBodyConsumerAborted(ptr);
 219     closeDataConnection();
 220 }
 221
 222 /// accept a new FTP control connection and hand it to a dedicated Server
 223 void
 224 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 225 {
 226     MasterXaction::Pointer xact = params.xaction;
 227     AnyP::PortCfgPointer s = xact->squidPort;
 228
 229     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 230
 231     if (params.flag != Comm::OK) {
 232         // Its possible the call was still queued when the client disconnected
 233         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 234         return;
 235     }
 236
 237     debugs(33, 4, params.conn << ": accepted");
 238     fd_note(params.conn->fd, "client ftp connect");
 239
 240     if (s->tcp_keepalive.enabled)
 241         commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
 242
 243     ++incoming_sockets_accepted;
 244
 245     AsyncJob::Start(new Server(xact));
 246 }
 247
 248 void
 249 Ftp::StartListening()
 250 {
 251     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 252         if (MAXTCPLISTENPORTS == NHttpSockets) {
 253             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 254                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 255             break;
 256         }
 257
 258         // direct new connections accepted by listenConn to Accept()
 259         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 260         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 261                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 262                                                           CommAcceptCbParams(NULL)));
 263         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 264     }
 265 }
 266
 267 void
 268 Ftp::StopListening()
 269 {
 270     for (AnyP::PortCfgPointer s = HttpPortList; s != NULL; s = s->next) {
 271         if (s->listenConn != NULL) {
 272             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 273             s->listenConn->close();
 274             s->listenConn = NULL;
 275         }
 276     }
 277 }
 278
 279 void
 280 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 281 {
 282     // find request
 283     ClientSocketContext::Pointer context = getCurrentContext();
 284     Must(context != NULL);
 285     ClientHttpRequest *const http = context->http;
 286     Must(http != NULL);
 287     HttpRequest *const request = http->request;
 288     Must(request != NULL);
 289
 290     // this is not an idle connection, so we do not want I/O monitoring
 291     const bool monitor = false;
 292
 293     // make FTP peer connection exclusive to our request
 294     pinConnection(conn, request, conn->getPeer(), false, monitor);
 295 }
 296
 297 void
 298 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 299 {
 300     ConnStateData::clientPinnedConnectionClosed(io);
 301
 302     // if the server control connection is gone, reset state to login again
 303     resetLogin("control connection closure");
 304
 305     // XXX: Reseting is not enough. FtpRelay::sendCommand() will not re-login
 306     // because FtpRelay::serverState() is not going to be fssConnected.
 307 }
 308
 309 /// clear client and server login-related state after the old login is gone
 310 void
 311 Ftp::Server::resetLogin(const char *reason)
 312 {
 313     debugs(33, 5, "will need to re-login due to " << reason);
 314     master->clientReadGreeting = false;
 315     changeState(fssBegin, reason);
 316 }
 317
 318 /// computes uri member from host and, if tracked, working dir with file name
 319 void
 320 Ftp::Server::calcUri(const SBuf *file)
 321 {
 322     uri = "ftp://";
 323     uri.append(host);
 324     if (port->ftp_track_dirs && master->workingDir.length()) {
 325         if (master->workingDir[0] != '/')
 326             uri.append("/");
 327         uri.append(master->workingDir);
 328     }
 329
 330     if (uri[uri.length() - 1] != '/')
 331         uri.append("/");
 332
 333     if (port->ftp_track_dirs && file) {
 334         static const CharacterSet Slash("/", "/");
 335         Parser::Tokenizer tok(*file);
 336         tok.skipAll(Slash);
 337         uri.append(tok.remaining());
 338     }
 339 }
 340
 341 /// Starts waiting for a data connection. Returns listening port.
 342 /// On errors, responds with an error and returns zero.
 343 unsigned int
 344 Ftp::Server::listenForDataConnection()
 345 {
 346     closeDataConnection();
 347
 348     Comm::ConnectionPointer conn = new Comm::Connection;
 349     conn->flags = COMM_NONBLOCKING;
 350     conn->local = transparent() ? port->s : clientConnection->local;
 351     conn->local.port(0);
 352     const char *const note = uri.c_str();
 353     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 354     if (!Comm::IsConnOpen(conn)) {
 355         debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " <<
 356                conn->local << " error: " << errno);
 357         writeCustomReply(451, "Internal error");
 358         return 0;
 359     }
 360
 361     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 362     typedef AsyncCallT<AcceptDialer> AcceptCall;
 363     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 364     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 365     listener = call.getRaw();
 366     dataListenConn = conn;
 367     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 368
 369     const unsigned int listeningPort = comm_local_port(conn->fd);
 370     conn->local.port(listeningPort);
 371     return listeningPort;
 372 }
 373
 374 void
 375 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 376 {
 377     if (params.flag != Comm::OK) {
 378         // Its possible the call was still queued when the client disconnected
 379         debugs(33, 2, dataListenConn << ": accept "
 380                "failure: " << xstrerr(params.xerrno));
 381         return;
 382     }
 383
 384     debugs(33, 4, "accepted " << params.conn);
 385     fd_note(params.conn->fd, "passive client ftp data");
 386     ++incoming_sockets_accepted;
 387
 388     if (!clientConnection) {
 389         debugs(33, 5, "late data connection?");
 390         closeDataConnection(); // in case we are still listening
 391         params.conn->close();
 392     } else if (params.conn->remote != clientConnection->remote) {
 393         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 394         params.conn->close();
 395         // Some FTP servers close control connection here, but it may make
 396         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 397     } else {
 398         closeDataConnection();
 399         dataConn = params.conn;
 400         uploadAvailSize = 0;
 401         debugs(33, 7, "ready for data");
 402         if (onDataAcceptCall != NULL) {
 403             AsyncCall::Pointer call = onDataAcceptCall;
 404             onDataAcceptCall = NULL;
 405             // If we got an upload request, start reading data from the client.
 406             if (master->serverState == fssHandleUploadRequest)
 407                 maybeReadUploadData();
 408             else
 409                 Must(master->serverState == fssHandleDataRequest);
 410             MemBuf mb;
 411             mb.init();
 412             mb.Printf("150 Data connection opened.\r\n");
 413             Comm::Write(clientConnection, &mb, call);
 414         }
 415     }
 416 }
 417
 418 void
 419 Ftp::Server::closeDataConnection()
 420 {
 421     if (listener != NULL) {
 422         listener->cancel("no longer needed");
 423         listener = NULL;
 424     }
 425
 426     if (Comm::IsConnOpen(dataListenConn)) {
 427         debugs(33, 5, "FTP closing client data listen socket: " <<
 428                *dataListenConn);
 429         dataListenConn->close();
 430     }
 431     dataListenConn = NULL;
 432
 433     if (reader != NULL) {
 434         // Comm::ReadCancel can deal with negative FDs
 435         Comm::ReadCancel(dataConn->fd, reader);
 436         reader = NULL;
 437     }
 438
 439     if (Comm::IsConnOpen(dataConn)) {
 440         debugs(33, 5, "FTP closing client data connection: " <<
 441                *dataConn);
 442         dataConn->close();
 443     }
 444     dataConn = NULL;
 445 }
 446
 447 /// Writes FTP [error] response before we fully parsed the FTP request and
 448 /// created the corresponding HTTP request wrapper for that FTP request.
 449 void
 450 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 451 {
 452     debugs(33, 7, code << ' ' << msg);
 453     assert(99 < code && code < 1000);
 454
 455     MemBuf mb;
 456     mb.init();
 457     mb.Printf("%i %s\r\n", code, msg);
 458
 459     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 460     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 461     Comm::Write(clientConnection, &mb, call);
 462
 463     flags.readMore = false;
 464
 465     // TODO: Create master transaction. Log it in wroteEarlyReply().
 466 }
 467
 468 void
 469 Ftp::Server::writeReply(MemBuf &mb)
 470 {
 471     debugs(9, 2, "FTP Client " << clientConnection);
 472     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 473            "\n----------");
 474
 475     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 476     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 477     Comm::Write(clientConnection, &mb, call);
 478 }
 479
 480 void
 481 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 482 {
 483     debugs(33, 7, code << ' ' << msg);
 484     assert(99 < code && code < 1000);
 485
 486     const bool sendDetails = reply != NULL &&
 487                              reply->header.has(HDR_FTP_STATUS) && reply->header.has(HDR_FTP_REASON);
 488
 489     MemBuf mb;
 490     mb.init();
 491     if (sendDetails) {
 492         mb.Printf("%i-%s\r\n", code, msg);
 493         mb.Printf(" Server reply:\r\n");
 494         Ftp::PrintReply(mb, reply, " ");
 495         mb.Printf("%i \r\n", code);
 496     } else
 497         mb.Printf("%i %s\r\n", code, msg);
 498
 499     writeReply(mb);
 500 }
 501
 502 void
 503 Ftp::Server::changeState(const ServerState newState, const char *reason)
 504 {
 505     if (master->serverState == newState) {
 506         debugs(33, 3, "client state unchanged at " << master->serverState <<
 507                " because " << reason);
 508         master->serverState = newState;
 509     } else {
 510         debugs(33, 3, "client state was " << master->serverState <<
 511                ", now " << newState << " because " << reason);
 512         master->serverState = newState;
 513     }
 514 }
 515
 516 /// whether the given FTP command has a pathname parameter
 517 static bool
 518 Ftp::CommandHasPathParameter(const SBuf &cmd)
 519 {
 520     static std::set<SBuf> PathedCommands;
 521     if (!PathedCommands.size()) {
 522         PathedCommands.insert(cmdMlst());
 523         PathedCommands.insert(cmdMlsd());
 524         PathedCommands.insert(cmdStat());
 525         PathedCommands.insert(cmdNlst());
 526         PathedCommands.insert(cmdList());
 527         PathedCommands.insert(cmdMkd());
 528         PathedCommands.insert(cmdRmd());
 529         PathedCommands.insert(cmdDele());
 530         PathedCommands.insert(cmdRnto());
 531         PathedCommands.insert(cmdRnfr());
 532         PathedCommands.insert(cmdAppe());
 533         PathedCommands.insert(cmdStor());
 534         PathedCommands.insert(cmdRetr());
 535         PathedCommands.insert(cmdSmnt());
 536         PathedCommands.insert(cmdCwd());
 537     }
 538
 539     return PathedCommands.find(cmd) != PathedCommands.end();
 540 }
 541
 542 /// creates a context filled with an error message for a given early error
 543 ClientSocketContext *
 544 Ftp::Server::earlyError(const EarlyErrorKind eek)
 545 {
 546     /* Default values, to be updated by the switch statement below */
 547     int scode = 421;
 548     const char *reason = "Internal error";
 549     const char *errUri = "error:ftp-internal-early-error";
 550
 551     switch (eek) {
 552     case eekHugeRequest:
 553         scode = 421;
 554         reason = "Huge request";
 555         errUri = "error:ftp-huge-request";
 556         break;
 557
 558     case eekMissingLogin:
 559         scode = 530;
 560         reason = "Must login first";
 561         errUri = "error:ftp-must-login-first";
 562         break;
 563
 564     case eekMissingUsername:
 565         scode = 501;
 566         reason = "Missing username";
 567         errUri = "error:ftp-missing-username";
 568         break;
 569
 570     case eekMissingHost:
 571         scode = 501;
 572         reason = "Missing host";
 573         errUri = "error:ftp-missing-host";
 574         break;
 575
 576     case eekUnsupportedCommand:
 577         scode = 502;
 578         reason = "Unknown or unsupported command";
 579         errUri = "error:ftp-unsupported-command";
 580         break;
 581
 582     case eekInvalidUri:
 583         scode = 501;
 584         reason = "Invalid URI";
 585         errUri = "error:ftp-invalid-uri";
 586         break;
 587
 588     case eekMalformedCommand:
 589         scode = 421;
 590         reason = "Malformed command";
 591         errUri = "error:ftp-malformed-command";
 592         break;
 593
 594         // no default so that a compiler can check that we have covered all cases
 595     }
 596
 597     ClientSocketContext *context = abortRequestParsing(errUri);
 598     clientStreamNode *node = context->getClientReplyContext();
 599     Must(node);
 600     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 601
 602     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 603     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 604     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 605     repContext->setReplyToReply(reply);
 606     return context;
 607 }
 608
 609 /// Parses a single FTP request on the control connection.
 610 /// Returns a new ClientSocketContext on valid requests and all errors.
 611 /// Returns NULL on incomplete requests that may still succeed given more data.
 612 ClientSocketContext *
 613 Ftp::Server::parseOneRequest()
 614 {
 615     flags.readMore = false; // common for all but one case below
 616
 617     // OWS <command> [ RWS <parameter> ] OWS LF
 618
 619     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 620     // for the LF character that we must exclude here (but see FullWhiteSpace).
 621     static const char * const InlineSpaceChars = " \f\r\t\v";
 622     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 623     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 624     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 625     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 626
 627     // This set is used to ignore empty commands without allowing an attacker
 628     // to keep us endlessly busy by feeding us whitespace or empty commands.
 629     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 630
 631     SBuf cmd;
 632     SBuf params;
 633
 634     Parser::Tokenizer tok(in.buf);
 635
 636     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 637     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 638
 639     // note that the condition below will eat either RWS or trailing OWS
 640     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 641         // now params may include trailing OWS
 642         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 643         static const SBuf bufWhiteSpace(InlineSpaceChars);
 644         params.trim(bufWhiteSpace, false, true);
 645     }
 646
 647     // Why limit command line and parameters size? Did not we just parse them?
 648     // XXX: Our good old String cannot handle very long strings.
 649     const SBuf::size_type tokenMax = min(
 650                                          static_cast<SBuf::size_type>(32*1024), // conservative
 651                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 652     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 653         changeState(fssError, "huge req token");
 654         quitAfterError(NULL);
 655         return earlyError(eekHugeRequest);
 656     }
 657
 658     // technically, we may skip multiple NLs below, but that is OK
 659     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 660         // we need more data, but can we buffer more?
 661         if (in.buf.length() >= Config.maxRequestHeaderSize) {
 662             changeState(fssError, "huge req");
 663             quitAfterError(NULL);
 664             return earlyError(eekHugeRequest);
 665         } else {
 666             flags.readMore = true;
 667             debugs(33, 5, "Waiting for more, up to " <<
 668                    (Config.maxRequestHeaderSize - in.buf.length()));
 669             return NULL;
 670         }
 671     }
 672
 673     Must(parsed && cmd.length());
 674     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 675
 676     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 677
 678     cmd.toUpper(); // this should speed up and simplify future comparisons
 679
 680     // interception cases do not need USER to calculate the uri
 681     if (!transparent()) {
 682         if (!master->clientReadGreeting) {
 683             // the first command must be USER
 684             if (!pinning.pinned && cmd != cmdUser())
 685                 return earlyError(eekMissingLogin);
 686         }
 687
 688         // process USER request now because it sets FTP peer host name
 689         if (cmd == cmdUser()) {
 690             if (ClientSocketContext *errCtx = handleUserRequest(cmd, params))
 691                 return errCtx;
 692         }
 693     }
 694
 695     if (!Ftp::SupportedCommand(cmd))
 696         return earlyError(eekUnsupportedCommand);
 697
 698     const HttpRequestMethod method =
 699         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 700         Http::METHOD_PUT : Http::METHOD_GET;
 701
 702     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 703                        &params : NULL;
 704     calcUri(path);
 705     char *newUri = xstrdup(uri.c_str());
 706     HttpRequest *const request = HttpRequest::CreateFromUrlAndMethod(newUri, method);
 707     if (!request) {
 708         debugs(33, 5, "Invalid FTP URL: " << uri);
 709         uri.clear();
 710         safe_free(newUri);
 711         return earlyError(eekInvalidUri);
 712     }
 713
 714     request->flags.ftpNative = true;
 715     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 716
 717     // Our fake Request-URIs are not distinctive enough for caching to work
 718     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 719     request->flags.noCache = true;
 720
 721     request->header.putStr(HDR_FTP_COMMAND, cmd.c_str());
 722     request->header.putStr(HDR_FTP_ARGUMENTS, params.c_str()); // may be ""
 723     if (method == Http::METHOD_PUT) {
 724         request->header.putStr(HDR_EXPECT, "100-continue");
 725         request->header.putStr(HDR_TRANSFER_ENCODING, "chunked");
 726     }
 727
 728     ClientHttpRequest *const http = new ClientHttpRequest(this);
 729     http->request = request;
 730     HTTPMSGLOCK(http->request);
 731     http->req_sz = tok.parsedSize();
 732     http->uri = newUri;
 733
 734     ClientSocketContext *const result =
 735         new ClientSocketContext(clientConnection, http);
 736
 737     StoreIOBuffer tempBuffer;
 738     tempBuffer.data = result->reqbuf;
 739     tempBuffer.length = HTTP_REQBUF_SZ;
 740
 741     ClientStreamData newServer = new clientReplyContext(http);
 742     ClientStreamData newClient = result;
 743     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 744                      clientReplyStatus, newServer, clientSocketRecipient,
 745                      clientSocketDetach, newClient, tempBuffer);
 746
 747     result->flags.parsed_ok = 1;
 748     return result;
 749 }
 750
 751 void
 752 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 753 {
 754     // the caller guarantees that we are dealing with the current context only
 755     ClientSocketContext::Pointer context = getCurrentContext();
 756     assert(context != NULL);
 757
 758     if (context->http && context->http->al != NULL &&
 759             !context->http->al->reply && reply) {
 760         context->http->al->reply = reply;
 761         HTTPMSGLOCK(context->http->al->reply);
 762     }
 763
 764     static ReplyHandler handlers[] = {
 765         NULL, // fssBegin
 766         NULL, // fssConnected
 767         &Ftp::Server::handleFeatReply, // fssHandleFeat
 768         &Ftp::Server::handlePasvReply, // fssHandlePasv
 769         &Ftp::Server::handlePortReply, // fssHandlePort
 770         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 771         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 772         &Ftp::Server::handleEprtReply,// fssHandleEprt
 773         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 774         NULL, // fssHandleCwd
 775         NULL, // fssHandlePass
 776         NULL, // fssHandleCdup
 777         &Ftp::Server::handleErrorReply // fssError
 778     };
 779     const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 780     if (const ReplyHandler handler = handlers[server.master->serverState])
 781         (this->*handler)(reply, data);
 782     else
 783         writeForwardedReply(reply);
 784 }
 785
 786 void
 787 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 788 {
 789     if (getCurrentContext()->http->request->errType != ERR_NONE) {
 790         writeCustomReply(502, "Server does not support FEAT", reply);
 791         return;
 792     }
 793
 794     HttpReply *filteredReply = reply->clone();
 795     HttpHeader &filteredHeader = filteredReply->header;
 796
 797     // Remove all unsupported commands from the response wrapper.
 798     int deletedCount = 0;
 799     HttpHeaderPos pos = HttpHeaderInitPos;
 800     bool hasEPRT = false;
 801     bool hasEPSV = false;
 802     int prependSpaces = 1;
 803     while (const HttpHeaderEntry *e = filteredHeader.getEntry(&pos)) {
 804         if (e->id == HDR_FTP_PRE) {
 805             // assume RFC 2389 FEAT response format, quoted by Squid:
 806             // <"> SP NAME [SP PARAMS] <">
 807             // but accommodate MS servers sending four SPs before NAME
 808
 809             // command name ends with (SP parameter) or quote
 810             static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 811             static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 812
 813             Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 814             if (!tok.skip('"') && !tok.skip(' '))
 815                 continue;
 816
 817             // optional spaces; remember their number to accomodate MS servers
 818             prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 819
 820             SBuf cmd;
 821             if (!tok.prefix(cmd, FeatNameChars))
 822                 continue;
 823             cmd.toUpper();
 824
 825             if (!Ftp::SupportedCommand(cmd))
 826                 filteredHeader.delAt(pos, deletedCount);
 827
 828             if (cmd == cmdEprt())
 829                 hasEPRT = true;
 830             else if (cmd == cmdEpsv())
 831                 hasEPSV = true;
 832         }
 833     }
 834
 835     char buf[256];
 836     int insertedCount = 0;
 837     if (!hasEPRT) {
 838         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 839         filteredHeader.putStr(HDR_FTP_PRE, buf);
 840         ++insertedCount;
 841     }
 842     if (!hasEPSV) {
 843         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 844         filteredHeader.putStr(HDR_FTP_PRE, buf);
 845         ++insertedCount;
 846     }
 847
 848     if (deletedCount || insertedCount) {
 849         filteredHeader.refreshMask();
 850         debugs(33, 5, "deleted " << deletedCount << " inserted " << insertedCount);
 851     }
 852
 853     writeForwardedReply(filteredReply);
 854 }
 855
 856 void
 857 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 858 {
 859     ClientSocketContext::Pointer context = getCurrentContext();
 860     assert(context != NULL);
 861
 862     if (context->http->request->errType != ERR_NONE) {
 863         writeCustomReply(502, "Server does not support PASV", reply);
 864         return;
 865     }
 866
 867     const unsigned short localPort = listenForDataConnection();
 868     if (!localPort)
 869         return;
 870
 871     char addr[MAX_IPSTRLEN];
 872     // remote server in interception setups and local address otherwise
 873     const Ip::Address &server = transparent() ?
 874                                 clientConnection->local : dataListenConn->local;
 875     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 876     addr[MAX_IPSTRLEN - 1] = '\0';
 877     for (char *c = addr; *c != '\0'; ++c) {
 878         if (*c == '.')
 879             *c = ',';
 880     }
 881
 882     // In interception setups, we combine remote server address with a
 883     // local port number and hope that traffic will be redirected to us.
 884     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 885     // versions block responses that use those alternative syntax rules!
 886     MemBuf mb;
 887     mb.init();
 888     mb.Printf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 889               addr,
 890               static_cast<int>(localPort / 256),
 891               static_cast<int>(localPort % 256));
 892     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 893     writeReply(mb);
 894 }
 895
 896 void
 897 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 898 {
 899     if (getCurrentContext()->http->request->errType != ERR_NONE) {
 900         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 901         return;
 902     }
 903
 904     writeCustomReply(200, "PORT successfully converted to PASV.");
 905
 906     // and wait for RETR
 907 }
 908
 909 void
 910 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 911 {
 912     if (!pinning.pinned) // we failed to connect to server
 913         uri.clear();
 914     // 421: we will close due to fssError
 915     writeErrorReply(reply, 421);
 916 }
 917
 918 void
 919 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 920 {
 921     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 922         writeForwardedReply(reply);
 923         if (Comm::IsConnOpen(dataConn)) {
 924             debugs(33, 3, "closing " << dataConn << " on KO reply");
 925             closeDataConnection();
 926         }
 927         return;
 928     }
 929
 930     if (!dataConn) {
 931         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 932         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 933         return;
 934     }
 935
 936     if (!checkDataConnPost()) {
 937         writeCustomReply(425, "Data connection is not established.");
 938         closeDataConnection();
 939         return;
 940     }
 941
 942     debugs(33, 7, data.length);
 943
 944     if (data.length <= 0) {
 945         replyDataWritingCheckpoint(); // skip the actual write call
 946         return;
 947     }
 948
 949     MemBuf mb;
 950     mb.init(data.length + 1, data.length + 1);
 951     mb.append(data.data, data.length);
 952
 953     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 954     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 955     Comm::Write(dataConn, &mb, call);
 956
 957     getCurrentContext()->noteSentBodyBytes(data.length);
 958 }
 959
 960 /// called when we are done writing a chunk of the response data
 961 void
 962 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 963 {
 964     if (io.flag == Comm::ERR_CLOSING)
 965         return;
 966
 967     if (io.flag != Comm::OK) {
 968         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 969         closeDataConnection();
 970         writeCustomReply(426, "Data connection error; transfer aborted");
 971         return;
 972     }
 973
 974     assert(getCurrentContext()->http);
 975     getCurrentContext()->http->out.size += io.size;
 976     replyDataWritingCheckpoint();
 977 }
 978
 979 /// ClientStream checks after (actual or skipped) reply data writing
 980 void
 981 Ftp::Server::replyDataWritingCheckpoint()
 982 {
 983     switch (getCurrentContext()->socketState()) {
 984     case STREAM_NONE:
 985         debugs(33, 3, "Keep going");
 986         getCurrentContext()->pullData();
 987         return;
 988     case STREAM_COMPLETE:
 989         debugs(33, 3, "FTP reply data transfer successfully complete");
 990         writeCustomReply(226, "Transfer complete");
 991         break;
 992     case STREAM_UNPLANNED_COMPLETE:
 993         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
 994         writeCustomReply(451, "Server error; transfer aborted");
 995         break;
 996     case STREAM_FAILED:
 997         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
 998         writeCustomReply(451, "Server error; transfer aborted");
 999         break;
1000     default:
1001         fatal("unreachable code");
1002     }
1003
1004     closeDataConnection();
1005 }
1006
1007 void
1008 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1009 {
1010     writeForwardedReply(reply);
1011     // note that the client data connection may already be closed by now
1012 }
1013
1014 void
1015 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1016 {
1017     assert(reply != NULL);
1018     const HttpHeader &header = reply->header;
1019     // adaptation and forwarding errors lack HDR_FTP_STATUS
1020     if (!header.has(HDR_FTP_STATUS)) {
1021         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1022         return;
1023     }
1024
1025     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1026     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1027     writeForwardedReplyAndCall(reply, call);
1028 }
1029
1030 void
1031 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1032 {
1033     if (getCurrentContext()->http->request->errType != ERR_NONE) {
1034         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1035         return;
1036     }
1037
1038     writeCustomReply(200, "EPRT successfully converted to PASV.");
1039
1040     // and wait for RETR
1041 }
1042
1043 void
1044 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1045 {
1046     if (getCurrentContext()->http->request->errType != ERR_NONE) {
1047         writeCustomReply(502, "Cannot connect to server", reply);
1048         return;
1049     }
1050
1051     const unsigned short localPort = listenForDataConnection();
1052     if (!localPort)
1053         return;
1054
1055     // In interception setups, we use a local port number and hope that data
1056     // traffic will be redirected to us.
1057     MemBuf mb;
1058     mb.init();
1059     mb.Printf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1060
1061     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1062     writeReply(mb);
1063 }
1064
1065 /// writes FTP error response with given status and reply-derived error details
1066 void
1067 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1068 {
1069     const HttpRequest *request = getCurrentContext()->http->request;
1070     assert(request);
1071
1072     MemBuf mb;
1073     mb.init();
1074
1075     if (request->errType != ERR_NONE)
1076         mb.Printf("%i-%s\r\n", scode, errorPageName(request->errType));
1077
1078     if (request->errDetail > 0) {
1079         // XXX: > 0 may not always mean that this is an errno
1080         mb.Printf("%i-Error: (%d) %s\r\n", scode,
1081                   request->errDetail,
1082                   strerror(request->errDetail));
1083     }
1084
1085 #if USE_ADAPTATION
1086     // XXX: Remove hard coded names. Use an error page template instead.
1087     const Adaptation::History::Pointer ah = request->adaptHistory();
1088     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1089         const String info = ah->allMeta.getByName("X-Response-Info");
1090         const String desc = ah->allMeta.getByName("X-Response-Desc");
1091         if (info.size())
1092             mb.Printf("%i-Information: %s\r\n", scode, info.termedBuf());
1093         if (desc.size())
1094             mb.Printf("%i-Description: %s\r\n", scode, desc.termedBuf());
1095     }
1096 #endif
1097
1098     assert(reply != NULL);
1099     const char *reason = reply->header.has(HDR_FTP_REASON) ?
1100                          reply->header.getStr(HDR_FTP_REASON):
1101                          reply->sline.reason();
1102
1103     mb.Printf("%i %s\r\n", scode, reason); // error terminating line
1104
1105     // TODO: errorpage.cc should detect FTP client and use
1106     // configurable FTP-friendly error templates which we should
1107     // write to the client "as is" instead of hiding most of the info
1108
1109     writeReply(mb);
1110 }
1111
1112 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1113 /// for example, internally-generated Squid "errorpages" end up here (for now)
1114 void
1115 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1116 {
1117     changeState(fssConnected, "foreign reply");
1118     closeDataConnection();
1119     // 451: We intend to keep the control connection open.
1120     writeErrorReply(reply, 451);
1121 }
1122
1123 void
1124 Ftp::Server::writeControlMsgAndCall(ClientSocketContext *context, HttpReply *reply, AsyncCall::Pointer &call)
1125 {
1126     // the caller guarantees that we are dealing with the current context only
1127     // the caller should also make sure reply->header.has(HDR_FTP_STATUS)
1128     writeForwardedReplyAndCall(reply, call);
1129 }
1130
1131 void
1132 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1133 {
1134     assert(reply != NULL);
1135     const HttpHeader &header = reply->header;
1136
1137     // without status, the caller must use the writeForwardedForeign() path
1138     Must(header.has(HDR_FTP_STATUS));
1139     Must(header.has(HDR_FTP_REASON));
1140     const int scode = header.getInt(HDR_FTP_STATUS);
1141     debugs(33, 7, "scode: " << scode);
1142
1143     // Status 125 or 150 implies upload or data request, but we still check
1144     // the state in case the server is buggy.
1145     if ((scode == 125 || scode == 150) &&
1146             (master->serverState == fssHandleUploadRequest ||
1147              master->serverState == fssHandleDataRequest)) {
1148         if (checkDataConnPost()) {
1149             // If the data connection is ready, start reading data (here)
1150             // and forward the response to client (further below).
1151             debugs(33, 7, "data connection established, start data transfer");
1152             if (master->serverState == fssHandleUploadRequest)
1153                 maybeReadUploadData();
1154         } else {
1155             // If we are waiting to accept the data connection, keep waiting.
1156             if (Comm::IsConnOpen(dataListenConn)) {
1157                 debugs(33, 7, "wait for the client to establish a data connection");
1158                 onDataAcceptCall = call;
1159                 // TODO: Add connect timeout for passive connections listener?
1160                 // TODO: Remember server response so that we can forward it?
1161             } else {
1162                 // Either the connection was establised and closed after the
1163                 // data was transferred OR we failed to establish an active
1164                 // data connection and already sent the error to the client.
1165                 // In either case, there is nothing more to do.
1166                 debugs(33, 7, "done with data OR active connection failed");
1167             }
1168             return;
1169         }
1170     }
1171
1172     MemBuf mb;
1173     mb.init();
1174     Ftp::PrintReply(mb, reply);
1175
1176     debugs(9, 2, "FTP Client " << clientConnection);
1177     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1178            "\n----------");
1179
1180     Comm::Write(clientConnection, &mb, call);
1181 }
1182
1183 static void
1184 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix)
1185 {
1186     const HttpHeader &header = reply->header;
1187
1188     HttpHeaderPos pos = HttpHeaderInitPos;
1189     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1190         if (e->id == HDR_FTP_PRE) {
1191             String raw;
1192             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1193                 mb.Printf("%s\r\n", raw.termedBuf());
1194         }
1195     }
1196
1197     if (header.has(HDR_FTP_STATUS)) {
1198         const char *reason = header.getStr(HDR_FTP_REASON);
1199         mb.Printf("%i %s\r\n", header.getInt(HDR_FTP_STATUS),
1200                   (reason ? reason : 0));
1201     }
1202 }
1203
1204 void
1205 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1206 {
1207     if (io.flag == Comm::ERR_CLOSING)
1208         return;
1209
1210     if (io.flag != Comm::OK) {
1211         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1212         io.conn->close();
1213         return;
1214     }
1215
1216     ClientSocketContext::Pointer context = getCurrentContext();
1217     if (context != NULL && context->http) {
1218         context->http->out.size += io.size;
1219         context->http->out.headers_sz += io.size;
1220     }
1221
1222     flags.readMore = true;
1223     readSomeData();
1224 }
1225
1226 void
1227 Ftp::Server::wroteReply(const CommIoCbParams &io)
1228 {
1229     if (io.flag == Comm::ERR_CLOSING)
1230         return;
1231
1232     if (io.flag != Comm::OK) {
1233         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1234         io.conn->close();
1235         return;
1236     }
1237
1238     ClientSocketContext::Pointer context = getCurrentContext();
1239     assert(context->http);
1240     context->http->out.size += io.size;
1241     context->http->out.headers_sz += io.size;
1242
1243     if (master->serverState == fssError) {
1244         debugs(33, 5, "closing on FTP server error");
1245         io.conn->close();
1246         return;
1247     }
1248
1249     const clientStream_status_t socketState = context->socketState();
1250     debugs(33, 5, "FTP client stream state " << socketState);
1251     switch (socketState) {
1252     case STREAM_UNPLANNED_COMPLETE:
1253     case STREAM_FAILED:
1254         io.conn->close();
1255         return;
1256
1257     case STREAM_NONE:
1258     case STREAM_COMPLETE:
1259         flags.readMore = true;
1260         changeState(fssConnected, "Ftp::Server::wroteReply");
1261         if (in.bodyParser)
1262             finishDechunkingRequest(false);
1263         context->keepaliveNextRequest();
1264         return;
1265     }
1266 }
1267
1268 bool
1269 Ftp::Server::handleRequest(HttpRequest *request)
1270 {
1271     debugs(33, 9, request);
1272     Must(request);
1273
1274     HttpHeader &header = request->header;
1275     Must(header.has(HDR_FTP_COMMAND));
1276     String &cmd = header.findEntry(HDR_FTP_COMMAND)->value;
1277     Must(header.has(HDR_FTP_ARGUMENTS));
1278     String &params = header.findEntry(HDR_FTP_ARGUMENTS)->value;
1279
1280     if (do_debug(9, 2)) {
1281         MemBuf mb;
1282         Packer p;
1283         mb.init();
1284         packerToMemInit(&p, &mb);
1285         request->pack(&p);
1286         packerClean(&p);
1287
1288         debugs(9, 2, "FTP Client " << clientConnection);
1289         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1290                "\n----------");
1291     }
1292
1293     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1294     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1295     static RequestHandlers handlers;
1296     if (!handlers.size()) {
1297         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1298         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1299         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1300         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1301         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1302         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1303         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1304         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1305         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1306         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1307         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1308         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1309     }
1310
1311     RequestHandler handler = NULL;
1312     if (request->method == Http::METHOD_PUT)
1313         handler = &Ftp::Server::handleUploadRequest;
1314     else {
1315         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1316         if (hi != handlers.end())
1317             handler = hi->second;
1318     }
1319
1320     if (!handler) {
1321         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1322         return true;
1323     }
1324
1325     return (this->*handler)(cmd, params);
1326 }
1327
1328 /// Called to parse USER command, which is required to create an HTTP request
1329 /// wrapper. W/o request, the errors are handled by returning earlyError().
1330 ClientSocketContext *
1331 Ftp::Server::handleUserRequest(const SBuf &cmd, SBuf &params)
1332 {
1333     if (params.isEmpty())
1334         return earlyError(eekMissingUsername);
1335
1336     // find the [end of] user name
1337     const SBuf::size_type eou = params.rfind('@');
1338     if (eou == SBuf::npos || eou + 1 >= params.length())
1339         return earlyError(eekMissingHost);
1340
1341     // Determine the intended destination.
1342     host = params.substr(eou + 1, params.length());
1343     // If we can parse it as raw IPv6 address, then surround with "[]".
1344     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1345     if (host.find(':') != SBuf::npos) {
1346         const Ip::Address ipa(host.c_str());
1347         if (!ipa.isAnyAddr()) {
1348             char ipBuf[MAX_IPSTRLEN];
1349             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1350             host = ipBuf;
1351         }
1352     }
1353
1354     // const SBuf login = params.substr(0, eou);
1355     params.chop(0, eou); // leave just the login part for the peer
1356
1357     SBuf oldUri;
1358     if (master->clientReadGreeting)
1359         oldUri = uri;
1360
1361     master->workingDir.clear();
1362     calcUri(NULL);
1363
1364     if (!master->clientReadGreeting) {
1365         debugs(9, 3, "set URI to " << uri);
1366     } else if (oldUri.caseCmp(uri) == 0) {
1367         debugs(9, 5, "kept URI as " << oldUri);
1368     } else {
1369         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1370         closeDataConnection();
1371         unpinConnection(true); // close control connection to peer
1372         resetLogin("URI reset");
1373     }
1374
1375     return NULL; // no early errors
1376 }
1377
1378 bool
1379 Ftp::Server::handleFeatRequest(String &cmd, String &params)
1380 {
1381     changeState(fssHandleFeat, "handleFeatRequest");
1382     return true;
1383 }
1384
1385 bool
1386 Ftp::Server::handlePasvRequest(String &cmd, String &params)
1387 {
1388     if (gotEpsvAll) {
1389         setReply(500, "Bad PASV command");
1390         return false;
1391     }
1392
1393     if (params.size() > 0) {
1394         setReply(501, "Unexpected parameter");
1395         return false;
1396     }
1397
1398     changeState(fssHandlePasv, "handlePasvRequest");
1399     // no need to fake PASV request via setDataCommand() in true PASV case
1400     return true;
1401 }
1402
1403 /// [Re]initializes dataConn for active data transfers. Does not connect.
1404 bool
1405 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1406 {
1407     assert(clientConnection != NULL);
1408     assert(!clientConnection->remote.isAnyAddr());
1409
1410     if (cltAddr != clientConnection->remote) {
1411         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1412         // Closing the control connection would not help with attacks because
1413         // the client is evidently able to connect to us. Besides, closing
1414         // makes retrials easier for the client and more damaging to us.
1415         setReply(501, "Prohibited parameter value");
1416         return false;
1417     }
1418
1419     closeDataConnection();
1420
1421     Comm::ConnectionPointer conn = new Comm::Connection();
1422     conn->flags |= COMM_DOBIND;
1423
1424     // Use local IP address of the control connection as the source address
1425     // of the active data connection, or some clients will refuse to accept.
1426     conn->setAddrs(clientConnection->local, cltAddr);
1427     // RFC 959 requires active FTP connections to originate from port 20
1428     // but that would preclude us from supporting concurrent transfers! (XXX?)
1429     conn->local.port(0);
1430
1431     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1432            conn->remote);
1433
1434     dataConn = conn;
1435     uploadAvailSize = 0;
1436     return true;
1437 }
1438
1439 bool
1440 Ftp::Server::handlePortRequest(String &cmd, String &params)
1441 {
1442     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1443
1444     if (gotEpsvAll) {
1445         setReply(500, "Rejecting PORT after EPSV ALL");
1446         return false;
1447     }
1448
1449     if (!params.size()) {
1450         setReply(501, "Missing parameter");
1451         return false;
1452     }
1453
1454     Ip::Address cltAddr;
1455     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1456         setReply(501, "Invalid parameter");
1457         return false;
1458     }
1459
1460     if (!createDataConnection(cltAddr))
1461         return false;
1462
1463     changeState(fssHandlePort, "handlePortRequest");
1464     setDataCommand();
1465     return true; // forward our fake PASV request
1466 }
1467
1468 bool
1469 Ftp::Server::handleDataRequest(String &cmd, String &params)
1470 {
1471     if (!checkDataConnPre())
1472         return false;
1473
1474     changeState(fssHandleDataRequest, "handleDataRequest");
1475
1476     return true;
1477 }
1478
1479 bool
1480 Ftp::Server::handleUploadRequest(String &cmd, String &params)
1481 {
1482     if (!checkDataConnPre())
1483         return false;
1484
1485     changeState(fssHandleUploadRequest, "handleDataRequest");
1486
1487     return true;
1488 }
1489
1490 bool
1491 Ftp::Server::handleEprtRequest(String &cmd, String &params)
1492 {
1493     debugs(9, 3, "Process an EPRT " << params);
1494
1495     if (gotEpsvAll) {
1496         setReply(500, "Rejecting EPRT after EPSV ALL");
1497         return false;
1498     }
1499
1500     if (!params.size()) {
1501         setReply(501, "Missing parameter");
1502         return false;
1503     }
1504
1505     Ip::Address cltAddr;
1506     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1507         setReply(501, "Invalid parameter");
1508         return false;
1509     }
1510
1511     if (!createDataConnection(cltAddr))
1512         return false;
1513
1514     changeState(fssHandleEprt, "handleEprtRequest");
1515     setDataCommand();
1516     return true; // forward our fake PASV request
1517 }
1518
1519 bool
1520 Ftp::Server::handleEpsvRequest(String &cmd, String &params)
1521 {
1522     debugs(9, 3, "Process an EPSV command with params: " << params);
1523     if (params.size() <= 0) {
1524         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1525     } else if (params.caseCmp("ALL") == 0) {
1526         setReply(200, "EPSV ALL ok");
1527         gotEpsvAll = true;
1528         return false;
1529     } else if (params.cmp("2") == 0) {
1530         if (!Ip::EnableIpv6) {
1531             setReply(522, "Network protocol not supported, use (1)");
1532             return false;
1533         }
1534     } else if (params.cmp("1") != 0) {
1535         setReply(501, "Unsupported EPSV parameter");
1536         return false;
1537     }
1538
1539     changeState(fssHandleEpsv, "handleEpsvRequest");
1540     setDataCommand();
1541     return true; // forward our fake PASV request
1542 }
1543
1544 bool
1545 Ftp::Server::handleCwdRequest(String &cmd, String &params)
1546 {
1547     changeState(fssHandleCwd, "handleCwdRequest");
1548     return true;
1549 }
1550
1551 bool
1552 Ftp::Server::handlePassRequest(String &cmd, String &params)
1553 {
1554     changeState(fssHandlePass, "handlePassRequest");
1555     return true;
1556 }
1557
1558 bool
1559 Ftp::Server::handleCdupRequest(String &cmd, String &params)
1560 {
1561     changeState(fssHandleCdup, "handleCdupRequest");
1562     return true;
1563 }
1564
1565 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1566 // Squid FTP client decides what data command to use with peers.
1567 void
1568 Ftp::Server::setDataCommand()
1569 {
1570     ClientHttpRequest *const http = getCurrentContext()->http;
1571     assert(http != NULL);
1572     HttpRequest *const request = http->request;
1573     assert(request != NULL);
1574     HttpHeader &header = request->header;
1575     header.delById(HDR_FTP_COMMAND);
1576     header.putStr(HDR_FTP_COMMAND, "PASV");
1577     header.delById(HDR_FTP_ARGUMENTS);
1578     header.putStr(HDR_FTP_ARGUMENTS, "");
1579     debugs(9, 5, "client data command converted to fake PASV");
1580 }
1581
1582 /// check that client data connection is ready for future I/O or at least
1583 /// has a chance of becoming ready soon.
1584 bool
1585 Ftp::Server::checkDataConnPre()
1586 {
1587     if (Comm::IsConnOpen(dataConn))
1588         return true;
1589
1590     if (Comm::IsConnOpen(dataListenConn)) {
1591         // We are still waiting for a client to connect to us after PASV.
1592         // Perhaps client's data conn handshake has not reached us yet.
1593         // After we talk to the server, checkDataConnPost() will recheck.
1594         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1595         return true;
1596     }
1597
1598     if (!dataConn || dataConn->remote.isAnyAddr()) {
1599         debugs(33, 5, "missing " << dataConn);
1600         // TODO: use client address and default port instead.
1601         setReply(425, "Use PORT or PASV first");
1602         return false;
1603     }
1604
1605     // active transfer: open a data connection from Squid to client
1606     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1607     connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1608     Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector,
1609             Config.Timeout.connect);
1610     AsyncJob::Start(cs);
1611     return false; // ConnStateData::processFtpRequest waits handleConnectDone
1612 }
1613
1614 /// Check that client data connection is ready for immediate I/O.
1615 bool
1616 Ftp::Server::checkDataConnPost() const
1617 {
1618     if (!Comm::IsConnOpen(dataConn)) {
1619         debugs(33, 3, "missing client data conn: " << dataConn);
1620         return false;
1621     }
1622     return true;
1623 }
1624
1625 /// Done establishing a data connection to the user.
1626 void
1627 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1628 {
1629     connector = NULL;
1630
1631     if (params.flag != Comm::OK) {
1632         /* it might have been a timeout with a partially open link */
1633         if (params.conn != NULL)
1634             params.conn->close();
1635         setReply(425, "Cannot open data connection.");
1636         ClientSocketContext::Pointer context = getCurrentContext();
1637         Must(context->http);
1638         Must(context->http->storeEntry() != NULL);
1639     } else {
1640         Must(dataConn == params.conn);
1641         Must(Comm::IsConnOpen(params.conn));
1642         fd_note(params.conn->fd, "active client ftp data");
1643     }
1644
1645     doProcessRequest();
1646 }
1647
1648 void
1649 Ftp::Server::setReply(const int code, const char *msg)
1650 {
1651     ClientSocketContext::Pointer context = getCurrentContext();
1652     ClientHttpRequest *const http = context->http;
1653     assert(http != NULL);
1654     assert(http->storeEntry() == NULL);
1655
1656     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1657
1658     setLogUri(http, urlCanonicalClean(http->request));
1659
1660     clientStreamNode *const node = context->getClientReplyContext();
1661     clientReplyContext *const repContext =
1662         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1663     assert(repContext != NULL);
1664
1665     RequestFlags reqFlags;
1666     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1667     reqFlags.noCache = true;
1668     repContext->createStoreEntry(http->request->method, reqFlags);
1669     http->storeEntry()->replaceHttpReply(reply);
1670 }
1671
1672 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1673 static bool
1674 Ftp::SupportedCommand(const SBuf &name)
1675 {
1676     static std::set<SBuf> BlackList;
1677     if (BlackList.empty()) {
1678         /* Add FTP commands that Squid cannot relay correctly. */
1679
1680         // We probably do not support AUTH TLS.* and AUTH SSL,
1681         // but let's disclaim all AUTH support to KISS, for now.
1682         BlackList.insert(cmdAuth());
1683     }
1684
1685     // we claim support for all commands that we do not know about
1686     return BlackList.find(name) == BlackList.end();
1687 }
1688