src/servers/FtpServer.cc

   1 /*
   2  * Copyright (C) 1996-2022 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Transfer protocol servers */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CharacterSet.h"
  14 #include "base/RefCount.h"
  15 #include "base/Subscription.h"
  16 #include "client_side_reply.h"
  17 #include "client_side_request.h"
  18 #include "clientStream.h"
  19 #include "comm/ConnOpener.h"
  20 #include "comm/Read.h"
  21 #include "comm/TcpAcceptor.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "ftp/Elements.h"
  26 #include "ftp/Parsing.h"
  27 #include "globals.h"
  28 #include "http/one/RequestParser.h"
  29 #include "http/Stream.h"
  30 #include "HttpHdrCc.h"
  31 #include "ip/tools.h"
  32 #include "ipc/FdNotes.h"
  33 #include "parser/Tokenizer.h"
  34 #include "servers/forward.h"
  35 #include "servers/FtpServer.h"
  36 #include "SquidConfig.h"
  37 #include "StatCounters.h"
  38 #include "tools.h"
  39
  40 #include <set>
  41 #include <map>
  42
  43 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  44
  45 namespace Ftp
  46 {
  47 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  48 static bool SupportedCommand(const SBuf &name);
  49 static bool CommandHasPathParameter(const SBuf &cmd);
  50 };
  51
  52 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  53     AsyncJob("Ftp::Server"),
  54     ConnStateData(xact),
  55     master(new MasterState),
  56     uri(),
  57     host(),
  58     gotEpsvAll(false),
  59     onDataAcceptCall(),
  60     dataListenConn(),
  61     dataConn(),
  62     uploadAvailSize(0),
  63     listener(),
  64     dataConnWait(),
  65     reader(),
  66     waitingForOrigin(false),
  67     originDataDownloadAbortedOnError(false)
  68 {
  69     flags.readMore = false; // we need to announce ourselves first
  70     *uploadBuf = 0;
  71 }
  72
  73 Ftp::Server::~Server()
  74 {
  75     closeDataConnection();
  76 }
  77
  78 int
  79 Ftp::Server::pipelinePrefetchMax() const
  80 {
  81     return 0; // no support for concurrent FTP requests
  82 }
  83
  84 time_t
  85 Ftp::Server::idleTimeout() const
  86 {
  87     return Config.Timeout.ftpClientIdle;
  88 }
  89
  90 void
  91 Ftp::Server::start()
  92 {
  93     ConnStateData::start();
  94
  95     if (transparent()) {
  96         char buf[MAX_IPSTRLEN];
  97         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  98         host = buf;
  99         calcUri(NULL);
 100         debugs(33, 5, "FTP transparent URL: " << uri);
 101     }
 102
 103     writeEarlyReply(220, "Service ready");
 104 }
 105
 106 /// schedules another data connection read if needed
 107 void
 108 Ftp::Server::maybeReadUploadData()
 109 {
 110     if (reader != NULL)
 111         return;
 112
 113     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 114     if (availSpace <= 0)
 115         return;
 116
 117     debugs(33, 4, dataConn << ": reading FTP data...");
 118
 119     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 120     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 121     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 122               reader);
 123 }
 124
 125 /// react to the freshly parsed request
 126 void
 127 Ftp::Server::doProcessRequest()
 128 {
 129     // zero pipelinePrefetchMax() ensures that there is only parsed request
 130     Must(pipeline.count() == 1);
 131     Http::StreamPointer context = pipeline.front();
 132     Must(context != nullptr);
 133
 134     ClientHttpRequest *const http = context->http;
 135     assert(http != NULL);
 136
 137     HttpRequest *const request = http->request;
 138     Must(http->storeEntry() || request);
 139     const bool mayForward = !http->storeEntry() && handleRequest(request);
 140
 141     if (http->storeEntry() != NULL) {
 142         debugs(33, 4, "got an immediate response");
 143         clientSetKeepaliveFlag(http);
 144         context->pullData();
 145     } else if (mayForward) {
 146         debugs(33, 4, "forwarding request to server side");
 147         assert(http->storeEntry() == NULL);
 148         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 149     } else {
 150         debugs(33, 4, "will resume processing later");
 151     }
 152 }
 153
 154 void
 155 Ftp::Server::processParsedRequest(Http::StreamPointer &)
 156 {
 157     Must(pipeline.count() == 1);
 158
 159     // Process FTP request asynchronously to make sure FTP
 160     // data connection accept callback is fired first.
 161     CallJobHere(33, 4, CbcPointer<Server>(this),
 162                 Ftp::Server, doProcessRequest);
 163 }
 164
 165 /// imports more upload data from the data connection
 166 void
 167 Ftp::Server::readUploadData(const CommIoCbParams &io)
 168 {
 169     debugs(33, 5, io.conn << " size " << io.size);
 170     Must(reader != NULL);
 171     reader = NULL;
 172
 173     assert(Comm::IsConnOpen(dataConn));
 174     assert(io.conn->fd == dataConn->fd);
 175
 176     if (io.flag == Comm::OK && bodyPipe != NULL) {
 177         if (io.size > 0) {
 178             statCounter.client_http.kbytes_in += io.size;
 179
 180             char *const current_buf = uploadBuf + uploadAvailSize;
 181             if (io.buf != current_buf)
 182                 memmove(current_buf, io.buf, io.size);
 183             uploadAvailSize += io.size;
 184             shovelUploadData();
 185         } else if (io.size == 0) {
 186             debugs(33, 5, io.conn << " closed");
 187             closeDataConnection();
 188             if (uploadAvailSize <= 0)
 189                 finishDechunkingRequest(true);
 190         }
 191     } else { // not Comm::Flags::OK or unexpected read
 192         debugs(33, 5, io.conn << " closed");
 193         closeDataConnection();
 194         finishDechunkingRequest(false);
 195     }
 196
 197 }
 198
 199 /// shovel upload data from the internal buffer to the body pipe if possible
 200 void
 201 Ftp::Server::shovelUploadData()
 202 {
 203     assert(bodyPipe != NULL);
 204
 205     debugs(33, 5, "handling FTP request data for " << clientConnection);
 206     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 207                            uploadAvailSize);
 208     if (putSize > 0) {
 209         uploadAvailSize -= putSize;
 210         if (uploadAvailSize > 0)
 211             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 212     }
 213
 214     if (Comm::IsConnOpen(dataConn))
 215         maybeReadUploadData();
 216     else if (uploadAvailSize <= 0)
 217         finishDechunkingRequest(true);
 218 }
 219
 220 void
 221 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 222 {
 223     if (!isOpen()) // if we are closing, nothing to do
 224         return;
 225
 226     shovelUploadData();
 227 }
 228
 229 void
 230 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 231 {
 232     if (!isOpen()) // if we are closing, nothing to do
 233         return;
 234
 235     ConnStateData::noteBodyConsumerAborted(ptr);
 236     closeDataConnection();
 237 }
 238
 239 /// accept a new FTP control connection and hand it to a dedicated Server
 240 void
 241 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 242 {
 243     MasterXaction::Pointer xact = params.xaction;
 244     AnyP::PortCfgPointer s = xact->squidPort;
 245
 246     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 247
 248     if (params.flag != Comm::OK) {
 249         // Its possible the call was still queued when the client disconnected
 250         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 251         return;
 252     }
 253
 254     debugs(33, 4, params.conn << ": accepted");
 255     fd_note(params.conn->fd, "client ftp connect");
 256
 257     AsyncJob::Start(new Server(xact));
 258 }
 259
 260 void
 261 Ftp::StartListening()
 262 {
 263     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 264         if (MAXTCPLISTENPORTS == NHttpSockets) {
 265             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 266                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 267             break;
 268         }
 269
 270         // direct new connections accepted by listenConn to Accept()
 271         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 272         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 273                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 274                                                CommAcceptCbParams(NULL)));
 275         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 276     }
 277 }
 278
 279 void
 280 Ftp::StopListening()
 281 {
 282     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 283         if (s->listenConn != NULL) {
 284             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 285             s->listenConn->close();
 286             s->listenConn = NULL;
 287         }
 288     }
 289 }
 290
 291 void
 292 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 293 {
 294     // find request
 295     Http::StreamPointer context = pipeline.front();
 296     Must(context != nullptr);
 297     ClientHttpRequest *const http = context->http;
 298     Must(http != NULL);
 299     HttpRequest *const request = http->request;
 300     Must(request != NULL);
 301     // make FTP peer connection exclusive to our request
 302     pinBusyConnection(conn, request);
 303 }
 304
 305 void
 306 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 307 {
 308     ConnStateData::clientPinnedConnectionClosed(io);
 309
 310     // TODO: Keep the control connection open after fixing the reset
 311     // problem below
 312     if (Comm::IsConnOpen(clientConnection))
 313         clientConnection->close();
 314
 315     // TODO: If the server control connection is gone, reset state to login
 316     // again. Resetting login alone is not enough: FtpRelay::sendCommand() will
 317     // not re-login because FtpRelay::serverState() is not going to be
 318     // fssConnected. Calling resetLogin() alone is also harmful because
 319     // it does not reset correctly the client-to-squid control connection (eg
 320     // respond if required with an error code, in all cases)
 321     // resetLogin("control connection closure");
 322 }
 323
 324 /// clear client and server login-related state after the old login is gone
 325 void
 326 Ftp::Server::resetLogin(const char *reason)
 327 {
 328     debugs(33, 5, "will need to re-login due to " << reason);
 329     master->clientReadGreeting = false;
 330     changeState(fssBegin, reason);
 331 }
 332
 333 /// computes uri member from host and, if tracked, working dir with file name
 334 void
 335 Ftp::Server::calcUri(const SBuf *file)
 336 {
 337     // TODO: fill a class AnyP::Uri instead of string
 338     uri = "ftp://";
 339     uri.append(host);
 340     if (port->ftp_track_dirs && master->workingDir.length()) {
 341         if (master->workingDir[0] != '/')
 342             uri.append("/", 1);
 343         uri.append(master->workingDir);
 344     }
 345
 346     if (uri[uri.length() - 1] != '/')
 347         uri.append("/", 1);
 348
 349     if (port->ftp_track_dirs && file) {
 350         static const CharacterSet Slash("/", "/");
 351         Parser::Tokenizer tok(*file);
 352         tok.skipAll(Slash);
 353         uri.append(tok.remaining());
 354     }
 355 }
 356
 357 /// Starts waiting for a data connection. Returns listening port.
 358 /// On errors, responds with an error and returns zero.
 359 unsigned int
 360 Ftp::Server::listenForDataConnection()
 361 {
 362     closeDataConnection();
 363
 364     Comm::ConnectionPointer conn = new Comm::Connection;
 365     conn->flags = COMM_NONBLOCKING;
 366     conn->local = transparent() ? port->s : clientConnection->local;
 367     conn->local.port(0);
 368     const char *const note = uri.c_str();
 369     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 370     if (!Comm::IsConnOpen(conn)) {
 371         debugs(5, DBG_CRITICAL, "ERROR: comm_open_listener failed for FTP data: " <<
 372                conn->local << " error: " << errno);
 373         writeCustomReply(451, "Internal error");
 374         return 0;
 375     }
 376
 377     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 378     typedef AsyncCallT<AcceptDialer> AcceptCall;
 379     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 380     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 381     listener = call.getRaw();
 382     dataListenConn = conn;
 383     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 384
 385     const unsigned int listeningPort = comm_local_port(conn->fd);
 386     conn->local.port(listeningPort);
 387     return listeningPort;
 388 }
 389
 390 void
 391 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 392 {
 393     if (params.flag != Comm::OK) {
 394         // Its possible the call was still queued when the client disconnected
 395         debugs(33, 2, dataListenConn << ": accept "
 396                "failure: " << xstrerr(params.xerrno));
 397         return;
 398     }
 399
 400     debugs(33, 4, "accepted " << params.conn);
 401     fd_note(params.conn->fd, "passive client ftp data");
 402
 403     if (!clientConnection) {
 404         debugs(33, 5, "late data connection?");
 405         closeDataConnection(); // in case we are still listening
 406         params.conn->close();
 407     } else if (params.conn->remote != clientConnection->remote) {
 408         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 409         params.conn->close();
 410         // Some FTP servers close control connection here, but it may make
 411         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 412     } else {
 413         closeDataConnection();
 414         dataConn = params.conn;
 415         dataConn->leaveOrphanage();
 416         uploadAvailSize = 0;
 417         debugs(33, 7, "ready for data");
 418         if (onDataAcceptCall != NULL) {
 419             AsyncCall::Pointer call = onDataAcceptCall;
 420             onDataAcceptCall = NULL;
 421             // If we got an upload request, start reading data from the client.
 422             if (master->serverState == fssHandleUploadRequest)
 423                 maybeReadUploadData();
 424             else
 425                 Must(master->serverState == fssHandleDataRequest);
 426             MemBuf mb;
 427             mb.init();
 428             mb.appendf("150 Data connection opened.\r\n");
 429             Comm::Write(clientConnection, &mb, call);
 430         }
 431     }
 432 }
 433
 434 void
 435 Ftp::Server::closeDataConnection()
 436 {
 437     if (listener != NULL) {
 438         listener->cancel("no longer needed");
 439         listener = NULL;
 440     }
 441
 442     if (Comm::IsConnOpen(dataListenConn)) {
 443         debugs(33, 5, "FTP closing client data listen socket: " <<
 444                *dataListenConn);
 445         dataListenConn->close();
 446     }
 447     dataListenConn = NULL;
 448
 449     if (reader != NULL) {
 450         // Comm::ReadCancel can deal with negative FDs
 451         Comm::ReadCancel(dataConn->fd, reader);
 452         reader = NULL;
 453     }
 454
 455     if (Comm::IsConnOpen(dataConn)) {
 456         debugs(33, 5, "FTP closing client data connection: " <<
 457                *dataConn);
 458         dataConn->close();
 459     }
 460     dataConn = NULL;
 461 }
 462
 463 /// Writes FTP [error] response before we fully parsed the FTP request and
 464 /// created the corresponding HTTP request wrapper for that FTP request.
 465 void
 466 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 467 {
 468     debugs(33, 7, code << ' ' << msg);
 469     assert(99 < code && code < 1000);
 470
 471     MemBuf mb;
 472     mb.init();
 473     mb.appendf("%i %s\r\n", code, msg);
 474
 475     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 476     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 477     Comm::Write(clientConnection, &mb, call);
 478
 479     flags.readMore = false;
 480
 481     // TODO: Create master transaction. Log it in wroteEarlyReply().
 482 }
 483
 484 void
 485 Ftp::Server::writeReply(MemBuf &mb)
 486 {
 487     debugs(9, 2, "FTP Client " << clientConnection);
 488     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 489            "\n----------");
 490
 491     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 492     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 493     Comm::Write(clientConnection, &mb, call);
 494 }
 495
 496 void
 497 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 498 {
 499     debugs(33, 7, code << ' ' << msg);
 500     assert(99 < code && code < 1000);
 501
 502     const bool sendDetails = reply != NULL &&
 503                              reply->header.has(Http::HdrType::FTP_STATUS) && reply->header.has(Http::HdrType::FTP_REASON);
 504
 505     MemBuf mb;
 506     mb.init();
 507     if (sendDetails) {
 508         mb.appendf("%i-%s\r\n", code, msg);
 509         mb.appendf(" Server reply:\r\n");
 510         Ftp::PrintReply(mb, reply, " ");
 511         mb.appendf("%i \r\n", code);
 512     } else
 513         mb.appendf("%i %s\r\n", code, msg);
 514
 515     writeReply(mb);
 516 }
 517
 518 void
 519 Ftp::Server::changeState(const ServerState newState, const char *reason)
 520 {
 521     if (master->serverState == newState) {
 522         debugs(33, 3, "client state unchanged at " << master->serverState <<
 523                " because " << reason);
 524         master->serverState = newState;
 525     } else {
 526         debugs(33, 3, "client state was " << master->serverState <<
 527                ", now " << newState << " because " << reason);
 528         master->serverState = newState;
 529     }
 530 }
 531
 532 /// whether the given FTP command has a pathname parameter
 533 static bool
 534 Ftp::CommandHasPathParameter(const SBuf &cmd)
 535 {
 536     static std::set<SBuf> PathedCommands;
 537     if (!PathedCommands.size()) {
 538         PathedCommands.insert(cmdMlst());
 539         PathedCommands.insert(cmdMlsd());
 540         PathedCommands.insert(cmdStat());
 541         PathedCommands.insert(cmdNlst());
 542         PathedCommands.insert(cmdList());
 543         PathedCommands.insert(cmdMkd());
 544         PathedCommands.insert(cmdRmd());
 545         PathedCommands.insert(cmdDele());
 546         PathedCommands.insert(cmdRnto());
 547         PathedCommands.insert(cmdRnfr());
 548         PathedCommands.insert(cmdAppe());
 549         PathedCommands.insert(cmdStor());
 550         PathedCommands.insert(cmdRetr());
 551         PathedCommands.insert(cmdSmnt());
 552         PathedCommands.insert(cmdCwd());
 553     }
 554
 555     return PathedCommands.find(cmd) != PathedCommands.end();
 556 }
 557
 558 /// creates a context filled with an error message for a given early error
 559 Http::Stream *
 560 Ftp::Server::earlyError(const EarlyErrorKind eek)
 561 {
 562     /* Default values, to be updated by the switch statement below */
 563     int scode = 421;
 564     const char *reason = "Internal error";
 565     const char *errUri = "error:ftp-internal-early-error";
 566
 567     switch (eek) {
 568     case EarlyErrorKind::HugeRequest:
 569         scode = 421;
 570         reason = "Huge request";
 571         errUri = "error:ftp-huge-request";
 572         break;
 573
 574     case EarlyErrorKind::MissingLogin:
 575         scode = 530;
 576         reason = "Must login first";
 577         errUri = "error:ftp-must-login-first";
 578         break;
 579
 580     case EarlyErrorKind::MissingUsername:
 581         scode = 501;
 582         reason = "Missing username";
 583         errUri = "error:ftp-missing-username";
 584         break;
 585
 586     case EarlyErrorKind::MissingHost:
 587         scode = 501;
 588         reason = "Missing host";
 589         errUri = "error:ftp-missing-host";
 590         break;
 591
 592     case EarlyErrorKind::UnsupportedCommand:
 593         scode = 502;
 594         reason = "Unknown or unsupported command";
 595         errUri = "error:ftp-unsupported-command";
 596         break;
 597
 598     case EarlyErrorKind::InvalidUri:
 599         scode = 501;
 600         reason = "Invalid URI";
 601         errUri = "error:ftp-invalid-uri";
 602         break;
 603
 604     case EarlyErrorKind::MalformedCommand:
 605         scode = 421;
 606         reason = "Malformed command";
 607         errUri = "error:ftp-malformed-command";
 608         break;
 609
 610         // no default so that a compiler can check that we have covered all cases
 611     }
 612
 613     Http::Stream *context = abortRequestParsing(errUri);
 614     clientStreamNode *node = context->getClientReplyContext();
 615     Must(node);
 616     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 617     Must(repContext);
 618
 619     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 620     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 621     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 622     repContext->setReplyToReply(reply);
 623     return context;
 624 }
 625
 626 /// Parses a single FTP request on the control connection.
 627 /// Returns a new Http::Stream on valid requests and all errors.
 628 /// Returns NULL on incomplete requests that may still succeed given more data.
 629 Http::Stream *
 630 Ftp::Server::parseOneRequest()
 631 {
 632     flags.readMore = false; // common for all but one case below
 633
 634     // OWS <command> [ RWS <parameter> ] OWS LF
 635
 636     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 637     // for the LF character that we must exclude here (but see FullWhiteSpace).
 638     static const char * const InlineSpaceChars = " \f\r\t\v";
 639     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 640     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 641     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 642     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 643
 644     // This set is used to ignore empty commands without allowing an attacker
 645     // to keep us endlessly busy by feeding us whitespace or empty commands.
 646     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 647
 648     SBuf cmd;
 649     SBuf params;
 650
 651     Parser::Tokenizer tok(inBuf);
 652
 653     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 654     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 655
 656     // note that the condition below will eat either RWS or trailing OWS
 657     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 658         // now params may include trailing OWS
 659         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 660         static const SBuf bufWhiteSpace(InlineSpaceChars);
 661         params.trim(bufWhiteSpace, false, true);
 662     }
 663
 664     // Why limit command line and parameters size? Did not we just parse them?
 665     // XXX: Our good old String cannot handle very long strings.
 666     const SBuf::size_type tokenMax = min(
 667                                          static_cast<SBuf::size_type>(32*1024), // conservative
 668                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 669     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 670         changeState(fssError, "huge req token");
 671         quitAfterError(NULL);
 672         return earlyError(EarlyErrorKind::HugeRequest);
 673     }
 674
 675     // technically, we may skip multiple NLs below, but that is OK
 676     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 677         // we need more data, but can we buffer more?
 678         if (inBuf.length() >= Config.maxRequestHeaderSize) {
 679             changeState(fssError, "huge req");
 680             quitAfterError(NULL);
 681             return earlyError(EarlyErrorKind::HugeRequest);
 682         } else {
 683             flags.readMore = true;
 684             debugs(33, 5, "Waiting for more, up to " <<
 685                    (Config.maxRequestHeaderSize - inBuf.length()));
 686             return NULL;
 687         }
 688     }
 689
 690     Must(parsed && cmd.length());
 691     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 692
 693     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 694
 695     cmd.toUpper(); // this should speed up and simplify future comparisons
 696
 697     // interception cases do not need USER to calculate the uri
 698     if (!transparent()) {
 699         if (!master->clientReadGreeting) {
 700             // the first command must be USER
 701             if (!pinning.pinned && cmd != cmdUser())
 702                 return earlyError(EarlyErrorKind::MissingLogin);
 703         }
 704
 705         // process USER request now because it sets FTP peer host name
 706         if (cmd == cmdUser()) {
 707             if (Http::Stream *errCtx = handleUserRequest(cmd, params))
 708                 return errCtx;
 709         }
 710     }
 711
 712     if (!Ftp::SupportedCommand(cmd))
 713         return earlyError(EarlyErrorKind::UnsupportedCommand);
 714
 715     const HttpRequestMethod method =
 716         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 717         Http::METHOD_PUT : Http::METHOD_GET;
 718
 719     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 720                        &params : NULL;
 721     calcUri(path);
 722     MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient);
 723     mx->tcpClient = clientConnection;
 724     auto * const request = HttpRequest::FromUrl(uri, mx, method);
 725     if (!request) {
 726         debugs(33, 5, "Invalid FTP URL: " << uri);
 727         uri.clear();
 728         return earlyError(EarlyErrorKind::InvalidUri);
 729     }
 730     char *newUri = xstrdup(uri.c_str());
 731
 732     request->flags.ftpNative = true;
 733     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 734
 735     // Our fake Request-URIs are not distinctive enough for caching to work
 736     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 737     request->flags.noCache = true;
 738
 739     request->header.putStr(Http::HdrType::FTP_COMMAND, cmd.c_str());
 740     request->header.putStr(Http::HdrType::FTP_ARGUMENTS, params.c_str()); // may be ""
 741     if (method == Http::METHOD_PUT) {
 742         request->header.putStr(Http::HdrType::EXPECT, "100-continue");
 743         request->header.putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
 744     }
 745
 746     ClientHttpRequest *const http = new ClientHttpRequest(this);
 747     http->req_sz = tok.parsedSize();
 748     http->uri = newUri;
 749     http->initRequest(request);
 750
 751     Http::Stream *const result =
 752         new Http::Stream(clientConnection, http);
 753
 754     StoreIOBuffer tempBuffer;
 755     tempBuffer.data = result->reqbuf;
 756     tempBuffer.length = HTTP_REQBUF_SZ;
 757
 758     ClientStreamData newServer = new clientReplyContext(http);
 759     ClientStreamData newClient = result;
 760     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 761                      clientReplyStatus, newServer, clientSocketRecipient,
 762                      clientSocketDetach, newClient, tempBuffer);
 763
 764     result->flags.parsed_ok = 1;
 765     return result;
 766 }
 767
 768 void
 769 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 770 {
 771     // the caller guarantees that we are dealing with the current context only
 772     Http::StreamPointer context = pipeline.front();
 773     assert(context != nullptr);
 774
 775     static ReplyHandler handlers[] = {
 776         NULL, // fssBegin
 777         NULL, // fssConnected
 778         &Ftp::Server::handleFeatReply, // fssHandleFeat
 779         &Ftp::Server::handlePasvReply, // fssHandlePasv
 780         &Ftp::Server::handlePortReply, // fssHandlePort
 781         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 782         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 783         &Ftp::Server::handleEprtReply,// fssHandleEprt
 784         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 785         NULL, // fssHandleCwd
 786         NULL, // fssHandlePass
 787         NULL, // fssHandleCdup
 788         &Ftp::Server::handleErrorReply // fssError
 789     };
 790     try {
 791         const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 792         if (const ReplyHandler handler = handlers[server.master->serverState])
 793             (this->*handler)(reply, data);
 794         else
 795             writeForwardedReply(reply);
 796     } catch (const std::exception &e) {
 797         callException(e);
 798         throw TexcHere(e.what());
 799     }
 800 }
 801
 802 void
 803 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 804 {
 805     if (pipeline.front()->http->request->error) {
 806         writeCustomReply(502, "Server does not support FEAT", reply);
 807         return;
 808     }
 809
 810     Must(reply);
 811     HttpReply::Pointer featReply = Ftp::HttpReplyWrapper(211, "End", Http::scNoContent, 0);
 812     HttpHeader const &serverReplyHeader = reply->header;
 813
 814     HttpHeaderPos pos = HttpHeaderInitPos;
 815     bool hasEPRT = false;
 816     bool hasEPSV = false;
 817     int prependSpaces = 1;
 818
 819     featReply->header.putStr(Http::HdrType::FTP_PRE, "\"211-Features:\"");
 820     const int scode = serverReplyHeader.getInt(Http::HdrType::FTP_STATUS);
 821     if (scode == 211) {
 822         while (const HttpHeaderEntry *e = serverReplyHeader.getEntry(&pos)) {
 823             if (e->id == Http::HdrType::FTP_PRE) {
 824                 // assume RFC 2389 FEAT response format, quoted by Squid:
 825                 // <"> SP NAME [SP PARAMS] <">
 826                 // but accommodate MS servers sending four SPs before NAME
 827
 828                 // command name ends with (SP parameter) or quote
 829                 static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 830                 static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 831
 832                 Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 833                 if (!tok.skip('"') || !tok.skip(' '))
 834                     continue;
 835
 836                 // optional spaces; remember their number to accommodate MS servers
 837                 prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 838
 839                 SBuf cmd;
 840                 if (!tok.prefix(cmd, FeatNameChars))
 841                     continue;
 842                 cmd.toUpper();
 843
 844                 if (Ftp::SupportedCommand(cmd)) {
 845                     featReply->header.addEntry(e->clone());
 846                 }
 847
 848                 if (cmd == cmdEprt())
 849                     hasEPRT = true;
 850                 else if (cmd == cmdEpsv())
 851                     hasEPSV = true;
 852             }
 853         }
 854     } // else we got a FEAT error and will only report Squid-supported features
 855
 856     char buf[256];
 857     if (!hasEPRT) {
 858         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 859         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 860     }
 861     if (!hasEPSV) {
 862         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 863         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 864     }
 865
 866     featReply->header.refreshMask();
 867
 868     writeForwardedReply(featReply.getRaw());
 869 }
 870
 871 void
 872 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 873 {
 874     const Http::StreamPointer context(pipeline.front());
 875     assert(context != nullptr);
 876
 877     if (context->http->request->error) {
 878         writeCustomReply(502, "Server does not support PASV", reply);
 879         return;
 880     }
 881
 882     const unsigned short localPort = listenForDataConnection();
 883     if (!localPort)
 884         return;
 885
 886     char addr[MAX_IPSTRLEN];
 887     // remote server in interception setups and local address otherwise
 888     const Ip::Address &server = transparent() ?
 889                                 clientConnection->local : dataListenConn->local;
 890     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 891     addr[MAX_IPSTRLEN - 1] = '\0';
 892     for (char *c = addr; *c != '\0'; ++c) {
 893         if (*c == '.')
 894             *c = ',';
 895     }
 896
 897     // In interception setups, we combine remote server address with a
 898     // local port number and hope that traffic will be redirected to us.
 899     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 900     // versions block responses that use those alternative syntax rules!
 901     MemBuf mb;
 902     mb.init();
 903     mb.appendf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 904                addr,
 905                static_cast<int>(localPort / 256),
 906                static_cast<int>(localPort % 256));
 907     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 908     writeReply(mb);
 909 }
 910
 911 void
 912 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 913 {
 914     if (pipeline.front()->http->request->error) {
 915         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 916         return;
 917     }
 918
 919     writeCustomReply(200, "PORT successfully converted to PASV.");
 920
 921     // and wait for RETR
 922 }
 923
 924 void
 925 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 926 {
 927     if (!pinning.pinned) // we failed to connect to server
 928         uri.clear();
 929     // 421: we will close due to fssError
 930     writeErrorReply(reply, 421);
 931 }
 932
 933 void
 934 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 935 {
 936     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 937         writeForwardedReply(reply);
 938         if (Comm::IsConnOpen(dataConn)) {
 939             debugs(33, 3, "closing " << dataConn << " on KO reply");
 940             closeDataConnection();
 941         }
 942         return;
 943     }
 944
 945     if (!dataConn) {
 946         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 947         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 948         return;
 949     }
 950
 951     if (!checkDataConnPost()) {
 952         writeCustomReply(425, "Data connection is not established.");
 953         closeDataConnection();
 954         return;
 955     }
 956
 957     debugs(33, 7, data.length);
 958
 959     if (data.length <= 0) {
 960         replyDataWritingCheckpoint(); // skip the actual write call
 961         return;
 962     }
 963
 964     MemBuf mb;
 965     mb.init(data.length + 1, data.length + 1);
 966     mb.append(data.data, data.length);
 967
 968     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 969     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 970     Comm::Write(dataConn, &mb, call);
 971
 972     pipeline.front()->noteSentBodyBytes(data.length);
 973 }
 974
 975 /// called when we are done writing a chunk of the response data
 976 void
 977 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 978 {
 979     if (io.flag == Comm::ERR_CLOSING)
 980         return;
 981
 982     if (io.flag != Comm::OK) {
 983         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 984         userDataCompletionCheckpoint(426);
 985         return;
 986     }
 987
 988     assert(pipeline.front()->http);
 989     pipeline.front()->http->out.size += io.size;
 990     replyDataWritingCheckpoint();
 991 }
 992
 993 /// ClientStream checks after (actual or skipped) reply data writing
 994 void
 995 Ftp::Server::replyDataWritingCheckpoint()
 996 {
 997     switch (pipeline.front()->socketState()) {
 998     case STREAM_NONE:
 999         debugs(33, 3, "Keep going");
1000         pipeline.front()->pullData();
1001         return;
1002     case STREAM_COMPLETE:
1003         debugs(33, 3, "FTP reply data transfer successfully complete");
1004         userDataCompletionCheckpoint(226);
1005         break;
1006     case STREAM_UNPLANNED_COMPLETE:
1007         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
1008         userDataCompletionCheckpoint(451);
1009         break;
1010     case STREAM_FAILED:
1011         userDataCompletionCheckpoint(451);
1012         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
1013         break;
1014     default:
1015         fatal("unreachable code");
1016     }
1017 }
1018
1019 void
1020 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1021 {
1022     writeForwardedReply(reply);
1023     // note that the client data connection may already be closed by now
1024 }
1025
1026 void
1027 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1028 {
1029     Must(reply);
1030
1031     if (waitingForOrigin) {
1032         Must(delayedReply == NULL);
1033         delayedReply = reply;
1034         return;
1035     }
1036
1037     const HttpHeader &header = reply->header;
1038     // adaptation and forwarding errors lack Http::HdrType::FTP_STATUS
1039     if (!header.has(Http::HdrType::FTP_STATUS)) {
1040         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1041         return;
1042     }
1043
1044     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1045     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1046     writeForwardedReplyAndCall(reply, call);
1047 }
1048
1049 void
1050 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1051 {
1052     if (pipeline.front()->http->request->error) {
1053         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1054         return;
1055     }
1056
1057     writeCustomReply(200, "EPRT successfully converted to PASV.");
1058
1059     // and wait for RETR
1060 }
1061
1062 void
1063 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1064 {
1065     if (pipeline.front()->http->request->error) {
1066         writeCustomReply(502, "Cannot connect to server", reply);
1067         return;
1068     }
1069
1070     const unsigned short localPort = listenForDataConnection();
1071     if (!localPort)
1072         return;
1073
1074     // In interception setups, we use a local port number and hope that data
1075     // traffic will be redirected to us.
1076     MemBuf mb;
1077     mb.init();
1078     mb.appendf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1079
1080     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1081     writeReply(mb);
1082 }
1083
1084 /// writes FTP error response with given status and reply-derived error details
1085 void
1086 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1087 {
1088     const HttpRequest *request = pipeline.front()->http->request;
1089     assert(request);
1090
1091     MemBuf mb;
1092     mb.init();
1093
1094     if (request->error)
1095         mb.appendf("%i-%s\r\n", scode, errorPageName(request->error.category));
1096
1097     if (const auto &detail = request->error.detail) {
1098         mb.appendf("%i-Error-Detail-Brief: " SQUIDSBUFPH "\r\n", scode, SQUIDSBUFPRINT(detail->brief()));
1099         mb.appendf("%i-Error-Detail-Verbose: " SQUIDSBUFPH "\r\n", scode, SQUIDSBUFPRINT(detail->verbose(request)));
1100     }
1101
1102 #if USE_ADAPTATION
1103     // XXX: Remove hard coded names. Use an error page template instead.
1104     const Adaptation::History::Pointer ah = request->adaptHistory();
1105     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1106         const String info = ah->allMeta.getByName("X-Response-Info");
1107         const String desc = ah->allMeta.getByName("X-Response-Desc");
1108         if (info.size())
1109             mb.appendf("%i-Information: %s\r\n", scode, info.termedBuf());
1110         if (desc.size())
1111             mb.appendf("%i-Description: %s\r\n", scode, desc.termedBuf());
1112     }
1113 #endif
1114
1115     const char *reason = "Lost Error";
1116     if (reply) {
1117         reason = reply->header.has(Http::HdrType::FTP_REASON) ?
1118                  reply->header.getStr(Http::HdrType::FTP_REASON):
1119                  reply->sline.reason();
1120     }
1121
1122     mb.appendf("%i %s\r\n", scode, reason); // error terminating line
1123
1124     // TODO: errorpage.cc should detect FTP client and use
1125     // configurable FTP-friendly error templates which we should
1126     // write to the client "as is" instead of hiding most of the info
1127
1128     writeReply(mb);
1129 }
1130
1131 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1132 /// for example, internally-generated Squid "errorpages" end up here (for now)
1133 void
1134 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1135 {
1136     changeState(fssConnected, "foreign reply");
1137     closeDataConnection();
1138     // 451: We intend to keep the control connection open.
1139     writeErrorReply(reply, 451);
1140 }
1141
1142 bool
1143 Ftp::Server::writeControlMsgAndCall(HttpReply *reply, AsyncCall::Pointer &call)
1144 {
1145     // the caller guarantees that we are dealing with the current context only
1146     // the caller should also make sure reply->header.has(Http::HdrType::FTP_STATUS)
1147     writeForwardedReplyAndCall(reply, call);
1148     return true;
1149 }
1150
1151 void
1152 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1153 {
1154     assert(reply != NULL);
1155     const HttpHeader &header = reply->header;
1156
1157     // without status, the caller must use the writeForwardedForeign() path
1158     Must(header.has(Http::HdrType::FTP_STATUS));
1159     Must(header.has(Http::HdrType::FTP_REASON));
1160     const int scode = header.getInt(Http::HdrType::FTP_STATUS);
1161     debugs(33, 7, "scode: " << scode);
1162
1163     // Status 125 or 150 implies upload or data request, but we still check
1164     // the state in case the server is buggy.
1165     if ((scode == 125 || scode == 150) &&
1166             (master->serverState == fssHandleUploadRequest ||
1167              master->serverState == fssHandleDataRequest)) {
1168         if (checkDataConnPost()) {
1169             // If the data connection is ready, start reading data (here)
1170             // and forward the response to client (further below).
1171             debugs(33, 7, "data connection established, start data transfer");
1172             if (master->serverState == fssHandleUploadRequest)
1173                 maybeReadUploadData();
1174         } else {
1175             // If we are waiting to accept the data connection, keep waiting.
1176             if (Comm::IsConnOpen(dataListenConn)) {
1177                 debugs(33, 7, "wait for the client to establish a data connection");
1178                 onDataAcceptCall = call;
1179                 // TODO: Add connect timeout for passive connections listener?
1180                 // TODO: Remember server response so that we can forward it?
1181             } else {
1182                 // Either the connection was establised and closed after the
1183                 // data was transferred OR we failed to establish an active
1184                 // data connection and already sent the error to the client.
1185                 // In either case, there is nothing more to do.
1186                 debugs(33, 7, "done with data OR active connection failed");
1187             }
1188             return;
1189         }
1190     }
1191
1192     MemBuf mb;
1193     mb.init();
1194     Ftp::PrintReply(mb, reply);
1195
1196     debugs(9, 2, "FTP Client " << clientConnection);
1197     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1198            "\n----------");
1199
1200     Comm::Write(clientConnection, &mb, call);
1201 }
1202
1203 static void
1204 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const)
1205 {
1206     const HttpHeader &header = reply->header;
1207
1208     HttpHeaderPos pos = HttpHeaderInitPos;
1209     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1210         if (e->id == Http::HdrType::FTP_PRE) {
1211             String raw;
1212             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1213                 mb.appendf("%s\r\n", raw.termedBuf());
1214         }
1215     }
1216
1217     if (header.has(Http::HdrType::FTP_STATUS)) {
1218         const char *reason = header.getStr(Http::HdrType::FTP_REASON);
1219         mb.appendf("%i %s\r\n", header.getInt(Http::HdrType::FTP_STATUS),
1220                    (reason ? reason : 0));
1221     }
1222 }
1223
1224 void
1225 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1226 {
1227     if (io.flag == Comm::ERR_CLOSING)
1228         return;
1229
1230     if (io.flag != Comm::OK) {
1231         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1232         io.conn->close();
1233         return;
1234     }
1235
1236     Http::StreamPointer context = pipeline.front();
1237     if (context != nullptr && context->http) {
1238         context->http->out.size += io.size;
1239         context->http->out.headers_sz += io.size;
1240     }
1241
1242     flags.readMore = true;
1243     readSomeData();
1244 }
1245
1246 void
1247 Ftp::Server::wroteReply(const CommIoCbParams &io)
1248 {
1249     if (io.flag == Comm::ERR_CLOSING)
1250         return;
1251
1252     if (io.flag != Comm::OK) {
1253         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1254         io.conn->close();
1255         return;
1256     }
1257
1258     Http::StreamPointer context = pipeline.front();
1259     assert(context->http);
1260     context->http->out.size += io.size;
1261     context->http->out.headers_sz += io.size;
1262
1263     if (master->serverState == fssError) {
1264         debugs(33, 5, "closing on FTP server error");
1265         io.conn->close();
1266         return;
1267     }
1268
1269     const clientStream_status_t socketState = context->socketState();
1270     debugs(33, 5, "FTP client stream state " << socketState);
1271     switch (socketState) {
1272     case STREAM_UNPLANNED_COMPLETE:
1273     case STREAM_FAILED:
1274         io.conn->close();
1275         return;
1276
1277     case STREAM_NONE:
1278     case STREAM_COMPLETE:
1279         flags.readMore = true;
1280         changeState(fssConnected, "Ftp::Server::wroteReply");
1281         if (bodyParser)
1282             finishDechunkingRequest(false);
1283         context->finished();
1284         kick();
1285         return;
1286     }
1287 }
1288
1289 bool
1290 Ftp::Server::handleRequest(HttpRequest *request)
1291 {
1292     debugs(33, 9, request);
1293     Must(request);
1294
1295     HttpHeader &header = request->header;
1296     Must(header.has(Http::HdrType::FTP_COMMAND));
1297     String &cmd = header.findEntry(Http::HdrType::FTP_COMMAND)->value;
1298     Must(header.has(Http::HdrType::FTP_ARGUMENTS));
1299     String &params = header.findEntry(Http::HdrType::FTP_ARGUMENTS)->value;
1300
1301     if (Debug::Enabled(9, 2)) {
1302         MemBuf mb;
1303         mb.init();
1304         request->pack(&mb);
1305
1306         debugs(9, 2, "FTP Client " << clientConnection);
1307         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1308                "\n----------");
1309     }
1310
1311     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1312     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1313     static RequestHandlers handlers;
1314     if (!handlers.size()) {
1315         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1316         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1317         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1318         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1319         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1320         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1321         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1322         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1323         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1324         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1325         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1326         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1327     }
1328
1329     RequestHandler handler = NULL;
1330     if (request->method == Http::METHOD_PUT)
1331         handler = &Ftp::Server::handleUploadRequest;
1332     else {
1333         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1334         if (hi != handlers.end())
1335             handler = hi->second;
1336     }
1337
1338     if (!handler) {
1339         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1340         return true;
1341     }
1342
1343     return (this->*handler)(cmd, params);
1344 }
1345
1346 /// Called to parse USER command, which is required to create an HTTP request
1347 /// wrapper. W/o request, the errors are handled by returning earlyError().
1348 Http::Stream *
1349 Ftp::Server::handleUserRequest(const SBuf &, SBuf &params)
1350 {
1351     if (params.isEmpty())
1352         return earlyError(EarlyErrorKind::MissingUsername);
1353
1354     // find the [end of] user name
1355     const SBuf::size_type eou = params.rfind('@');
1356     if (eou == SBuf::npos || eou + 1 >= params.length())
1357         return earlyError(EarlyErrorKind::MissingHost);
1358
1359     // Determine the intended destination.
1360     host = params.substr(eou + 1, params.length());
1361     // If we can parse it as raw IPv6 address, then surround with "[]".
1362     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1363     if (host.find(':') != SBuf::npos) {
1364         const Ip::Address ipa(host.c_str());
1365         if (!ipa.isAnyAddr()) {
1366             char ipBuf[MAX_IPSTRLEN];
1367             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1368             host = ipBuf;
1369         }
1370     }
1371
1372     // const SBuf login = params.substr(0, eou);
1373     params.chop(0, eou); // leave just the login part for the peer
1374
1375     SBuf oldUri;
1376     if (master->clientReadGreeting)
1377         oldUri = uri;
1378
1379     master->workingDir.clear();
1380     calcUri(NULL);
1381
1382     if (!master->clientReadGreeting) {
1383         debugs(9, 3, "set URI to " << uri);
1384     } else if (oldUri.caseCmp(uri) == 0) {
1385         debugs(9, 5, "kept URI as " << oldUri);
1386     } else {
1387         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1388         closeDataConnection();
1389         unpinConnection(true); // close control connection to peer
1390         resetLogin("URI reset");
1391     }
1392
1393     return NULL; // no early errors
1394 }
1395
1396 bool
1397 Ftp::Server::handleFeatRequest(String &, String &)
1398 {
1399     changeState(fssHandleFeat, "handleFeatRequest");
1400     return true;
1401 }
1402
1403 bool
1404 Ftp::Server::handlePasvRequest(String &, String &params)
1405 {
1406     if (gotEpsvAll) {
1407         setReply(500, "Bad PASV command");
1408         return false;
1409     }
1410
1411     if (params.size() > 0) {
1412         setReply(501, "Unexpected parameter");
1413         return false;
1414     }
1415
1416     changeState(fssHandlePasv, "handlePasvRequest");
1417     // no need to fake PASV request via setDataCommand() in true PASV case
1418     return true;
1419 }
1420
1421 /// [Re]initializes dataConn for active data transfers. Does not connect.
1422 bool
1423 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1424 {
1425     assert(clientConnection != NULL);
1426     assert(!clientConnection->remote.isAnyAddr());
1427
1428     if (cltAddr != clientConnection->remote) {
1429         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1430         // Closing the control connection would not help with attacks because
1431         // the client is evidently able to connect to us. Besides, closing
1432         // makes retrials easier for the client and more damaging to us.
1433         setReply(501, "Prohibited parameter value");
1434         return false;
1435     }
1436
1437     closeDataConnection();
1438
1439     Comm::ConnectionPointer conn = new Comm::Connection();
1440     conn->flags |= COMM_DOBIND;
1441
1442     if (clientConnection->flags & COMM_INTERCEPTION) {
1443         // In the case of NAT interception conn->local value is not set
1444         // because the TCP stack will automatically pick correct source
1445         // address for the data connection. We must only ensure that IP
1446         // version matches client's address.
1447         conn->local.setAnyAddr();
1448
1449         if (cltAddr.isIPv4())
1450             conn->local.setIPv4();
1451
1452         conn->remote = cltAddr;
1453     } else {
1454         // In the case of explicit-proxy the local IP of the control connection
1455         // is the Squid IP the client is knowingly talking to.
1456         //
1457         // In the case of TPROXY the IP address of the control connection is
1458         // server IP the client is connecting to, it can be spoofed by Squid.
1459         //
1460         // In both cases some clients may refuse to accept data connections if
1461         // these control connectin local-IP's are not used.
1462         conn->setAddrs(clientConnection->local, cltAddr);
1463
1464         // Using non-local addresses in TPROXY mode requires appropriate socket option.
1465         if (clientConnection->flags & COMM_TRANSPARENT)
1466             conn->flags |= COMM_TRANSPARENT;
1467     }
1468
1469     // RFC 959 requires active FTP connections to originate from port 20
1470     // but that would preclude us from supporting concurrent transfers! (XXX?)
1471     conn->local.port(0);
1472
1473     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1474            conn->remote);
1475
1476     dataConn = conn;
1477     uploadAvailSize = 0;
1478     return true;
1479 }
1480
1481 bool
1482 Ftp::Server::handlePortRequest(String &, String &params)
1483 {
1484     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1485
1486     if (gotEpsvAll) {
1487         setReply(500, "Rejecting PORT after EPSV ALL");
1488         return false;
1489     }
1490
1491     if (!params.size()) {
1492         setReply(501, "Missing parameter");
1493         return false;
1494     }
1495
1496     Ip::Address cltAddr;
1497     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1498         setReply(501, "Invalid parameter");
1499         return false;
1500     }
1501
1502     if (!createDataConnection(cltAddr))
1503         return false;
1504
1505     changeState(fssHandlePort, "handlePortRequest");
1506     setDataCommand();
1507     return true; // forward our fake PASV request
1508 }
1509
1510 bool
1511 Ftp::Server::handleDataRequest(String &, String &)
1512 {
1513     if (!checkDataConnPre())
1514         return false;
1515
1516     master->userDataDone = 0;
1517     originDataDownloadAbortedOnError = false;
1518
1519     changeState(fssHandleDataRequest, "handleDataRequest");
1520
1521     return true;
1522 }
1523
1524 bool
1525 Ftp::Server::handleUploadRequest(String &, String &)
1526 {
1527     if (!checkDataConnPre())
1528         return false;
1529
1530     if (Config.accessList.forceRequestBodyContinuation) {
1531         ClientHttpRequest *http = pipeline.front()->http;
1532         HttpRequest *request = http->request;
1533         ACLFilledChecklist bodyContinuationCheck(Config.accessList.forceRequestBodyContinuation, request, NULL);
1534         bodyContinuationCheck.al = http->al;
1535         bodyContinuationCheck.syncAle(request, http->log_uri);
1536         if (bodyContinuationCheck.fastCheck().allowed()) {
1537             request->forcedBodyContinuation = true;
1538             if (checkDataConnPost()) {
1539                 // Write control Msg
1540                 writeEarlyReply(150, "Data connection opened");
1541                 maybeReadUploadData();
1542             } else {
1543                 // wait for acceptDataConnection but tell it to call wroteEarlyReply
1544                 // after writing "150 Data connection opened"
1545                 typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1546                 AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
1547                 onDataAcceptCall = call;
1548             }
1549         }
1550     }
1551
1552     changeState(fssHandleUploadRequest, "handleDataRequest");
1553
1554     return true;
1555 }
1556
1557 bool
1558 Ftp::Server::handleEprtRequest(String &, String &params)
1559 {
1560     debugs(9, 3, "Process an EPRT " << params);
1561
1562     if (gotEpsvAll) {
1563         setReply(500, "Rejecting EPRT after EPSV ALL");
1564         return false;
1565     }
1566
1567     if (!params.size()) {
1568         setReply(501, "Missing parameter");
1569         return false;
1570     }
1571
1572     Ip::Address cltAddr;
1573     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1574         setReply(501, "Invalid parameter");
1575         return false;
1576     }
1577
1578     if (!createDataConnection(cltAddr))
1579         return false;
1580
1581     changeState(fssHandleEprt, "handleEprtRequest");
1582     setDataCommand();
1583     return true; // forward our fake PASV request
1584 }
1585
1586 bool
1587 Ftp::Server::handleEpsvRequest(String &, String &params)
1588 {
1589     debugs(9, 3, "Process an EPSV command with params: " << params);
1590     if (params.size() <= 0) {
1591         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1592     } else if (params.caseCmp("ALL") == 0) {
1593         setReply(200, "EPSV ALL ok");
1594         gotEpsvAll = true;
1595         return false;
1596     } else if (params.cmp("2") == 0) {
1597         if (!Ip::EnableIpv6) {
1598             setReply(522, "Network protocol not supported, use (1)");
1599             return false;
1600         }
1601     } else if (params.cmp("1") != 0) {
1602         setReply(501, "Unsupported EPSV parameter");
1603         return false;
1604     }
1605
1606     changeState(fssHandleEpsv, "handleEpsvRequest");
1607     setDataCommand();
1608     return true; // forward our fake PASV request
1609 }
1610
1611 bool
1612 Ftp::Server::handleCwdRequest(String &, String &)
1613 {
1614     changeState(fssHandleCwd, "handleCwdRequest");
1615     return true;
1616 }
1617
1618 bool
1619 Ftp::Server::handlePassRequest(String &, String &)
1620 {
1621     changeState(fssHandlePass, "handlePassRequest");
1622     return true;
1623 }
1624
1625 bool
1626 Ftp::Server::handleCdupRequest(String &, String &)
1627 {
1628     changeState(fssHandleCdup, "handleCdupRequest");
1629     return true;
1630 }
1631
1632 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1633 // Squid FTP client decides what data command to use with peers.
1634 void
1635 Ftp::Server::setDataCommand()
1636 {
1637     ClientHttpRequest *const http = pipeline.front()->http;
1638     assert(http != NULL);
1639     HttpRequest *const request = http->request;
1640     assert(request != NULL);
1641     HttpHeader &header = request->header;
1642     header.delById(Http::HdrType::FTP_COMMAND);
1643     header.putStr(Http::HdrType::FTP_COMMAND, "PASV");
1644     header.delById(Http::HdrType::FTP_ARGUMENTS);
1645     header.putStr(Http::HdrType::FTP_ARGUMENTS, "");
1646     debugs(9, 5, "client data command converted to fake PASV");
1647 }
1648
1649 /// check that client data connection is ready for future I/O or at least
1650 /// has a chance of becoming ready soon.
1651 bool
1652 Ftp::Server::checkDataConnPre()
1653 {
1654     if (Comm::IsConnOpen(dataConn))
1655         return true;
1656
1657     if (Comm::IsConnOpen(dataListenConn)) {
1658         // We are still waiting for a client to connect to us after PASV.
1659         // Perhaps client's data conn handshake has not reached us yet.
1660         // After we talk to the server, checkDataConnPost() will recheck.
1661         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1662         return true;
1663     }
1664
1665     if (!dataConn || dataConn->remote.isAnyAddr()) {
1666         debugs(33, 5, "missing " << dataConn);
1667         // TODO: use client address and default port instead.
1668         setReply(425, "Use PORT or PASV first");
1669         return false;
1670     }
1671
1672     // active transfer: open a data connection from Squid to client
1673     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1674     AsyncCall::Pointer callback = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1675     const auto cs = new Comm::ConnOpener(dataConn->cloneProfile(), callback,
1676                                          Config.Timeout.connect);
1677     dataConnWait.start(cs, callback);
1678     return false;
1679 }
1680
1681 /// Check that client data connection is ready for immediate I/O.
1682 bool
1683 Ftp::Server::checkDataConnPost() const
1684 {
1685     if (!Comm::IsConnOpen(dataConn)) {
1686         debugs(33, 3, "missing client data conn: " << dataConn);
1687         return false;
1688     }
1689     return true;
1690 }
1691
1692 /// Done establishing a data connection to the user.
1693 void
1694 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1695 {
1696     dataConnWait.finish();
1697
1698     if (params.flag != Comm::OK) {
1699         setReply(425, "Cannot open data connection.");
1700         Http::StreamPointer context = pipeline.front();
1701         Must(context->http);
1702         Must(context->http->storeEntry() != NULL);
1703         // TODO: call closeDataConnection() to reset data conn processing?
1704     } else {
1705         // Finalize the details and start owning the supplied connection.
1706         assert(params.conn);
1707         assert(dataConn);
1708         assert(!dataConn->isOpen());
1709         dataConn = params.conn;
1710         // XXX: Missing comm_add_close_handler() to track external closures.
1711
1712         Must(Comm::IsConnOpen(params.conn));
1713         fd_note(params.conn->fd, "active client ftp data");
1714     }
1715
1716     doProcessRequest();
1717 }
1718
1719 void
1720 Ftp::Server::setReply(const int code, const char *msg)
1721 {
1722     Http::StreamPointer context = pipeline.front();
1723     ClientHttpRequest *const http = context->http;
1724     assert(http != NULL);
1725     assert(http->storeEntry() == NULL);
1726
1727     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1728
1729     clientStreamNode *const node = context->getClientReplyContext();
1730     clientReplyContext *const repContext =
1731         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1732     assert(repContext != NULL);
1733
1734     RequestFlags reqFlags;
1735     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1736     reqFlags.noCache = true;
1737     repContext->createStoreEntry(http->request->method, reqFlags);
1738     http->storeEntry()->replaceHttpReply(reply);
1739 }
1740
1741 void
1742 Ftp::Server::callException(const std::exception &e)
1743 {
1744     debugs(33, 2, "FTP::Server job caught: " << e.what());
1745     closeDataConnection();
1746     unpinConnection(true);
1747     if (Comm::IsConnOpen(clientConnection))
1748         clientConnection->close();
1749     AsyncJob::callException(e);
1750 }
1751
1752 void
1753 Ftp::Server::startWaitingForOrigin()
1754 {
1755     if (!isOpen()) // if we are closing, nothing to do
1756         return;
1757
1758     debugs(33, 5, "waiting for Ftp::Client data transfer to end");
1759     waitingForOrigin = true;
1760 }
1761
1762 void
1763 Ftp::Server::stopWaitingForOrigin(int originStatus)
1764 {
1765     Must(waitingForOrigin);
1766     waitingForOrigin = false;
1767
1768     if (!isOpen()) // if we are closing, nothing to do
1769         return;
1770
1771     // if we have already decided how to respond, respond now
1772     if (delayedReply) {
1773         HttpReply::Pointer reply = delayedReply;
1774         delayedReply = nullptr;
1775         writeForwardedReply(reply.getRaw());
1776         return; // do not completeDataDownload() after an earlier response
1777     }
1778
1779     if (master->serverState != fssHandleDataRequest)
1780         return;
1781
1782     // completeDataDownload() could be waitingForOrigin in fssHandleDataRequest
1783     // Depending on which side has finished downloading first, either trust
1784     // master->userDataDone status or set originDataDownloadAbortedOnError:
1785     if (master->userDataDone) {
1786         // We finished downloading before Ftp::Client. Most likely, the
1787         // adaptation shortened the origin response or we hit an error.
1788         // Our status (stored in master->userDataDone) is more informative.
1789         // Use master->userDataDone; avoid originDataDownloadAbortedOnError.
1790         completeDataDownload();
1791     } else {
1792         debugs(33, 5, "too early to write the response");
1793         // Ftp::Client naturally finished downloading before us. Set
1794         // originDataDownloadAbortedOnError to overwrite future
1795         // master->userDataDone and relay Ftp::Client error, if there was
1796         // any, to the user.
1797         originDataDownloadAbortedOnError = (originStatus >= 400);
1798     }
1799 }
1800
1801 void Ftp::Server::userDataCompletionCheckpoint(int finalStatusCode)
1802 {
1803     Must(!master->userDataDone);
1804     master->userDataDone = finalStatusCode;
1805
1806     if (bodyParser)
1807         finishDechunkingRequest(false);
1808
1809     if (waitingForOrigin) {
1810         // The completeDataDownload() is not called here unconditionally
1811         // because we want to signal the FTP user that we are not fully
1812         // done processing its data stream, even though all data bytes
1813         // have been sent or received already.
1814         debugs(33, 5, "Transferring from FTP server is not complete");
1815         return;
1816     }
1817
1818     // Adjust our reply if the server aborted with an error before we are done.
1819     if (master->userDataDone == 226 && originDataDownloadAbortedOnError) {
1820         debugs(33, 5, "Transferring from FTP server terminated with an error, adjust status code");
1821         master->userDataDone = 451;
1822     }
1823     completeDataDownload();
1824 }
1825
1826 void Ftp::Server::completeDataDownload()
1827 {
1828     writeCustomReply(master->userDataDone, master->userDataDone == 226 ? "Transfer complete" : "Server error; transfer aborted");
1829     closeDataConnection();
1830 }
1831
1832 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1833 static bool
1834 Ftp::SupportedCommand(const SBuf &name)
1835 {
1836     static std::set<SBuf> BlockList;
1837     if (BlockList.empty()) {
1838         /* Add FTP commands that Squid cannot relay correctly. */
1839
1840         // We probably do not support AUTH TLS.* and AUTH SSL,
1841         // but let's disclaim all AUTH support to KISS, for now.
1842         BlockList.insert(cmdAuth());
1843     }
1844
1845     // we claim support for all commands that we do not know about
1846     return BlockList.find(name) == BlockList.end();
1847 }
1848