src/servers/FtpServer.cc

   1 /*
   2  * Copyright (C) 1996-2020 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Transfer protocol servers */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CharacterSet.h"
  14 #include "base/RefCount.h"
  15 #include "base/Subscription.h"
  16 #include "client_side_reply.h"
  17 #include "client_side_request.h"
  18 #include "clientStream.h"
  19 #include "comm/ConnOpener.h"
  20 #include "comm/Read.h"
  21 #include "comm/TcpAcceptor.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "ftp/Elements.h"
  26 #include "ftp/Parsing.h"
  27 #include "globals.h"
  28 #include "http/one/RequestParser.h"
  29 #include "http/Stream.h"
  30 #include "HttpHdrCc.h"
  31 #include "ip/tools.h"
  32 #include "ipc/FdNotes.h"
  33 #include "parser/Tokenizer.h"
  34 #include "servers/forward.h"
  35 #include "servers/FtpServer.h"
  36 #include "SquidConfig.h"
  37 #include "StatCounters.h"
  38 #include "tools.h"
  39
  40 #include <set>
  41 #include <map>
  42
  43 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  44
  45 namespace Ftp
  46 {
  47 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  48 static bool SupportedCommand(const SBuf &name);
  49 static bool CommandHasPathParameter(const SBuf &cmd);
  50 };
  51
  52 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  53     AsyncJob("Ftp::Server"),
  54     ConnStateData(xact),
  55     master(new MasterState),
  56     uri(),
  57     host(),
  58     gotEpsvAll(false),
  59     onDataAcceptCall(),
  60     dataListenConn(),
  61     dataConn(),
  62     uploadAvailSize(0),
  63     listener(),
  64     connector(),
  65     reader(),
  66     waitingForOrigin(false),
  67     originDataDownloadAbortedOnError(false)
  68 {
  69     flags.readMore = false; // we need to announce ourselves first
  70     *uploadBuf = 0;
  71 }
  72
  73 Ftp::Server::~Server()
  74 {
  75     closeDataConnection();
  76 }
  77
  78 int
  79 Ftp::Server::pipelinePrefetchMax() const
  80 {
  81     return 0; // no support for concurrent FTP requests
  82 }
  83
  84 time_t
  85 Ftp::Server::idleTimeout() const
  86 {
  87     return Config.Timeout.ftpClientIdle;
  88 }
  89
  90 void
  91 Ftp::Server::start()
  92 {
  93     ConnStateData::start();
  94
  95     if (transparent()) {
  96         char buf[MAX_IPSTRLEN];
  97         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  98         host = buf;
  99         calcUri(NULL);
 100         debugs(33, 5, "FTP transparent URL: " << uri);
 101     }
 102
 103     writeEarlyReply(220, "Service ready");
 104 }
 105
 106 /// schedules another data connection read if needed
 107 void
 108 Ftp::Server::maybeReadUploadData()
 109 {
 110     if (reader != NULL)
 111         return;
 112
 113     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 114     if (availSpace <= 0)
 115         return;
 116
 117     debugs(33, 4, dataConn << ": reading FTP data...");
 118
 119     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 120     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 121     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 122               reader);
 123 }
 124
 125 /// react to the freshly parsed request
 126 void
 127 Ftp::Server::doProcessRequest()
 128 {
 129     // zero pipelinePrefetchMax() ensures that there is only parsed request
 130     Must(pipeline.count() == 1);
 131     Http::StreamPointer context = pipeline.front();
 132     Must(context != nullptr);
 133
 134     ClientHttpRequest *const http = context->http;
 135     assert(http != NULL);
 136
 137     HttpRequest *const request = http->request;
 138     Must(http->storeEntry() || request);
 139     const bool mayForward = !http->storeEntry() && handleRequest(request);
 140
 141     if (http->storeEntry() != NULL) {
 142         debugs(33, 4, "got an immediate response");
 143         clientSetKeepaliveFlag(http);
 144         context->pullData();
 145     } else if (mayForward) {
 146         debugs(33, 4, "forwarding request to server side");
 147         assert(http->storeEntry() == NULL);
 148         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 149     } else {
 150         debugs(33, 4, "will resume processing later");
 151     }
 152 }
 153
 154 void
 155 Ftp::Server::processParsedRequest(Http::StreamPointer &)
 156 {
 157     Must(pipeline.count() == 1);
 158
 159     // Process FTP request asynchronously to make sure FTP
 160     // data connection accept callback is fired first.
 161     CallJobHere(33, 4, CbcPointer<Server>(this),
 162                 Ftp::Server, doProcessRequest);
 163 }
 164
 165 /// imports more upload data from the data connection
 166 void
 167 Ftp::Server::readUploadData(const CommIoCbParams &io)
 168 {
 169     debugs(33, 5, io.conn << " size " << io.size);
 170     Must(reader != NULL);
 171     reader = NULL;
 172
 173     assert(Comm::IsConnOpen(dataConn));
 174     assert(io.conn->fd == dataConn->fd);
 175
 176     if (io.flag == Comm::OK && bodyPipe != NULL) {
 177         if (io.size > 0) {
 178             statCounter.client_http.kbytes_in += io.size;
 179
 180             char *const current_buf = uploadBuf + uploadAvailSize;
 181             if (io.buf != current_buf)
 182                 memmove(current_buf, io.buf, io.size);
 183             uploadAvailSize += io.size;
 184             shovelUploadData();
 185         } else if (io.size == 0) {
 186             debugs(33, 5, io.conn << " closed");
 187             closeDataConnection();
 188             if (uploadAvailSize <= 0)
 189                 finishDechunkingRequest(true);
 190         }
 191     } else { // not Comm::Flags::OK or unexpected read
 192         debugs(33, 5, io.conn << " closed");
 193         closeDataConnection();
 194         finishDechunkingRequest(false);
 195     }
 196
 197 }
 198
 199 /// shovel upload data from the internal buffer to the body pipe if possible
 200 void
 201 Ftp::Server::shovelUploadData()
 202 {
 203     assert(bodyPipe != NULL);
 204
 205     debugs(33, 5, "handling FTP request data for " << clientConnection);
 206     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 207                            uploadAvailSize);
 208     if (putSize > 0) {
 209         uploadAvailSize -= putSize;
 210         if (uploadAvailSize > 0)
 211             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 212     }
 213
 214     if (Comm::IsConnOpen(dataConn))
 215         maybeReadUploadData();
 216     else if (uploadAvailSize <= 0)
 217         finishDechunkingRequest(true);
 218 }
 219
 220 void
 221 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 222 {
 223     if (!isOpen()) // if we are closing, nothing to do
 224         return;
 225
 226     shovelUploadData();
 227 }
 228
 229 void
 230 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 231 {
 232     if (!isOpen()) // if we are closing, nothing to do
 233         return;
 234
 235     ConnStateData::noteBodyConsumerAborted(ptr);
 236     closeDataConnection();
 237 }
 238
 239 /// accept a new FTP control connection and hand it to a dedicated Server
 240 void
 241 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 242 {
 243     MasterXaction::Pointer xact = params.xaction;
 244     AnyP::PortCfgPointer s = xact->squidPort;
 245
 246     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 247
 248     if (params.flag != Comm::OK) {
 249         // Its possible the call was still queued when the client disconnected
 250         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 251         return;
 252     }
 253
 254     debugs(33, 4, params.conn << ": accepted");
 255     fd_note(params.conn->fd, "client ftp connect");
 256
 257     if (s->tcp_keepalive.enabled)
 258         commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
 259
 260     ++incoming_sockets_accepted;
 261
 262     AsyncJob::Start(new Server(xact));
 263 }
 264
 265 void
 266 Ftp::StartListening()
 267 {
 268     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 269         if (MAXTCPLISTENPORTS == NHttpSockets) {
 270             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 271                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 272             break;
 273         }
 274
 275         // direct new connections accepted by listenConn to Accept()
 276         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 277         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 278                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 279                                                CommAcceptCbParams(NULL)));
 280         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 281     }
 282 }
 283
 284 void
 285 Ftp::StopListening()
 286 {
 287     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 288         if (s->listenConn != NULL) {
 289             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 290             s->listenConn->close();
 291             s->listenConn = NULL;
 292         }
 293     }
 294 }
 295
 296 void
 297 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 298 {
 299     // find request
 300     Http::StreamPointer context = pipeline.front();
 301     Must(context != nullptr);
 302     ClientHttpRequest *const http = context->http;
 303     Must(http != NULL);
 304     HttpRequest *const request = http->request;
 305     Must(request != NULL);
 306     // make FTP peer connection exclusive to our request
 307     pinBusyConnection(conn, request);
 308 }
 309
 310 void
 311 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 312 {
 313     ConnStateData::clientPinnedConnectionClosed(io);
 314
 315     // TODO: Keep the control connection open after fixing the reset
 316     // problem below
 317     if (Comm::IsConnOpen(clientConnection))
 318         clientConnection->close();
 319
 320     // TODO: If the server control connection is gone, reset state to login
 321     // again. Resetting login alone is not enough: FtpRelay::sendCommand() will
 322     // not re-login because FtpRelay::serverState() is not going to be
 323     // fssConnected. Calling resetLogin() alone is also harmful because
 324     // it does not reset correctly the client-to-squid control connection (eg
 325     // respond if required with an error code, in all cases)
 326     // resetLogin("control connection closure");
 327 }
 328
 329 /// clear client and server login-related state after the old login is gone
 330 void
 331 Ftp::Server::resetLogin(const char *reason)
 332 {
 333     debugs(33, 5, "will need to re-login due to " << reason);
 334     master->clientReadGreeting = false;
 335     changeState(fssBegin, reason);
 336 }
 337
 338 /// computes uri member from host and, if tracked, working dir with file name
 339 void
 340 Ftp::Server::calcUri(const SBuf *file)
 341 {
 342     // TODO: fill a class AnyP::Uri instead of string
 343     uri = "ftp://";
 344     uri.append(host);
 345     if (port->ftp_track_dirs && master->workingDir.length()) {
 346         if (master->workingDir[0] != '/')
 347             uri.append("/", 1);
 348         uri.append(master->workingDir);
 349     }
 350
 351     if (uri[uri.length() - 1] != '/')
 352         uri.append("/", 1);
 353
 354     if (port->ftp_track_dirs && file) {
 355         static const CharacterSet Slash("/", "/");
 356         Parser::Tokenizer tok(*file);
 357         tok.skipAll(Slash);
 358         uri.append(tok.remaining());
 359     }
 360 }
 361
 362 /// Starts waiting for a data connection. Returns listening port.
 363 /// On errors, responds with an error and returns zero.
 364 unsigned int
 365 Ftp::Server::listenForDataConnection()
 366 {
 367     closeDataConnection();
 368
 369     Comm::ConnectionPointer conn = new Comm::Connection;
 370     conn->flags = COMM_NONBLOCKING;
 371     conn->local = transparent() ? port->s : clientConnection->local;
 372     conn->local.port(0);
 373     const char *const note = uri.c_str();
 374     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 375     if (!Comm::IsConnOpen(conn)) {
 376         debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " <<
 377                conn->local << " error: " << errno);
 378         writeCustomReply(451, "Internal error");
 379         return 0;
 380     }
 381
 382     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 383     typedef AsyncCallT<AcceptDialer> AcceptCall;
 384     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 385     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 386     listener = call.getRaw();
 387     dataListenConn = conn;
 388     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 389
 390     const unsigned int listeningPort = comm_local_port(conn->fd);
 391     conn->local.port(listeningPort);
 392     return listeningPort;
 393 }
 394
 395 void
 396 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 397 {
 398     if (params.flag != Comm::OK) {
 399         // Its possible the call was still queued when the client disconnected
 400         debugs(33, 2, dataListenConn << ": accept "
 401                "failure: " << xstrerr(params.xerrno));
 402         return;
 403     }
 404
 405     debugs(33, 4, "accepted " << params.conn);
 406     fd_note(params.conn->fd, "passive client ftp data");
 407     ++incoming_sockets_accepted;
 408
 409     if (!clientConnection) {
 410         debugs(33, 5, "late data connection?");
 411         closeDataConnection(); // in case we are still listening
 412         params.conn->close();
 413     } else if (params.conn->remote != clientConnection->remote) {
 414         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 415         params.conn->close();
 416         // Some FTP servers close control connection here, but it may make
 417         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 418     } else {
 419         closeDataConnection();
 420         dataConn = params.conn;
 421         uploadAvailSize = 0;
 422         debugs(33, 7, "ready for data");
 423         if (onDataAcceptCall != NULL) {
 424             AsyncCall::Pointer call = onDataAcceptCall;
 425             onDataAcceptCall = NULL;
 426             // If we got an upload request, start reading data from the client.
 427             if (master->serverState == fssHandleUploadRequest)
 428                 maybeReadUploadData();
 429             else
 430                 Must(master->serverState == fssHandleDataRequest);
 431             MemBuf mb;
 432             mb.init();
 433             mb.appendf("150 Data connection opened.\r\n");
 434             Comm::Write(clientConnection, &mb, call);
 435         }
 436     }
 437 }
 438
 439 void
 440 Ftp::Server::closeDataConnection()
 441 {
 442     if (listener != NULL) {
 443         listener->cancel("no longer needed");
 444         listener = NULL;
 445     }
 446
 447     if (Comm::IsConnOpen(dataListenConn)) {
 448         debugs(33, 5, "FTP closing client data listen socket: " <<
 449                *dataListenConn);
 450         dataListenConn->close();
 451     }
 452     dataListenConn = NULL;
 453
 454     if (reader != NULL) {
 455         // Comm::ReadCancel can deal with negative FDs
 456         Comm::ReadCancel(dataConn->fd, reader);
 457         reader = NULL;
 458     }
 459
 460     if (Comm::IsConnOpen(dataConn)) {
 461         debugs(33, 5, "FTP closing client data connection: " <<
 462                *dataConn);
 463         dataConn->close();
 464     }
 465     dataConn = NULL;
 466 }
 467
 468 /// Writes FTP [error] response before we fully parsed the FTP request and
 469 /// created the corresponding HTTP request wrapper for that FTP request.
 470 void
 471 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 472 {
 473     debugs(33, 7, code << ' ' << msg);
 474     assert(99 < code && code < 1000);
 475
 476     MemBuf mb;
 477     mb.init();
 478     mb.appendf("%i %s\r\n", code, msg);
 479
 480     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 481     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 482     Comm::Write(clientConnection, &mb, call);
 483
 484     flags.readMore = false;
 485
 486     // TODO: Create master transaction. Log it in wroteEarlyReply().
 487 }
 488
 489 void
 490 Ftp::Server::writeReply(MemBuf &mb)
 491 {
 492     debugs(9, 2, "FTP Client " << clientConnection);
 493     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 494            "\n----------");
 495
 496     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 497     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 498     Comm::Write(clientConnection, &mb, call);
 499 }
 500
 501 void
 502 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 503 {
 504     debugs(33, 7, code << ' ' << msg);
 505     assert(99 < code && code < 1000);
 506
 507     const bool sendDetails = reply != NULL &&
 508                              reply->header.has(Http::HdrType::FTP_STATUS) && reply->header.has(Http::HdrType::FTP_REASON);
 509
 510     MemBuf mb;
 511     mb.init();
 512     if (sendDetails) {
 513         mb.appendf("%i-%s\r\n", code, msg);
 514         mb.appendf(" Server reply:\r\n");
 515         Ftp::PrintReply(mb, reply, " ");
 516         mb.appendf("%i \r\n", code);
 517     } else
 518         mb.appendf("%i %s\r\n", code, msg);
 519
 520     writeReply(mb);
 521 }
 522
 523 void
 524 Ftp::Server::changeState(const ServerState newState, const char *reason)
 525 {
 526     if (master->serverState == newState) {
 527         debugs(33, 3, "client state unchanged at " << master->serverState <<
 528                " because " << reason);
 529         master->serverState = newState;
 530     } else {
 531         debugs(33, 3, "client state was " << master->serverState <<
 532                ", now " << newState << " because " << reason);
 533         master->serverState = newState;
 534     }
 535 }
 536
 537 /// whether the given FTP command has a pathname parameter
 538 static bool
 539 Ftp::CommandHasPathParameter(const SBuf &cmd)
 540 {
 541     static std::set<SBuf> PathedCommands;
 542     if (!PathedCommands.size()) {
 543         PathedCommands.insert(cmdMlst());
 544         PathedCommands.insert(cmdMlsd());
 545         PathedCommands.insert(cmdStat());
 546         PathedCommands.insert(cmdNlst());
 547         PathedCommands.insert(cmdList());
 548         PathedCommands.insert(cmdMkd());
 549         PathedCommands.insert(cmdRmd());
 550         PathedCommands.insert(cmdDele());
 551         PathedCommands.insert(cmdRnto());
 552         PathedCommands.insert(cmdRnfr());
 553         PathedCommands.insert(cmdAppe());
 554         PathedCommands.insert(cmdStor());
 555         PathedCommands.insert(cmdRetr());
 556         PathedCommands.insert(cmdSmnt());
 557         PathedCommands.insert(cmdCwd());
 558     }
 559
 560     return PathedCommands.find(cmd) != PathedCommands.end();
 561 }
 562
 563 /// creates a context filled with an error message for a given early error
 564 Http::Stream *
 565 Ftp::Server::earlyError(const EarlyErrorKind eek)
 566 {
 567     /* Default values, to be updated by the switch statement below */
 568     int scode = 421;
 569     const char *reason = "Internal error";
 570     const char *errUri = "error:ftp-internal-early-error";
 571
 572     switch (eek) {
 573     case EarlyErrorKind::HugeRequest:
 574         scode = 421;
 575         reason = "Huge request";
 576         errUri = "error:ftp-huge-request";
 577         break;
 578
 579     case EarlyErrorKind::MissingLogin:
 580         scode = 530;
 581         reason = "Must login first";
 582         errUri = "error:ftp-must-login-first";
 583         break;
 584
 585     case EarlyErrorKind::MissingUsername:
 586         scode = 501;
 587         reason = "Missing username";
 588         errUri = "error:ftp-missing-username";
 589         break;
 590
 591     case EarlyErrorKind::MissingHost:
 592         scode = 501;
 593         reason = "Missing host";
 594         errUri = "error:ftp-missing-host";
 595         break;
 596
 597     case EarlyErrorKind::UnsupportedCommand:
 598         scode = 502;
 599         reason = "Unknown or unsupported command";
 600         errUri = "error:ftp-unsupported-command";
 601         break;
 602
 603     case EarlyErrorKind::InvalidUri:
 604         scode = 501;
 605         reason = "Invalid URI";
 606         errUri = "error:ftp-invalid-uri";
 607         break;
 608
 609     case EarlyErrorKind::MalformedCommand:
 610         scode = 421;
 611         reason = "Malformed command";
 612         errUri = "error:ftp-malformed-command";
 613         break;
 614
 615         // no default so that a compiler can check that we have covered all cases
 616     }
 617
 618     Http::Stream *context = abortRequestParsing(errUri);
 619     clientStreamNode *node = context->getClientReplyContext();
 620     Must(node);
 621     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 622     Must(repContext);
 623
 624     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 625     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 626     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 627     repContext->setReplyToReply(reply);
 628     return context;
 629 }
 630
 631 /// Parses a single FTP request on the control connection.
 632 /// Returns a new Http::Stream on valid requests and all errors.
 633 /// Returns NULL on incomplete requests that may still succeed given more data.
 634 Http::Stream *
 635 Ftp::Server::parseOneRequest()
 636 {
 637     flags.readMore = false; // common for all but one case below
 638
 639     // OWS <command> [ RWS <parameter> ] OWS LF
 640
 641     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 642     // for the LF character that we must exclude here (but see FullWhiteSpace).
 643     static const char * const InlineSpaceChars = " \f\r\t\v";
 644     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 645     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 646     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 647     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 648
 649     // This set is used to ignore empty commands without allowing an attacker
 650     // to keep us endlessly busy by feeding us whitespace or empty commands.
 651     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 652
 653     SBuf cmd;
 654     SBuf params;
 655
 656     Parser::Tokenizer tok(inBuf);
 657
 658     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 659     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 660
 661     // note that the condition below will eat either RWS or trailing OWS
 662     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 663         // now params may include trailing OWS
 664         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 665         static const SBuf bufWhiteSpace(InlineSpaceChars);
 666         params.trim(bufWhiteSpace, false, true);
 667     }
 668
 669     // Why limit command line and parameters size? Did not we just parse them?
 670     // XXX: Our good old String cannot handle very long strings.
 671     const SBuf::size_type tokenMax = min(
 672                                          static_cast<SBuf::size_type>(32*1024), // conservative
 673                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 674     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 675         changeState(fssError, "huge req token");
 676         quitAfterError(NULL);
 677         return earlyError(EarlyErrorKind::HugeRequest);
 678     }
 679
 680     // technically, we may skip multiple NLs below, but that is OK
 681     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 682         // we need more data, but can we buffer more?
 683         if (inBuf.length() >= Config.maxRequestHeaderSize) {
 684             changeState(fssError, "huge req");
 685             quitAfterError(NULL);
 686             return earlyError(EarlyErrorKind::HugeRequest);
 687         } else {
 688             flags.readMore = true;
 689             debugs(33, 5, "Waiting for more, up to " <<
 690                    (Config.maxRequestHeaderSize - inBuf.length()));
 691             return NULL;
 692         }
 693     }
 694
 695     Must(parsed && cmd.length());
 696     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 697
 698     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 699
 700     cmd.toUpper(); // this should speed up and simplify future comparisons
 701
 702     // interception cases do not need USER to calculate the uri
 703     if (!transparent()) {
 704         if (!master->clientReadGreeting) {
 705             // the first command must be USER
 706             if (!pinning.pinned && cmd != cmdUser())
 707                 return earlyError(EarlyErrorKind::MissingLogin);
 708         }
 709
 710         // process USER request now because it sets FTP peer host name
 711         if (cmd == cmdUser()) {
 712             if (Http::Stream *errCtx = handleUserRequest(cmd, params))
 713                 return errCtx;
 714         }
 715     }
 716
 717     if (!Ftp::SupportedCommand(cmd))
 718         return earlyError(EarlyErrorKind::UnsupportedCommand);
 719
 720     const HttpRequestMethod method =
 721         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 722         Http::METHOD_PUT : Http::METHOD_GET;
 723
 724     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 725                        &params : NULL;
 726     calcUri(path);
 727     MasterXaction::Pointer mx = new MasterXaction(XactionInitiator::initClient);
 728     mx->tcpClient = clientConnection;
 729     auto * const request = HttpRequest::FromUrl(uri, mx, method);
 730     if (!request) {
 731         debugs(33, 5, "Invalid FTP URL: " << uri);
 732         uri.clear();
 733         return earlyError(EarlyErrorKind::InvalidUri);
 734     }
 735     char *newUri = xstrdup(uri.c_str());
 736
 737     request->flags.ftpNative = true;
 738     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 739
 740     // Our fake Request-URIs are not distinctive enough for caching to work
 741     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 742     request->flags.noCache = true;
 743
 744     request->header.putStr(Http::HdrType::FTP_COMMAND, cmd.c_str());
 745     request->header.putStr(Http::HdrType::FTP_ARGUMENTS, params.c_str()); // may be ""
 746     if (method == Http::METHOD_PUT) {
 747         request->header.putStr(Http::HdrType::EXPECT, "100-continue");
 748         request->header.putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
 749     }
 750
 751     ClientHttpRequest *const http = new ClientHttpRequest(this);
 752     http->req_sz = tok.parsedSize();
 753     http->uri = newUri;
 754     http->initRequest(request);
 755
 756     Http::Stream *const result =
 757         new Http::Stream(clientConnection, http);
 758
 759     StoreIOBuffer tempBuffer;
 760     tempBuffer.data = result->reqbuf;
 761     tempBuffer.length = HTTP_REQBUF_SZ;
 762
 763     ClientStreamData newServer = new clientReplyContext(http);
 764     ClientStreamData newClient = result;
 765     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 766                      clientReplyStatus, newServer, clientSocketRecipient,
 767                      clientSocketDetach, newClient, tempBuffer);
 768
 769     result->flags.parsed_ok = 1;
 770     return result;
 771 }
 772
 773 void
 774 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 775 {
 776     // the caller guarantees that we are dealing with the current context only
 777     Http::StreamPointer context = pipeline.front();
 778     assert(context != nullptr);
 779
 780     static ReplyHandler handlers[] = {
 781         NULL, // fssBegin
 782         NULL, // fssConnected
 783         &Ftp::Server::handleFeatReply, // fssHandleFeat
 784         &Ftp::Server::handlePasvReply, // fssHandlePasv
 785         &Ftp::Server::handlePortReply, // fssHandlePort
 786         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 787         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 788         &Ftp::Server::handleEprtReply,// fssHandleEprt
 789         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 790         NULL, // fssHandleCwd
 791         NULL, // fssHandlePass
 792         NULL, // fssHandleCdup
 793         &Ftp::Server::handleErrorReply // fssError
 794     };
 795     try {
 796         const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 797         if (const ReplyHandler handler = handlers[server.master->serverState])
 798             (this->*handler)(reply, data);
 799         else
 800             writeForwardedReply(reply);
 801     } catch (const std::exception &e) {
 802         callException(e);
 803         throw TexcHere(e.what());
 804     }
 805 }
 806
 807 void
 808 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 809 {
 810     if (pipeline.front()->http->request->error) {
 811         writeCustomReply(502, "Server does not support FEAT", reply);
 812         return;
 813     }
 814
 815     Must(reply);
 816     HttpReply::Pointer featReply = Ftp::HttpReplyWrapper(211, "End", Http::scNoContent, 0);
 817     HttpHeader const &serverReplyHeader = reply->header;
 818
 819     HttpHeaderPos pos = HttpHeaderInitPos;
 820     bool hasEPRT = false;
 821     bool hasEPSV = false;
 822     int prependSpaces = 1;
 823
 824     featReply->header.putStr(Http::HdrType::FTP_PRE, "\"211-Features:\"");
 825     const int scode = serverReplyHeader.getInt(Http::HdrType::FTP_STATUS);
 826     if (scode == 211) {
 827         while (const HttpHeaderEntry *e = serverReplyHeader.getEntry(&pos)) {
 828             if (e->id == Http::HdrType::FTP_PRE) {
 829                 // assume RFC 2389 FEAT response format, quoted by Squid:
 830                 // <"> SP NAME [SP PARAMS] <">
 831                 // but accommodate MS servers sending four SPs before NAME
 832
 833                 // command name ends with (SP parameter) or quote
 834                 static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 835                 static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 836
 837                 Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 838                 if (!tok.skip('"') || !tok.skip(' '))
 839                     continue;
 840
 841                 // optional spaces; remember their number to accommodate MS servers
 842                 prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 843
 844                 SBuf cmd;
 845                 if (!tok.prefix(cmd, FeatNameChars))
 846                     continue;
 847                 cmd.toUpper();
 848
 849                 if (Ftp::SupportedCommand(cmd)) {
 850                     featReply->header.addEntry(e->clone());
 851                 }
 852
 853                 if (cmd == cmdEprt())
 854                     hasEPRT = true;
 855                 else if (cmd == cmdEpsv())
 856                     hasEPSV = true;
 857             }
 858         }
 859     } // else we got a FEAT error and will only report Squid-supported features
 860
 861     char buf[256];
 862     if (!hasEPRT) {
 863         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 864         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 865     }
 866     if (!hasEPSV) {
 867         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 868         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 869     }
 870
 871     featReply->header.refreshMask();
 872
 873     writeForwardedReply(featReply.getRaw());
 874 }
 875
 876 void
 877 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 878 {
 879     const Http::StreamPointer context(pipeline.front());
 880     assert(context != nullptr);
 881
 882     if (context->http->request->error) {
 883         writeCustomReply(502, "Server does not support PASV", reply);
 884         return;
 885     }
 886
 887     const unsigned short localPort = listenForDataConnection();
 888     if (!localPort)
 889         return;
 890
 891     char addr[MAX_IPSTRLEN];
 892     // remote server in interception setups and local address otherwise
 893     const Ip::Address &server = transparent() ?
 894                                 clientConnection->local : dataListenConn->local;
 895     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 896     addr[MAX_IPSTRLEN - 1] = '\0';
 897     for (char *c = addr; *c != '\0'; ++c) {
 898         if (*c == '.')
 899             *c = ',';
 900     }
 901
 902     // In interception setups, we combine remote server address with a
 903     // local port number and hope that traffic will be redirected to us.
 904     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 905     // versions block responses that use those alternative syntax rules!
 906     MemBuf mb;
 907     mb.init();
 908     mb.appendf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 909                addr,
 910                static_cast<int>(localPort / 256),
 911                static_cast<int>(localPort % 256));
 912     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 913     writeReply(mb);
 914 }
 915
 916 void
 917 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 918 {
 919     if (pipeline.front()->http->request->error) {
 920         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 921         return;
 922     }
 923
 924     writeCustomReply(200, "PORT successfully converted to PASV.");
 925
 926     // and wait for RETR
 927 }
 928
 929 void
 930 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 931 {
 932     if (!pinning.pinned) // we failed to connect to server
 933         uri.clear();
 934     // 421: we will close due to fssError
 935     writeErrorReply(reply, 421);
 936 }
 937
 938 void
 939 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 940 {
 941     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 942         writeForwardedReply(reply);
 943         if (Comm::IsConnOpen(dataConn)) {
 944             debugs(33, 3, "closing " << dataConn << " on KO reply");
 945             closeDataConnection();
 946         }
 947         return;
 948     }
 949
 950     if (!dataConn) {
 951         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 952         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 953         return;
 954     }
 955
 956     if (!checkDataConnPost()) {
 957         writeCustomReply(425, "Data connection is not established.");
 958         closeDataConnection();
 959         return;
 960     }
 961
 962     debugs(33, 7, data.length);
 963
 964     if (data.length <= 0) {
 965         replyDataWritingCheckpoint(); // skip the actual write call
 966         return;
 967     }
 968
 969     MemBuf mb;
 970     mb.init(data.length + 1, data.length + 1);
 971     mb.append(data.data, data.length);
 972
 973     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 974     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 975     Comm::Write(dataConn, &mb, call);
 976
 977     pipeline.front()->noteSentBodyBytes(data.length);
 978 }
 979
 980 /// called when we are done writing a chunk of the response data
 981 void
 982 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 983 {
 984     if (io.flag == Comm::ERR_CLOSING)
 985         return;
 986
 987     if (io.flag != Comm::OK) {
 988         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 989         userDataCompletionCheckpoint(426);
 990         return;
 991     }
 992
 993     assert(pipeline.front()->http);
 994     pipeline.front()->http->out.size += io.size;
 995     replyDataWritingCheckpoint();
 996 }
 997
 998 /// ClientStream checks after (actual or skipped) reply data writing
 999 void
1000 Ftp::Server::replyDataWritingCheckpoint()
1001 {
1002     switch (pipeline.front()->socketState()) {
1003     case STREAM_NONE:
1004         debugs(33, 3, "Keep going");
1005         pipeline.front()->pullData();
1006         return;
1007     case STREAM_COMPLETE:
1008         debugs(33, 3, "FTP reply data transfer successfully complete");
1009         userDataCompletionCheckpoint(226);
1010         break;
1011     case STREAM_UNPLANNED_COMPLETE:
1012         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
1013         userDataCompletionCheckpoint(451);
1014         break;
1015     case STREAM_FAILED:
1016         userDataCompletionCheckpoint(451);
1017         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
1018         break;
1019     default:
1020         fatal("unreachable code");
1021     }
1022 }
1023
1024 void
1025 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1026 {
1027     writeForwardedReply(reply);
1028     // note that the client data connection may already be closed by now
1029 }
1030
1031 void
1032 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1033 {
1034     Must(reply);
1035
1036     if (waitingForOrigin) {
1037         Must(delayedReply == NULL);
1038         delayedReply = reply;
1039         return;
1040     }
1041
1042     const HttpHeader &header = reply->header;
1043     // adaptation and forwarding errors lack Http::HdrType::FTP_STATUS
1044     if (!header.has(Http::HdrType::FTP_STATUS)) {
1045         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1046         return;
1047     }
1048
1049     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1050     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1051     writeForwardedReplyAndCall(reply, call);
1052 }
1053
1054 void
1055 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1056 {
1057     if (pipeline.front()->http->request->error) {
1058         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1059         return;
1060     }
1061
1062     writeCustomReply(200, "EPRT successfully converted to PASV.");
1063
1064     // and wait for RETR
1065 }
1066
1067 void
1068 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1069 {
1070     if (pipeline.front()->http->request->error) {
1071         writeCustomReply(502, "Cannot connect to server", reply);
1072         return;
1073     }
1074
1075     const unsigned short localPort = listenForDataConnection();
1076     if (!localPort)
1077         return;
1078
1079     // In interception setups, we use a local port number and hope that data
1080     // traffic will be redirected to us.
1081     MemBuf mb;
1082     mb.init();
1083     mb.appendf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1084
1085     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1086     writeReply(mb);
1087 }
1088
1089 /// writes FTP error response with given status and reply-derived error details
1090 void
1091 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1092 {
1093     const HttpRequest *request = pipeline.front()->http->request;
1094     assert(request);
1095
1096     MemBuf mb;
1097     mb.init();
1098
1099     if (request->error)
1100         mb.appendf("%i-%s\r\n", scode, errorPageName(request->error.category));
1101
1102     if (const auto &detail = request->error.detail) {
1103         mb.appendf("%i-Error-Detail-Brief: " SQUIDSBUFPH "\r\n", scode, SQUIDSBUFPRINT(detail->brief()));
1104         mb.appendf("%i-Error-Detail-Verbose: " SQUIDSBUFPH "\r\n", scode, SQUIDSBUFPRINT(detail->verbose(request)));
1105     }
1106
1107 #if USE_ADAPTATION
1108     // XXX: Remove hard coded names. Use an error page template instead.
1109     const Adaptation::History::Pointer ah = request->adaptHistory();
1110     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1111         const String info = ah->allMeta.getByName("X-Response-Info");
1112         const String desc = ah->allMeta.getByName("X-Response-Desc");
1113         if (info.size())
1114             mb.appendf("%i-Information: %s\r\n", scode, info.termedBuf());
1115         if (desc.size())
1116             mb.appendf("%i-Description: %s\r\n", scode, desc.termedBuf());
1117     }
1118 #endif
1119
1120     const char *reason = "Lost Error";
1121     if (reply) {
1122         reason = reply->header.has(Http::HdrType::FTP_REASON) ?
1123                  reply->header.getStr(Http::HdrType::FTP_REASON):
1124                  reply->sline.reason();
1125     }
1126
1127     mb.appendf("%i %s\r\n", scode, reason); // error terminating line
1128
1129     // TODO: errorpage.cc should detect FTP client and use
1130     // configurable FTP-friendly error templates which we should
1131     // write to the client "as is" instead of hiding most of the info
1132
1133     writeReply(mb);
1134 }
1135
1136 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1137 /// for example, internally-generated Squid "errorpages" end up here (for now)
1138 void
1139 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1140 {
1141     changeState(fssConnected, "foreign reply");
1142     closeDataConnection();
1143     // 451: We intend to keep the control connection open.
1144     writeErrorReply(reply, 451);
1145 }
1146
1147 bool
1148 Ftp::Server::writeControlMsgAndCall(HttpReply *reply, AsyncCall::Pointer &call)
1149 {
1150     // the caller guarantees that we are dealing with the current context only
1151     // the caller should also make sure reply->header.has(Http::HdrType::FTP_STATUS)
1152     writeForwardedReplyAndCall(reply, call);
1153     return true;
1154 }
1155
1156 void
1157 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1158 {
1159     assert(reply != NULL);
1160     const HttpHeader &header = reply->header;
1161
1162     // without status, the caller must use the writeForwardedForeign() path
1163     Must(header.has(Http::HdrType::FTP_STATUS));
1164     Must(header.has(Http::HdrType::FTP_REASON));
1165     const int scode = header.getInt(Http::HdrType::FTP_STATUS);
1166     debugs(33, 7, "scode: " << scode);
1167
1168     // Status 125 or 150 implies upload or data request, but we still check
1169     // the state in case the server is buggy.
1170     if ((scode == 125 || scode == 150) &&
1171             (master->serverState == fssHandleUploadRequest ||
1172              master->serverState == fssHandleDataRequest)) {
1173         if (checkDataConnPost()) {
1174             // If the data connection is ready, start reading data (here)
1175             // and forward the response to client (further below).
1176             debugs(33, 7, "data connection established, start data transfer");
1177             if (master->serverState == fssHandleUploadRequest)
1178                 maybeReadUploadData();
1179         } else {
1180             // If we are waiting to accept the data connection, keep waiting.
1181             if (Comm::IsConnOpen(dataListenConn)) {
1182                 debugs(33, 7, "wait for the client to establish a data connection");
1183                 onDataAcceptCall = call;
1184                 // TODO: Add connect timeout for passive connections listener?
1185                 // TODO: Remember server response so that we can forward it?
1186             } else {
1187                 // Either the connection was establised and closed after the
1188                 // data was transferred OR we failed to establish an active
1189                 // data connection and already sent the error to the client.
1190                 // In either case, there is nothing more to do.
1191                 debugs(33, 7, "done with data OR active connection failed");
1192             }
1193             return;
1194         }
1195     }
1196
1197     MemBuf mb;
1198     mb.init();
1199     Ftp::PrintReply(mb, reply);
1200
1201     debugs(9, 2, "FTP Client " << clientConnection);
1202     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1203            "\n----------");
1204
1205     Comm::Write(clientConnection, &mb, call);
1206 }
1207
1208 static void
1209 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const)
1210 {
1211     const HttpHeader &header = reply->header;
1212
1213     HttpHeaderPos pos = HttpHeaderInitPos;
1214     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1215         if (e->id == Http::HdrType::FTP_PRE) {
1216             String raw;
1217             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1218                 mb.appendf("%s\r\n", raw.termedBuf());
1219         }
1220     }
1221
1222     if (header.has(Http::HdrType::FTP_STATUS)) {
1223         const char *reason = header.getStr(Http::HdrType::FTP_REASON);
1224         mb.appendf("%i %s\r\n", header.getInt(Http::HdrType::FTP_STATUS),
1225                    (reason ? reason : 0));
1226     }
1227 }
1228
1229 void
1230 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1231 {
1232     if (io.flag == Comm::ERR_CLOSING)
1233         return;
1234
1235     if (io.flag != Comm::OK) {
1236         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1237         io.conn->close();
1238         return;
1239     }
1240
1241     Http::StreamPointer context = pipeline.front();
1242     if (context != nullptr && context->http) {
1243         context->http->out.size += io.size;
1244         context->http->out.headers_sz += io.size;
1245     }
1246
1247     flags.readMore = true;
1248     readSomeData();
1249 }
1250
1251 void
1252 Ftp::Server::wroteReply(const CommIoCbParams &io)
1253 {
1254     if (io.flag == Comm::ERR_CLOSING)
1255         return;
1256
1257     if (io.flag != Comm::OK) {
1258         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1259         io.conn->close();
1260         return;
1261     }
1262
1263     Http::StreamPointer context = pipeline.front();
1264     assert(context->http);
1265     context->http->out.size += io.size;
1266     context->http->out.headers_sz += io.size;
1267
1268     if (master->serverState == fssError) {
1269         debugs(33, 5, "closing on FTP server error");
1270         io.conn->close();
1271         return;
1272     }
1273
1274     const clientStream_status_t socketState = context->socketState();
1275     debugs(33, 5, "FTP client stream state " << socketState);
1276     switch (socketState) {
1277     case STREAM_UNPLANNED_COMPLETE:
1278     case STREAM_FAILED:
1279         io.conn->close();
1280         return;
1281
1282     case STREAM_NONE:
1283     case STREAM_COMPLETE:
1284         flags.readMore = true;
1285         changeState(fssConnected, "Ftp::Server::wroteReply");
1286         if (bodyParser)
1287             finishDechunkingRequest(false);
1288         context->finished();
1289         kick();
1290         return;
1291     }
1292 }
1293
1294 bool
1295 Ftp::Server::handleRequest(HttpRequest *request)
1296 {
1297     debugs(33, 9, request);
1298     Must(request);
1299
1300     HttpHeader &header = request->header;
1301     Must(header.has(Http::HdrType::FTP_COMMAND));
1302     String &cmd = header.findEntry(Http::HdrType::FTP_COMMAND)->value;
1303     Must(header.has(Http::HdrType::FTP_ARGUMENTS));
1304     String &params = header.findEntry(Http::HdrType::FTP_ARGUMENTS)->value;
1305
1306     if (Debug::Enabled(9, 2)) {
1307         MemBuf mb;
1308         mb.init();
1309         request->pack(&mb);
1310
1311         debugs(9, 2, "FTP Client " << clientConnection);
1312         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1313                "\n----------");
1314     }
1315
1316     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1317     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1318     static RequestHandlers handlers;
1319     if (!handlers.size()) {
1320         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1321         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1322         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1323         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1324         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1325         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1326         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1327         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1328         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1329         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1330         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1331         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1332     }
1333
1334     RequestHandler handler = NULL;
1335     if (request->method == Http::METHOD_PUT)
1336         handler = &Ftp::Server::handleUploadRequest;
1337     else {
1338         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1339         if (hi != handlers.end())
1340             handler = hi->second;
1341     }
1342
1343     if (!handler) {
1344         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1345         return true;
1346     }
1347
1348     return (this->*handler)(cmd, params);
1349 }
1350
1351 /// Called to parse USER command, which is required to create an HTTP request
1352 /// wrapper. W/o request, the errors are handled by returning earlyError().
1353 Http::Stream *
1354 Ftp::Server::handleUserRequest(const SBuf &, SBuf &params)
1355 {
1356     if (params.isEmpty())
1357         return earlyError(EarlyErrorKind::MissingUsername);
1358
1359     // find the [end of] user name
1360     const SBuf::size_type eou = params.rfind('@');
1361     if (eou == SBuf::npos || eou + 1 >= params.length())
1362         return earlyError(EarlyErrorKind::MissingHost);
1363
1364     // Determine the intended destination.
1365     host = params.substr(eou + 1, params.length());
1366     // If we can parse it as raw IPv6 address, then surround with "[]".
1367     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1368     if (host.find(':') != SBuf::npos) {
1369         const Ip::Address ipa(host.c_str());
1370         if (!ipa.isAnyAddr()) {
1371             char ipBuf[MAX_IPSTRLEN];
1372             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1373             host = ipBuf;
1374         }
1375     }
1376
1377     // const SBuf login = params.substr(0, eou);
1378     params.chop(0, eou); // leave just the login part for the peer
1379
1380     SBuf oldUri;
1381     if (master->clientReadGreeting)
1382         oldUri = uri;
1383
1384     master->workingDir.clear();
1385     calcUri(NULL);
1386
1387     if (!master->clientReadGreeting) {
1388         debugs(9, 3, "set URI to " << uri);
1389     } else if (oldUri.caseCmp(uri) == 0) {
1390         debugs(9, 5, "kept URI as " << oldUri);
1391     } else {
1392         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1393         closeDataConnection();
1394         unpinConnection(true); // close control connection to peer
1395         resetLogin("URI reset");
1396     }
1397
1398     return NULL; // no early errors
1399 }
1400
1401 bool
1402 Ftp::Server::handleFeatRequest(String &, String &)
1403 {
1404     changeState(fssHandleFeat, "handleFeatRequest");
1405     return true;
1406 }
1407
1408 bool
1409 Ftp::Server::handlePasvRequest(String &, String &params)
1410 {
1411     if (gotEpsvAll) {
1412         setReply(500, "Bad PASV command");
1413         return false;
1414     }
1415
1416     if (params.size() > 0) {
1417         setReply(501, "Unexpected parameter");
1418         return false;
1419     }
1420
1421     changeState(fssHandlePasv, "handlePasvRequest");
1422     // no need to fake PASV request via setDataCommand() in true PASV case
1423     return true;
1424 }
1425
1426 /// [Re]initializes dataConn for active data transfers. Does not connect.
1427 bool
1428 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1429 {
1430     assert(clientConnection != NULL);
1431     assert(!clientConnection->remote.isAnyAddr());
1432
1433     if (cltAddr != clientConnection->remote) {
1434         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1435         // Closing the control connection would not help with attacks because
1436         // the client is evidently able to connect to us. Besides, closing
1437         // makes retrials easier for the client and more damaging to us.
1438         setReply(501, "Prohibited parameter value");
1439         return false;
1440     }
1441
1442     closeDataConnection();
1443
1444     Comm::ConnectionPointer conn = new Comm::Connection();
1445     conn->flags |= COMM_DOBIND;
1446
1447     if (clientConnection->flags & COMM_INTERCEPTION) {
1448         // In the case of NAT interception conn->local value is not set
1449         // because the TCP stack will automatically pick correct source
1450         // address for the data connection. We must only ensure that IP
1451         // version matches client's address.
1452         conn->local.setAnyAddr();
1453
1454         if (cltAddr.isIPv4())
1455             conn->local.setIPv4();
1456
1457         conn->remote = cltAddr;
1458     } else {
1459         // In the case of explicit-proxy the local IP of the control connection
1460         // is the Squid IP the client is knowingly talking to.
1461         //
1462         // In the case of TPROXY the IP address of the control connection is
1463         // server IP the client is connecting to, it can be spoofed by Squid.
1464         //
1465         // In both cases some clients may refuse to accept data connections if
1466         // these control connectin local-IP's are not used.
1467         conn->setAddrs(clientConnection->local, cltAddr);
1468
1469         // Using non-local addresses in TPROXY mode requires appropriate socket option.
1470         if (clientConnection->flags & COMM_TRANSPARENT)
1471             conn->flags |= COMM_TRANSPARENT;
1472     }
1473
1474     // RFC 959 requires active FTP connections to originate from port 20
1475     // but that would preclude us from supporting concurrent transfers! (XXX?)
1476     conn->local.port(0);
1477
1478     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1479            conn->remote);
1480
1481     dataConn = conn;
1482     uploadAvailSize = 0;
1483     return true;
1484 }
1485
1486 bool
1487 Ftp::Server::handlePortRequest(String &, String &params)
1488 {
1489     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1490
1491     if (gotEpsvAll) {
1492         setReply(500, "Rejecting PORT after EPSV ALL");
1493         return false;
1494     }
1495
1496     if (!params.size()) {
1497         setReply(501, "Missing parameter");
1498         return false;
1499     }
1500
1501     Ip::Address cltAddr;
1502     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1503         setReply(501, "Invalid parameter");
1504         return false;
1505     }
1506
1507     if (!createDataConnection(cltAddr))
1508         return false;
1509
1510     changeState(fssHandlePort, "handlePortRequest");
1511     setDataCommand();
1512     return true; // forward our fake PASV request
1513 }
1514
1515 bool
1516 Ftp::Server::handleDataRequest(String &, String &)
1517 {
1518     if (!checkDataConnPre())
1519         return false;
1520
1521     master->userDataDone = 0;
1522     originDataDownloadAbortedOnError = false;
1523
1524     changeState(fssHandleDataRequest, "handleDataRequest");
1525
1526     return true;
1527 }
1528
1529 bool
1530 Ftp::Server::handleUploadRequest(String &, String &)
1531 {
1532     if (!checkDataConnPre())
1533         return false;
1534
1535     if (Config.accessList.forceRequestBodyContinuation) {
1536         ClientHttpRequest *http = pipeline.front()->http;
1537         HttpRequest *request = http->request;
1538         ACLFilledChecklist bodyContinuationCheck(Config.accessList.forceRequestBodyContinuation, request, NULL);
1539         bodyContinuationCheck.al = http->al;
1540         bodyContinuationCheck.syncAle(request, http->log_uri);
1541         if (bodyContinuationCheck.fastCheck().allowed()) {
1542             request->forcedBodyContinuation = true;
1543             if (checkDataConnPost()) {
1544                 // Write control Msg
1545                 writeEarlyReply(150, "Data connection opened");
1546                 maybeReadUploadData();
1547             } else {
1548                 // wait for acceptDataConnection but tell it to call wroteEarlyReply
1549                 // after writing "150 Data connection opened"
1550                 typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1551                 AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
1552                 onDataAcceptCall = call;
1553             }
1554         }
1555     }
1556
1557     changeState(fssHandleUploadRequest, "handleDataRequest");
1558
1559     return true;
1560 }
1561
1562 bool
1563 Ftp::Server::handleEprtRequest(String &, String &params)
1564 {
1565     debugs(9, 3, "Process an EPRT " << params);
1566
1567     if (gotEpsvAll) {
1568         setReply(500, "Rejecting EPRT after EPSV ALL");
1569         return false;
1570     }
1571
1572     if (!params.size()) {
1573         setReply(501, "Missing parameter");
1574         return false;
1575     }
1576
1577     Ip::Address cltAddr;
1578     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1579         setReply(501, "Invalid parameter");
1580         return false;
1581     }
1582
1583     if (!createDataConnection(cltAddr))
1584         return false;
1585
1586     changeState(fssHandleEprt, "handleEprtRequest");
1587     setDataCommand();
1588     return true; // forward our fake PASV request
1589 }
1590
1591 bool
1592 Ftp::Server::handleEpsvRequest(String &, String &params)
1593 {
1594     debugs(9, 3, "Process an EPSV command with params: " << params);
1595     if (params.size() <= 0) {
1596         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1597     } else if (params.caseCmp("ALL") == 0) {
1598         setReply(200, "EPSV ALL ok");
1599         gotEpsvAll = true;
1600         return false;
1601     } else if (params.cmp("2") == 0) {
1602         if (!Ip::EnableIpv6) {
1603             setReply(522, "Network protocol not supported, use (1)");
1604             return false;
1605         }
1606     } else if (params.cmp("1") != 0) {
1607         setReply(501, "Unsupported EPSV parameter");
1608         return false;
1609     }
1610
1611     changeState(fssHandleEpsv, "handleEpsvRequest");
1612     setDataCommand();
1613     return true; // forward our fake PASV request
1614 }
1615
1616 bool
1617 Ftp::Server::handleCwdRequest(String &, String &)
1618 {
1619     changeState(fssHandleCwd, "handleCwdRequest");
1620     return true;
1621 }
1622
1623 bool
1624 Ftp::Server::handlePassRequest(String &, String &)
1625 {
1626     changeState(fssHandlePass, "handlePassRequest");
1627     return true;
1628 }
1629
1630 bool
1631 Ftp::Server::handleCdupRequest(String &, String &)
1632 {
1633     changeState(fssHandleCdup, "handleCdupRequest");
1634     return true;
1635 }
1636
1637 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1638 // Squid FTP client decides what data command to use with peers.
1639 void
1640 Ftp::Server::setDataCommand()
1641 {
1642     ClientHttpRequest *const http = pipeline.front()->http;
1643     assert(http != NULL);
1644     HttpRequest *const request = http->request;
1645     assert(request != NULL);
1646     HttpHeader &header = request->header;
1647     header.delById(Http::HdrType::FTP_COMMAND);
1648     header.putStr(Http::HdrType::FTP_COMMAND, "PASV");
1649     header.delById(Http::HdrType::FTP_ARGUMENTS);
1650     header.putStr(Http::HdrType::FTP_ARGUMENTS, "");
1651     debugs(9, 5, "client data command converted to fake PASV");
1652 }
1653
1654 /// check that client data connection is ready for future I/O or at least
1655 /// has a chance of becoming ready soon.
1656 bool
1657 Ftp::Server::checkDataConnPre()
1658 {
1659     if (Comm::IsConnOpen(dataConn))
1660         return true;
1661
1662     if (Comm::IsConnOpen(dataListenConn)) {
1663         // We are still waiting for a client to connect to us after PASV.
1664         // Perhaps client's data conn handshake has not reached us yet.
1665         // After we talk to the server, checkDataConnPost() will recheck.
1666         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1667         return true;
1668     }
1669
1670     if (!dataConn || dataConn->remote.isAnyAddr()) {
1671         debugs(33, 5, "missing " << dataConn);
1672         // TODO: use client address and default port instead.
1673         setReply(425, "Use PORT or PASV first");
1674         return false;
1675     }
1676
1677     // active transfer: open a data connection from Squid to client
1678     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1679     connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1680     Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector,
1681             Config.Timeout.connect);
1682     AsyncJob::Start(cs);
1683     return false; // ConnStateData::processFtpRequest waits handleConnectDone
1684 }
1685
1686 /// Check that client data connection is ready for immediate I/O.
1687 bool
1688 Ftp::Server::checkDataConnPost() const
1689 {
1690     if (!Comm::IsConnOpen(dataConn)) {
1691         debugs(33, 3, "missing client data conn: " << dataConn);
1692         return false;
1693     }
1694     return true;
1695 }
1696
1697 /// Done establishing a data connection to the user.
1698 void
1699 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1700 {
1701     connector = NULL;
1702
1703     if (params.flag != Comm::OK) {
1704         /* it might have been a timeout with a partially open link */
1705         if (params.conn != NULL)
1706             params.conn->close();
1707         setReply(425, "Cannot open data connection.");
1708         Http::StreamPointer context = pipeline.front();
1709         Must(context->http);
1710         Must(context->http->storeEntry() != NULL);
1711     } else {
1712         Must(dataConn == params.conn);
1713         Must(Comm::IsConnOpen(params.conn));
1714         fd_note(params.conn->fd, "active client ftp data");
1715     }
1716
1717     doProcessRequest();
1718 }
1719
1720 void
1721 Ftp::Server::setReply(const int code, const char *msg)
1722 {
1723     Http::StreamPointer context = pipeline.front();
1724     ClientHttpRequest *const http = context->http;
1725     assert(http != NULL);
1726     assert(http->storeEntry() == NULL);
1727
1728     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1729
1730     clientStreamNode *const node = context->getClientReplyContext();
1731     clientReplyContext *const repContext =
1732         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1733     assert(repContext != NULL);
1734
1735     RequestFlags reqFlags;
1736     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1737     reqFlags.noCache = true;
1738     repContext->createStoreEntry(http->request->method, reqFlags);
1739     http->storeEntry()->replaceHttpReply(reply);
1740 }
1741
1742 void
1743 Ftp::Server::callException(const std::exception &e)
1744 {
1745     debugs(33, 2, "FTP::Server job caught: " << e.what());
1746     closeDataConnection();
1747     unpinConnection(true);
1748     if (Comm::IsConnOpen(clientConnection))
1749         clientConnection->close();
1750     AsyncJob::callException(e);
1751 }
1752
1753 void
1754 Ftp::Server::startWaitingForOrigin()
1755 {
1756     if (!isOpen()) // if we are closing, nothing to do
1757         return;
1758
1759     debugs(33, 5, "waiting for Ftp::Client data transfer to end");
1760     waitingForOrigin = true;
1761 }
1762
1763 void
1764 Ftp::Server::stopWaitingForOrigin(int originStatus)
1765 {
1766     Must(waitingForOrigin);
1767     waitingForOrigin = false;
1768
1769     if (!isOpen()) // if we are closing, nothing to do
1770         return;
1771
1772     // if we have already decided how to respond, respond now
1773     if (delayedReply) {
1774         HttpReply::Pointer reply = delayedReply;
1775         delayedReply = nullptr;
1776         writeForwardedReply(reply.getRaw());
1777         return; // do not completeDataDownload() after an earlier response
1778     }
1779
1780     if (master->serverState != fssHandleDataRequest)
1781         return;
1782
1783     // completeDataDownload() could be waitingForOrigin in fssHandleDataRequest
1784     // Depending on which side has finished downloading first, either trust
1785     // master->userDataDone status or set originDataDownloadAbortedOnError:
1786     if (master->userDataDone) {
1787         // We finished downloading before Ftp::Client. Most likely, the
1788         // adaptation shortened the origin response or we hit an error.
1789         // Our status (stored in master->userDataDone) is more informative.
1790         // Use master->userDataDone; avoid originDataDownloadAbortedOnError.
1791         completeDataDownload();
1792     } else {
1793         debugs(33, 5, "too early to write the response");
1794         // Ftp::Client naturally finished downloading before us. Set
1795         // originDataDownloadAbortedOnError to overwrite future
1796         // master->userDataDone and relay Ftp::Client error, if there was
1797         // any, to the user.
1798         originDataDownloadAbortedOnError = (originStatus >= 400);
1799     }
1800 }
1801
1802 void Ftp::Server::userDataCompletionCheckpoint(int finalStatusCode)
1803 {
1804     Must(!master->userDataDone);
1805     master->userDataDone = finalStatusCode;
1806
1807     if (bodyParser)
1808         finishDechunkingRequest(false);
1809
1810     if (waitingForOrigin) {
1811         // The completeDataDownload() is not called here unconditionally
1812         // because we want to signal the FTP user that we are not fully
1813         // done processing its data stream, even though all data bytes
1814         // have been sent or received already.
1815         debugs(33, 5, "Transferring from FTP server is not complete");
1816         return;
1817     }
1818
1819     // Adjust our reply if the server aborted with an error before we are done.
1820     if (master->userDataDone == 226 && originDataDownloadAbortedOnError) {
1821         debugs(33, 5, "Transferring from FTP server terminated with an error, adjust status code");
1822         master->userDataDone = 451;
1823     }
1824     completeDataDownload();
1825 }
1826
1827 void Ftp::Server::completeDataDownload()
1828 {
1829     writeCustomReply(master->userDataDone, master->userDataDone == 226 ? "Transfer complete" : "Server error; transfer aborted");
1830     closeDataConnection();
1831 }
1832
1833 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1834 static bool
1835 Ftp::SupportedCommand(const SBuf &name)
1836 {
1837     static std::set<SBuf> BlackList;
1838     if (BlackList.empty()) {
1839         /* Add FTP commands that Squid cannot relay correctly. */
1840
1841         // We probably do not support AUTH TLS.* and AUTH SSL,
1842         // but let's disclaim all AUTH support to KISS, for now.
1843         BlackList.insert(cmdAuth());
1844     }
1845
1846     // we claim support for all commands that we do not know about
1847     return BlackList.find(name) == BlackList.end();
1848 }
1849