src/servers/FtpServer.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Transfer protocol servers */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CharacterSet.h"
  14 #include "base/RefCount.h"
  15 #include "base/Subscription.h"
  16 #include "client_side_reply.h"
  17 #include "client_side_request.h"
  18 #include "clientStream.h"
  19 #include "comm/ConnOpener.h"
  20 #include "comm/Read.h"
  21 #include "comm/TcpAcceptor.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "ftp/Elements.h"
  26 #include "ftp/Parsing.h"
  27 #include "globals.h"
  28 #include "http/one/RequestParser.h"
  29 #include "http/Stream.h"
  30 #include "HttpHdrCc.h"
  31 #include "ip/tools.h"
  32 #include "ipc/FdNotes.h"
  33 #include "parser/Tokenizer.h"
  34 #include "servers/forward.h"
  35 #include "servers/FtpServer.h"
  36 #include "SquidConfig.h"
  37 #include "StatCounters.h"
  38 #include "tools.h"
  39
  40 #include <set>
  41 #include <map>
  42
  43 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  44
  45 namespace Ftp
  46 {
  47 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  48 static bool SupportedCommand(const SBuf &name);
  49 static bool CommandHasPathParameter(const SBuf &cmd);
  50 };
  51
  52 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  53     AsyncJob("Ftp::Server"),
  54     ConnStateData(xact),
  55     master(new MasterState),
  56     uri(),
  57     host(),
  58     gotEpsvAll(false),
  59     onDataAcceptCall(),
  60     dataListenConn(),
  61     dataConn(),
  62     uploadAvailSize(0),
  63     listener(),
  64     connector(),
  65     reader(),
  66     waitingForOrigin(false),
  67     originDataDownloadAbortedOnError(false)
  68 {
  69     flags.readMore = false; // we need to announce ourselves first
  70     *uploadBuf = 0;
  71 }
  72
  73 Ftp::Server::~Server()
  74 {
  75     closeDataConnection();
  76 }
  77
  78 int
  79 Ftp::Server::pipelinePrefetchMax() const
  80 {
  81     return 0; // no support for concurrent FTP requests
  82 }
  83
  84 time_t
  85 Ftp::Server::idleTimeout() const
  86 {
  87     return Config.Timeout.ftpClientIdle;
  88 }
  89
  90 void
  91 Ftp::Server::start()
  92 {
  93     ConnStateData::start();
  94
  95     if (transparent()) {
  96         char buf[MAX_IPSTRLEN];
  97         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  98         host = buf;
  99         calcUri(NULL);
 100         debugs(33, 5, "FTP transparent URL: " << uri);
 101     }
 102
 103     writeEarlyReply(220, "Service ready");
 104 }
 105
 106 /// schedules another data connection read if needed
 107 void
 108 Ftp::Server::maybeReadUploadData()
 109 {
 110     if (reader != NULL)
 111         return;
 112
 113     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 114     if (availSpace <= 0)
 115         return;
 116
 117     debugs(33, 4, dataConn << ": reading FTP data...");
 118
 119     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 120     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 121     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 122               reader);
 123 }
 124
 125 /// react to the freshly parsed request
 126 void
 127 Ftp::Server::doProcessRequest()
 128 {
 129     // zero pipelinePrefetchMax() ensures that there is only parsed request
 130     Must(pipeline.count() == 1);
 131     Http::StreamPointer context = pipeline.front();
 132     Must(context != nullptr);
 133
 134     ClientHttpRequest *const http = context->http;
 135     assert(http != NULL);
 136
 137     HttpRequest *const request = http->request;
 138     Must(http->storeEntry() || request);
 139     const bool mayForward = !http->storeEntry() && handleRequest(request);
 140
 141     if (http->storeEntry() != NULL) {
 142         debugs(33, 4, "got an immediate response");
 143         clientSetKeepaliveFlag(http);
 144         context->pullData();
 145     } else if (mayForward) {
 146         debugs(33, 4, "forwarding request to server side");
 147         assert(http->storeEntry() == NULL);
 148         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 149     } else {
 150         debugs(33, 4, "will resume processing later");
 151     }
 152 }
 153
 154 void
 155 Ftp::Server::processParsedRequest(Http::Stream *)
 156 {
 157     Must(pipeline.count() == 1);
 158
 159     // Process FTP request asynchronously to make sure FTP
 160     // data connection accept callback is fired first.
 161     CallJobHere(33, 4, CbcPointer<Server>(this),
 162                 Ftp::Server, doProcessRequest);
 163 }
 164
 165 /// imports more upload data from the data connection
 166 void
 167 Ftp::Server::readUploadData(const CommIoCbParams &io)
 168 {
 169     debugs(33, 5, io.conn << " size " << io.size);
 170     Must(reader != NULL);
 171     reader = NULL;
 172
 173     assert(Comm::IsConnOpen(dataConn));
 174     assert(io.conn->fd == dataConn->fd);
 175
 176     if (io.flag == Comm::OK && bodyPipe != NULL) {
 177         if (io.size > 0) {
 178             statCounter.client_http.kbytes_in += io.size;
 179
 180             char *const current_buf = uploadBuf + uploadAvailSize;
 181             if (io.buf != current_buf)
 182                 memmove(current_buf, io.buf, io.size);
 183             uploadAvailSize += io.size;
 184             shovelUploadData();
 185         } else if (io.size == 0) {
 186             debugs(33, 5, io.conn << " closed");
 187             closeDataConnection();
 188             if (uploadAvailSize <= 0)
 189                 finishDechunkingRequest(true);
 190         }
 191     } else { // not Comm::Flags::OK or unexpected read
 192         debugs(33, 5, io.conn << " closed");
 193         closeDataConnection();
 194         finishDechunkingRequest(false);
 195     }
 196
 197 }
 198
 199 /// shovel upload data from the internal buffer to the body pipe if possible
 200 void
 201 Ftp::Server::shovelUploadData()
 202 {
 203     assert(bodyPipe != NULL);
 204
 205     debugs(33, 5, "handling FTP request data for " << clientConnection);
 206     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 207                            uploadAvailSize);
 208     if (putSize > 0) {
 209         uploadAvailSize -= putSize;
 210         if (uploadAvailSize > 0)
 211             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 212     }
 213
 214     if (Comm::IsConnOpen(dataConn))
 215         maybeReadUploadData();
 216     else if (uploadAvailSize <= 0)
 217         finishDechunkingRequest(true);
 218 }
 219
 220 void
 221 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 222 {
 223     if (!isOpen()) // if we are closing, nothing to do
 224         return;
 225
 226     shovelUploadData();
 227 }
 228
 229 void
 230 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 231 {
 232     if (!isOpen()) // if we are closing, nothing to do
 233         return;
 234
 235     ConnStateData::noteBodyConsumerAborted(ptr);
 236     closeDataConnection();
 237 }
 238
 239 /// accept a new FTP control connection and hand it to a dedicated Server
 240 void
 241 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 242 {
 243     MasterXaction::Pointer xact = params.xaction;
 244     AnyP::PortCfgPointer s = xact->squidPort;
 245
 246     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 247
 248     if (params.flag != Comm::OK) {
 249         // Its possible the call was still queued when the client disconnected
 250         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 251         return;
 252     }
 253
 254     debugs(33, 4, params.conn << ": accepted");
 255     fd_note(params.conn->fd, "client ftp connect");
 256
 257     if (s->tcp_keepalive.enabled)
 258         commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
 259
 260     ++incoming_sockets_accepted;
 261
 262     AsyncJob::Start(new Server(xact));
 263 }
 264
 265 void
 266 Ftp::StartListening()
 267 {
 268     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 269         if (MAXTCPLISTENPORTS == NHttpSockets) {
 270             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 271                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 272             break;
 273         }
 274
 275         // direct new connections accepted by listenConn to Accept()
 276         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 277         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 278                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 279                                                CommAcceptCbParams(NULL)));
 280         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 281     }
 282 }
 283
 284 void
 285 Ftp::StopListening()
 286 {
 287     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 288         if (s->listenConn != NULL) {
 289             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 290             s->listenConn->close();
 291             s->listenConn = NULL;
 292         }
 293     }
 294 }
 295
 296 void
 297 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 298 {
 299     // find request
 300     Http::StreamPointer context = pipeline.front();
 301     Must(context != nullptr);
 302     ClientHttpRequest *const http = context->http;
 303     Must(http != NULL);
 304     HttpRequest *const request = http->request;
 305     Must(request != NULL);
 306
 307     // this is not an idle connection, so we do not want I/O monitoring
 308     const bool monitor = false;
 309
 310     // make FTP peer connection exclusive to our request
 311     pinConnection(conn, request, conn->getPeer(), false, monitor);
 312 }
 313
 314 void
 315 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 316 {
 317     ConnStateData::clientPinnedConnectionClosed(io);
 318
 319     // TODO: Keep the control connection open after fixing the reset
 320     // problem below
 321     if (Comm::IsConnOpen(clientConnection))
 322         clientConnection->close();
 323
 324     // TODO: If the server control connection is gone, reset state to login
 325     // again. Reseting login alone is not enough: FtpRelay::sendCommand() will
 326     // not re-login because FtpRelay::serverState() is not going to be
 327     // fssConnected. Calling resetLogin() alone is also harmful because
 328     // it does not reset correctly the client-to-squid control connection (eg
 329     // respond if required with an error code, in all cases)
 330     // resetLogin("control connection closure");
 331 }
 332
 333 /// clear client and server login-related state after the old login is gone
 334 void
 335 Ftp::Server::resetLogin(const char *reason)
 336 {
 337     debugs(33, 5, "will need to re-login due to " << reason);
 338     master->clientReadGreeting = false;
 339     changeState(fssBegin, reason);
 340 }
 341
 342 /// computes uri member from host and, if tracked, working dir with file name
 343 void
 344 Ftp::Server::calcUri(const SBuf *file)
 345 {
 346     uri = "ftp://";
 347     uri.append(host);
 348     if (port->ftp_track_dirs && master->workingDir.length()) {
 349         if (master->workingDir[0] != '/')
 350             uri.append("/", 1);
 351         uri.append(master->workingDir);
 352     }
 353
 354     if (uri[uri.length() - 1] != '/')
 355         uri.append("/", 1);
 356
 357     if (port->ftp_track_dirs && file) {
 358         static const CharacterSet Slash("/", "/");
 359         Parser::Tokenizer tok(*file);
 360         tok.skipAll(Slash);
 361         uri.append(tok.remaining());
 362     }
 363 }
 364
 365 /// Starts waiting for a data connection. Returns listening port.
 366 /// On errors, responds with an error and returns zero.
 367 unsigned int
 368 Ftp::Server::listenForDataConnection()
 369 {
 370     closeDataConnection();
 371
 372     Comm::ConnectionPointer conn = new Comm::Connection;
 373     conn->flags = COMM_NONBLOCKING;
 374     conn->local = transparent() ? port->s : clientConnection->local;
 375     conn->local.port(0);
 376     const char *const note = uri.c_str();
 377     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 378     if (!Comm::IsConnOpen(conn)) {
 379         debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " <<
 380                conn->local << " error: " << errno);
 381         writeCustomReply(451, "Internal error");
 382         return 0;
 383     }
 384
 385     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 386     typedef AsyncCallT<AcceptDialer> AcceptCall;
 387     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 388     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 389     listener = call.getRaw();
 390     dataListenConn = conn;
 391     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 392
 393     const unsigned int listeningPort = comm_local_port(conn->fd);
 394     conn->local.port(listeningPort);
 395     return listeningPort;
 396 }
 397
 398 void
 399 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 400 {
 401     if (params.flag != Comm::OK) {
 402         // Its possible the call was still queued when the client disconnected
 403         debugs(33, 2, dataListenConn << ": accept "
 404                "failure: " << xstrerr(params.xerrno));
 405         return;
 406     }
 407
 408     debugs(33, 4, "accepted " << params.conn);
 409     fd_note(params.conn->fd, "passive client ftp data");
 410     ++incoming_sockets_accepted;
 411
 412     if (!clientConnection) {
 413         debugs(33, 5, "late data connection?");
 414         closeDataConnection(); // in case we are still listening
 415         params.conn->close();
 416     } else if (params.conn->remote != clientConnection->remote) {
 417         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 418         params.conn->close();
 419         // Some FTP servers close control connection here, but it may make
 420         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 421     } else {
 422         closeDataConnection();
 423         dataConn = params.conn;
 424         uploadAvailSize = 0;
 425         debugs(33, 7, "ready for data");
 426         if (onDataAcceptCall != NULL) {
 427             AsyncCall::Pointer call = onDataAcceptCall;
 428             onDataAcceptCall = NULL;
 429             // If we got an upload request, start reading data from the client.
 430             if (master->serverState == fssHandleUploadRequest)
 431                 maybeReadUploadData();
 432             else
 433                 Must(master->serverState == fssHandleDataRequest);
 434             MemBuf mb;
 435             mb.init();
 436             mb.appendf("150 Data connection opened.\r\n");
 437             Comm::Write(clientConnection, &mb, call);
 438         }
 439     }
 440 }
 441
 442 void
 443 Ftp::Server::closeDataConnection()
 444 {
 445     if (listener != NULL) {
 446         listener->cancel("no longer needed");
 447         listener = NULL;
 448     }
 449
 450     if (Comm::IsConnOpen(dataListenConn)) {
 451         debugs(33, 5, "FTP closing client data listen socket: " <<
 452                *dataListenConn);
 453         dataListenConn->close();
 454     }
 455     dataListenConn = NULL;
 456
 457     if (reader != NULL) {
 458         // Comm::ReadCancel can deal with negative FDs
 459         Comm::ReadCancel(dataConn->fd, reader);
 460         reader = NULL;
 461     }
 462
 463     if (Comm::IsConnOpen(dataConn)) {
 464         debugs(33, 5, "FTP closing client data connection: " <<
 465                *dataConn);
 466         dataConn->close();
 467     }
 468     dataConn = NULL;
 469 }
 470
 471 /// Writes FTP [error] response before we fully parsed the FTP request and
 472 /// created the corresponding HTTP request wrapper for that FTP request.
 473 void
 474 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 475 {
 476     debugs(33, 7, code << ' ' << msg);
 477     assert(99 < code && code < 1000);
 478
 479     MemBuf mb;
 480     mb.init();
 481     mb.appendf("%i %s\r\n", code, msg);
 482
 483     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 484     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 485     Comm::Write(clientConnection, &mb, call);
 486
 487     flags.readMore = false;
 488
 489     // TODO: Create master transaction. Log it in wroteEarlyReply().
 490 }
 491
 492 void
 493 Ftp::Server::writeReply(MemBuf &mb)
 494 {
 495     debugs(9, 2, "FTP Client " << clientConnection);
 496     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 497            "\n----------");
 498
 499     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 500     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 501     Comm::Write(clientConnection, &mb, call);
 502 }
 503
 504 void
 505 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 506 {
 507     debugs(33, 7, code << ' ' << msg);
 508     assert(99 < code && code < 1000);
 509
 510     const bool sendDetails = reply != NULL &&
 511                              reply->header.has(Http::HdrType::FTP_STATUS) && reply->header.has(Http::HdrType::FTP_REASON);
 512
 513     MemBuf mb;
 514     mb.init();
 515     if (sendDetails) {
 516         mb.appendf("%i-%s\r\n", code, msg);
 517         mb.appendf(" Server reply:\r\n");
 518         Ftp::PrintReply(mb, reply, " ");
 519         mb.appendf("%i \r\n", code);
 520     } else
 521         mb.appendf("%i %s\r\n", code, msg);
 522
 523     writeReply(mb);
 524 }
 525
 526 void
 527 Ftp::Server::changeState(const ServerState newState, const char *reason)
 528 {
 529     if (master->serverState == newState) {
 530         debugs(33, 3, "client state unchanged at " << master->serverState <<
 531                " because " << reason);
 532         master->serverState = newState;
 533     } else {
 534         debugs(33, 3, "client state was " << master->serverState <<
 535                ", now " << newState << " because " << reason);
 536         master->serverState = newState;
 537     }
 538 }
 539
 540 /// whether the given FTP command has a pathname parameter
 541 static bool
 542 Ftp::CommandHasPathParameter(const SBuf &cmd)
 543 {
 544     static std::set<SBuf> PathedCommands;
 545     if (!PathedCommands.size()) {
 546         PathedCommands.insert(cmdMlst());
 547         PathedCommands.insert(cmdMlsd());
 548         PathedCommands.insert(cmdStat());
 549         PathedCommands.insert(cmdNlst());
 550         PathedCommands.insert(cmdList());
 551         PathedCommands.insert(cmdMkd());
 552         PathedCommands.insert(cmdRmd());
 553         PathedCommands.insert(cmdDele());
 554         PathedCommands.insert(cmdRnto());
 555         PathedCommands.insert(cmdRnfr());
 556         PathedCommands.insert(cmdAppe());
 557         PathedCommands.insert(cmdStor());
 558         PathedCommands.insert(cmdRetr());
 559         PathedCommands.insert(cmdSmnt());
 560         PathedCommands.insert(cmdCwd());
 561     }
 562
 563     return PathedCommands.find(cmd) != PathedCommands.end();
 564 }
 565
 566 /// creates a context filled with an error message for a given early error
 567 Http::Stream *
 568 Ftp::Server::earlyError(const EarlyErrorKind eek)
 569 {
 570     /* Default values, to be updated by the switch statement below */
 571     int scode = 421;
 572     const char *reason = "Internal error";
 573     const char *errUri = "error:ftp-internal-early-error";
 574
 575     switch (eek) {
 576     case EarlyErrorKind::HugeRequest:
 577         scode = 421;
 578         reason = "Huge request";
 579         errUri = "error:ftp-huge-request";
 580         break;
 581
 582     case EarlyErrorKind::MissingLogin:
 583         scode = 530;
 584         reason = "Must login first";
 585         errUri = "error:ftp-must-login-first";
 586         break;
 587
 588     case EarlyErrorKind::MissingUsername:
 589         scode = 501;
 590         reason = "Missing username";
 591         errUri = "error:ftp-missing-username";
 592         break;
 593
 594     case EarlyErrorKind::MissingHost:
 595         scode = 501;
 596         reason = "Missing host";
 597         errUri = "error:ftp-missing-host";
 598         break;
 599
 600     case EarlyErrorKind::UnsupportedCommand:
 601         scode = 502;
 602         reason = "Unknown or unsupported command";
 603         errUri = "error:ftp-unsupported-command";
 604         break;
 605
 606     case EarlyErrorKind::InvalidUri:
 607         scode = 501;
 608         reason = "Invalid URI";
 609         errUri = "error:ftp-invalid-uri";
 610         break;
 611
 612     case EarlyErrorKind::MalformedCommand:
 613         scode = 421;
 614         reason = "Malformed command";
 615         errUri = "error:ftp-malformed-command";
 616         break;
 617
 618         // no default so that a compiler can check that we have covered all cases
 619     }
 620
 621     Http::Stream *context = abortRequestParsing(errUri);
 622     clientStreamNode *node = context->getClientReplyContext();
 623     Must(node);
 624     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 625     Must(repContext);
 626
 627     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 628     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 629     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 630     repContext->setReplyToReply(reply);
 631     return context;
 632 }
 633
 634 /// Parses a single FTP request on the control connection.
 635 /// Returns a new Http::Stream on valid requests and all errors.
 636 /// Returns NULL on incomplete requests that may still succeed given more data.
 637 Http::Stream *
 638 Ftp::Server::parseOneRequest()
 639 {
 640     flags.readMore = false; // common for all but one case below
 641
 642     // OWS <command> [ RWS <parameter> ] OWS LF
 643
 644     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 645     // for the LF character that we must exclude here (but see FullWhiteSpace).
 646     static const char * const InlineSpaceChars = " \f\r\t\v";
 647     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 648     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 649     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 650     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 651
 652     // This set is used to ignore empty commands without allowing an attacker
 653     // to keep us endlessly busy by feeding us whitespace or empty commands.
 654     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 655
 656     SBuf cmd;
 657     SBuf params;
 658
 659     Parser::Tokenizer tok(inBuf);
 660
 661     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 662     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 663
 664     // note that the condition below will eat either RWS or trailing OWS
 665     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 666         // now params may include trailing OWS
 667         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 668         static const SBuf bufWhiteSpace(InlineSpaceChars);
 669         params.trim(bufWhiteSpace, false, true);
 670     }
 671
 672     // Why limit command line and parameters size? Did not we just parse them?
 673     // XXX: Our good old String cannot handle very long strings.
 674     const SBuf::size_type tokenMax = min(
 675                                          static_cast<SBuf::size_type>(32*1024), // conservative
 676                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 677     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 678         changeState(fssError, "huge req token");
 679         quitAfterError(NULL);
 680         return earlyError(EarlyErrorKind::HugeRequest);
 681     }
 682
 683     // technically, we may skip multiple NLs below, but that is OK
 684     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 685         // we need more data, but can we buffer more?
 686         if (inBuf.length() >= Config.maxRequestHeaderSize) {
 687             changeState(fssError, "huge req");
 688             quitAfterError(NULL);
 689             return earlyError(EarlyErrorKind::HugeRequest);
 690         } else {
 691             flags.readMore = true;
 692             debugs(33, 5, "Waiting for more, up to " <<
 693                    (Config.maxRequestHeaderSize - inBuf.length()));
 694             return NULL;
 695         }
 696     }
 697
 698     Must(parsed && cmd.length());
 699     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 700
 701     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 702
 703     cmd.toUpper(); // this should speed up and simplify future comparisons
 704
 705     // interception cases do not need USER to calculate the uri
 706     if (!transparent()) {
 707         if (!master->clientReadGreeting) {
 708             // the first command must be USER
 709             if (!pinning.pinned && cmd != cmdUser())
 710                 return earlyError(EarlyErrorKind::MissingLogin);
 711         }
 712
 713         // process USER request now because it sets FTP peer host name
 714         if (cmd == cmdUser()) {
 715             if (Http::Stream *errCtx = handleUserRequest(cmd, params))
 716                 return errCtx;
 717         }
 718     }
 719
 720     if (!Ftp::SupportedCommand(cmd))
 721         return earlyError(EarlyErrorKind::UnsupportedCommand);
 722
 723     const HttpRequestMethod method =
 724         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 725         Http::METHOD_PUT : Http::METHOD_GET;
 726
 727     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 728                        &params : NULL;
 729     calcUri(path);
 730     char *newUri = xstrdup(uri.c_str());
 731     HttpRequest *const request = HttpRequest::CreateFromUrl(newUri, method);
 732     if (!request) {
 733         debugs(33, 5, "Invalid FTP URL: " << uri);
 734         uri.clear();
 735         safe_free(newUri);
 736         return earlyError(EarlyErrorKind::InvalidUri);
 737     }
 738
 739     request->flags.ftpNative = true;
 740     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 741
 742     // Our fake Request-URIs are not distinctive enough for caching to work
 743     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 744     request->flags.noCache = true;
 745
 746     request->header.putStr(Http::HdrType::FTP_COMMAND, cmd.c_str());
 747     request->header.putStr(Http::HdrType::FTP_ARGUMENTS, params.c_str()); // may be ""
 748     if (method == Http::METHOD_PUT) {
 749         request->header.putStr(Http::HdrType::EXPECT, "100-continue");
 750         request->header.putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
 751     }
 752
 753     ClientHttpRequest *const http = new ClientHttpRequest(this);
 754     http->request = request;
 755     HTTPMSGLOCK(http->request);
 756     http->req_sz = tok.parsedSize();
 757     http->uri = newUri;
 758
 759     Http::Stream *const result =
 760         new Http::Stream(clientConnection, http);
 761
 762     StoreIOBuffer tempBuffer;
 763     tempBuffer.data = result->reqbuf;
 764     tempBuffer.length = HTTP_REQBUF_SZ;
 765
 766     ClientStreamData newServer = new clientReplyContext(http);
 767     ClientStreamData newClient = result;
 768     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 769                      clientReplyStatus, newServer, clientSocketRecipient,
 770                      clientSocketDetach, newClient, tempBuffer);
 771
 772     result->flags.parsed_ok = 1;
 773     return result;
 774 }
 775
 776 void
 777 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 778 {
 779     // the caller guarantees that we are dealing with the current context only
 780     Http::StreamPointer context = pipeline.front();
 781     assert(context != nullptr);
 782
 783     if (context->http && context->http->al != NULL &&
 784             !context->http->al->reply && reply) {
 785         context->http->al->reply = reply;
 786         HTTPMSGLOCK(context->http->al->reply);
 787     }
 788
 789     static ReplyHandler handlers[] = {
 790         NULL, // fssBegin
 791         NULL, // fssConnected
 792         &Ftp::Server::handleFeatReply, // fssHandleFeat
 793         &Ftp::Server::handlePasvReply, // fssHandlePasv
 794         &Ftp::Server::handlePortReply, // fssHandlePort
 795         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 796         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 797         &Ftp::Server::handleEprtReply,// fssHandleEprt
 798         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 799         NULL, // fssHandleCwd
 800         NULL, // fssHandlePass
 801         NULL, // fssHandleCdup
 802         &Ftp::Server::handleErrorReply // fssError
 803     };
 804     try {
 805         const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 806         if (const ReplyHandler handler = handlers[server.master->serverState])
 807             (this->*handler)(reply, data);
 808         else
 809             writeForwardedReply(reply);
 810     } catch (const std::exception &e) {
 811         callException(e);
 812         throw TexcHere(e.what());
 813     }
 814 }
 815
 816 void
 817 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 818 {
 819     if (pipeline.front()->http->request->errType != ERR_NONE) {
 820         writeCustomReply(502, "Server does not support FEAT", reply);
 821         return;
 822     }
 823
 824     Must(reply);
 825     HttpReply::Pointer featReply = Ftp::HttpReplyWrapper(211, "End", Http::scNoContent, 0);
 826     HttpHeader const &serverReplyHeader = reply->header;
 827
 828     HttpHeaderPos pos = HttpHeaderInitPos;
 829     bool hasEPRT = false;
 830     bool hasEPSV = false;
 831     int prependSpaces = 1;
 832
 833     featReply->header.putStr(Http::HdrType::FTP_PRE, "\"211-Features:\"");
 834     const int scode = serverReplyHeader.getInt(Http::HdrType::FTP_STATUS);
 835     if (scode == 211) {
 836         while (const HttpHeaderEntry *e = serverReplyHeader.getEntry(&pos)) {
 837             if (e->id == Http::HdrType::FTP_PRE) {
 838                 // assume RFC 2389 FEAT response format, quoted by Squid:
 839                 // <"> SP NAME [SP PARAMS] <">
 840                 // but accommodate MS servers sending four SPs before NAME
 841
 842                 // command name ends with (SP parameter) or quote
 843                 static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 844                 static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 845
 846                 Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 847                 if (!tok.skip('"') || !tok.skip(' '))
 848                     continue;
 849
 850                 // optional spaces; remember their number to accomodate MS servers
 851                 prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 852
 853                 SBuf cmd;
 854                 if (!tok.prefix(cmd, FeatNameChars))
 855                     continue;
 856                 cmd.toUpper();
 857
 858                 if (Ftp::SupportedCommand(cmd)) {
 859                     featReply->header.addEntry(e->clone());
 860                 }
 861
 862                 if (cmd == cmdEprt())
 863                     hasEPRT = true;
 864                 else if (cmd == cmdEpsv())
 865                     hasEPSV = true;
 866             }
 867         }
 868     } // else we got a FEAT error and will only report Squid-supported features
 869
 870     char buf[256];
 871     if (!hasEPRT) {
 872         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 873         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 874     }
 875     if (!hasEPSV) {
 876         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 877         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 878     }
 879
 880     featReply->header.refreshMask();
 881
 882     writeForwardedReply(featReply.getRaw());
 883 }
 884
 885 void
 886 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 887 {
 888     const Http::StreamPointer context(pipeline.front());
 889     assert(context != nullptr);
 890
 891     if (context->http->request->errType != ERR_NONE) {
 892         writeCustomReply(502, "Server does not support PASV", reply);
 893         return;
 894     }
 895
 896     const unsigned short localPort = listenForDataConnection();
 897     if (!localPort)
 898         return;
 899
 900     char addr[MAX_IPSTRLEN];
 901     // remote server in interception setups and local address otherwise
 902     const Ip::Address &server = transparent() ?
 903                                 clientConnection->local : dataListenConn->local;
 904     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 905     addr[MAX_IPSTRLEN - 1] = '\0';
 906     for (char *c = addr; *c != '\0'; ++c) {
 907         if (*c == '.')
 908             *c = ',';
 909     }
 910
 911     // In interception setups, we combine remote server address with a
 912     // local port number and hope that traffic will be redirected to us.
 913     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 914     // versions block responses that use those alternative syntax rules!
 915     MemBuf mb;
 916     mb.init();
 917     mb.appendf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 918                addr,
 919                static_cast<int>(localPort / 256),
 920                static_cast<int>(localPort % 256));
 921     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 922     writeReply(mb);
 923 }
 924
 925 void
 926 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 927 {
 928     if (pipeline.front()->http->request->errType != ERR_NONE) {
 929         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 930         return;
 931     }
 932
 933     writeCustomReply(200, "PORT successfully converted to PASV.");
 934
 935     // and wait for RETR
 936 }
 937
 938 void
 939 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 940 {
 941     if (!pinning.pinned) // we failed to connect to server
 942         uri.clear();
 943     // 421: we will close due to fssError
 944     writeErrorReply(reply, 421);
 945 }
 946
 947 void
 948 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 949 {
 950     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 951         writeForwardedReply(reply);
 952         if (Comm::IsConnOpen(dataConn)) {
 953             debugs(33, 3, "closing " << dataConn << " on KO reply");
 954             closeDataConnection();
 955         }
 956         return;
 957     }
 958
 959     if (!dataConn) {
 960         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 961         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 962         return;
 963     }
 964
 965     if (!checkDataConnPost()) {
 966         writeCustomReply(425, "Data connection is not established.");
 967         closeDataConnection();
 968         return;
 969     }
 970
 971     debugs(33, 7, data.length);
 972
 973     if (data.length <= 0) {
 974         replyDataWritingCheckpoint(); // skip the actual write call
 975         return;
 976     }
 977
 978     MemBuf mb;
 979     mb.init(data.length + 1, data.length + 1);
 980     mb.append(data.data, data.length);
 981
 982     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 983     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 984     Comm::Write(dataConn, &mb, call);
 985
 986     pipeline.front()->noteSentBodyBytes(data.length);
 987 }
 988
 989 /// called when we are done writing a chunk of the response data
 990 void
 991 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 992 {
 993     if (io.flag == Comm::ERR_CLOSING)
 994         return;
 995
 996     if (io.flag != Comm::OK) {
 997         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 998         userDataCompletionCheckpoint(426);
 999         return;
1000     }
1001
1002     assert(pipeline.front()->http);
1003     pipeline.front()->http->out.size += io.size;
1004     replyDataWritingCheckpoint();
1005 }
1006
1007 /// ClientStream checks after (actual or skipped) reply data writing
1008 void
1009 Ftp::Server::replyDataWritingCheckpoint()
1010 {
1011     switch (pipeline.front()->socketState()) {
1012     case STREAM_NONE:
1013         debugs(33, 3, "Keep going");
1014         pipeline.front()->pullData();
1015         return;
1016     case STREAM_COMPLETE:
1017         debugs(33, 3, "FTP reply data transfer successfully complete");
1018         userDataCompletionCheckpoint(226);
1019         break;
1020     case STREAM_UNPLANNED_COMPLETE:
1021         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
1022         userDataCompletionCheckpoint(451);
1023         break;
1024     case STREAM_FAILED:
1025         userDataCompletionCheckpoint(451);
1026         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
1027         break;
1028     default:
1029         fatal("unreachable code");
1030     }
1031 }
1032
1033 void
1034 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1035 {
1036     writeForwardedReply(reply);
1037     // note that the client data connection may already be closed by now
1038 }
1039
1040 void
1041 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1042 {
1043     Must(reply);
1044
1045     if (waitingForOrigin) {
1046         Must(delayedReply == NULL);
1047         delayedReply = reply;
1048         return;
1049     }
1050
1051     const HttpHeader &header = reply->header;
1052     // adaptation and forwarding errors lack Http::HdrType::FTP_STATUS
1053     if (!header.has(Http::HdrType::FTP_STATUS)) {
1054         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1055         return;
1056     }
1057
1058     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1059     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1060     writeForwardedReplyAndCall(reply, call);
1061 }
1062
1063 void
1064 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1065 {
1066     if (pipeline.front()->http->request->errType != ERR_NONE) {
1067         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1068         return;
1069     }
1070
1071     writeCustomReply(200, "EPRT successfully converted to PASV.");
1072
1073     // and wait for RETR
1074 }
1075
1076 void
1077 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1078 {
1079     if (pipeline.front()->http->request->errType != ERR_NONE) {
1080         writeCustomReply(502, "Cannot connect to server", reply);
1081         return;
1082     }
1083
1084     const unsigned short localPort = listenForDataConnection();
1085     if (!localPort)
1086         return;
1087
1088     // In interception setups, we use a local port number and hope that data
1089     // traffic will be redirected to us.
1090     MemBuf mb;
1091     mb.init();
1092     mb.appendf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1093
1094     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1095     writeReply(mb);
1096 }
1097
1098 /// writes FTP error response with given status and reply-derived error details
1099 void
1100 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1101 {
1102     const HttpRequest *request = pipeline.front()->http->request;
1103     assert(request);
1104
1105     MemBuf mb;
1106     mb.init();
1107
1108     if (request->errType != ERR_NONE)
1109         mb.appendf("%i-%s\r\n", scode, errorPageName(request->errType));
1110
1111     if (request->errDetail > 0) {
1112         // XXX: > 0 may not always mean that this is an errno
1113         mb.appendf("%i-Error: (%d) %s\r\n", scode,
1114                    request->errDetail,
1115                    strerror(request->errDetail));
1116     }
1117
1118 #if USE_ADAPTATION
1119     // XXX: Remove hard coded names. Use an error page template instead.
1120     const Adaptation::History::Pointer ah = request->adaptHistory();
1121     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1122         const String info = ah->allMeta.getByName("X-Response-Info");
1123         const String desc = ah->allMeta.getByName("X-Response-Desc");
1124         if (info.size())
1125             mb.appendf("%i-Information: %s\r\n", scode, info.termedBuf());
1126         if (desc.size())
1127             mb.appendf("%i-Description: %s\r\n", scode, desc.termedBuf());
1128     }
1129 #endif
1130
1131     Must(reply);
1132     const char *reason = reply->header.has(Http::HdrType::FTP_REASON) ?
1133                          reply->header.getStr(Http::HdrType::FTP_REASON):
1134                          reply->sline.reason();
1135
1136     mb.appendf("%i %s\r\n", scode, reason); // error terminating line
1137
1138     // TODO: errorpage.cc should detect FTP client and use
1139     // configurable FTP-friendly error templates which we should
1140     // write to the client "as is" instead of hiding most of the info
1141
1142     writeReply(mb);
1143 }
1144
1145 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1146 /// for example, internally-generated Squid "errorpages" end up here (for now)
1147 void
1148 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1149 {
1150     changeState(fssConnected, "foreign reply");
1151     closeDataConnection();
1152     // 451: We intend to keep the control connection open.
1153     writeErrorReply(reply, 451);
1154 }
1155
1156 void
1157 Ftp::Server::writeControlMsgAndCall(HttpReply *reply, AsyncCall::Pointer &call)
1158 {
1159     // the caller guarantees that we are dealing with the current context only
1160     // the caller should also make sure reply->header.has(Http::HdrType::FTP_STATUS)
1161     writeForwardedReplyAndCall(reply, call);
1162 }
1163
1164 void
1165 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1166 {
1167     assert(reply != NULL);
1168     const HttpHeader &header = reply->header;
1169
1170     // without status, the caller must use the writeForwardedForeign() path
1171     Must(header.has(Http::HdrType::FTP_STATUS));
1172     Must(header.has(Http::HdrType::FTP_REASON));
1173     const int scode = header.getInt(Http::HdrType::FTP_STATUS);
1174     debugs(33, 7, "scode: " << scode);
1175
1176     // Status 125 or 150 implies upload or data request, but we still check
1177     // the state in case the server is buggy.
1178     if ((scode == 125 || scode == 150) &&
1179             (master->serverState == fssHandleUploadRequest ||
1180              master->serverState == fssHandleDataRequest)) {
1181         if (checkDataConnPost()) {
1182             // If the data connection is ready, start reading data (here)
1183             // and forward the response to client (further below).
1184             debugs(33, 7, "data connection established, start data transfer");
1185             if (master->serverState == fssHandleUploadRequest)
1186                 maybeReadUploadData();
1187         } else {
1188             // If we are waiting to accept the data connection, keep waiting.
1189             if (Comm::IsConnOpen(dataListenConn)) {
1190                 debugs(33, 7, "wait for the client to establish a data connection");
1191                 onDataAcceptCall = call;
1192                 // TODO: Add connect timeout for passive connections listener?
1193                 // TODO: Remember server response so that we can forward it?
1194             } else {
1195                 // Either the connection was establised and closed after the
1196                 // data was transferred OR we failed to establish an active
1197                 // data connection and already sent the error to the client.
1198                 // In either case, there is nothing more to do.
1199                 debugs(33, 7, "done with data OR active connection failed");
1200             }
1201             return;
1202         }
1203     }
1204
1205     MemBuf mb;
1206     mb.init();
1207     Ftp::PrintReply(mb, reply);
1208
1209     debugs(9, 2, "FTP Client " << clientConnection);
1210     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1211            "\n----------");
1212
1213     Comm::Write(clientConnection, &mb, call);
1214 }
1215
1216 static void
1217 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const)
1218 {
1219     const HttpHeader &header = reply->header;
1220
1221     HttpHeaderPos pos = HttpHeaderInitPos;
1222     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1223         if (e->id == Http::HdrType::FTP_PRE) {
1224             String raw;
1225             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1226                 mb.appendf("%s\r\n", raw.termedBuf());
1227         }
1228     }
1229
1230     if (header.has(Http::HdrType::FTP_STATUS)) {
1231         const char *reason = header.getStr(Http::HdrType::FTP_REASON);
1232         mb.appendf("%i %s\r\n", header.getInt(Http::HdrType::FTP_STATUS),
1233                    (reason ? reason : 0));
1234     }
1235 }
1236
1237 void
1238 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1239 {
1240     if (io.flag == Comm::ERR_CLOSING)
1241         return;
1242
1243     if (io.flag != Comm::OK) {
1244         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1245         io.conn->close();
1246         return;
1247     }
1248
1249     Http::StreamPointer context = pipeline.front();
1250     if (context != nullptr && context->http) {
1251         context->http->out.size += io.size;
1252         context->http->out.headers_sz += io.size;
1253     }
1254
1255     flags.readMore = true;
1256     readSomeData();
1257 }
1258
1259 void
1260 Ftp::Server::wroteReply(const CommIoCbParams &io)
1261 {
1262     if (io.flag == Comm::ERR_CLOSING)
1263         return;
1264
1265     if (io.flag != Comm::OK) {
1266         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1267         io.conn->close();
1268         return;
1269     }
1270
1271     Http::StreamPointer context = pipeline.front();
1272     assert(context->http);
1273     context->http->out.size += io.size;
1274     context->http->out.headers_sz += io.size;
1275
1276     if (master->serverState == fssError) {
1277         debugs(33, 5, "closing on FTP server error");
1278         io.conn->close();
1279         return;
1280     }
1281
1282     const clientStream_status_t socketState = context->socketState();
1283     debugs(33, 5, "FTP client stream state " << socketState);
1284     switch (socketState) {
1285     case STREAM_UNPLANNED_COMPLETE:
1286     case STREAM_FAILED:
1287         io.conn->close();
1288         return;
1289
1290     case STREAM_NONE:
1291     case STREAM_COMPLETE:
1292         flags.readMore = true;
1293         changeState(fssConnected, "Ftp::Server::wroteReply");
1294         if (bodyParser)
1295             finishDechunkingRequest(false);
1296         context->finished();
1297         kick();
1298         return;
1299     }
1300 }
1301
1302 bool
1303 Ftp::Server::handleRequest(HttpRequest *request)
1304 {
1305     debugs(33, 9, request);
1306     Must(request);
1307
1308     HttpHeader &header = request->header;
1309     Must(header.has(Http::HdrType::FTP_COMMAND));
1310     String &cmd = header.findEntry(Http::HdrType::FTP_COMMAND)->value;
1311     Must(header.has(Http::HdrType::FTP_ARGUMENTS));
1312     String &params = header.findEntry(Http::HdrType::FTP_ARGUMENTS)->value;
1313
1314     if (Debug::Enabled(9, 2)) {
1315         MemBuf mb;
1316         mb.init();
1317         request->pack(&mb);
1318
1319         debugs(9, 2, "FTP Client " << clientConnection);
1320         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1321                "\n----------");
1322     }
1323
1324     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1325     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1326     static RequestHandlers handlers;
1327     if (!handlers.size()) {
1328         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1329         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1330         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1331         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1332         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1333         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1334         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1335         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1336         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1337         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1338         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1339         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1340     }
1341
1342     RequestHandler handler = NULL;
1343     if (request->method == Http::METHOD_PUT)
1344         handler = &Ftp::Server::handleUploadRequest;
1345     else {
1346         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1347         if (hi != handlers.end())
1348             handler = hi->second;
1349     }
1350
1351     if (!handler) {
1352         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1353         return true;
1354     }
1355
1356     return (this->*handler)(cmd, params);
1357 }
1358
1359 /// Called to parse USER command, which is required to create an HTTP request
1360 /// wrapper. W/o request, the errors are handled by returning earlyError().
1361 Http::Stream *
1362 Ftp::Server::handleUserRequest(const SBuf &, SBuf &params)
1363 {
1364     if (params.isEmpty())
1365         return earlyError(EarlyErrorKind::MissingUsername);
1366
1367     // find the [end of] user name
1368     const SBuf::size_type eou = params.rfind('@');
1369     if (eou == SBuf::npos || eou + 1 >= params.length())
1370         return earlyError(EarlyErrorKind::MissingHost);
1371
1372     // Determine the intended destination.
1373     host = params.substr(eou + 1, params.length());
1374     // If we can parse it as raw IPv6 address, then surround with "[]".
1375     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1376     if (host.find(':') != SBuf::npos) {
1377         const Ip::Address ipa(host.c_str());
1378         if (!ipa.isAnyAddr()) {
1379             char ipBuf[MAX_IPSTRLEN];
1380             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1381             host = ipBuf;
1382         }
1383     }
1384
1385     // const SBuf login = params.substr(0, eou);
1386     params.chop(0, eou); // leave just the login part for the peer
1387
1388     SBuf oldUri;
1389     if (master->clientReadGreeting)
1390         oldUri = uri;
1391
1392     master->workingDir.clear();
1393     calcUri(NULL);
1394
1395     if (!master->clientReadGreeting) {
1396         debugs(9, 3, "set URI to " << uri);
1397     } else if (oldUri.caseCmp(uri) == 0) {
1398         debugs(9, 5, "kept URI as " << oldUri);
1399     } else {
1400         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1401         closeDataConnection();
1402         unpinConnection(true); // close control connection to peer
1403         resetLogin("URI reset");
1404     }
1405
1406     return NULL; // no early errors
1407 }
1408
1409 bool
1410 Ftp::Server::handleFeatRequest(String &, String &)
1411 {
1412     changeState(fssHandleFeat, "handleFeatRequest");
1413     return true;
1414 }
1415
1416 bool
1417 Ftp::Server::handlePasvRequest(String &, String &params)
1418 {
1419     if (gotEpsvAll) {
1420         setReply(500, "Bad PASV command");
1421         return false;
1422     }
1423
1424     if (params.size() > 0) {
1425         setReply(501, "Unexpected parameter");
1426         return false;
1427     }
1428
1429     changeState(fssHandlePasv, "handlePasvRequest");
1430     // no need to fake PASV request via setDataCommand() in true PASV case
1431     return true;
1432 }
1433
1434 /// [Re]initializes dataConn for active data transfers. Does not connect.
1435 bool
1436 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1437 {
1438     assert(clientConnection != NULL);
1439     assert(!clientConnection->remote.isAnyAddr());
1440
1441     if (cltAddr != clientConnection->remote) {
1442         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1443         // Closing the control connection would not help with attacks because
1444         // the client is evidently able to connect to us. Besides, closing
1445         // makes retrials easier for the client and more damaging to us.
1446         setReply(501, "Prohibited parameter value");
1447         return false;
1448     }
1449
1450     closeDataConnection();
1451
1452     Comm::ConnectionPointer conn = new Comm::Connection();
1453     conn->flags |= COMM_DOBIND;
1454
1455     // Use local IP address of the control connection as the source address
1456     // of the active data connection, or some clients will refuse to accept.
1457     conn->setAddrs(clientConnection->local, cltAddr);
1458     // RFC 959 requires active FTP connections to originate from port 20
1459     // but that would preclude us from supporting concurrent transfers! (XXX?)
1460     conn->local.port(0);
1461
1462     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1463            conn->remote);
1464
1465     dataConn = conn;
1466     uploadAvailSize = 0;
1467     return true;
1468 }
1469
1470 bool
1471 Ftp::Server::handlePortRequest(String &, String &params)
1472 {
1473     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1474
1475     if (gotEpsvAll) {
1476         setReply(500, "Rejecting PORT after EPSV ALL");
1477         return false;
1478     }
1479
1480     if (!params.size()) {
1481         setReply(501, "Missing parameter");
1482         return false;
1483     }
1484
1485     Ip::Address cltAddr;
1486     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1487         setReply(501, "Invalid parameter");
1488         return false;
1489     }
1490
1491     if (!createDataConnection(cltAddr))
1492         return false;
1493
1494     changeState(fssHandlePort, "handlePortRequest");
1495     setDataCommand();
1496     return true; // forward our fake PASV request
1497 }
1498
1499 bool
1500 Ftp::Server::handleDataRequest(String &, String &)
1501 {
1502     if (!checkDataConnPre())
1503         return false;
1504
1505     master->userDataDone = 0;
1506     originDataDownloadAbortedOnError = false;
1507
1508     changeState(fssHandleDataRequest, "handleDataRequest");
1509
1510     return true;
1511 }
1512
1513 bool
1514 Ftp::Server::handleUploadRequest(String &, String &)
1515 {
1516     if (!checkDataConnPre())
1517         return false;
1518
1519     if (Config.accessList.forceRequestBodyContinuation) {
1520         ClientHttpRequest *http = pipeline.front()->http;
1521         HttpRequest *request = http->request;
1522         ACLFilledChecklist bodyContinuationCheck(Config.accessList.forceRequestBodyContinuation, request, NULL);
1523         if (bodyContinuationCheck.fastCheck() == ACCESS_ALLOWED) {
1524             request->forcedBodyContinuation = true;
1525             if (checkDataConnPost()) {
1526                 // Write control Msg
1527                 writeEarlyReply(150, "Data connection opened");
1528                 maybeReadUploadData();
1529             } else {
1530                 // wait for acceptDataConnection but tell it to call wroteEarlyReply
1531                 // after writing "150 Data connection opened"
1532                 typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1533                 AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
1534                 onDataAcceptCall = call;
1535             }
1536         }
1537     }
1538
1539     changeState(fssHandleUploadRequest, "handleDataRequest");
1540
1541     return true;
1542 }
1543
1544 bool
1545 Ftp::Server::handleEprtRequest(String &, String &params)
1546 {
1547     debugs(9, 3, "Process an EPRT " << params);
1548
1549     if (gotEpsvAll) {
1550         setReply(500, "Rejecting EPRT after EPSV ALL");
1551         return false;
1552     }
1553
1554     if (!params.size()) {
1555         setReply(501, "Missing parameter");
1556         return false;
1557     }
1558
1559     Ip::Address cltAddr;
1560     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1561         setReply(501, "Invalid parameter");
1562         return false;
1563     }
1564
1565     if (!createDataConnection(cltAddr))
1566         return false;
1567
1568     changeState(fssHandleEprt, "handleEprtRequest");
1569     setDataCommand();
1570     return true; // forward our fake PASV request
1571 }
1572
1573 bool
1574 Ftp::Server::handleEpsvRequest(String &, String &params)
1575 {
1576     debugs(9, 3, "Process an EPSV command with params: " << params);
1577     if (params.size() <= 0) {
1578         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1579     } else if (params.caseCmp("ALL") == 0) {
1580         setReply(200, "EPSV ALL ok");
1581         gotEpsvAll = true;
1582         return false;
1583     } else if (params.cmp("2") == 0) {
1584         if (!Ip::EnableIpv6) {
1585             setReply(522, "Network protocol not supported, use (1)");
1586             return false;
1587         }
1588     } else if (params.cmp("1") != 0) {
1589         setReply(501, "Unsupported EPSV parameter");
1590         return false;
1591     }
1592
1593     changeState(fssHandleEpsv, "handleEpsvRequest");
1594     setDataCommand();
1595     return true; // forward our fake PASV request
1596 }
1597
1598 bool
1599 Ftp::Server::handleCwdRequest(String &, String &)
1600 {
1601     changeState(fssHandleCwd, "handleCwdRequest");
1602     return true;
1603 }
1604
1605 bool
1606 Ftp::Server::handlePassRequest(String &, String &)
1607 {
1608     changeState(fssHandlePass, "handlePassRequest");
1609     return true;
1610 }
1611
1612 bool
1613 Ftp::Server::handleCdupRequest(String &, String &)
1614 {
1615     changeState(fssHandleCdup, "handleCdupRequest");
1616     return true;
1617 }
1618
1619 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1620 // Squid FTP client decides what data command to use with peers.
1621 void
1622 Ftp::Server::setDataCommand()
1623 {
1624     ClientHttpRequest *const http = pipeline.front()->http;
1625     assert(http != NULL);
1626     HttpRequest *const request = http->request;
1627     assert(request != NULL);
1628     HttpHeader &header = request->header;
1629     header.delById(Http::HdrType::FTP_COMMAND);
1630     header.putStr(Http::HdrType::FTP_COMMAND, "PASV");
1631     header.delById(Http::HdrType::FTP_ARGUMENTS);
1632     header.putStr(Http::HdrType::FTP_ARGUMENTS, "");
1633     debugs(9, 5, "client data command converted to fake PASV");
1634 }
1635
1636 /// check that client data connection is ready for future I/O or at least
1637 /// has a chance of becoming ready soon.
1638 bool
1639 Ftp::Server::checkDataConnPre()
1640 {
1641     if (Comm::IsConnOpen(dataConn))
1642         return true;
1643
1644     if (Comm::IsConnOpen(dataListenConn)) {
1645         // We are still waiting for a client to connect to us after PASV.
1646         // Perhaps client's data conn handshake has not reached us yet.
1647         // After we talk to the server, checkDataConnPost() will recheck.
1648         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1649         return true;
1650     }
1651
1652     if (!dataConn || dataConn->remote.isAnyAddr()) {
1653         debugs(33, 5, "missing " << dataConn);
1654         // TODO: use client address and default port instead.
1655         setReply(425, "Use PORT or PASV first");
1656         return false;
1657     }
1658
1659     // active transfer: open a data connection from Squid to client
1660     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1661     connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1662     Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector,
1663             Config.Timeout.connect);
1664     AsyncJob::Start(cs);
1665     return false; // ConnStateData::processFtpRequest waits handleConnectDone
1666 }
1667
1668 /// Check that client data connection is ready for immediate I/O.
1669 bool
1670 Ftp::Server::checkDataConnPost() const
1671 {
1672     if (!Comm::IsConnOpen(dataConn)) {
1673         debugs(33, 3, "missing client data conn: " << dataConn);
1674         return false;
1675     }
1676     return true;
1677 }
1678
1679 /// Done establishing a data connection to the user.
1680 void
1681 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1682 {
1683     connector = NULL;
1684
1685     if (params.flag != Comm::OK) {
1686         /* it might have been a timeout with a partially open link */
1687         if (params.conn != NULL)
1688             params.conn->close();
1689         setReply(425, "Cannot open data connection.");
1690         Http::StreamPointer context = pipeline.front();
1691         Must(context->http);
1692         Must(context->http->storeEntry() != NULL);
1693     } else {
1694         Must(dataConn == params.conn);
1695         Must(Comm::IsConnOpen(params.conn));
1696         fd_note(params.conn->fd, "active client ftp data");
1697     }
1698
1699     doProcessRequest();
1700 }
1701
1702 void
1703 Ftp::Server::setReply(const int code, const char *msg)
1704 {
1705     Http::StreamPointer context = pipeline.front();
1706     ClientHttpRequest *const http = context->http;
1707     assert(http != NULL);
1708     assert(http->storeEntry() == NULL);
1709
1710     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1711
1712     setLogUri(http, urlCanonicalClean(http->request));
1713
1714     clientStreamNode *const node = context->getClientReplyContext();
1715     clientReplyContext *const repContext =
1716         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1717     assert(repContext != NULL);
1718
1719     RequestFlags reqFlags;
1720     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1721     reqFlags.noCache = true;
1722     repContext->createStoreEntry(http->request->method, reqFlags);
1723     http->storeEntry()->replaceHttpReply(reply);
1724 }
1725
1726 void
1727 Ftp::Server::callException(const std::exception &e)
1728 {
1729     debugs(33, 2, "FTP::Server job caught: " << e.what());
1730     closeDataConnection();
1731     unpinConnection(true);
1732     if (Comm::IsConnOpen(clientConnection))
1733         clientConnection->close();
1734     AsyncJob::callException(e);
1735 }
1736
1737 void
1738 Ftp::Server::startWaitingForOrigin()
1739 {
1740      if (!isOpen()) // if we are closing, nothing to do
1741         return;
1742
1743     debugs(33, 5, "waiting for Ftp::Client data transfer to end");
1744     waitingForOrigin = true;
1745 }
1746
1747 void
1748 Ftp::Server::stopWaitingForOrigin(int originStatus)
1749 {
1750     Must(waitingForOrigin);
1751     waitingForOrigin = false;
1752
1753     if (!isOpen()) // if we are closing, nothing to do
1754         return;
1755
1756     // if we have already decided how to respond, respond now
1757     if (delayedReply) {
1758         HttpReply::Pointer reply = delayedReply;
1759         delayedReply = nullptr;
1760         writeForwardedReply(reply.getRaw());
1761         return; // do not completeDataDownload() after an earlier response
1762     }
1763
1764     if (master->serverState != fssHandleDataRequest)
1765         return;
1766
1767     // completeDataDownload() could be waitingForOrigin in fssHandleDataRequest
1768     // Depending on which side has finished downloading first, either trust
1769     // master->userDataDone status or set originDataDownloadAbortedOnError:
1770     if (master->userDataDone) {
1771         // We finished downloading before Ftp::Client. Most likely, the
1772         // adaptation shortened the origin response or we hit an error.
1773         // Our status (stored in master->userDataDone) is more informative.
1774         // Use master->userDataDone; avoid originDataDownloadAbortedOnError.
1775         completeDataDownload();
1776     } else {
1777         debugs(33, 5, "too early to write the response");
1778         // Ftp::Client naturally finished downloading before us. Set
1779         // originDataDownloadAbortedOnError to overwrite future
1780         // master->userDataDone and relay Ftp::Client error, if there was
1781         // any, to the user.
1782         originDataDownloadAbortedOnError = (originStatus >= 400);
1783     }
1784 }
1785
1786 void Ftp::Server::userDataCompletionCheckpoint(int finalStatusCode)
1787 {
1788     Must(!master->userDataDone);
1789     master->userDataDone = finalStatusCode;
1790
1791     if (bodyParser)
1792         finishDechunkingRequest(false);
1793
1794     if (waitingForOrigin) {
1795         // The completeDataDownload() is not called here unconditionally
1796         // because we want to signal the FTP user that we are not fully
1797         // done processing its data stream, even though all data bytes
1798         // have been sent or received already.
1799         debugs(33, 5, "Transfering from FTP server is not complete");
1800         return;
1801     }
1802
1803     // Adjust our reply if the server aborted with an error before we are done.
1804     if (master->userDataDone == 226 && originDataDownloadAbortedOnError) {
1805         debugs(33, 5, "Transfering from FTP server terminated with an error, adjust status code");
1806         master->userDataDone = 451;
1807     }
1808     completeDataDownload();
1809 }
1810
1811 void Ftp::Server::completeDataDownload()
1812 {
1813     writeCustomReply(master->userDataDone, master->userDataDone == 226 ? "Transfer complete" : "Server error; transfer aborted");
1814     closeDataConnection();
1815 }
1816
1817 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1818 static bool
1819 Ftp::SupportedCommand(const SBuf &name)
1820 {
1821     static std::set<SBuf> BlackList;
1822     if (BlackList.empty()) {
1823         /* Add FTP commands that Squid cannot relay correctly. */
1824
1825         // We probably do not support AUTH TLS.* and AUTH SSL,
1826         // but let's disclaim all AUTH support to KISS, for now.
1827         BlackList.insert(cmdAuth());
1828     }
1829
1830     // we claim support for all commands that we do not know about
1831     return BlackList.find(name) == BlackList.end();
1832 }
1833