src/servers/FtpServer.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Transfer protocol servers */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CharacterSet.h"
  14 #include "base/RefCount.h"
  15 #include "base/Subscription.h"
  16 #include "client_side_reply.h"
  17 #include "client_side_request.h"
  18 #include "clientStream.h"
  19 #include "comm/ConnOpener.h"
  20 #include "comm/Read.h"
  21 #include "comm/TcpAcceptor.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "ftp/Elements.h"
  26 #include "ftp/Parsing.h"
  27 #include "globals.h"
  28 #include "http/one/RequestParser.h"
  29 #include "http/Stream.h"
  30 #include "HttpHdrCc.h"
  31 #include "ip/tools.h"
  32 #include "ipc/FdNotes.h"
  33 #include "parser/Tokenizer.h"
  34 #include "servers/forward.h"
  35 #include "servers/FtpServer.h"
  36 #include "SquidConfig.h"
  37 #include "StatCounters.h"
  38 #include "tools.h"
  39
  40 #include <set>
  41 #include <map>
  42
  43 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  44
  45 namespace Ftp
  46 {
  47 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  48 static bool SupportedCommand(const SBuf &name);
  49 static bool CommandHasPathParameter(const SBuf &cmd);
  50 };
  51
  52 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  53     AsyncJob("Ftp::Server"),
  54     ConnStateData(xact),
  55     master(new MasterState),
  56     uri(),
  57     host(),
  58     gotEpsvAll(false),
  59     onDataAcceptCall(),
  60     dataListenConn(),
  61     dataConn(),
  62     uploadAvailSize(0),
  63     listener(),
  64     connector(),
  65     reader(),
  66     waitingForOrigin(false),
  67     originDataDownloadAbortedOnError(false)
  68 {
  69     flags.readMore = false; // we need to announce ourselves first
  70     *uploadBuf = 0;
  71 }
  72
  73 Ftp::Server::~Server()
  74 {
  75     closeDataConnection();
  76 }
  77
  78 int
  79 Ftp::Server::pipelinePrefetchMax() const
  80 {
  81     return 0; // no support for concurrent FTP requests
  82 }
  83
  84 time_t
  85 Ftp::Server::idleTimeout() const
  86 {
  87     return Config.Timeout.ftpClientIdle;
  88 }
  89
  90 void
  91 Ftp::Server::start()
  92 {
  93     ConnStateData::start();
  94
  95     if (transparent()) {
  96         char buf[MAX_IPSTRLEN];
  97         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  98         host = buf;
  99         calcUri(NULL);
 100         debugs(33, 5, "FTP transparent URL: " << uri);
 101     }
 102
 103     writeEarlyReply(220, "Service ready");
 104 }
 105
 106 /// schedules another data connection read if needed
 107 void
 108 Ftp::Server::maybeReadUploadData()
 109 {
 110     if (reader != NULL)
 111         return;
 112
 113     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 114     if (availSpace <= 0)
 115         return;
 116
 117     debugs(33, 4, dataConn << ": reading FTP data...");
 118
 119     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 120     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 121     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 122               reader);
 123 }
 124
 125 /// react to the freshly parsed request
 126 void
 127 Ftp::Server::doProcessRequest()
 128 {
 129     // zero pipelinePrefetchMax() ensures that there is only parsed request
 130     Must(pipeline.count() == 1);
 131     Http::StreamPointer context = pipeline.front();
 132     Must(context != nullptr);
 133
 134     ClientHttpRequest *const http = context->http;
 135     assert(http != NULL);
 136
 137     HttpRequest *const request = http->request;
 138     Must(http->storeEntry() || request);
 139     const bool mayForward = !http->storeEntry() && handleRequest(request);
 140
 141     if (http->storeEntry() != NULL) {
 142         debugs(33, 4, "got an immediate response");
 143         clientSetKeepaliveFlag(http);
 144         context->pullData();
 145     } else if (mayForward) {
 146         debugs(33, 4, "forwarding request to server side");
 147         assert(http->storeEntry() == NULL);
 148         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 149     } else {
 150         debugs(33, 4, "will resume processing later");
 151     }
 152 }
 153
 154 void
 155 Ftp::Server::processParsedRequest(Http::Stream *)
 156 {
 157     Must(pipeline.count() == 1);
 158
 159     // Process FTP request asynchronously to make sure FTP
 160     // data connection accept callback is fired first.
 161     CallJobHere(33, 4, CbcPointer<Server>(this),
 162                 Ftp::Server, doProcessRequest);
 163 }
 164
 165 /// imports more upload data from the data connection
 166 void
 167 Ftp::Server::readUploadData(const CommIoCbParams &io)
 168 {
 169     debugs(33, 5, io.conn << " size " << io.size);
 170     Must(reader != NULL);
 171     reader = NULL;
 172
 173     assert(Comm::IsConnOpen(dataConn));
 174     assert(io.conn->fd == dataConn->fd);
 175
 176     if (io.flag == Comm::OK && bodyPipe != NULL) {
 177         if (io.size > 0) {
 178             statCounter.client_http.kbytes_in += io.size;
 179
 180             char *const current_buf = uploadBuf + uploadAvailSize;
 181             if (io.buf != current_buf)
 182                 memmove(current_buf, io.buf, io.size);
 183             uploadAvailSize += io.size;
 184             shovelUploadData();
 185         } else if (io.size == 0) {
 186             debugs(33, 5, io.conn << " closed");
 187             closeDataConnection();
 188             if (uploadAvailSize <= 0)
 189                 finishDechunkingRequest(true);
 190         }
 191     } else { // not Comm::Flags::OK or unexpected read
 192         debugs(33, 5, io.conn << " closed");
 193         closeDataConnection();
 194         finishDechunkingRequest(false);
 195     }
 196
 197 }
 198
 199 /// shovel upload data from the internal buffer to the body pipe if possible
 200 void
 201 Ftp::Server::shovelUploadData()
 202 {
 203     assert(bodyPipe != NULL);
 204
 205     debugs(33, 5, "handling FTP request data for " << clientConnection);
 206     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 207                            uploadAvailSize);
 208     if (putSize > 0) {
 209         uploadAvailSize -= putSize;
 210         if (uploadAvailSize > 0)
 211             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 212     }
 213
 214     if (Comm::IsConnOpen(dataConn))
 215         maybeReadUploadData();
 216     else if (uploadAvailSize <= 0)
 217         finishDechunkingRequest(true);
 218 }
 219
 220 void
 221 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 222 {
 223     shovelUploadData();
 224 }
 225
 226 void
 227 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 228 {
 229     ConnStateData::noteBodyConsumerAborted(ptr);
 230     closeDataConnection();
 231 }
 232
 233 /// accept a new FTP control connection and hand it to a dedicated Server
 234 void
 235 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 236 {
 237     MasterXaction::Pointer xact = params.xaction;
 238     AnyP::PortCfgPointer s = xact->squidPort;
 239
 240     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 241
 242     if (params.flag != Comm::OK) {
 243         // Its possible the call was still queued when the client disconnected
 244         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 245         return;
 246     }
 247
 248     debugs(33, 4, params.conn << ": accepted");
 249     fd_note(params.conn->fd, "client ftp connect");
 250
 251     if (s->tcp_keepalive.enabled)
 252         commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
 253
 254     ++incoming_sockets_accepted;
 255
 256     AsyncJob::Start(new Server(xact));
 257 }
 258
 259 void
 260 Ftp::StartListening()
 261 {
 262     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 263         if (MAXTCPLISTENPORTS == NHttpSockets) {
 264             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 265                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 266             break;
 267         }
 268
 269         // direct new connections accepted by listenConn to Accept()
 270         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 271         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 272                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 273                                                CommAcceptCbParams(NULL)));
 274         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 275     }
 276 }
 277
 278 void
 279 Ftp::StopListening()
 280 {
 281     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 282         if (s->listenConn != NULL) {
 283             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 284             s->listenConn->close();
 285             s->listenConn = NULL;
 286         }
 287     }
 288 }
 289
 290 void
 291 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 292 {
 293     // find request
 294     Http::StreamPointer context = pipeline.front();
 295     Must(context != nullptr);
 296     ClientHttpRequest *const http = context->http;
 297     Must(http != NULL);
 298     HttpRequest *const request = http->request;
 299     Must(request != NULL);
 300
 301     // this is not an idle connection, so we do not want I/O monitoring
 302     const bool monitor = false;
 303
 304     // make FTP peer connection exclusive to our request
 305     pinConnection(conn, request, conn->getPeer(), false, monitor);
 306 }
 307
 308 void
 309 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 310 {
 311     ConnStateData::clientPinnedConnectionClosed(io);
 312
 313     // TODO: Keep the control connection open after fixing the reset
 314     // problem below
 315     if (Comm::IsConnOpen(clientConnection))
 316         clientConnection->close();
 317
 318     // TODO: If the server control connection is gone, reset state to login
 319     // again. Reseting login alone is not enough: FtpRelay::sendCommand() will
 320     // not re-login because FtpRelay::serverState() is not going to be
 321     // fssConnected. Calling resetLogin() alone is also harmful because
 322     // it does not reset correctly the client-to-squid control connection (eg
 323     // respond if required with an error code, in all cases)
 324     // resetLogin("control connection closure");
 325 }
 326
 327 /// clear client and server login-related state after the old login is gone
 328 void
 329 Ftp::Server::resetLogin(const char *reason)
 330 {
 331     debugs(33, 5, "will need to re-login due to " << reason);
 332     master->clientReadGreeting = false;
 333     changeState(fssBegin, reason);
 334 }
 335
 336 /// computes uri member from host and, if tracked, working dir with file name
 337 void
 338 Ftp::Server::calcUri(const SBuf *file)
 339 {
 340     uri = "ftp://";
 341     uri.append(host);
 342     if (port->ftp_track_dirs && master->workingDir.length()) {
 343         if (master->workingDir[0] != '/')
 344             uri.append("/", 1);
 345         uri.append(master->workingDir);
 346     }
 347
 348     if (uri[uri.length() - 1] != '/')
 349         uri.append("/", 1);
 350
 351     if (port->ftp_track_dirs && file) {
 352         static const CharacterSet Slash("/", "/");
 353         Parser::Tokenizer tok(*file);
 354         tok.skipAll(Slash);
 355         uri.append(tok.remaining());
 356     }
 357 }
 358
 359 /// Starts waiting for a data connection. Returns listening port.
 360 /// On errors, responds with an error and returns zero.
 361 unsigned int
 362 Ftp::Server::listenForDataConnection()
 363 {
 364     closeDataConnection();
 365
 366     Comm::ConnectionPointer conn = new Comm::Connection;
 367     conn->flags = COMM_NONBLOCKING;
 368     conn->local = transparent() ? port->s : clientConnection->local;
 369     conn->local.port(0);
 370     const char *const note = uri.c_str();
 371     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 372     if (!Comm::IsConnOpen(conn)) {
 373         debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " <<
 374                conn->local << " error: " << errno);
 375         writeCustomReply(451, "Internal error");
 376         return 0;
 377     }
 378
 379     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 380     typedef AsyncCallT<AcceptDialer> AcceptCall;
 381     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 382     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 383     listener = call.getRaw();
 384     dataListenConn = conn;
 385     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 386
 387     const unsigned int listeningPort = comm_local_port(conn->fd);
 388     conn->local.port(listeningPort);
 389     return listeningPort;
 390 }
 391
 392 void
 393 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 394 {
 395     if (params.flag != Comm::OK) {
 396         // Its possible the call was still queued when the client disconnected
 397         debugs(33, 2, dataListenConn << ": accept "
 398                "failure: " << xstrerr(params.xerrno));
 399         return;
 400     }
 401
 402     debugs(33, 4, "accepted " << params.conn);
 403     fd_note(params.conn->fd, "passive client ftp data");
 404     ++incoming_sockets_accepted;
 405
 406     if (!clientConnection) {
 407         debugs(33, 5, "late data connection?");
 408         closeDataConnection(); // in case we are still listening
 409         params.conn->close();
 410     } else if (params.conn->remote != clientConnection->remote) {
 411         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 412         params.conn->close();
 413         // Some FTP servers close control connection here, but it may make
 414         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 415     } else {
 416         closeDataConnection();
 417         dataConn = params.conn;
 418         uploadAvailSize = 0;
 419         debugs(33, 7, "ready for data");
 420         if (onDataAcceptCall != NULL) {
 421             AsyncCall::Pointer call = onDataAcceptCall;
 422             onDataAcceptCall = NULL;
 423             // If we got an upload request, start reading data from the client.
 424             if (master->serverState == fssHandleUploadRequest)
 425                 maybeReadUploadData();
 426             else
 427                 Must(master->serverState == fssHandleDataRequest);
 428             MemBuf mb;
 429             mb.init();
 430             mb.appendf("150 Data connection opened.\r\n");
 431             Comm::Write(clientConnection, &mb, call);
 432         }
 433     }
 434 }
 435
 436 void
 437 Ftp::Server::closeDataConnection()
 438 {
 439     if (listener != NULL) {
 440         listener->cancel("no longer needed");
 441         listener = NULL;
 442     }
 443
 444     if (Comm::IsConnOpen(dataListenConn)) {
 445         debugs(33, 5, "FTP closing client data listen socket: " <<
 446                *dataListenConn);
 447         dataListenConn->close();
 448     }
 449     dataListenConn = NULL;
 450
 451     if (reader != NULL) {
 452         // Comm::ReadCancel can deal with negative FDs
 453         Comm::ReadCancel(dataConn->fd, reader);
 454         reader = NULL;
 455     }
 456
 457     if (Comm::IsConnOpen(dataConn)) {
 458         debugs(33, 5, "FTP closing client data connection: " <<
 459                *dataConn);
 460         dataConn->close();
 461     }
 462     dataConn = NULL;
 463 }
 464
 465 /// Writes FTP [error] response before we fully parsed the FTP request and
 466 /// created the corresponding HTTP request wrapper for that FTP request.
 467 void
 468 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 469 {
 470     debugs(33, 7, code << ' ' << msg);
 471     assert(99 < code && code < 1000);
 472
 473     MemBuf mb;
 474     mb.init();
 475     mb.appendf("%i %s\r\n", code, msg);
 476
 477     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 478     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 479     Comm::Write(clientConnection, &mb, call);
 480
 481     flags.readMore = false;
 482
 483     // TODO: Create master transaction. Log it in wroteEarlyReply().
 484 }
 485
 486 void
 487 Ftp::Server::writeReply(MemBuf &mb)
 488 {
 489     debugs(9, 2, "FTP Client " << clientConnection);
 490     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 491            "\n----------");
 492
 493     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 494     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 495     Comm::Write(clientConnection, &mb, call);
 496 }
 497
 498 void
 499 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 500 {
 501     debugs(33, 7, code << ' ' << msg);
 502     assert(99 < code && code < 1000);
 503
 504     const bool sendDetails = reply != NULL &&
 505                              reply->header.has(Http::HdrType::FTP_STATUS) && reply->header.has(Http::HdrType::FTP_REASON);
 506
 507     MemBuf mb;
 508     mb.init();
 509     if (sendDetails) {
 510         mb.appendf("%i-%s\r\n", code, msg);
 511         mb.appendf(" Server reply:\r\n");
 512         Ftp::PrintReply(mb, reply, " ");
 513         mb.appendf("%i \r\n", code);
 514     } else
 515         mb.appendf("%i %s\r\n", code, msg);
 516
 517     writeReply(mb);
 518 }
 519
 520 void
 521 Ftp::Server::changeState(const ServerState newState, const char *reason)
 522 {
 523     if (master->serverState == newState) {
 524         debugs(33, 3, "client state unchanged at " << master->serverState <<
 525                " because " << reason);
 526         master->serverState = newState;
 527     } else {
 528         debugs(33, 3, "client state was " << master->serverState <<
 529                ", now " << newState << " because " << reason);
 530         master->serverState = newState;
 531     }
 532 }
 533
 534 /// whether the given FTP command has a pathname parameter
 535 static bool
 536 Ftp::CommandHasPathParameter(const SBuf &cmd)
 537 {
 538     static std::set<SBuf> PathedCommands;
 539     if (!PathedCommands.size()) {
 540         PathedCommands.insert(cmdMlst());
 541         PathedCommands.insert(cmdMlsd());
 542         PathedCommands.insert(cmdStat());
 543         PathedCommands.insert(cmdNlst());
 544         PathedCommands.insert(cmdList());
 545         PathedCommands.insert(cmdMkd());
 546         PathedCommands.insert(cmdRmd());
 547         PathedCommands.insert(cmdDele());
 548         PathedCommands.insert(cmdRnto());
 549         PathedCommands.insert(cmdRnfr());
 550         PathedCommands.insert(cmdAppe());
 551         PathedCommands.insert(cmdStor());
 552         PathedCommands.insert(cmdRetr());
 553         PathedCommands.insert(cmdSmnt());
 554         PathedCommands.insert(cmdCwd());
 555     }
 556
 557     return PathedCommands.find(cmd) != PathedCommands.end();
 558 }
 559
 560 /// creates a context filled with an error message for a given early error
 561 Http::Stream *
 562 Ftp::Server::earlyError(const EarlyErrorKind eek)
 563 {
 564     /* Default values, to be updated by the switch statement below */
 565     int scode = 421;
 566     const char *reason = "Internal error";
 567     const char *errUri = "error:ftp-internal-early-error";
 568
 569     switch (eek) {
 570     case EarlyErrorKind::HugeRequest:
 571         scode = 421;
 572         reason = "Huge request";
 573         errUri = "error:ftp-huge-request";
 574         break;
 575
 576     case EarlyErrorKind::MissingLogin:
 577         scode = 530;
 578         reason = "Must login first";
 579         errUri = "error:ftp-must-login-first";
 580         break;
 581
 582     case EarlyErrorKind::MissingUsername:
 583         scode = 501;
 584         reason = "Missing username";
 585         errUri = "error:ftp-missing-username";
 586         break;
 587
 588     case EarlyErrorKind::MissingHost:
 589         scode = 501;
 590         reason = "Missing host";
 591         errUri = "error:ftp-missing-host";
 592         break;
 593
 594     case EarlyErrorKind::UnsupportedCommand:
 595         scode = 502;
 596         reason = "Unknown or unsupported command";
 597         errUri = "error:ftp-unsupported-command";
 598         break;
 599
 600     case EarlyErrorKind::InvalidUri:
 601         scode = 501;
 602         reason = "Invalid URI";
 603         errUri = "error:ftp-invalid-uri";
 604         break;
 605
 606     case EarlyErrorKind::MalformedCommand:
 607         scode = 421;
 608         reason = "Malformed command";
 609         errUri = "error:ftp-malformed-command";
 610         break;
 611
 612         // no default so that a compiler can check that we have covered all cases
 613     }
 614
 615     Http::Stream *context = abortRequestParsing(errUri);
 616     clientStreamNode *node = context->getClientReplyContext();
 617     Must(node);
 618     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 619     Must(repContext);
 620
 621     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 622     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 623     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 624     repContext->setReplyToReply(reply);
 625     return context;
 626 }
 627
 628 /// Parses a single FTP request on the control connection.
 629 /// Returns a new Http::Stream on valid requests and all errors.
 630 /// Returns NULL on incomplete requests that may still succeed given more data.
 631 Http::Stream *
 632 Ftp::Server::parseOneRequest()
 633 {
 634     flags.readMore = false; // common for all but one case below
 635
 636     // OWS <command> [ RWS <parameter> ] OWS LF
 637
 638     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 639     // for the LF character that we must exclude here (but see FullWhiteSpace).
 640     static const char * const InlineSpaceChars = " \f\r\t\v";
 641     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 642     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 643     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 644     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 645
 646     // This set is used to ignore empty commands without allowing an attacker
 647     // to keep us endlessly busy by feeding us whitespace or empty commands.
 648     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 649
 650     SBuf cmd;
 651     SBuf params;
 652
 653     Parser::Tokenizer tok(inBuf);
 654
 655     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 656     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 657
 658     // note that the condition below will eat either RWS or trailing OWS
 659     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 660         // now params may include trailing OWS
 661         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 662         static const SBuf bufWhiteSpace(InlineSpaceChars);
 663         params.trim(bufWhiteSpace, false, true);
 664     }
 665
 666     // Why limit command line and parameters size? Did not we just parse them?
 667     // XXX: Our good old String cannot handle very long strings.
 668     const SBuf::size_type tokenMax = min(
 669                                          static_cast<SBuf::size_type>(32*1024), // conservative
 670                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 671     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 672         changeState(fssError, "huge req token");
 673         quitAfterError(NULL);
 674         return earlyError(EarlyErrorKind::HugeRequest);
 675     }
 676
 677     // technically, we may skip multiple NLs below, but that is OK
 678     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 679         // we need more data, but can we buffer more?
 680         if (inBuf.length() >= Config.maxRequestHeaderSize) {
 681             changeState(fssError, "huge req");
 682             quitAfterError(NULL);
 683             return earlyError(EarlyErrorKind::HugeRequest);
 684         } else {
 685             flags.readMore = true;
 686             debugs(33, 5, "Waiting for more, up to " <<
 687                    (Config.maxRequestHeaderSize - inBuf.length()));
 688             return NULL;
 689         }
 690     }
 691
 692     Must(parsed && cmd.length());
 693     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 694
 695     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 696
 697     cmd.toUpper(); // this should speed up and simplify future comparisons
 698
 699     // interception cases do not need USER to calculate the uri
 700     if (!transparent()) {
 701         if (!master->clientReadGreeting) {
 702             // the first command must be USER
 703             if (!pinning.pinned && cmd != cmdUser())
 704                 return earlyError(EarlyErrorKind::MissingLogin);
 705         }
 706
 707         // process USER request now because it sets FTP peer host name
 708         if (cmd == cmdUser()) {
 709             if (Http::Stream *errCtx = handleUserRequest(cmd, params))
 710                 return errCtx;
 711         }
 712     }
 713
 714     if (!Ftp::SupportedCommand(cmd))
 715         return earlyError(EarlyErrorKind::UnsupportedCommand);
 716
 717     const HttpRequestMethod method =
 718         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 719         Http::METHOD_PUT : Http::METHOD_GET;
 720
 721     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 722                        &params : NULL;
 723     calcUri(path);
 724     char *newUri = xstrdup(uri.c_str());
 725     HttpRequest *const request = HttpRequest::CreateFromUrl(newUri, method);
 726     if (!request) {
 727         debugs(33, 5, "Invalid FTP URL: " << uri);
 728         uri.clear();
 729         safe_free(newUri);
 730         return earlyError(EarlyErrorKind::InvalidUri);
 731     }
 732
 733     request->flags.ftpNative = true;
 734     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 735
 736     // Our fake Request-URIs are not distinctive enough for caching to work
 737     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 738     request->flags.noCache = true;
 739
 740     request->header.putStr(Http::HdrType::FTP_COMMAND, cmd.c_str());
 741     request->header.putStr(Http::HdrType::FTP_ARGUMENTS, params.c_str()); // may be ""
 742     if (method == Http::METHOD_PUT) {
 743         request->header.putStr(Http::HdrType::EXPECT, "100-continue");
 744         request->header.putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
 745     }
 746
 747     ClientHttpRequest *const http = new ClientHttpRequest(this);
 748     http->request = request;
 749     HTTPMSGLOCK(http->request);
 750     http->req_sz = tok.parsedSize();
 751     http->uri = newUri;
 752
 753     Http::Stream *const result =
 754         new Http::Stream(clientConnection, http);
 755
 756     StoreIOBuffer tempBuffer;
 757     tempBuffer.data = result->reqbuf;
 758     tempBuffer.length = HTTP_REQBUF_SZ;
 759
 760     ClientStreamData newServer = new clientReplyContext(http);
 761     ClientStreamData newClient = result;
 762     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 763                      clientReplyStatus, newServer, clientSocketRecipient,
 764                      clientSocketDetach, newClient, tempBuffer);
 765
 766     result->flags.parsed_ok = 1;
 767     return result;
 768 }
 769
 770 void
 771 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 772 {
 773     // the caller guarantees that we are dealing with the current context only
 774     Http::StreamPointer context = pipeline.front();
 775     assert(context != nullptr);
 776
 777     if (context->http && context->http->al != NULL &&
 778             !context->http->al->reply && reply) {
 779         context->http->al->reply = reply;
 780         HTTPMSGLOCK(context->http->al->reply);
 781     }
 782
 783     static ReplyHandler handlers[] = {
 784         NULL, // fssBegin
 785         NULL, // fssConnected
 786         &Ftp::Server::handleFeatReply, // fssHandleFeat
 787         &Ftp::Server::handlePasvReply, // fssHandlePasv
 788         &Ftp::Server::handlePortReply, // fssHandlePort
 789         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 790         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 791         &Ftp::Server::handleEprtReply,// fssHandleEprt
 792         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 793         NULL, // fssHandleCwd
 794         NULL, // fssHandlePass
 795         NULL, // fssHandleCdup
 796         &Ftp::Server::handleErrorReply // fssError
 797     };
 798     try {
 799         const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 800         if (const ReplyHandler handler = handlers[server.master->serverState])
 801             (this->*handler)(reply, data);
 802         else
 803             writeForwardedReply(reply);
 804     } catch (const std::exception &e) {
 805         callException(e);
 806         throw TexcHere(e.what());
 807     }
 808 }
 809
 810 void
 811 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 812 {
 813     if (pipeline.front()->http->request->errType != ERR_NONE) {
 814         writeCustomReply(502, "Server does not support FEAT", reply);
 815         return;
 816     }
 817
 818     Must(reply);
 819     HttpReply::Pointer featReply = Ftp::HttpReplyWrapper(211, "End", Http::scNoContent, 0);
 820     HttpHeader const &serverReplyHeader = reply->header;
 821
 822     HttpHeaderPos pos = HttpHeaderInitPos;
 823     bool hasEPRT = false;
 824     bool hasEPSV = false;
 825     int prependSpaces = 1;
 826
 827     featReply->header.putStr(Http::HdrType::FTP_PRE, "\"211-Features:\"");
 828     const int scode = serverReplyHeader.getInt(Http::HdrType::FTP_STATUS);
 829     if (scode == 211) {
 830         while (const HttpHeaderEntry *e = serverReplyHeader.getEntry(&pos)) {
 831             if (e->id == Http::HdrType::FTP_PRE) {
 832                 // assume RFC 2389 FEAT response format, quoted by Squid:
 833                 // <"> SP NAME [SP PARAMS] <">
 834                 // but accommodate MS servers sending four SPs before NAME
 835
 836                 // command name ends with (SP parameter) or quote
 837                 static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 838                 static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 839
 840                 Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 841                 if (!tok.skip('"') || !tok.skip(' '))
 842                     continue;
 843
 844                 // optional spaces; remember their number to accomodate MS servers
 845                 prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 846
 847                 SBuf cmd;
 848                 if (!tok.prefix(cmd, FeatNameChars))
 849                     continue;
 850                 cmd.toUpper();
 851
 852                 if (Ftp::SupportedCommand(cmd)) {
 853                     featReply->header.addEntry(e->clone());
 854                 }
 855
 856                 if (cmd == cmdEprt())
 857                     hasEPRT = true;
 858                 else if (cmd == cmdEpsv())
 859                     hasEPSV = true;
 860             }
 861         }
 862     } // else we got a FEAT error and will only report Squid-supported features
 863
 864     char buf[256];
 865     if (!hasEPRT) {
 866         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 867         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 868     }
 869     if (!hasEPSV) {
 870         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 871         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 872     }
 873
 874     featReply->header.refreshMask();
 875
 876     writeForwardedReply(featReply.getRaw());
 877 }
 878
 879 void
 880 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 881 {
 882     const Http::StreamPointer context(pipeline.front());
 883     assert(context != nullptr);
 884
 885     if (context->http->request->errType != ERR_NONE) {
 886         writeCustomReply(502, "Server does not support PASV", reply);
 887         return;
 888     }
 889
 890     const unsigned short localPort = listenForDataConnection();
 891     if (!localPort)
 892         return;
 893
 894     char addr[MAX_IPSTRLEN];
 895     // remote server in interception setups and local address otherwise
 896     const Ip::Address &server = transparent() ?
 897                                 clientConnection->local : dataListenConn->local;
 898     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 899     addr[MAX_IPSTRLEN - 1] = '\0';
 900     for (char *c = addr; *c != '\0'; ++c) {
 901         if (*c == '.')
 902             *c = ',';
 903     }
 904
 905     // In interception setups, we combine remote server address with a
 906     // local port number and hope that traffic will be redirected to us.
 907     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 908     // versions block responses that use those alternative syntax rules!
 909     MemBuf mb;
 910     mb.init();
 911     mb.appendf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 912                addr,
 913                static_cast<int>(localPort / 256),
 914                static_cast<int>(localPort % 256));
 915     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 916     writeReply(mb);
 917 }
 918
 919 void
 920 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 921 {
 922     if (pipeline.front()->http->request->errType != ERR_NONE) {
 923         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 924         return;
 925     }
 926
 927     writeCustomReply(200, "PORT successfully converted to PASV.");
 928
 929     // and wait for RETR
 930 }
 931
 932 void
 933 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 934 {
 935     if (!pinning.pinned) // we failed to connect to server
 936         uri.clear();
 937     // 421: we will close due to fssError
 938     writeErrorReply(reply, 421);
 939 }
 940
 941 void
 942 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 943 {
 944     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 945         writeForwardedReply(reply);
 946         if (Comm::IsConnOpen(dataConn)) {
 947             debugs(33, 3, "closing " << dataConn << " on KO reply");
 948             closeDataConnection();
 949         }
 950         return;
 951     }
 952
 953     if (!dataConn) {
 954         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 955         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 956         return;
 957     }
 958
 959     if (!checkDataConnPost()) {
 960         writeCustomReply(425, "Data connection is not established.");
 961         closeDataConnection();
 962         return;
 963     }
 964
 965     debugs(33, 7, data.length);
 966
 967     if (data.length <= 0) {
 968         replyDataWritingCheckpoint(); // skip the actual write call
 969         return;
 970     }
 971
 972     MemBuf mb;
 973     mb.init(data.length + 1, data.length + 1);
 974     mb.append(data.data, data.length);
 975
 976     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 977     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 978     Comm::Write(dataConn, &mb, call);
 979
 980     pipeline.front()->noteSentBodyBytes(data.length);
 981 }
 982
 983 /// called when we are done writing a chunk of the response data
 984 void
 985 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 986 {
 987     if (io.flag == Comm::ERR_CLOSING)
 988         return;
 989
 990     if (io.flag != Comm::OK) {
 991         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 992         userDataCompletionCheckpoint(426);
 993         return;
 994     }
 995
 996     assert(pipeline.front()->http);
 997     pipeline.front()->http->out.size += io.size;
 998     replyDataWritingCheckpoint();
 999 }
1000
1001 /// ClientStream checks after (actual or skipped) reply data writing
1002 void
1003 Ftp::Server::replyDataWritingCheckpoint()
1004 {
1005     switch (pipeline.front()->socketState()) {
1006     case STREAM_NONE:
1007         debugs(33, 3, "Keep going");
1008         pipeline.front()->pullData();
1009         return;
1010     case STREAM_COMPLETE:
1011         debugs(33, 3, "FTP reply data transfer successfully complete");
1012         userDataCompletionCheckpoint(226);
1013         break;
1014     case STREAM_UNPLANNED_COMPLETE:
1015         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
1016         userDataCompletionCheckpoint(451);
1017         break;
1018     case STREAM_FAILED:
1019         userDataCompletionCheckpoint(451);
1020         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
1021         break;
1022     default:
1023         fatal("unreachable code");
1024     }
1025 }
1026
1027 void
1028 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1029 {
1030     writeForwardedReply(reply);
1031     // note that the client data connection may already be closed by now
1032 }
1033
1034 void
1035 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1036 {
1037     Must(reply);
1038
1039     if (waitingForOrigin) {
1040         Must(delayedReply == NULL);
1041         delayedReply = reply;
1042         return;
1043     }
1044
1045     const HttpHeader &header = reply->header;
1046     // adaptation and forwarding errors lack Http::HdrType::FTP_STATUS
1047     if (!header.has(Http::HdrType::FTP_STATUS)) {
1048         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1049         return;
1050     }
1051
1052     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1053     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1054     writeForwardedReplyAndCall(reply, call);
1055 }
1056
1057 void
1058 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1059 {
1060     if (pipeline.front()->http->request->errType != ERR_NONE) {
1061         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1062         return;
1063     }
1064
1065     writeCustomReply(200, "EPRT successfully converted to PASV.");
1066
1067     // and wait for RETR
1068 }
1069
1070 void
1071 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1072 {
1073     if (pipeline.front()->http->request->errType != ERR_NONE) {
1074         writeCustomReply(502, "Cannot connect to server", reply);
1075         return;
1076     }
1077
1078     const unsigned short localPort = listenForDataConnection();
1079     if (!localPort)
1080         return;
1081
1082     // In interception setups, we use a local port number and hope that data
1083     // traffic will be redirected to us.
1084     MemBuf mb;
1085     mb.init();
1086     mb.appendf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1087
1088     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1089     writeReply(mb);
1090 }
1091
1092 /// writes FTP error response with given status and reply-derived error details
1093 void
1094 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1095 {
1096     const HttpRequest *request = pipeline.front()->http->request;
1097     assert(request);
1098
1099     MemBuf mb;
1100     mb.init();
1101
1102     if (request->errType != ERR_NONE)
1103         mb.appendf("%i-%s\r\n", scode, errorPageName(request->errType));
1104
1105     if (request->errDetail > 0) {
1106         // XXX: > 0 may not always mean that this is an errno
1107         mb.appendf("%i-Error: (%d) %s\r\n", scode,
1108                    request->errDetail,
1109                    strerror(request->errDetail));
1110     }
1111
1112 #if USE_ADAPTATION
1113     // XXX: Remove hard coded names. Use an error page template instead.
1114     const Adaptation::History::Pointer ah = request->adaptHistory();
1115     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1116         const String info = ah->allMeta.getByName("X-Response-Info");
1117         const String desc = ah->allMeta.getByName("X-Response-Desc");
1118         if (info.size())
1119             mb.appendf("%i-Information: %s\r\n", scode, info.termedBuf());
1120         if (desc.size())
1121             mb.appendf("%i-Description: %s\r\n", scode, desc.termedBuf());
1122     }
1123 #endif
1124
1125     Must(reply);
1126     const char *reason = reply->header.has(Http::HdrType::FTP_REASON) ?
1127                          reply->header.getStr(Http::HdrType::FTP_REASON):
1128                          reply->sline.reason();
1129
1130     mb.appendf("%i %s\r\n", scode, reason); // error terminating line
1131
1132     // TODO: errorpage.cc should detect FTP client and use
1133     // configurable FTP-friendly error templates which we should
1134     // write to the client "as is" instead of hiding most of the info
1135
1136     writeReply(mb);
1137 }
1138
1139 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1140 /// for example, internally-generated Squid "errorpages" end up here (for now)
1141 void
1142 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1143 {
1144     changeState(fssConnected, "foreign reply");
1145     closeDataConnection();
1146     // 451: We intend to keep the control connection open.
1147     writeErrorReply(reply, 451);
1148 }
1149
1150 void
1151 Ftp::Server::writeControlMsgAndCall(HttpReply *reply, AsyncCall::Pointer &call)
1152 {
1153     // the caller guarantees that we are dealing with the current context only
1154     // the caller should also make sure reply->header.has(Http::HdrType::FTP_STATUS)
1155     writeForwardedReplyAndCall(reply, call);
1156 }
1157
1158 void
1159 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1160 {
1161     assert(reply != NULL);
1162     const HttpHeader &header = reply->header;
1163
1164     // without status, the caller must use the writeForwardedForeign() path
1165     Must(header.has(Http::HdrType::FTP_STATUS));
1166     Must(header.has(Http::HdrType::FTP_REASON));
1167     const int scode = header.getInt(Http::HdrType::FTP_STATUS);
1168     debugs(33, 7, "scode: " << scode);
1169
1170     // Status 125 or 150 implies upload or data request, but we still check
1171     // the state in case the server is buggy.
1172     if ((scode == 125 || scode == 150) &&
1173             (master->serverState == fssHandleUploadRequest ||
1174              master->serverState == fssHandleDataRequest)) {
1175         if (checkDataConnPost()) {
1176             // If the data connection is ready, start reading data (here)
1177             // and forward the response to client (further below).
1178             debugs(33, 7, "data connection established, start data transfer");
1179             if (master->serverState == fssHandleUploadRequest)
1180                 maybeReadUploadData();
1181         } else {
1182             // If we are waiting to accept the data connection, keep waiting.
1183             if (Comm::IsConnOpen(dataListenConn)) {
1184                 debugs(33, 7, "wait for the client to establish a data connection");
1185                 onDataAcceptCall = call;
1186                 // TODO: Add connect timeout for passive connections listener?
1187                 // TODO: Remember server response so that we can forward it?
1188             } else {
1189                 // Either the connection was establised and closed after the
1190                 // data was transferred OR we failed to establish an active
1191                 // data connection and already sent the error to the client.
1192                 // In either case, there is nothing more to do.
1193                 debugs(33, 7, "done with data OR active connection failed");
1194             }
1195             return;
1196         }
1197     }
1198
1199     MemBuf mb;
1200     mb.init();
1201     Ftp::PrintReply(mb, reply);
1202
1203     debugs(9, 2, "FTP Client " << clientConnection);
1204     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1205            "\n----------");
1206
1207     Comm::Write(clientConnection, &mb, call);
1208 }
1209
1210 static void
1211 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const)
1212 {
1213     const HttpHeader &header = reply->header;
1214
1215     HttpHeaderPos pos = HttpHeaderInitPos;
1216     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1217         if (e->id == Http::HdrType::FTP_PRE) {
1218             String raw;
1219             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1220                 mb.appendf("%s\r\n", raw.termedBuf());
1221         }
1222     }
1223
1224     if (header.has(Http::HdrType::FTP_STATUS)) {
1225         const char *reason = header.getStr(Http::HdrType::FTP_REASON);
1226         mb.appendf("%i %s\r\n", header.getInt(Http::HdrType::FTP_STATUS),
1227                    (reason ? reason : 0));
1228     }
1229 }
1230
1231 void
1232 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1233 {
1234     if (io.flag == Comm::ERR_CLOSING)
1235         return;
1236
1237     if (io.flag != Comm::OK) {
1238         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1239         io.conn->close();
1240         return;
1241     }
1242
1243     Http::StreamPointer context = pipeline.front();
1244     if (context != nullptr && context->http) {
1245         context->http->out.size += io.size;
1246         context->http->out.headers_sz += io.size;
1247     }
1248
1249     flags.readMore = true;
1250     readSomeData();
1251 }
1252
1253 void
1254 Ftp::Server::wroteReply(const CommIoCbParams &io)
1255 {
1256     if (io.flag == Comm::ERR_CLOSING)
1257         return;
1258
1259     if (io.flag != Comm::OK) {
1260         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1261         io.conn->close();
1262         return;
1263     }
1264
1265     Http::StreamPointer context = pipeline.front();
1266     assert(context->http);
1267     context->http->out.size += io.size;
1268     context->http->out.headers_sz += io.size;
1269
1270     if (master->serverState == fssError) {
1271         debugs(33, 5, "closing on FTP server error");
1272         io.conn->close();
1273         return;
1274     }
1275
1276     const clientStream_status_t socketState = context->socketState();
1277     debugs(33, 5, "FTP client stream state " << socketState);
1278     switch (socketState) {
1279     case STREAM_UNPLANNED_COMPLETE:
1280     case STREAM_FAILED:
1281         io.conn->close();
1282         return;
1283
1284     case STREAM_NONE:
1285     case STREAM_COMPLETE:
1286         flags.readMore = true;
1287         changeState(fssConnected, "Ftp::Server::wroteReply");
1288         if (bodyParser)
1289             finishDechunkingRequest(false);
1290         context->finished();
1291         kick();
1292         return;
1293     }
1294 }
1295
1296 bool
1297 Ftp::Server::handleRequest(HttpRequest *request)
1298 {
1299     debugs(33, 9, request);
1300     Must(request);
1301
1302     HttpHeader &header = request->header;
1303     Must(header.has(Http::HdrType::FTP_COMMAND));
1304     String &cmd = header.findEntry(Http::HdrType::FTP_COMMAND)->value;
1305     Must(header.has(Http::HdrType::FTP_ARGUMENTS));
1306     String &params = header.findEntry(Http::HdrType::FTP_ARGUMENTS)->value;
1307
1308     if (Debug::Enabled(9, 2)) {
1309         MemBuf mb;
1310         mb.init();
1311         request->pack(&mb);
1312
1313         debugs(9, 2, "FTP Client " << clientConnection);
1314         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1315                "\n----------");
1316     }
1317
1318     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1319     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1320     static RequestHandlers handlers;
1321     if (!handlers.size()) {
1322         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1323         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1324         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1325         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1326         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1327         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1328         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1329         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1330         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1331         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1332         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1333         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1334     }
1335
1336     RequestHandler handler = NULL;
1337     if (request->method == Http::METHOD_PUT)
1338         handler = &Ftp::Server::handleUploadRequest;
1339     else {
1340         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1341         if (hi != handlers.end())
1342             handler = hi->second;
1343     }
1344
1345     if (!handler) {
1346         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1347         return true;
1348     }
1349
1350     return (this->*handler)(cmd, params);
1351 }
1352
1353 /// Called to parse USER command, which is required to create an HTTP request
1354 /// wrapper. W/o request, the errors are handled by returning earlyError().
1355 Http::Stream *
1356 Ftp::Server::handleUserRequest(const SBuf &, SBuf &params)
1357 {
1358     if (params.isEmpty())
1359         return earlyError(EarlyErrorKind::MissingUsername);
1360
1361     // find the [end of] user name
1362     const SBuf::size_type eou = params.rfind('@');
1363     if (eou == SBuf::npos || eou + 1 >= params.length())
1364         return earlyError(EarlyErrorKind::MissingHost);
1365
1366     // Determine the intended destination.
1367     host = params.substr(eou + 1, params.length());
1368     // If we can parse it as raw IPv6 address, then surround with "[]".
1369     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1370     if (host.find(':') != SBuf::npos) {
1371         const Ip::Address ipa(host.c_str());
1372         if (!ipa.isAnyAddr()) {
1373             char ipBuf[MAX_IPSTRLEN];
1374             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1375             host = ipBuf;
1376         }
1377     }
1378
1379     // const SBuf login = params.substr(0, eou);
1380     params.chop(0, eou); // leave just the login part for the peer
1381
1382     SBuf oldUri;
1383     if (master->clientReadGreeting)
1384         oldUri = uri;
1385
1386     master->workingDir.clear();
1387     calcUri(NULL);
1388
1389     if (!master->clientReadGreeting) {
1390         debugs(9, 3, "set URI to " << uri);
1391     } else if (oldUri.caseCmp(uri) == 0) {
1392         debugs(9, 5, "kept URI as " << oldUri);
1393     } else {
1394         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1395         closeDataConnection();
1396         unpinConnection(true); // close control connection to peer
1397         resetLogin("URI reset");
1398     }
1399
1400     return NULL; // no early errors
1401 }
1402
1403 bool
1404 Ftp::Server::handleFeatRequest(String &, String &)
1405 {
1406     changeState(fssHandleFeat, "handleFeatRequest");
1407     return true;
1408 }
1409
1410 bool
1411 Ftp::Server::handlePasvRequest(String &, String &params)
1412 {
1413     if (gotEpsvAll) {
1414         setReply(500, "Bad PASV command");
1415         return false;
1416     }
1417
1418     if (params.size() > 0) {
1419         setReply(501, "Unexpected parameter");
1420         return false;
1421     }
1422
1423     changeState(fssHandlePasv, "handlePasvRequest");
1424     // no need to fake PASV request via setDataCommand() in true PASV case
1425     return true;
1426 }
1427
1428 /// [Re]initializes dataConn for active data transfers. Does not connect.
1429 bool
1430 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1431 {
1432     assert(clientConnection != NULL);
1433     assert(!clientConnection->remote.isAnyAddr());
1434
1435     if (cltAddr != clientConnection->remote) {
1436         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1437         // Closing the control connection would not help with attacks because
1438         // the client is evidently able to connect to us. Besides, closing
1439         // makes retrials easier for the client and more damaging to us.
1440         setReply(501, "Prohibited parameter value");
1441         return false;
1442     }
1443
1444     closeDataConnection();
1445
1446     Comm::ConnectionPointer conn = new Comm::Connection();
1447     conn->flags |= COMM_DOBIND;
1448
1449     // Use local IP address of the control connection as the source address
1450     // of the active data connection, or some clients will refuse to accept.
1451     conn->setAddrs(clientConnection->local, cltAddr);
1452     // RFC 959 requires active FTP connections to originate from port 20
1453     // but that would preclude us from supporting concurrent transfers! (XXX?)
1454     conn->local.port(0);
1455
1456     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1457            conn->remote);
1458
1459     dataConn = conn;
1460     uploadAvailSize = 0;
1461     return true;
1462 }
1463
1464 bool
1465 Ftp::Server::handlePortRequest(String &, String &params)
1466 {
1467     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1468
1469     if (gotEpsvAll) {
1470         setReply(500, "Rejecting PORT after EPSV ALL");
1471         return false;
1472     }
1473
1474     if (!params.size()) {
1475         setReply(501, "Missing parameter");
1476         return false;
1477     }
1478
1479     Ip::Address cltAddr;
1480     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1481         setReply(501, "Invalid parameter");
1482         return false;
1483     }
1484
1485     if (!createDataConnection(cltAddr))
1486         return false;
1487
1488     changeState(fssHandlePort, "handlePortRequest");
1489     setDataCommand();
1490     return true; // forward our fake PASV request
1491 }
1492
1493 bool
1494 Ftp::Server::handleDataRequest(String &, String &)
1495 {
1496     if (!checkDataConnPre())
1497         return false;
1498
1499     master->userDataDone = 0;
1500     originDataDownloadAbortedOnError = false;
1501
1502     changeState(fssHandleDataRequest, "handleDataRequest");
1503
1504     return true;
1505 }
1506
1507 bool
1508 Ftp::Server::handleUploadRequest(String &, String &)
1509 {
1510     if (!checkDataConnPre())
1511         return false;
1512
1513     if (Config.accessList.forceRequestBodyContinuation) {
1514         ClientHttpRequest *http = pipeline.front()->http;
1515         HttpRequest *request = http->request;
1516         ACLFilledChecklist bodyContinuationCheck(Config.accessList.forceRequestBodyContinuation, request, NULL);
1517         if (bodyContinuationCheck.fastCheck() == ACCESS_ALLOWED) {
1518             request->forcedBodyContinuation = true;
1519             if (checkDataConnPost()) {
1520                 // Write control Msg
1521                 writeEarlyReply(150, "Data connection opened");
1522                 maybeReadUploadData();
1523             } else {
1524                 // wait for acceptDataConnection but tell it to call wroteEarlyReply
1525                 // after writing "150 Data connection opened"
1526                 typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1527                 AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
1528                 onDataAcceptCall = call;
1529             }
1530         }
1531     }
1532
1533     changeState(fssHandleUploadRequest, "handleDataRequest");
1534
1535     return true;
1536 }
1537
1538 bool
1539 Ftp::Server::handleEprtRequest(String &, String &params)
1540 {
1541     debugs(9, 3, "Process an EPRT " << params);
1542
1543     if (gotEpsvAll) {
1544         setReply(500, "Rejecting EPRT after EPSV ALL");
1545         return false;
1546     }
1547
1548     if (!params.size()) {
1549         setReply(501, "Missing parameter");
1550         return false;
1551     }
1552
1553     Ip::Address cltAddr;
1554     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1555         setReply(501, "Invalid parameter");
1556         return false;
1557     }
1558
1559     if (!createDataConnection(cltAddr))
1560         return false;
1561
1562     changeState(fssHandleEprt, "handleEprtRequest");
1563     setDataCommand();
1564     return true; // forward our fake PASV request
1565 }
1566
1567 bool
1568 Ftp::Server::handleEpsvRequest(String &, String &params)
1569 {
1570     debugs(9, 3, "Process an EPSV command with params: " << params);
1571     if (params.size() <= 0) {
1572         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1573     } else if (params.caseCmp("ALL") == 0) {
1574         setReply(200, "EPSV ALL ok");
1575         gotEpsvAll = true;
1576         return false;
1577     } else if (params.cmp("2") == 0) {
1578         if (!Ip::EnableIpv6) {
1579             setReply(522, "Network protocol not supported, use (1)");
1580             return false;
1581         }
1582     } else if (params.cmp("1") != 0) {
1583         setReply(501, "Unsupported EPSV parameter");
1584         return false;
1585     }
1586
1587     changeState(fssHandleEpsv, "handleEpsvRequest");
1588     setDataCommand();
1589     return true; // forward our fake PASV request
1590 }
1591
1592 bool
1593 Ftp::Server::handleCwdRequest(String &, String &)
1594 {
1595     changeState(fssHandleCwd, "handleCwdRequest");
1596     return true;
1597 }
1598
1599 bool
1600 Ftp::Server::handlePassRequest(String &, String &)
1601 {
1602     changeState(fssHandlePass, "handlePassRequest");
1603     return true;
1604 }
1605
1606 bool
1607 Ftp::Server::handleCdupRequest(String &, String &)
1608 {
1609     changeState(fssHandleCdup, "handleCdupRequest");
1610     return true;
1611 }
1612
1613 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1614 // Squid FTP client decides what data command to use with peers.
1615 void
1616 Ftp::Server::setDataCommand()
1617 {
1618     ClientHttpRequest *const http = pipeline.front()->http;
1619     assert(http != NULL);
1620     HttpRequest *const request = http->request;
1621     assert(request != NULL);
1622     HttpHeader &header = request->header;
1623     header.delById(Http::HdrType::FTP_COMMAND);
1624     header.putStr(Http::HdrType::FTP_COMMAND, "PASV");
1625     header.delById(Http::HdrType::FTP_ARGUMENTS);
1626     header.putStr(Http::HdrType::FTP_ARGUMENTS, "");
1627     debugs(9, 5, "client data command converted to fake PASV");
1628 }
1629
1630 /// check that client data connection is ready for future I/O or at least
1631 /// has a chance of becoming ready soon.
1632 bool
1633 Ftp::Server::checkDataConnPre()
1634 {
1635     if (Comm::IsConnOpen(dataConn))
1636         return true;
1637
1638     if (Comm::IsConnOpen(dataListenConn)) {
1639         // We are still waiting for a client to connect to us after PASV.
1640         // Perhaps client's data conn handshake has not reached us yet.
1641         // After we talk to the server, checkDataConnPost() will recheck.
1642         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1643         return true;
1644     }
1645
1646     if (!dataConn || dataConn->remote.isAnyAddr()) {
1647         debugs(33, 5, "missing " << dataConn);
1648         // TODO: use client address and default port instead.
1649         setReply(425, "Use PORT or PASV first");
1650         return false;
1651     }
1652
1653     // active transfer: open a data connection from Squid to client
1654     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1655     connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1656     Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector,
1657             Config.Timeout.connect);
1658     AsyncJob::Start(cs);
1659     return false; // ConnStateData::processFtpRequest waits handleConnectDone
1660 }
1661
1662 /// Check that client data connection is ready for immediate I/O.
1663 bool
1664 Ftp::Server::checkDataConnPost() const
1665 {
1666     if (!Comm::IsConnOpen(dataConn)) {
1667         debugs(33, 3, "missing client data conn: " << dataConn);
1668         return false;
1669     }
1670     return true;
1671 }
1672
1673 /// Done establishing a data connection to the user.
1674 void
1675 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1676 {
1677     connector = NULL;
1678
1679     if (params.flag != Comm::OK) {
1680         /* it might have been a timeout with a partially open link */
1681         if (params.conn != NULL)
1682             params.conn->close();
1683         setReply(425, "Cannot open data connection.");
1684         Http::StreamPointer context = pipeline.front();
1685         Must(context->http);
1686         Must(context->http->storeEntry() != NULL);
1687     } else {
1688         Must(dataConn == params.conn);
1689         Must(Comm::IsConnOpen(params.conn));
1690         fd_note(params.conn->fd, "active client ftp data");
1691     }
1692
1693     doProcessRequest();
1694 }
1695
1696 void
1697 Ftp::Server::setReply(const int code, const char *msg)
1698 {
1699     Http::StreamPointer context = pipeline.front();
1700     ClientHttpRequest *const http = context->http;
1701     assert(http != NULL);
1702     assert(http->storeEntry() == NULL);
1703
1704     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1705
1706     setLogUri(http, urlCanonicalClean(http->request));
1707
1708     clientStreamNode *const node = context->getClientReplyContext();
1709     clientReplyContext *const repContext =
1710         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1711     assert(repContext != NULL);
1712
1713     RequestFlags reqFlags;
1714     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1715     reqFlags.noCache = true;
1716     repContext->createStoreEntry(http->request->method, reqFlags);
1717     http->storeEntry()->replaceHttpReply(reply);
1718 }
1719
1720 void
1721 Ftp::Server::callException(const std::exception &e)
1722 {
1723     debugs(33, 2, "FTP::Server job caught: " << e.what());
1724     closeDataConnection();
1725     unpinConnection(true);
1726     if (Comm::IsConnOpen(clientConnection))
1727         clientConnection->close();
1728     AsyncJob::callException(e);
1729 }
1730
1731 void
1732 Ftp::Server::startWaitingForOrigin()
1733 {
1734     debugs(33, 5, "waiting for Ftp::Client data transfer to end");
1735     waitingForOrigin = true;
1736 }
1737
1738 void
1739 Ftp::Server::stopWaitingForOrigin(int originStatus)
1740 {
1741     Must(waitingForOrigin);
1742     waitingForOrigin = false;
1743
1744     // if we have already decided how to respond, respond now
1745     if (delayedReply) {
1746         HttpReply::Pointer reply = delayedReply;
1747         delayedReply = nullptr;
1748         writeForwardedReply(reply.getRaw());
1749         return; // do not completeDataDownload() after an earlier response
1750     }
1751
1752     if (master->serverState != fssHandleDataRequest)
1753         return;
1754
1755     // completeDataDownload() could be waitingForOrigin in fssHandleDataRequest
1756     // Depending on which side has finished downloading first, either trust
1757     // master->userDataDone status or set originDataDownloadAbortedOnError:
1758     if (master->userDataDone) {
1759         // We finished downloading before Ftp::Client. Most likely, the
1760         // adaptation shortened the origin response or we hit an error.
1761         // Our status (stored in master->userDataDone) is more informative.
1762         // Use master->userDataDone; avoid originDataDownloadAbortedOnError.
1763         completeDataDownload();
1764     } else {
1765         debugs(33, 5, "too early to write the response");
1766         // Ftp::Client naturally finished downloading before us. Set
1767         // originDataDownloadAbortedOnError to overwrite future
1768         // master->userDataDone and relay Ftp::Client error, if there was
1769         // any, to the user.
1770         originDataDownloadAbortedOnError = (originStatus >= 400);
1771     }
1772 }
1773
1774 void Ftp::Server::userDataCompletionCheckpoint(int finalStatusCode)
1775 {
1776     Must(!master->userDataDone);
1777     master->userDataDone = finalStatusCode;
1778
1779     if (bodyParser)
1780         finishDechunkingRequest(false);
1781
1782     if (waitingForOrigin) {
1783         // The completeDataDownload() is not called here unconditionally
1784         // because we want to signal the FTP user that we are not fully
1785         // done processing its data stream, even though all data bytes
1786         // have been sent or received already.
1787         debugs(33, 5, "Transfering from FTP server is not complete");
1788         return;
1789     }
1790
1791     // Adjust our reply if the server aborted with an error before we are done.
1792     if (master->userDataDone == 226 && originDataDownloadAbortedOnError) {
1793         debugs(33, 5, "Transfering from FTP server terminated with an error, adjust status code");
1794         master->userDataDone = 451;
1795     }
1796     completeDataDownload();
1797 }
1798
1799 void Ftp::Server::completeDataDownload()
1800 {
1801     writeCustomReply(master->userDataDone, master->userDataDone == 226 ? "Transfer complete" : "Server error; transfer aborted");
1802     closeDataConnection();
1803 }
1804
1805 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1806 static bool
1807 Ftp::SupportedCommand(const SBuf &name)
1808 {
1809     static std::set<SBuf> BlackList;
1810     if (BlackList.empty()) {
1811         /* Add FTP commands that Squid cannot relay correctly. */
1812
1813         // We probably do not support AUTH TLS.* and AUTH SSL,
1814         // but let's disclaim all AUTH support to KISS, for now.
1815         BlackList.insert(cmdAuth());
1816     }
1817
1818     // we claim support for all commands that we do not know about
1819     return BlackList.find(name) == BlackList.end();
1820 }
1821