src/servers/FtpServer.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 33    Transfer protocol servers */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "base/CharacterSet.h"
  14 #include "base/RefCount.h"
  15 #include "base/Subscription.h"
  16 #include "client_side_reply.h"
  17 #include "client_side_request.h"
  18 #include "clientStream.h"
  19 #include "comm/ConnOpener.h"
  20 #include "comm/Read.h"
  21 #include "comm/TcpAcceptor.h"
  22 #include "comm/Write.h"
  23 #include "errorpage.h"
  24 #include "fd.h"
  25 #include "ftp/Elements.h"
  26 #include "ftp/Parsing.h"
  27 #include "globals.h"
  28 #include "http/one/RequestParser.h"
  29 #include "HttpHdrCc.h"
  30 #include "ip/tools.h"
  31 #include "ipc/FdNotes.h"
  32 #include "parser/Tokenizer.h"
  33 #include "servers/forward.h"
  34 #include "servers/FtpServer.h"
  35 #include "SquidConfig.h"
  36 #include "StatCounters.h"
  37 #include "tools.h"
  38
  39 #include <set>
  40 #include <map>
  41
  42 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Server);
  43
  44 namespace Ftp
  45 {
  46 static void PrintReply(MemBuf &mb, const HttpReply *reply, const char *const prefix = "");
  47 static bool SupportedCommand(const SBuf &name);
  48 static bool CommandHasPathParameter(const SBuf &cmd);
  49 };
  50
  51 Ftp::Server::Server(const MasterXaction::Pointer &xact):
  52     AsyncJob("Ftp::Server"),
  53     ConnStateData(xact),
  54     master(new MasterState),
  55     uri(),
  56     host(),
  57     gotEpsvAll(false),
  58     onDataAcceptCall(),
  59     dataListenConn(),
  60     dataConn(),
  61     uploadAvailSize(0),
  62     listener(),
  63     connector(),
  64     reader()
  65 {
  66     flags.readMore = false; // we need to announce ourselves first
  67     *uploadBuf = 0;
  68 }
  69
  70 Ftp::Server::~Server()
  71 {
  72     closeDataConnection();
  73 }
  74
  75 int
  76 Ftp::Server::pipelinePrefetchMax() const
  77 {
  78     return 0; // no support for concurrent FTP requests
  79 }
  80
  81 time_t
  82 Ftp::Server::idleTimeout() const
  83 {
  84     return Config.Timeout.ftpClientIdle;
  85 }
  86
  87 void
  88 Ftp::Server::start()
  89 {
  90     ConnStateData::start();
  91
  92     if (transparent()) {
  93         char buf[MAX_IPSTRLEN];
  94         clientConnection->local.toUrl(buf, MAX_IPSTRLEN);
  95         host = buf;
  96         calcUri(NULL);
  97         debugs(33, 5, "FTP transparent URL: " << uri);
  98     }
  99
 100     writeEarlyReply(220, "Service ready");
 101 }
 102
 103 /// schedules another data connection read if needed
 104 void
 105 Ftp::Server::maybeReadUploadData()
 106 {
 107     if (reader != NULL)
 108         return;
 109
 110     const size_t availSpace = sizeof(uploadBuf) - uploadAvailSize;
 111     if (availSpace <= 0)
 112         return;
 113
 114     debugs(33, 4, dataConn << ": reading FTP data...");
 115
 116     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 117     reader = JobCallback(33, 5, Dialer, this, Ftp::Server::readUploadData);
 118     comm_read(dataConn, uploadBuf + uploadAvailSize, availSpace,
 119               reader);
 120 }
 121
 122 /// react to the freshly parsed request
 123 void
 124 Ftp::Server::doProcessRequest()
 125 {
 126     // zero pipelinePrefetchMax() ensures that there is only parsed request
 127     Must(pipeline.count() == 1);
 128     ClientSocketContext::Pointer context = pipeline.front();
 129     Must(context != nullptr);
 130
 131     ClientHttpRequest *const http = context->http;
 132     assert(http != NULL);
 133
 134     HttpRequest *const request = http->request;
 135     Must(http->storeEntry() || request);
 136     const bool mayForward = !http->storeEntry() && handleRequest(request);
 137
 138     if (http->storeEntry() != NULL) {
 139         debugs(33, 4, "got an immediate response");
 140         clientSetKeepaliveFlag(http);
 141         context->pullData();
 142     } else if (mayForward) {
 143         debugs(33, 4, "forwarding request to server side");
 144         assert(http->storeEntry() == NULL);
 145         clientProcessRequest(this, Http1::RequestParserPointer(), context.getRaw());
 146     } else {
 147         debugs(33, 4, "will resume processing later");
 148     }
 149 }
 150
 151 void
 152 Ftp::Server::processParsedRequest(ClientSocketContext *)
 153 {
 154     Must(pipeline.count() == 1);
 155
 156     // Process FTP request asynchronously to make sure FTP
 157     // data connection accept callback is fired first.
 158     CallJobHere(33, 4, CbcPointer<Server>(this),
 159                 Ftp::Server, doProcessRequest);
 160 }
 161
 162 /// imports more upload data from the data connection
 163 void
 164 Ftp::Server::readUploadData(const CommIoCbParams &io)
 165 {
 166     debugs(33, 5, io.conn << " size " << io.size);
 167     Must(reader != NULL);
 168     reader = NULL;
 169
 170     assert(Comm::IsConnOpen(dataConn));
 171     assert(io.conn->fd == dataConn->fd);
 172
 173     if (io.flag == Comm::OK && bodyPipe != NULL) {
 174         if (io.size > 0) {
 175             statCounter.client_http.kbytes_in += io.size;
 176
 177             char *const current_buf = uploadBuf + uploadAvailSize;
 178             if (io.buf != current_buf)
 179                 memmove(current_buf, io.buf, io.size);
 180             uploadAvailSize += io.size;
 181             shovelUploadData();
 182         } else if (io.size == 0) {
 183             debugs(33, 5, io.conn << " closed");
 184             closeDataConnection();
 185             if (uploadAvailSize <= 0)
 186                 finishDechunkingRequest(true);
 187         }
 188     } else { // not Comm::Flags::OK or unexpected read
 189         debugs(33, 5, io.conn << " closed");
 190         closeDataConnection();
 191         finishDechunkingRequest(false);
 192     }
 193
 194 }
 195
 196 /// shovel upload data from the internal buffer to the body pipe if possible
 197 void
 198 Ftp::Server::shovelUploadData()
 199 {
 200     assert(bodyPipe != NULL);
 201
 202     debugs(33, 5, "handling FTP request data for " << clientConnection);
 203     const size_t putSize = bodyPipe->putMoreData(uploadBuf,
 204                            uploadAvailSize);
 205     if (putSize > 0) {
 206         uploadAvailSize -= putSize;
 207         if (uploadAvailSize > 0)
 208             memmove(uploadBuf, uploadBuf + putSize, uploadAvailSize);
 209     }
 210
 211     if (Comm::IsConnOpen(dataConn))
 212         maybeReadUploadData();
 213     else if (uploadAvailSize <= 0)
 214         finishDechunkingRequest(true);
 215 }
 216
 217 void
 218 Ftp::Server::noteMoreBodySpaceAvailable(BodyPipe::Pointer)
 219 {
 220     shovelUploadData();
 221 }
 222
 223 void
 224 Ftp::Server::noteBodyConsumerAborted(BodyPipe::Pointer ptr)
 225 {
 226     ConnStateData::noteBodyConsumerAborted(ptr);
 227     closeDataConnection();
 228 }
 229
 230 /// accept a new FTP control connection and hand it to a dedicated Server
 231 void
 232 Ftp::Server::AcceptCtrlConnection(const CommAcceptCbParams &params)
 233 {
 234     MasterXaction::Pointer xact = params.xaction;
 235     AnyP::PortCfgPointer s = xact->squidPort;
 236
 237     // NP: it is possible the port was reconfigured when the call or accept() was queued.
 238
 239     if (params.flag != Comm::OK) {
 240         // Its possible the call was still queued when the client disconnected
 241         debugs(33, 2, s->listenConn << ": FTP accept failure: " << xstrerr(params.xerrno));
 242         return;
 243     }
 244
 245     debugs(33, 4, params.conn << ": accepted");
 246     fd_note(params.conn->fd, "client ftp connect");
 247
 248     if (s->tcp_keepalive.enabled)
 249         commSetTcpKeepalive(params.conn->fd, s->tcp_keepalive.idle, s->tcp_keepalive.interval, s->tcp_keepalive.timeout);
 250
 251     ++incoming_sockets_accepted;
 252
 253     AsyncJob::Start(new Server(xact));
 254 }
 255
 256 void
 257 Ftp::StartListening()
 258 {
 259     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 260         if (MAXTCPLISTENPORTS == NHttpSockets) {
 261             debugs(1, DBG_IMPORTANT, "Ignoring ftp_port lines exceeding the" <<
 262                    " limit of " << MAXTCPLISTENPORTS << " ports.");
 263             break;
 264         }
 265
 266         // direct new connections accepted by listenConn to Accept()
 267         typedef CommCbFunPtrCallT<CommAcceptCbPtrFun> AcceptCall;
 268         RefCount<AcceptCall> subCall = commCbCall(5, 5, "Ftp::Server::AcceptCtrlConnection",
 269                                        CommAcceptCbPtrFun(Ftp::Server::AcceptCtrlConnection,
 270                                                CommAcceptCbParams(NULL)));
 271         clientStartListeningOn(s, subCall, Ipc::fdnFtpSocket);
 272     }
 273 }
 274
 275 void
 276 Ftp::StopListening()
 277 {
 278     for (AnyP::PortCfgPointer s = FtpPortList; s != NULL; s = s->next) {
 279         if (s->listenConn != NULL) {
 280             debugs(1, DBG_IMPORTANT, "Closing FTP port " << s->listenConn->local);
 281             s->listenConn->close();
 282             s->listenConn = NULL;
 283         }
 284     }
 285 }
 286
 287 void
 288 Ftp::Server::notePeerConnection(Comm::ConnectionPointer conn)
 289 {
 290     // find request
 291     ClientSocketContext::Pointer context = pipeline.front();
 292     Must(context != nullptr);
 293     ClientHttpRequest *const http = context->http;
 294     Must(http != NULL);
 295     HttpRequest *const request = http->request;
 296     Must(request != NULL);
 297
 298     // this is not an idle connection, so we do not want I/O monitoring
 299     const bool monitor = false;
 300
 301     // make FTP peer connection exclusive to our request
 302     pinConnection(conn, request, conn->getPeer(), false, monitor);
 303 }
 304
 305 void
 306 Ftp::Server::clientPinnedConnectionClosed(const CommCloseCbParams &io)
 307 {
 308     ConnStateData::clientPinnedConnectionClosed(io);
 309
 310     // if the server control connection is gone, reset state to login again
 311     resetLogin("control connection closure");
 312
 313     // XXX: Reseting is not enough. FtpRelay::sendCommand() will not re-login
 314     // because FtpRelay::serverState() is not going to be fssConnected.
 315 }
 316
 317 /// clear client and server login-related state after the old login is gone
 318 void
 319 Ftp::Server::resetLogin(const char *reason)
 320 {
 321     debugs(33, 5, "will need to re-login due to " << reason);
 322     master->clientReadGreeting = false;
 323     changeState(fssBegin, reason);
 324 }
 325
 326 /// computes uri member from host and, if tracked, working dir with file name
 327 void
 328 Ftp::Server::calcUri(const SBuf *file)
 329 {
 330     uri = "ftp://";
 331     uri.append(host);
 332     if (port->ftp_track_dirs && master->workingDir.length()) {
 333         if (master->workingDir[0] != '/')
 334             uri.append("/", 1);
 335         uri.append(master->workingDir);
 336     }
 337
 338     if (uri[uri.length() - 1] != '/')
 339         uri.append("/", 1);
 340
 341     if (port->ftp_track_dirs && file) {
 342         static const CharacterSet Slash("/", "/");
 343         Parser::Tokenizer tok(*file);
 344         tok.skipAll(Slash);
 345         uri.append(tok.remaining());
 346     }
 347 }
 348
 349 /// Starts waiting for a data connection. Returns listening port.
 350 /// On errors, responds with an error and returns zero.
 351 unsigned int
 352 Ftp::Server::listenForDataConnection()
 353 {
 354     closeDataConnection();
 355
 356     Comm::ConnectionPointer conn = new Comm::Connection;
 357     conn->flags = COMM_NONBLOCKING;
 358     conn->local = transparent() ? port->s : clientConnection->local;
 359     conn->local.port(0);
 360     const char *const note = uri.c_str();
 361     comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn, note);
 362     if (!Comm::IsConnOpen(conn)) {
 363         debugs(5, DBG_CRITICAL, "comm_open_listener failed for FTP data: " <<
 364                conn->local << " error: " << errno);
 365         writeCustomReply(451, "Internal error");
 366         return 0;
 367     }
 368
 369     typedef CommCbMemFunT<Server, CommAcceptCbParams> AcceptDialer;
 370     typedef AsyncCallT<AcceptDialer> AcceptCall;
 371     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(5, 5, AcceptDialer, this, Ftp::Server::acceptDataConnection));
 372     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 373     listener = call.getRaw();
 374     dataListenConn = conn;
 375     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 376
 377     const unsigned int listeningPort = comm_local_port(conn->fd);
 378     conn->local.port(listeningPort);
 379     return listeningPort;
 380 }
 381
 382 void
 383 Ftp::Server::acceptDataConnection(const CommAcceptCbParams &params)
 384 {
 385     if (params.flag != Comm::OK) {
 386         // Its possible the call was still queued when the client disconnected
 387         debugs(33, 2, dataListenConn << ": accept "
 388                "failure: " << xstrerr(params.xerrno));
 389         return;
 390     }
 391
 392     debugs(33, 4, "accepted " << params.conn);
 393     fd_note(params.conn->fd, "passive client ftp data");
 394     ++incoming_sockets_accepted;
 395
 396     if (!clientConnection) {
 397         debugs(33, 5, "late data connection?");
 398         closeDataConnection(); // in case we are still listening
 399         params.conn->close();
 400     } else if (params.conn->remote != clientConnection->remote) {
 401         debugs(33, 2, "rogue data conn? ctrl: " << clientConnection->remote);
 402         params.conn->close();
 403         // Some FTP servers close control connection here, but it may make
 404         // things worse from DoS p.o.v. and no better from data stealing p.o.v.
 405     } else {
 406         closeDataConnection();
 407         dataConn = params.conn;
 408         uploadAvailSize = 0;
 409         debugs(33, 7, "ready for data");
 410         if (onDataAcceptCall != NULL) {
 411             AsyncCall::Pointer call = onDataAcceptCall;
 412             onDataAcceptCall = NULL;
 413             // If we got an upload request, start reading data from the client.
 414             if (master->serverState == fssHandleUploadRequest)
 415                 maybeReadUploadData();
 416             else
 417                 Must(master->serverState == fssHandleDataRequest);
 418             MemBuf mb;
 419             mb.init();
 420             mb.appendf("150 Data connection opened.\r\n");
 421             Comm::Write(clientConnection, &mb, call);
 422         }
 423     }
 424 }
 425
 426 void
 427 Ftp::Server::closeDataConnection()
 428 {
 429     if (listener != NULL) {
 430         listener->cancel("no longer needed");
 431         listener = NULL;
 432     }
 433
 434     if (Comm::IsConnOpen(dataListenConn)) {
 435         debugs(33, 5, "FTP closing client data listen socket: " <<
 436                *dataListenConn);
 437         dataListenConn->close();
 438     }
 439     dataListenConn = NULL;
 440
 441     if (reader != NULL) {
 442         // Comm::ReadCancel can deal with negative FDs
 443         Comm::ReadCancel(dataConn->fd, reader);
 444         reader = NULL;
 445     }
 446
 447     if (Comm::IsConnOpen(dataConn)) {
 448         debugs(33, 5, "FTP closing client data connection: " <<
 449                *dataConn);
 450         dataConn->close();
 451     }
 452     dataConn = NULL;
 453 }
 454
 455 /// Writes FTP [error] response before we fully parsed the FTP request and
 456 /// created the corresponding HTTP request wrapper for that FTP request.
 457 void
 458 Ftp::Server::writeEarlyReply(const int code, const char *msg)
 459 {
 460     debugs(33, 7, code << ' ' << msg);
 461     assert(99 < code && code < 1000);
 462
 463     MemBuf mb;
 464     mb.init();
 465     mb.appendf("%i %s\r\n", code, msg);
 466
 467     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 468     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
 469     Comm::Write(clientConnection, &mb, call);
 470
 471     flags.readMore = false;
 472
 473     // TODO: Create master transaction. Log it in wroteEarlyReply().
 474 }
 475
 476 void
 477 Ftp::Server::writeReply(MemBuf &mb)
 478 {
 479     debugs(9, 2, "FTP Client " << clientConnection);
 480     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
 481            "\n----------");
 482
 483     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 484     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
 485     Comm::Write(clientConnection, &mb, call);
 486 }
 487
 488 void
 489 Ftp::Server::writeCustomReply(const int code, const char *msg, const HttpReply *reply)
 490 {
 491     debugs(33, 7, code << ' ' << msg);
 492     assert(99 < code && code < 1000);
 493
 494     const bool sendDetails = reply != NULL &&
 495                              reply->header.has(Http::HdrType::FTP_STATUS) && reply->header.has(Http::HdrType::FTP_REASON);
 496
 497     MemBuf mb;
 498     mb.init();
 499     if (sendDetails) {
 500         mb.appendf("%i-%s\r\n", code, msg);
 501         mb.appendf(" Server reply:\r\n");
 502         Ftp::PrintReply(mb, reply, " ");
 503         mb.appendf("%i \r\n", code);
 504     } else
 505         mb.appendf("%i %s\r\n", code, msg);
 506
 507     writeReply(mb);
 508 }
 509
 510 void
 511 Ftp::Server::changeState(const ServerState newState, const char *reason)
 512 {
 513     if (master->serverState == newState) {
 514         debugs(33, 3, "client state unchanged at " << master->serverState <<
 515                " because " << reason);
 516         master->serverState = newState;
 517     } else {
 518         debugs(33, 3, "client state was " << master->serverState <<
 519                ", now " << newState << " because " << reason);
 520         master->serverState = newState;
 521     }
 522 }
 523
 524 /// whether the given FTP command has a pathname parameter
 525 static bool
 526 Ftp::CommandHasPathParameter(const SBuf &cmd)
 527 {
 528     static std::set<SBuf> PathedCommands;
 529     if (!PathedCommands.size()) {
 530         PathedCommands.insert(cmdMlst());
 531         PathedCommands.insert(cmdMlsd());
 532         PathedCommands.insert(cmdStat());
 533         PathedCommands.insert(cmdNlst());
 534         PathedCommands.insert(cmdList());
 535         PathedCommands.insert(cmdMkd());
 536         PathedCommands.insert(cmdRmd());
 537         PathedCommands.insert(cmdDele());
 538         PathedCommands.insert(cmdRnto());
 539         PathedCommands.insert(cmdRnfr());
 540         PathedCommands.insert(cmdAppe());
 541         PathedCommands.insert(cmdStor());
 542         PathedCommands.insert(cmdRetr());
 543         PathedCommands.insert(cmdSmnt());
 544         PathedCommands.insert(cmdCwd());
 545     }
 546
 547     return PathedCommands.find(cmd) != PathedCommands.end();
 548 }
 549
 550 /// creates a context filled with an error message for a given early error
 551 ClientSocketContext *
 552 Ftp::Server::earlyError(const EarlyErrorKind eek)
 553 {
 554     /* Default values, to be updated by the switch statement below */
 555     int scode = 421;
 556     const char *reason = "Internal error";
 557     const char *errUri = "error:ftp-internal-early-error";
 558
 559     switch (eek) {
 560     case EarlyErrorKind::HugeRequest:
 561         scode = 421;
 562         reason = "Huge request";
 563         errUri = "error:ftp-huge-request";
 564         break;
 565
 566     case EarlyErrorKind::MissingLogin:
 567         scode = 530;
 568         reason = "Must login first";
 569         errUri = "error:ftp-must-login-first";
 570         break;
 571
 572     case EarlyErrorKind::MissingUsername:
 573         scode = 501;
 574         reason = "Missing username";
 575         errUri = "error:ftp-missing-username";
 576         break;
 577
 578     case EarlyErrorKind::MissingHost:
 579         scode = 501;
 580         reason = "Missing host";
 581         errUri = "error:ftp-missing-host";
 582         break;
 583
 584     case EarlyErrorKind::UnsupportedCommand:
 585         scode = 502;
 586         reason = "Unknown or unsupported command";
 587         errUri = "error:ftp-unsupported-command";
 588         break;
 589
 590     case EarlyErrorKind::InvalidUri:
 591         scode = 501;
 592         reason = "Invalid URI";
 593         errUri = "error:ftp-invalid-uri";
 594         break;
 595
 596     case EarlyErrorKind::MalformedCommand:
 597         scode = 421;
 598         reason = "Malformed command";
 599         errUri = "error:ftp-malformed-command";
 600         break;
 601
 602         // no default so that a compiler can check that we have covered all cases
 603     }
 604
 605     ClientSocketContext *context = abortRequestParsing(errUri);
 606     clientStreamNode *node = context->getClientReplyContext();
 607     Must(node);
 608     clientReplyContext *repContext = dynamic_cast<clientReplyContext *>(node->data.getRaw());
 609     Must(repContext);
 610
 611     // We cannot relay FTP scode/reason via HTTP-specific ErrorState.
 612     // TODO: When/if ErrorState can handle native FTP errors, use it instead.
 613     HttpReply *reply = Ftp::HttpReplyWrapper(scode, reason, Http::scBadRequest, -1);
 614     repContext->setReplyToReply(reply);
 615     return context;
 616 }
 617
 618 /// Parses a single FTP request on the control connection.
 619 /// Returns a new ClientSocketContext on valid requests and all errors.
 620 /// Returns NULL on incomplete requests that may still succeed given more data.
 621 ClientSocketContext *
 622 Ftp::Server::parseOneRequest()
 623 {
 624     flags.readMore = false; // common for all but one case below
 625
 626     // OWS <command> [ RWS <parameter> ] OWS LF
 627
 628     // InlineSpaceChars are isspace(3) or RFC 959 Section 3.1.1.5.2, except
 629     // for the LF character that we must exclude here (but see FullWhiteSpace).
 630     static const char * const InlineSpaceChars = " \f\r\t\v";
 631     static const CharacterSet InlineSpace = CharacterSet("Ftp::Inline", InlineSpaceChars);
 632     static const CharacterSet FullWhiteSpace = (InlineSpace + CharacterSet::LF).rename("Ftp::FWS");
 633     static const CharacterSet CommandChars = FullWhiteSpace.complement("Ftp::Command");
 634     static const CharacterSet TailChars = CharacterSet::LF.complement("Ftp::Tail");
 635
 636     // This set is used to ignore empty commands without allowing an attacker
 637     // to keep us endlessly busy by feeding us whitespace or empty commands.
 638     static const CharacterSet &LeadingSpace = FullWhiteSpace;
 639
 640     SBuf cmd;
 641     SBuf params;
 642
 643     Parser::Tokenizer tok(inBuf);
 644
 645     (void)tok.skipAll(LeadingSpace); // leading OWS and empty commands
 646     const bool parsed = tok.prefix(cmd, CommandChars); // required command
 647
 648     // note that the condition below will eat either RWS or trailing OWS
 649     if (parsed && tok.skipAll(InlineSpace) && tok.prefix(params, TailChars)) {
 650         // now params may include trailing OWS
 651         // TODO: Support right-trimming using CharacterSet in Tokenizer instead
 652         static const SBuf bufWhiteSpace(InlineSpaceChars);
 653         params.trim(bufWhiteSpace, false, true);
 654     }
 655
 656     // Why limit command line and parameters size? Did not we just parse them?
 657     // XXX: Our good old String cannot handle very long strings.
 658     const SBuf::size_type tokenMax = min(
 659                                          static_cast<SBuf::size_type>(32*1024), // conservative
 660                                          static_cast<SBuf::size_type>(Config.maxRequestHeaderSize));
 661     if (cmd.length() > tokenMax || params.length() > tokenMax) {
 662         changeState(fssError, "huge req token");
 663         quitAfterError(NULL);
 664         return earlyError(EarlyErrorKind::HugeRequest);
 665     }
 666
 667     // technically, we may skip multiple NLs below, but that is OK
 668     if (!parsed || !tok.skipAll(CharacterSet::LF)) { // did not find terminating LF yet
 669         // we need more data, but can we buffer more?
 670         if (inBuf.length() >= Config.maxRequestHeaderSize) {
 671             changeState(fssError, "huge req");
 672             quitAfterError(NULL);
 673             return earlyError(EarlyErrorKind::HugeRequest);
 674         } else {
 675             flags.readMore = true;
 676             debugs(33, 5, "Waiting for more, up to " <<
 677                    (Config.maxRequestHeaderSize - inBuf.length()));
 678             return NULL;
 679         }
 680     }
 681
 682     Must(parsed && cmd.length());
 683     consumeInput(tok.parsedSize()); // TODO: Would delaying optimize copying?
 684
 685     debugs(33, 2, ">>ftp " << cmd << (params.isEmpty() ? "" : " ") << params);
 686
 687     cmd.toUpper(); // this should speed up and simplify future comparisons
 688
 689     // interception cases do not need USER to calculate the uri
 690     if (!transparent()) {
 691         if (!master->clientReadGreeting) {
 692             // the first command must be USER
 693             if (!pinning.pinned && cmd != cmdUser())
 694                 return earlyError(EarlyErrorKind::MissingLogin);
 695         }
 696
 697         // process USER request now because it sets FTP peer host name
 698         if (cmd == cmdUser()) {
 699             if (ClientSocketContext *errCtx = handleUserRequest(cmd, params))
 700                 return errCtx;
 701         }
 702     }
 703
 704     if (!Ftp::SupportedCommand(cmd))
 705         return earlyError(EarlyErrorKind::UnsupportedCommand);
 706
 707     const HttpRequestMethod method =
 708         cmd == cmdAppe() || cmd == cmdStor() || cmd == cmdStou() ?
 709         Http::METHOD_PUT : Http::METHOD_GET;
 710
 711     const SBuf *path = (params.length() && CommandHasPathParameter(cmd)) ?
 712                        &params : NULL;
 713     calcUri(path);
 714     char *newUri = xstrdup(uri.c_str());
 715     HttpRequest *const request = HttpRequest::CreateFromUrlAndMethod(newUri, method);
 716     if (!request) {
 717         debugs(33, 5, "Invalid FTP URL: " << uri);
 718         uri.clear();
 719         safe_free(newUri);
 720         return earlyError(EarlyErrorKind::InvalidUri);
 721     }
 722
 723     request->flags.ftpNative = true;
 724     request->http_ver = Http::ProtocolVersion(Ftp::ProtocolVersion().major, Ftp::ProtocolVersion().minor);
 725
 726     // Our fake Request-URIs are not distinctive enough for caching to work
 727     request->flags.cachable = false; // XXX: reset later by maybeCacheable()
 728     request->flags.noCache = true;
 729
 730     request->header.putStr(Http::HdrType::FTP_COMMAND, cmd.c_str());
 731     request->header.putStr(Http::HdrType::FTP_ARGUMENTS, params.c_str()); // may be ""
 732     if (method == Http::METHOD_PUT) {
 733         request->header.putStr(Http::HdrType::EXPECT, "100-continue");
 734         request->header.putStr(Http::HdrType::TRANSFER_ENCODING, "chunked");
 735     }
 736
 737     ClientHttpRequest *const http = new ClientHttpRequest(this);
 738     http->request = request;
 739     HTTPMSGLOCK(http->request);
 740     http->req_sz = tok.parsedSize();
 741     http->uri = newUri;
 742
 743     ClientSocketContext *const result =
 744         new ClientSocketContext(clientConnection, http);
 745
 746     StoreIOBuffer tempBuffer;
 747     tempBuffer.data = result->reqbuf;
 748     tempBuffer.length = HTTP_REQBUF_SZ;
 749
 750     ClientStreamData newServer = new clientReplyContext(http);
 751     ClientStreamData newClient = result;
 752     clientStreamInit(&http->client_stream, clientGetMoreData, clientReplyDetach,
 753                      clientReplyStatus, newServer, clientSocketRecipient,
 754                      clientSocketDetach, newClient, tempBuffer);
 755
 756     result->flags.parsed_ok = 1;
 757     return result;
 758 }
 759
 760 void
 761 Ftp::Server::handleReply(HttpReply *reply, StoreIOBuffer data)
 762 {
 763     // the caller guarantees that we are dealing with the current context only
 764     ClientSocketContext::Pointer context = pipeline.front();
 765     assert(context != nullptr);
 766
 767     if (context->http && context->http->al != NULL &&
 768             !context->http->al->reply && reply) {
 769         context->http->al->reply = reply;
 770         HTTPMSGLOCK(context->http->al->reply);
 771     }
 772
 773     static ReplyHandler handlers[] = {
 774         NULL, // fssBegin
 775         NULL, // fssConnected
 776         &Ftp::Server::handleFeatReply, // fssHandleFeat
 777         &Ftp::Server::handlePasvReply, // fssHandlePasv
 778         &Ftp::Server::handlePortReply, // fssHandlePort
 779         &Ftp::Server::handleDataReply, // fssHandleDataRequest
 780         &Ftp::Server::handleUploadReply, // fssHandleUploadRequest
 781         &Ftp::Server::handleEprtReply,// fssHandleEprt
 782         &Ftp::Server::handleEpsvReply,// fssHandleEpsv
 783         NULL, // fssHandleCwd
 784         NULL, // fssHandlePass
 785         NULL, // fssHandleCdup
 786         &Ftp::Server::handleErrorReply // fssError
 787     };
 788     try {
 789         const Server &server = dynamic_cast<const Ftp::Server&>(*context->getConn());
 790         if (const ReplyHandler handler = handlers[server.master->serverState])
 791             (this->*handler)(reply, data);
 792         else
 793             writeForwardedReply(reply);
 794     } catch (const std::exception &e) {
 795         callException(e);
 796         throw TexcHere(e.what());
 797     }
 798 }
 799
 800 void
 801 Ftp::Server::handleFeatReply(const HttpReply *reply, StoreIOBuffer)
 802 {
 803     if (pipeline.front()->http->request->errType != ERR_NONE) {
 804         writeCustomReply(502, "Server does not support FEAT", reply);
 805         return;
 806     }
 807
 808     Must(reply);
 809     HttpReply::Pointer featReply = Ftp::HttpReplyWrapper(211, "End", Http::scNoContent, 0);
 810     HttpHeader const &serverReplyHeader = reply->header;
 811
 812     HttpHeaderPos pos = HttpHeaderInitPos;
 813     bool hasEPRT = false;
 814     bool hasEPSV = false;
 815     int prependSpaces = 1;
 816
 817     featReply->header.putStr(Http::HdrType::FTP_PRE, "\"211-Features:\"");
 818     const int scode = serverReplyHeader.getInt(Http::HdrType::FTP_STATUS);
 819     if (scode == 211) {
 820         while (const HttpHeaderEntry *e = serverReplyHeader.getEntry(&pos)) {
 821             if (e->id == Http::HdrType::FTP_PRE) {
 822                 // assume RFC 2389 FEAT response format, quoted by Squid:
 823                 // <"> SP NAME [SP PARAMS] <">
 824                 // but accommodate MS servers sending four SPs before NAME
 825
 826                 // command name ends with (SP parameter) or quote
 827                 static const CharacterSet AfterFeatNameChars("AfterFeatName", " \"");
 828                 static const CharacterSet FeatNameChars = AfterFeatNameChars.complement("FeatName");
 829
 830                 Parser::Tokenizer tok(SBuf(e->value.termedBuf()));
 831                 if (!tok.skip('"') || !tok.skip(' '))
 832                     continue;
 833
 834                 // optional spaces; remember their number to accomodate MS servers
 835                 prependSpaces = 1 + tok.skipAll(CharacterSet::SP);
 836
 837                 SBuf cmd;
 838                 if (!tok.prefix(cmd, FeatNameChars))
 839                     continue;
 840                 cmd.toUpper();
 841
 842                 if (Ftp::SupportedCommand(cmd)) {
 843                     featReply->header.addEntry(e->clone());
 844                 }
 845
 846                 if (cmd == cmdEprt())
 847                     hasEPRT = true;
 848                 else if (cmd == cmdEpsv())
 849                     hasEPSV = true;
 850             }
 851         }
 852     } // else we got a FEAT error and will only report Squid-supported features
 853
 854     char buf[256];
 855     if (!hasEPRT) {
 856         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPRT");
 857         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 858     }
 859     if (!hasEPSV) {
 860         snprintf(buf, sizeof(buf), "\"%*s\"", prependSpaces + 4, "EPSV");
 861         featReply->header.putStr(Http::HdrType::FTP_PRE, buf);
 862     }
 863
 864     featReply->header.refreshMask();
 865
 866     writeForwardedReply(featReply.getRaw());
 867 }
 868
 869 void
 870 Ftp::Server::handlePasvReply(const HttpReply *reply, StoreIOBuffer)
 871 {
 872     const ClientSocketContext::Pointer context(pipeline.front());
 873     assert(context != nullptr);
 874
 875     if (context->http->request->errType != ERR_NONE) {
 876         writeCustomReply(502, "Server does not support PASV", reply);
 877         return;
 878     }
 879
 880     const unsigned short localPort = listenForDataConnection();
 881     if (!localPort)
 882         return;
 883
 884     char addr[MAX_IPSTRLEN];
 885     // remote server in interception setups and local address otherwise
 886     const Ip::Address &server = transparent() ?
 887                                 clientConnection->local : dataListenConn->local;
 888     server.toStr(addr, MAX_IPSTRLEN, AF_INET);
 889     addr[MAX_IPSTRLEN - 1] = '\0';
 890     for (char *c = addr; *c != '\0'; ++c) {
 891         if (*c == '.')
 892             *c = ',';
 893     }
 894
 895     // In interception setups, we combine remote server address with a
 896     // local port number and hope that traffic will be redirected to us.
 897     // Do not use "227 =a,b,c,d,p1,p2" format or omit parens: some nf_ct_ftp
 898     // versions block responses that use those alternative syntax rules!
 899     MemBuf mb;
 900     mb.init();
 901     mb.appendf("227 Entering Passive Mode (%s,%i,%i).\r\n",
 902                addr,
 903                static_cast<int>(localPort / 256),
 904                static_cast<int>(localPort % 256));
 905     debugs(9, 3, Raw("writing", mb.buf, mb.size));
 906     writeReply(mb);
 907 }
 908
 909 void
 910 Ftp::Server::handlePortReply(const HttpReply *reply, StoreIOBuffer)
 911 {
 912     if (pipeline.front()->http->request->errType != ERR_NONE) {
 913         writeCustomReply(502, "Server does not support PASV (converted from PORT)", reply);
 914         return;
 915     }
 916
 917     writeCustomReply(200, "PORT successfully converted to PASV.");
 918
 919     // and wait for RETR
 920 }
 921
 922 void
 923 Ftp::Server::handleErrorReply(const HttpReply *reply, StoreIOBuffer)
 924 {
 925     if (!pinning.pinned) // we failed to connect to server
 926         uri.clear();
 927     // 421: we will close due to fssError
 928     writeErrorReply(reply, 421);
 929 }
 930
 931 void
 932 Ftp::Server::handleDataReply(const HttpReply *reply, StoreIOBuffer data)
 933 {
 934     if (reply != NULL && reply->sline.status() != Http::scOkay) {
 935         writeForwardedReply(reply);
 936         if (Comm::IsConnOpen(dataConn)) {
 937             debugs(33, 3, "closing " << dataConn << " on KO reply");
 938             closeDataConnection();
 939         }
 940         return;
 941     }
 942
 943     if (!dataConn) {
 944         // We got STREAM_COMPLETE (or error) and closed the client data conn.
 945         debugs(33, 3, "ignoring FTP srv data response after clt data closure");
 946         return;
 947     }
 948
 949     if (!checkDataConnPost()) {
 950         writeCustomReply(425, "Data connection is not established.");
 951         closeDataConnection();
 952         return;
 953     }
 954
 955     debugs(33, 7, data.length);
 956
 957     if (data.length <= 0) {
 958         replyDataWritingCheckpoint(); // skip the actual write call
 959         return;
 960     }
 961
 962     MemBuf mb;
 963     mb.init(data.length + 1, data.length + 1);
 964     mb.append(data.data, data.length);
 965
 966     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
 967     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReplyData);
 968     Comm::Write(dataConn, &mb, call);
 969
 970     pipeline.front()->noteSentBodyBytes(data.length);
 971 }
 972
 973 /// called when we are done writing a chunk of the response data
 974 void
 975 Ftp::Server::wroteReplyData(const CommIoCbParams &io)
 976 {
 977     if (io.flag == Comm::ERR_CLOSING)
 978         return;
 979
 980     if (io.flag != Comm::OK) {
 981         debugs(33, 3, "FTP reply data writing failed: " << xstrerr(io.xerrno));
 982         closeDataConnection();
 983         writeCustomReply(426, "Data connection error; transfer aborted");
 984         return;
 985     }
 986
 987     assert(pipeline.front()->http);
 988     pipeline.front()->http->out.size += io.size;
 989     replyDataWritingCheckpoint();
 990 }
 991
 992 /// ClientStream checks after (actual or skipped) reply data writing
 993 void
 994 Ftp::Server::replyDataWritingCheckpoint()
 995 {
 996     switch (pipeline.front()->socketState()) {
 997     case STREAM_NONE:
 998         debugs(33, 3, "Keep going");
 999         pipeline.front()->pullData();
1000         return;
1001     case STREAM_COMPLETE:
1002         debugs(33, 3, "FTP reply data transfer successfully complete");
1003         writeCustomReply(226, "Transfer complete");
1004         break;
1005     case STREAM_UNPLANNED_COMPLETE:
1006         debugs(33, 3, "FTP reply data transfer failed: STREAM_UNPLANNED_COMPLETE");
1007         writeCustomReply(451, "Server error; transfer aborted");
1008         break;
1009     case STREAM_FAILED:
1010         debugs(33, 3, "FTP reply data transfer failed: STREAM_FAILED");
1011         writeCustomReply(451, "Server error; transfer aborted");
1012         break;
1013     default:
1014         fatal("unreachable code");
1015     }
1016
1017     closeDataConnection();
1018 }
1019
1020 void
1021 Ftp::Server::handleUploadReply(const HttpReply *reply, StoreIOBuffer)
1022 {
1023     writeForwardedReply(reply);
1024     // note that the client data connection may already be closed by now
1025 }
1026
1027 void
1028 Ftp::Server::writeForwardedReply(const HttpReply *reply)
1029 {
1030     Must(reply);
1031
1032     const HttpHeader &header = reply->header;
1033     // adaptation and forwarding errors lack Http::HdrType::FTP_STATUS
1034     if (!header.has(Http::HdrType::FTP_STATUS)) {
1035         writeForwardedForeign(reply); // will get to Ftp::Server::wroteReply
1036         return;
1037     }
1038
1039     typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1040     AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteReply);
1041     writeForwardedReplyAndCall(reply, call);
1042 }
1043
1044 void
1045 Ftp::Server::handleEprtReply(const HttpReply *reply, StoreIOBuffer)
1046 {
1047     if (pipeline.front()->http->request->errType != ERR_NONE) {
1048         writeCustomReply(502, "Server does not support PASV (converted from EPRT)", reply);
1049         return;
1050     }
1051
1052     writeCustomReply(200, "EPRT successfully converted to PASV.");
1053
1054     // and wait for RETR
1055 }
1056
1057 void
1058 Ftp::Server::handleEpsvReply(const HttpReply *reply, StoreIOBuffer)
1059 {
1060     if (pipeline.front()->http->request->errType != ERR_NONE) {
1061         writeCustomReply(502, "Cannot connect to server", reply);
1062         return;
1063     }
1064
1065     const unsigned short localPort = listenForDataConnection();
1066     if (!localPort)
1067         return;
1068
1069     // In interception setups, we use a local port number and hope that data
1070     // traffic will be redirected to us.
1071     MemBuf mb;
1072     mb.init();
1073     mb.appendf("229 Entering Extended Passive Mode (|||%u|)\r\n", localPort);
1074
1075     debugs(9, 3, Raw("writing", mb.buf, mb.size));
1076     writeReply(mb);
1077 }
1078
1079 /// writes FTP error response with given status and reply-derived error details
1080 void
1081 Ftp::Server::writeErrorReply(const HttpReply *reply, const int scode)
1082 {
1083     const HttpRequest *request = pipeline.front()->http->request;
1084     assert(request);
1085
1086     MemBuf mb;
1087     mb.init();
1088
1089     if (request->errType != ERR_NONE)
1090         mb.appendf("%i-%s\r\n", scode, errorPageName(request->errType));
1091
1092     if (request->errDetail > 0) {
1093         // XXX: > 0 may not always mean that this is an errno
1094         mb.appendf("%i-Error: (%d) %s\r\n", scode,
1095                    request->errDetail,
1096                    strerror(request->errDetail));
1097     }
1098
1099 #if USE_ADAPTATION
1100     // XXX: Remove hard coded names. Use an error page template instead.
1101     const Adaptation::History::Pointer ah = request->adaptHistory();
1102     if (ah != NULL) { // XXX: add adapt::<all_h but use lastMeta here
1103         const String info = ah->allMeta.getByName("X-Response-Info");
1104         const String desc = ah->allMeta.getByName("X-Response-Desc");
1105         if (info.size())
1106             mb.appendf("%i-Information: %s\r\n", scode, info.termedBuf());
1107         if (desc.size())
1108             mb.appendf("%i-Description: %s\r\n", scode, desc.termedBuf());
1109     }
1110 #endif
1111
1112     Must(reply);
1113     const char *reason = reply->header.has(Http::HdrType::FTP_REASON) ?
1114                          reply->header.getStr(Http::HdrType::FTP_REASON):
1115                          reply->sline.reason();
1116
1117     mb.appendf("%i %s\r\n", scode, reason); // error terminating line
1118
1119     // TODO: errorpage.cc should detect FTP client and use
1120     // configurable FTP-friendly error templates which we should
1121     // write to the client "as is" instead of hiding most of the info
1122
1123     writeReply(mb);
1124 }
1125
1126 /// writes FTP response based on HTTP reply that is not an FTP-response wrapper
1127 /// for example, internally-generated Squid "errorpages" end up here (for now)
1128 void
1129 Ftp::Server::writeForwardedForeign(const HttpReply *reply)
1130 {
1131     changeState(fssConnected, "foreign reply");
1132     closeDataConnection();
1133     // 451: We intend to keep the control connection open.
1134     writeErrorReply(reply, 451);
1135 }
1136
1137 void
1138 Ftp::Server::writeControlMsgAndCall(ClientSocketContext *, HttpReply *reply, AsyncCall::Pointer &call)
1139 {
1140     // the caller guarantees that we are dealing with the current context only
1141     // the caller should also make sure reply->header.has(Http::HdrType::FTP_STATUS)
1142     writeForwardedReplyAndCall(reply, call);
1143 }
1144
1145 void
1146 Ftp::Server::writeForwardedReplyAndCall(const HttpReply *reply, AsyncCall::Pointer &call)
1147 {
1148     assert(reply != NULL);
1149     const HttpHeader &header = reply->header;
1150
1151     // without status, the caller must use the writeForwardedForeign() path
1152     Must(header.has(Http::HdrType::FTP_STATUS));
1153     Must(header.has(Http::HdrType::FTP_REASON));
1154     const int scode = header.getInt(Http::HdrType::FTP_STATUS);
1155     debugs(33, 7, "scode: " << scode);
1156
1157     // Status 125 or 150 implies upload or data request, but we still check
1158     // the state in case the server is buggy.
1159     if ((scode == 125 || scode == 150) &&
1160             (master->serverState == fssHandleUploadRequest ||
1161              master->serverState == fssHandleDataRequest)) {
1162         if (checkDataConnPost()) {
1163             // If the data connection is ready, start reading data (here)
1164             // and forward the response to client (further below).
1165             debugs(33, 7, "data connection established, start data transfer");
1166             if (master->serverState == fssHandleUploadRequest)
1167                 maybeReadUploadData();
1168         } else {
1169             // If we are waiting to accept the data connection, keep waiting.
1170             if (Comm::IsConnOpen(dataListenConn)) {
1171                 debugs(33, 7, "wait for the client to establish a data connection");
1172                 onDataAcceptCall = call;
1173                 // TODO: Add connect timeout for passive connections listener?
1174                 // TODO: Remember server response so that we can forward it?
1175             } else {
1176                 // Either the connection was establised and closed after the
1177                 // data was transferred OR we failed to establish an active
1178                 // data connection and already sent the error to the client.
1179                 // In either case, there is nothing more to do.
1180                 debugs(33, 7, "done with data OR active connection failed");
1181             }
1182             return;
1183         }
1184     }
1185
1186     MemBuf mb;
1187     mb.init();
1188     Ftp::PrintReply(mb, reply);
1189
1190     debugs(9, 2, "FTP Client " << clientConnection);
1191     debugs(9, 2, "FTP Client REPLY:\n---------\n" << mb.buf <<
1192            "\n----------");
1193
1194     Comm::Write(clientConnection, &mb, call);
1195 }
1196
1197 static void
1198 Ftp::PrintReply(MemBuf &mb, const HttpReply *reply, const char *const)
1199 {
1200     const HttpHeader &header = reply->header;
1201
1202     HttpHeaderPos pos = HttpHeaderInitPos;
1203     while (const HttpHeaderEntry *e = header.getEntry(&pos)) {
1204         if (e->id == Http::HdrType::FTP_PRE) {
1205             String raw;
1206             if (httpHeaderParseQuotedString(e->value.rawBuf(), e->value.size(), &raw))
1207                 mb.appendf("%s\r\n", raw.termedBuf());
1208         }
1209     }
1210
1211     if (header.has(Http::HdrType::FTP_STATUS)) {
1212         const char *reason = header.getStr(Http::HdrType::FTP_REASON);
1213         mb.appendf("%i %s\r\n", header.getInt(Http::HdrType::FTP_STATUS),
1214                    (reason ? reason : 0));
1215     }
1216 }
1217
1218 void
1219 Ftp::Server::wroteEarlyReply(const CommIoCbParams &io)
1220 {
1221     if (io.flag == Comm::ERR_CLOSING)
1222         return;
1223
1224     if (io.flag != Comm::OK) {
1225         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1226         io.conn->close();
1227         return;
1228     }
1229
1230     ClientSocketContext::Pointer context = pipeline.front();
1231     if (context != nullptr && context->http) {
1232         context->http->out.size += io.size;
1233         context->http->out.headers_sz += io.size;
1234     }
1235
1236     flags.readMore = true;
1237     readSomeData();
1238 }
1239
1240 void
1241 Ftp::Server::wroteReply(const CommIoCbParams &io)
1242 {
1243     if (io.flag == Comm::ERR_CLOSING)
1244         return;
1245
1246     if (io.flag != Comm::OK) {
1247         debugs(33, 3, "FTP reply writing failed: " << xstrerr(io.xerrno));
1248         io.conn->close();
1249         return;
1250     }
1251
1252     ClientSocketContext::Pointer context = pipeline.front();
1253     assert(context->http);
1254     context->http->out.size += io.size;
1255     context->http->out.headers_sz += io.size;
1256
1257     if (master->serverState == fssError) {
1258         debugs(33, 5, "closing on FTP server error");
1259         io.conn->close();
1260         return;
1261     }
1262
1263     const clientStream_status_t socketState = context->socketState();
1264     debugs(33, 5, "FTP client stream state " << socketState);
1265     switch (socketState) {
1266     case STREAM_UNPLANNED_COMPLETE:
1267     case STREAM_FAILED:
1268         io.conn->close();
1269         return;
1270
1271     case STREAM_NONE:
1272     case STREAM_COMPLETE:
1273         flags.readMore = true;
1274         changeState(fssConnected, "Ftp::Server::wroteReply");
1275         if (bodyParser)
1276             finishDechunkingRequest(false);
1277         context->finished();
1278         return;
1279     }
1280 }
1281
1282 bool
1283 Ftp::Server::handleRequest(HttpRequest *request)
1284 {
1285     debugs(33, 9, request);
1286     Must(request);
1287
1288     HttpHeader &header = request->header;
1289     Must(header.has(Http::HdrType::FTP_COMMAND));
1290     String &cmd = header.findEntry(Http::HdrType::FTP_COMMAND)->value;
1291     Must(header.has(Http::HdrType::FTP_ARGUMENTS));
1292     String &params = header.findEntry(Http::HdrType::FTP_ARGUMENTS)->value;
1293
1294     if (do_debug(9, 2)) {
1295         MemBuf mb;
1296         mb.init();
1297         request->pack(&mb);
1298
1299         debugs(9, 2, "FTP Client " << clientConnection);
1300         debugs(9, 2, "FTP Client REQUEST:\n---------\n" << mb.buf <<
1301                "\n----------");
1302     }
1303
1304     // TODO: When HttpHeader uses SBuf, change keys to SBuf
1305     typedef std::map<const std::string, RequestHandler> RequestHandlers;
1306     static RequestHandlers handlers;
1307     if (!handlers.size()) {
1308         handlers["LIST"] = &Ftp::Server::handleDataRequest;
1309         handlers["NLST"] = &Ftp::Server::handleDataRequest;
1310         handlers["MLSD"] = &Ftp::Server::handleDataRequest;
1311         handlers["FEAT"] = &Ftp::Server::handleFeatRequest;
1312         handlers["PASV"] = &Ftp::Server::handlePasvRequest;
1313         handlers["PORT"] = &Ftp::Server::handlePortRequest;
1314         handlers["RETR"] = &Ftp::Server::handleDataRequest;
1315         handlers["EPRT"] = &Ftp::Server::handleEprtRequest;
1316         handlers["EPSV"] = &Ftp::Server::handleEpsvRequest;
1317         handlers["CWD"] = &Ftp::Server::handleCwdRequest;
1318         handlers["PASS"] = &Ftp::Server::handlePassRequest;
1319         handlers["CDUP"] = &Ftp::Server::handleCdupRequest;
1320     }
1321
1322     RequestHandler handler = NULL;
1323     if (request->method == Http::METHOD_PUT)
1324         handler = &Ftp::Server::handleUploadRequest;
1325     else {
1326         const RequestHandlers::const_iterator hi = handlers.find(cmd.termedBuf());
1327         if (hi != handlers.end())
1328             handler = hi->second;
1329     }
1330
1331     if (!handler) {
1332         debugs(9, 7, "forwarding " << cmd << " as is, no post-processing");
1333         return true;
1334     }
1335
1336     return (this->*handler)(cmd, params);
1337 }
1338
1339 /// Called to parse USER command, which is required to create an HTTP request
1340 /// wrapper. W/o request, the errors are handled by returning earlyError().
1341 ClientSocketContext *
1342 Ftp::Server::handleUserRequest(const SBuf &, SBuf &params)
1343 {
1344     if (params.isEmpty())
1345         return earlyError(EarlyErrorKind::MissingUsername);
1346
1347     // find the [end of] user name
1348     const SBuf::size_type eou = params.rfind('@');
1349     if (eou == SBuf::npos || eou + 1 >= params.length())
1350         return earlyError(EarlyErrorKind::MissingHost);
1351
1352     // Determine the intended destination.
1353     host = params.substr(eou + 1, params.length());
1354     // If we can parse it as raw IPv6 address, then surround with "[]".
1355     // Otherwise (domain, IPv4, [bracketed] IPv6, garbage, etc), use as is.
1356     if (host.find(':') != SBuf::npos) {
1357         const Ip::Address ipa(host.c_str());
1358         if (!ipa.isAnyAddr()) {
1359             char ipBuf[MAX_IPSTRLEN];
1360             ipa.toHostStr(ipBuf, MAX_IPSTRLEN);
1361             host = ipBuf;
1362         }
1363     }
1364
1365     // const SBuf login = params.substr(0, eou);
1366     params.chop(0, eou); // leave just the login part for the peer
1367
1368     SBuf oldUri;
1369     if (master->clientReadGreeting)
1370         oldUri = uri;
1371
1372     master->workingDir.clear();
1373     calcUri(NULL);
1374
1375     if (!master->clientReadGreeting) {
1376         debugs(9, 3, "set URI to " << uri);
1377     } else if (oldUri.caseCmp(uri) == 0) {
1378         debugs(9, 5, "kept URI as " << oldUri);
1379     } else {
1380         debugs(9, 3, "reset URI from " << oldUri << " to " << uri);
1381         closeDataConnection();
1382         unpinConnection(true); // close control connection to peer
1383         resetLogin("URI reset");
1384     }
1385
1386     return NULL; // no early errors
1387 }
1388
1389 bool
1390 Ftp::Server::handleFeatRequest(String &, String &)
1391 {
1392     changeState(fssHandleFeat, "handleFeatRequest");
1393     return true;
1394 }
1395
1396 bool
1397 Ftp::Server::handlePasvRequest(String &, String &params)
1398 {
1399     if (gotEpsvAll) {
1400         setReply(500, "Bad PASV command");
1401         return false;
1402     }
1403
1404     if (params.size() > 0) {
1405         setReply(501, "Unexpected parameter");
1406         return false;
1407     }
1408
1409     changeState(fssHandlePasv, "handlePasvRequest");
1410     // no need to fake PASV request via setDataCommand() in true PASV case
1411     return true;
1412 }
1413
1414 /// [Re]initializes dataConn for active data transfers. Does not connect.
1415 bool
1416 Ftp::Server::createDataConnection(Ip::Address cltAddr)
1417 {
1418     assert(clientConnection != NULL);
1419     assert(!clientConnection->remote.isAnyAddr());
1420
1421     if (cltAddr != clientConnection->remote) {
1422         debugs(33, 2, "rogue PORT " << cltAddr << " request? ctrl: " << clientConnection->remote);
1423         // Closing the control connection would not help with attacks because
1424         // the client is evidently able to connect to us. Besides, closing
1425         // makes retrials easier for the client and more damaging to us.
1426         setReply(501, "Prohibited parameter value");
1427         return false;
1428     }
1429
1430     closeDataConnection();
1431
1432     Comm::ConnectionPointer conn = new Comm::Connection();
1433     conn->flags |= COMM_DOBIND;
1434
1435     // Use local IP address of the control connection as the source address
1436     // of the active data connection, or some clients will refuse to accept.
1437     conn->setAddrs(clientConnection->local, cltAddr);
1438     // RFC 959 requires active FTP connections to originate from port 20
1439     // but that would preclude us from supporting concurrent transfers! (XXX?)
1440     conn->local.port(0);
1441
1442     debugs(9, 3, "will actively connect from " << conn->local << " to " <<
1443            conn->remote);
1444
1445     dataConn = conn;
1446     uploadAvailSize = 0;
1447     return true;
1448 }
1449
1450 bool
1451 Ftp::Server::handlePortRequest(String &, String &params)
1452 {
1453     // TODO: Should PORT errors trigger closeDataConnection() cleanup?
1454
1455     if (gotEpsvAll) {
1456         setReply(500, "Rejecting PORT after EPSV ALL");
1457         return false;
1458     }
1459
1460     if (!params.size()) {
1461         setReply(501, "Missing parameter");
1462         return false;
1463     }
1464
1465     Ip::Address cltAddr;
1466     if (!Ftp::ParseIpPort(params.termedBuf(), NULL, cltAddr)) {
1467         setReply(501, "Invalid parameter");
1468         return false;
1469     }
1470
1471     if (!createDataConnection(cltAddr))
1472         return false;
1473
1474     changeState(fssHandlePort, "handlePortRequest");
1475     setDataCommand();
1476     return true; // forward our fake PASV request
1477 }
1478
1479 bool
1480 Ftp::Server::handleDataRequest(String &, String &)
1481 {
1482     if (!checkDataConnPre())
1483         return false;
1484
1485     changeState(fssHandleDataRequest, "handleDataRequest");
1486
1487     return true;
1488 }
1489
1490 bool
1491 Ftp::Server::handleUploadRequest(String &, String &)
1492 {
1493     if (!checkDataConnPre())
1494         return false;
1495
1496     if (Config.accessList.forceRequestBodyContinuation) {
1497         ClientHttpRequest *http = pipeline.front()->http;
1498         HttpRequest *request = http->request;
1499         ACLFilledChecklist bodyContinuationCheck(Config.accessList.forceRequestBodyContinuation, request, NULL);
1500         if (bodyContinuationCheck.fastCheck() == ACCESS_ALLOWED) {
1501             request->forcedBodyContinuation = true;
1502             if (checkDataConnPost()) {
1503                 // Write control Msg
1504                 writeEarlyReply(150, "Data connection opened");
1505                 maybeReadUploadData();
1506             } else {
1507                 // wait for acceptDataConnection but tell it to call wroteEarlyReply
1508                 // after writing "150 Data connection opened"
1509                 typedef CommCbMemFunT<Server, CommIoCbParams> Dialer;
1510                 AsyncCall::Pointer call = JobCallback(33, 5, Dialer, this, Ftp::Server::wroteEarlyReply);
1511                 onDataAcceptCall = call;
1512             }
1513         }
1514     }
1515
1516     changeState(fssHandleUploadRequest, "handleDataRequest");
1517
1518     return true;
1519 }
1520
1521 bool
1522 Ftp::Server::handleEprtRequest(String &, String &params)
1523 {
1524     debugs(9, 3, "Process an EPRT " << params);
1525
1526     if (gotEpsvAll) {
1527         setReply(500, "Rejecting EPRT after EPSV ALL");
1528         return false;
1529     }
1530
1531     if (!params.size()) {
1532         setReply(501, "Missing parameter");
1533         return false;
1534     }
1535
1536     Ip::Address cltAddr;
1537     if (!Ftp::ParseProtoIpPort(params.termedBuf(), cltAddr)) {
1538         setReply(501, "Invalid parameter");
1539         return false;
1540     }
1541
1542     if (!createDataConnection(cltAddr))
1543         return false;
1544
1545     changeState(fssHandleEprt, "handleEprtRequest");
1546     setDataCommand();
1547     return true; // forward our fake PASV request
1548 }
1549
1550 bool
1551 Ftp::Server::handleEpsvRequest(String &, String &params)
1552 {
1553     debugs(9, 3, "Process an EPSV command with params: " << params);
1554     if (params.size() <= 0) {
1555         // treat parameterless EPSV as "use the protocol of the ctrl conn"
1556     } else if (params.caseCmp("ALL") == 0) {
1557         setReply(200, "EPSV ALL ok");
1558         gotEpsvAll = true;
1559         return false;
1560     } else if (params.cmp("2") == 0) {
1561         if (!Ip::EnableIpv6) {
1562             setReply(522, "Network protocol not supported, use (1)");
1563             return false;
1564         }
1565     } else if (params.cmp("1") != 0) {
1566         setReply(501, "Unsupported EPSV parameter");
1567         return false;
1568     }
1569
1570     changeState(fssHandleEpsv, "handleEpsvRequest");
1571     setDataCommand();
1572     return true; // forward our fake PASV request
1573 }
1574
1575 bool
1576 Ftp::Server::handleCwdRequest(String &, String &)
1577 {
1578     changeState(fssHandleCwd, "handleCwdRequest");
1579     return true;
1580 }
1581
1582 bool
1583 Ftp::Server::handlePassRequest(String &, String &)
1584 {
1585     changeState(fssHandlePass, "handlePassRequest");
1586     return true;
1587 }
1588
1589 bool
1590 Ftp::Server::handleCdupRequest(String &, String &)
1591 {
1592     changeState(fssHandleCdup, "handleCdupRequest");
1593     return true;
1594 }
1595
1596 // Convert user PORT, EPRT, PASV, or EPSV data command to Squid PASV command.
1597 // Squid FTP client decides what data command to use with peers.
1598 void
1599 Ftp::Server::setDataCommand()
1600 {
1601     ClientHttpRequest *const http = pipeline.front()->http;
1602     assert(http != NULL);
1603     HttpRequest *const request = http->request;
1604     assert(request != NULL);
1605     HttpHeader &header = request->header;
1606     header.delById(Http::HdrType::FTP_COMMAND);
1607     header.putStr(Http::HdrType::FTP_COMMAND, "PASV");
1608     header.delById(Http::HdrType::FTP_ARGUMENTS);
1609     header.putStr(Http::HdrType::FTP_ARGUMENTS, "");
1610     debugs(9, 5, "client data command converted to fake PASV");
1611 }
1612
1613 /// check that client data connection is ready for future I/O or at least
1614 /// has a chance of becoming ready soon.
1615 bool
1616 Ftp::Server::checkDataConnPre()
1617 {
1618     if (Comm::IsConnOpen(dataConn))
1619         return true;
1620
1621     if (Comm::IsConnOpen(dataListenConn)) {
1622         // We are still waiting for a client to connect to us after PASV.
1623         // Perhaps client's data conn handshake has not reached us yet.
1624         // After we talk to the server, checkDataConnPost() will recheck.
1625         debugs(33, 3, "expecting clt data conn " << dataListenConn);
1626         return true;
1627     }
1628
1629     if (!dataConn || dataConn->remote.isAnyAddr()) {
1630         debugs(33, 5, "missing " << dataConn);
1631         // TODO: use client address and default port instead.
1632         setReply(425, "Use PORT or PASV first");
1633         return false;
1634     }
1635
1636     // active transfer: open a data connection from Squid to client
1637     typedef CommCbMemFunT<Server, CommConnectCbParams> Dialer;
1638     connector = JobCallback(17, 3, Dialer, this, Ftp::Server::connectedForData);
1639     Comm::ConnOpener *cs = new Comm::ConnOpener(dataConn, connector,
1640             Config.Timeout.connect);
1641     AsyncJob::Start(cs);
1642     return false; // ConnStateData::processFtpRequest waits handleConnectDone
1643 }
1644
1645 /// Check that client data connection is ready for immediate I/O.
1646 bool
1647 Ftp::Server::checkDataConnPost() const
1648 {
1649     if (!Comm::IsConnOpen(dataConn)) {
1650         debugs(33, 3, "missing client data conn: " << dataConn);
1651         return false;
1652     }
1653     return true;
1654 }
1655
1656 /// Done establishing a data connection to the user.
1657 void
1658 Ftp::Server::connectedForData(const CommConnectCbParams &params)
1659 {
1660     connector = NULL;
1661
1662     if (params.flag != Comm::OK) {
1663         /* it might have been a timeout with a partially open link */
1664         if (params.conn != NULL)
1665             params.conn->close();
1666         setReply(425, "Cannot open data connection.");
1667         ClientSocketContext::Pointer context = pipeline.front();
1668         Must(context->http);
1669         Must(context->http->storeEntry() != NULL);
1670     } else {
1671         Must(dataConn == params.conn);
1672         Must(Comm::IsConnOpen(params.conn));
1673         fd_note(params.conn->fd, "active client ftp data");
1674     }
1675
1676     doProcessRequest();
1677 }
1678
1679 void
1680 Ftp::Server::setReply(const int code, const char *msg)
1681 {
1682     ClientSocketContext::Pointer context = pipeline.front();
1683     ClientHttpRequest *const http = context->http;
1684     assert(http != NULL);
1685     assert(http->storeEntry() == NULL);
1686
1687     HttpReply *const reply = Ftp::HttpReplyWrapper(code, msg, Http::scNoContent, 0);
1688
1689     setLogUri(http, urlCanonicalClean(http->request));
1690
1691     clientStreamNode *const node = context->getClientReplyContext();
1692     clientReplyContext *const repContext =
1693         dynamic_cast<clientReplyContext *>(node->data.getRaw());
1694     assert(repContext != NULL);
1695
1696     RequestFlags reqFlags;
1697     reqFlags.cachable = false; // force releaseRequest() in storeCreateEntry()
1698     reqFlags.noCache = true;
1699     repContext->createStoreEntry(http->request->method, reqFlags);
1700     http->storeEntry()->replaceHttpReply(reply);
1701 }
1702
1703 void
1704 Ftp::Server::callException(const std::exception &e)
1705 {
1706     debugs(33, 2, "FTP::Server job caught: " << e.what());
1707     closeDataConnection();
1708     unpinConnection(true);
1709     if (Comm::IsConnOpen(clientConnection))
1710         clientConnection->close();
1711     AsyncJob::callException(e);
1712 }
1713
1714 /// Whether Squid FTP Relay supports a named feature (e.g., a command).
1715 static bool
1716 Ftp::SupportedCommand(const SBuf &name)
1717 {
1718     static std::set<SBuf> BlackList;
1719     if (BlackList.empty()) {
1720         /* Add FTP commands that Squid cannot relay correctly. */
1721
1722         // We probably do not support AUTH TLS.* and AUTH SSL,
1723         // but let's disclaim all AUTH support to KISS, for now.
1724         BlackList.insert(cmdAuth());
1725     }
1726
1727     // we claim support for all commands that we do not know about
1728     return BlackList.find(name) == BlackList.end();
1729 }
1730