src/clients/FtpClient.cc

   1 /*
   2  * Copyright (C) 1996-2021 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "client_side.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm/ConnOpener.h"
  16 #include "comm/Read.h"
  17 #include "comm/TcpAcceptor.h"
  18 #include "comm/Write.h"
  19 #include "error/SysErrorDetail.h"
  20 #include "errorpage.h"
  21 #include "fd.h"
  22 #include "ftp/Parsing.h"
  23 #include "http/Stream.h"
  24 #include "ip/tools.h"
  25 #include "sbuf/SBuf.h"
  26 #include "sbuf/Stream.h"
  27 #include "SquidConfig.h"
  28 #include "SquidString.h"
  29 #include "StatCounters.h"
  30 #include "tools.h"
  31 #include "wordlist.h"
  32
  33 #include <set>
  34
  35 namespace Ftp
  36 {
  37
  38 const char *const crlf = "\r\n";
  39
  40 static char *
  41 escapeIAC(const char *buf)
  42 {
  43     int n;
  44     char *ret;
  45     unsigned const char *p;
  46     unsigned char *r;
  47
  48     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
  49         if (*p == 255)
  50             ++n;
  51
  52     ret = (char *)xmalloc(n);
  53
  54     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
  55         *r = *p;
  56         ++r;
  57
  58         if (*p == 255) {
  59             *r = 255;
  60             ++r;
  61         }
  62     }
  63
  64     *r = '\0';
  65     ++r;
  66     assert((r - (unsigned char *)ret) == n );
  67     return ret;
  68 }
  69
  70 /* Ftp::ErrorDetail */
  71
  72 SBuf
  73 Ftp::ErrorDetail::brief() const
  74 {
  75     return ToSBuf("FTP_REPLY_CODE=", completionCode);
  76 }
  77
  78 SBuf
  79 Ftp::ErrorDetail::verbose(const HttpRequest::Pointer &) const
  80 {
  81     return ToSBuf("FTP reply with completion code ", completionCode);
  82 }
  83
  84 /* Ftp::Channel */
  85
  86 /// configures the channel with a descriptor and registers a close handler
  87 void
  88 Ftp::Channel::opened(const Comm::ConnectionPointer &newConn,
  89                      const AsyncCall::Pointer &aCloser)
  90 {
  91     assert(!Comm::IsConnOpen(conn));
  92     assert(closer == NULL);
  93
  94     assert(Comm::IsConnOpen(newConn));
  95     assert(aCloser != NULL);
  96
  97     conn = newConn;
  98     closer = aCloser;
  99     comm_add_close_handler(conn->fd, closer);
 100 }
 101
 102 /// planned close: removes the close handler and calls comm_close
 103 void
 104 Ftp::Channel::close()
 105 {
 106     // channels with active listeners will be closed when the listener handler dies.
 107     if (Comm::IsConnOpen(conn)) {
 108         comm_remove_close_handler(conn->fd, closer);
 109         conn->close(); // we do not expect to be called back
 110     }
 111     clear();
 112 }
 113
 114 void
 115 Ftp::Channel::forget()
 116 {
 117     if (Comm::IsConnOpen(conn)) {
 118         commUnsetConnTimeout(conn);
 119         comm_remove_close_handler(conn->fd, closer);
 120     }
 121     clear();
 122 }
 123
 124 void
 125 Ftp::Channel::clear()
 126 {
 127     conn = NULL;
 128     closer = NULL;
 129 }
 130
 131 /* Ftp::CtrlChannel */
 132
 133 Ftp::CtrlChannel::CtrlChannel():
 134     buf(NULL),
 135     size(0),
 136     offset(0),
 137     message(NULL),
 138     last_command(NULL),
 139     last_reply(NULL),
 140     replycode(0)
 141 {
 142     buf = static_cast<char*>(memAllocBuf(4096, &size));
 143 }
 144
 145 Ftp::CtrlChannel::~CtrlChannel()
 146 {
 147     memFreeBuf(size, buf);
 148     if (message)
 149         wordlistDestroy(&message);
 150     safe_free(last_command);
 151     safe_free(last_reply);
 152 }
 153
 154 /* Ftp::DataChannel */
 155
 156 Ftp::DataChannel::DataChannel():
 157     readBuf(NULL),
 158     host(NULL),
 159     port(0),
 160     read_pending(false)
 161 {
 162 }
 163
 164 Ftp::DataChannel::~DataChannel()
 165 {
 166     delete readBuf;
 167 }
 168
 169 void
 170 Ftp::DataChannel::addr(const Ip::Address &import)
 171 {
 172     static char addrBuf[MAX_IPSTRLEN];
 173     import.toStr(addrBuf, sizeof(addrBuf));
 174     xfree(host);
 175     host = xstrdup(addrBuf);
 176     port = import.port();
 177 }
 178
 179 /* Ftp::Client */
 180
 181 Ftp::Client::Client(FwdState *fwdState):
 182     AsyncJob("Ftp::Client"),
 183     ::Client(fwdState),
 184      ctrl(),
 185      data(),
 186      state(BEGIN),
 187      old_request(NULL),
 188      old_reply(NULL),
 189      shortenReadTimeout(false)
 190 {
 191     ++statCounter.server.all.requests;
 192     ++statCounter.server.ftp.requests;
 193
 194     ctrl.last_command = xstrdup("Connect to server");
 195
 196     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 197     const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this,
 198                                       Ftp::Client::ctrlClosed);
 199     ctrl.opened(fwdState->serverConnection(), closer);
 200 }
 201
 202 Ftp::Client::~Client()
 203 {
 204     if (data.opener != NULL) {
 205         data.opener->cancel("Ftp::Client destructed");
 206         data.opener = NULL;
 207     }
 208     data.close();
 209
 210     safe_free(old_request);
 211     safe_free(old_reply);
 212     fwd = NULL; // refcounted
 213 }
 214
 215 void
 216 Ftp::Client::start()
 217 {
 218     scheduleReadControlReply(0);
 219 }
 220
 221 void
 222 Ftp::Client::initReadBuf()
 223 {
 224     if (data.readBuf == NULL) {
 225         data.readBuf = new MemBuf;
 226         data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 227     }
 228 }
 229
 230 /**
 231  * Close the FTP server connection(s). Used by serverComplete().
 232  */
 233 void
 234 Ftp::Client::closeServer()
 235 {
 236     if (Comm::IsConnOpen(ctrl.conn)) {
 237         debugs(9, 3, "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
 238         fwd->unregister(ctrl.conn);
 239         ctrl.close();
 240     }
 241
 242     if (Comm::IsConnOpen(data.conn)) {
 243         debugs(9, 3, "closing FTP data FD " << data.conn->fd << ", this " << this);
 244         data.close();
 245     }
 246
 247     debugs(9, 3, "FTP ctrl and data connections closed. this " << this);
 248 }
 249
 250 /**
 251  * Did we close all FTP server connection(s)?
 252  *
 253  \retval true  Both server control and data channels are closed. And not waiting for a new data connection to open.
 254  \retval false Either control channel or data is still active.
 255  */
 256 bool
 257 Ftp::Client::doneWithServer() const
 258 {
 259     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
 260 }
 261
 262 void
 263 Ftp::Client::failed(err_type error, int xerrno, ErrorState *err)
 264 {
 265     debugs(9, 3, "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
 266
 267     const char *command, *reply;
 268     ErrorState *ftperr;
 269
 270     if (err) {
 271         debugs(9, 6, "error=" << err->type << ", code=" << xerrno <<
 272                ", status=" << err->httpStatus);
 273         error = err->type;
 274         ftperr = err;
 275     } else {
 276         Http::StatusCode httpStatus = failedHttpStatus(error);
 277         ftperr = new ErrorState(error, httpStatus, fwd->request, fwd->al);
 278     }
 279
 280     ftperr->xerrno = xerrno;
 281
 282     ftperr->ftp.server_msg = ctrl.message;
 283     ctrl.message = NULL;
 284
 285     if (old_request)
 286         command = old_request;
 287     else
 288         command = ctrl.last_command;
 289
 290     if (command && strncmp(command, "PASS", 4) == 0)
 291         command = "PASS <yourpassword>";
 292
 293     if (old_reply)
 294         reply = old_reply;
 295     else
 296         reply = ctrl.last_reply;
 297
 298     if (command)
 299         ftperr->ftp.request = xstrdup(command);
 300
 301     if (reply)
 302         ftperr->ftp.reply = xstrdup(reply);
 303
 304     if (!err) {
 305         fwd->request->detailError(error, SysErrorDetail::NewIfAny(xerrno));
 306         fwd->fail(ftperr);
 307         closeServer(); // we failed, so no serverComplete()
 308     }
 309 }
 310
 311 Http::StatusCode
 312 Ftp::Client::failedHttpStatus(err_type &error)
 313 {
 314     if (error == ERR_NONE)
 315         error = ERR_FTP_FAILURE;
 316     return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout :
 317            Http::scBadGateway;
 318 }
 319
 320 /**
 321  * DPW 2007-04-23
 322  * Looks like there are no longer anymore callers that set
 323  * buffered_ok=1.  Perhaps it can be removed at some point.
 324  */
 325 void
 326 Ftp::Client::scheduleReadControlReply(int buffered_ok)
 327 {
 328     debugs(9, 3, ctrl.conn);
 329
 330     if (buffered_ok && ctrl.offset > 0) {
 331         /* We've already read some reply data */
 332         handleControlReply();
 333     } else {
 334
 335         if (!Comm::IsConnOpen(ctrl.conn)) {
 336             debugs(9, 3, "cannot read without ctrl " << ctrl.conn);
 337             return;
 338         }
 339         /*
 340          * Cancel the timeout on the Data socket (if any) and
 341          * establish one on the control socket.
 342          */
 343         if (Comm::IsConnOpen(data.conn)) {
 344             commUnsetConnTimeout(data.conn);
 345         }
 346
 347         const time_t tout = shortenReadTimeout ?
 348                             min(Config.Timeout.connect, Config.Timeout.read):
 349                             Config.Timeout.read;
 350         shortenReadTimeout = false; // we only need to do this once, after PASV
 351
 352         typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 353         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, Ftp::Client::timeout);
 354         commSetConnTimeout(ctrl.conn, tout, timeoutCall);
 355
 356         typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 357         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, Ftp::Client::readControlReply);
 358         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
 359     }
 360 }
 361
 362 void
 363 Ftp::Client::readControlReply(const CommIoCbParams &io)
 364 {
 365     debugs(9, 3, "FD " << io.fd << ", Read " << io.size << " bytes");
 366
 367     if (io.size > 0) {
 368         statCounter.server.all.kbytes_in += io.size;
 369         statCounter.server.ftp.kbytes_in += io.size;
 370     }
 371
 372     if (io.flag == Comm::ERR_CLOSING)
 373         return;
 374
 375     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 376         if (abortOnData("entry aborted during control reply read"))
 377             return;
 378     }
 379
 380     assert(ctrl.offset < ctrl.size);
 381
 382     if (io.flag == Comm::OK && io.size > 0) {
 383         fd_bytes(io.fd, io.size, FD_READ);
 384     }
 385
 386     if (io.flag != Comm::OK) {
 387         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 388                "FTP control reply read error: " << xstrerr(io.xerrno));
 389
 390         if (ignoreErrno(io.xerrno)) {
 391             scheduleReadControlReply(0);
 392         } else {
 393             failed(ERR_READ_ERROR, io.xerrno);
 394             /* failed closes ctrl.conn and frees ftpState */
 395         }
 396         return;
 397     }
 398
 399     if (io.size == 0) {
 400         if (entry->store_status == STORE_PENDING) {
 401             failed(ERR_FTP_FAILURE, 0);
 402             /* failed closes ctrl.conn and frees ftpState */
 403             return;
 404         }
 405
 406         /* XXX this may end up having to be serverComplete() .. */
 407         abortAll("zero control reply read");
 408         return;
 409     }
 410
 411     unsigned int len =io.size + ctrl.offset;
 412     ctrl.offset = len;
 413     assert(len <= ctrl.size);
 414     if (Comm::IsConnOpen(ctrl.conn))
 415         commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply
 416     handleControlReply();
 417 }
 418
 419 void
 420 Ftp::Client::handleControlReply()
 421 {
 422     debugs(9, 3, status());
 423
 424     size_t bytes_used = 0;
 425     wordlistDestroy(&ctrl.message);
 426
 427     if (!parseControlReply(bytes_used)) {
 428         /* didn't get complete reply yet */
 429
 430         if (ctrl.offset == ctrl.size) {
 431             ctrl.buf = static_cast<char*>(memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size));
 432         }
 433
 434         scheduleReadControlReply(0);
 435         return;
 436     }
 437
 438     assert(ctrl.message); // the entire FTP server response, line by line
 439     assert(ctrl.replycode >= 0); // FTP status code (from the last line)
 440     assert(ctrl.last_reply); // FTP reason (from the last line)
 441
 442     if (ctrl.offset == bytes_used) {
 443         /* used it all up */
 444         ctrl.offset = 0;
 445     } else {
 446         /* Got some data past the complete reply */
 447         assert(bytes_used < ctrl.offset);
 448         ctrl.offset -= bytes_used;
 449         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
 450     }
 451
 452     debugs(9, 3, "state=" << state << ", code=" << ctrl.replycode);
 453 }
 454
 455 bool
 456 Ftp::Client::handlePasvReply(Ip::Address &srvAddr)
 457 {
 458     int code = ctrl.replycode;
 459     char *buf;
 460     debugs(9, 3, status());
 461
 462     if (!Comm::IsConnOpen(ctrl.conn)) {
 463         debugs(9, 5, "The control connection to the remote end is closed");
 464         return false;
 465     }
 466
 467     if (code != 227) {
 468         debugs(9, 2, "PASV not supported by remote end");
 469         return false;
 470     }
 471
 472     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
 473     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 474     debugs(9, 5, "scanning: " << ctrl.last_reply);
 475
 476     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789");
 477
 478     const char *forceIp = Config.Ftp.sanitycheck ?
 479                           fd_table[ctrl.conn->fd].ipaddr : NULL;
 480     if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) {
 481         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
 482                ctrl.conn->remote << ": " << ctrl.last_reply);
 483         return false;
 484     }
 485
 486     data.addr(srvAddr);
 487
 488     return true;
 489 }
 490
 491 bool
 492 Ftp::Client::handleEpsvReply(Ip::Address &remoteAddr)
 493 {
 494     int code = ctrl.replycode;
 495     char *buf;
 496     debugs(9, 3, status());
 497
 498     if (!Comm::IsConnOpen(ctrl.conn)) {
 499         debugs(9, 5, "The control connection to the remote end is closed");
 500         return false;
 501     }
 502
 503     if (code != 229 && code != 522) {
 504         if (code == 200) {
 505             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
 506             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
 507              * Its okay to re-send EPSV 1/2 but nothing else. */
 508             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV");
 509         } else {
 510             debugs(9, 2, "EPSV not supported by remote end");
 511         }
 512         return sendPassive();
 513     }
 514
 515     if (code == 522) {
 516         /* Peer responded with a list of supported methods:
 517          *   522 Network protocol not supported, use (1)
 518          *   522 Network protocol not supported, use (1,2)
 519          *   522 Network protocol not supported, use (2)
 520          * TODO: Handle the (1,2) case which may happen after EPSV ALL. Close
 521          * data + control without self-destructing and re-open from scratch.
 522          */
 523         debugs(9, 5, "scanning: " << ctrl.last_reply);
 524         buf = ctrl.last_reply;
 525         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
 526             ++buf;
 527         if (buf != NULL && *buf == '\n')
 528             ++buf;
 529
 530         if (buf == NULL || *buf == '\0') {
 531             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 532             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
 533             return sendPassive();
 534         } else if (strcmp(buf, "(1)") == 0) {
 535             state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
 536             return sendPassive();
 537         } else if (strcmp(buf, "(2)") == 0) {
 538             if (Ip::EnableIpv6) {
 539                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
 540                 if (state == SENT_EPSV_2) {
 541                     return sendEprt();
 542                 } else {
 543                     /* or try the next Passive mode down the chain. */
 544                     return sendPassive();
 545                 }
 546             } else {
 547                 /* Server only accept EPSV in IPv6 traffic. */
 548                 state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
 549                 return sendPassive();
 550             }
 551         } else {
 552             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 553             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
 554             return sendPassive();
 555         }
 556         /* coverity[unreachable] */
 557         /* safeguard against possible future bugs in above conditions */
 558         failed(ERR_FTP_FAILURE, 0);
 559         return false;
 560     }
 561
 562     /*  229 Entering Extended Passive Mode (|||port|) */
 563     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 564     debugs(9, 5, "scanning: " << ctrl.last_reply);
 565
 566     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "(");
 567
 568     char h1, h2, h3, h4;
 569     unsigned short port;
 570     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
 571
 572     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
 573         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
 574                ctrl.conn->remote << ": " <<
 575                ctrl.last_reply);
 576
 577         return sendPassive();
 578     }
 579
 580     if (0 == port) {
 581         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 582                ctrl.conn->remote << ": " <<
 583                ctrl.last_reply);
 584
 585         return sendPassive();
 586     }
 587
 588     if (Config.Ftp.sanitycheck) {
 589         if (port < 1024) {
 590             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 591                    ctrl.conn->remote << ": " <<
 592                    ctrl.last_reply);
 593
 594             return sendPassive();
 595         }
 596     }
 597
 598     remoteAddr = ctrl.conn->remote;
 599     remoteAddr.port(port);
 600     data.addr(remoteAddr);
 601     return true;
 602 }
 603
 604 // FTP clients do not support EPRT and PORT commands yet.
 605 // The Ftp::Client::sendEprt() will fail because of the unimplemented
 606 // openListenSocket() or sendPort() methods
 607 bool
 608 Ftp::Client::sendEprt()
 609 {
 610     if (!Config.Ftp.eprt) {
 611         /* Disabled. Switch immediately to attempting old PORT command. */
 612         debugs(9, 3, "EPRT disabled by local administrator");
 613         return sendPort();
 614     }
 615
 616     debugs(9, 3, status());
 617
 618     if (!openListenSocket()) {
 619         failed(ERR_FTP_FAILURE, 0);
 620         return false;
 621     }
 622
 623     debugs(9, 3, "Listening for FTP data connection with FD " << data.conn);
 624     if (!Comm::IsConnOpen(data.conn)) {
 625         // TODO: Set error message.
 626         failed(ERR_FTP_FAILURE, 0);
 627         return false;
 628     }
 629
 630     static MemBuf mb;
 631     mb.reset();
 632     char buf[MAX_IPSTRLEN];
 633     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
 634     /* Which can be used by EITHER protocol. */
 635     debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port());
 636     mb.appendf("EPRT |%d|%s|%d|%s",
 637                ( data.conn->local.isIPv6() ? 2 : 1 ),
 638                data.conn->local.toStr(buf,MAX_IPSTRLEN),
 639                comm_local_port(data.conn->fd), Ftp::crlf );
 640
 641     state = SENT_EPRT;
 642     writeCommand(mb.content());
 643     return true;
 644 }
 645
 646 bool
 647 Ftp::Client::sendPort()
 648 {
 649     failed(ERR_FTP_FAILURE, 0);
 650     return false;
 651 }
 652
 653 bool
 654 Ftp::Client::sendPassive()
 655 {
 656     debugs(9, 3, status());
 657
 658     /** \par
 659       * Checks for EPSV ALL special conditions:
 660       * If enabled to be sent, squid MUST NOT request any other connect methods.
 661       * If 'ALL' is sent and fails the entire FTP Session fails.
 662       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
 663     if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) {
 664         // We are here because the last "EPSV 1" failed, but because of epsv_all
 665         // no other method allowed.
 666         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 667         failed(ERR_FTP_FAILURE, 0);
 668         return false;
 669     }
 670
 671     /// Closes any old FTP-Data connection which may exist. */
 672     data.close();
 673
 674     /** \par
 675       * Checks for previous EPSV/PASV failures on this server/session.
 676       * Diverts to EPRT immediately if they are not working. */
 677     if (!Config.Ftp.passive || state == SENT_PASV) {
 678         sendEprt();
 679         return true;
 680     }
 681
 682     static MemBuf mb;
 683     mb.reset();
 684     /** \par
 685       * Send EPSV (ALL,2,1) or PASV on the control channel.
 686       *
 687       *  - EPSV ALL  is used if enabled.
 688       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
 689       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
 690       *  - PASV      is used if EPSV 1 fails.
 691       */
 692     switch (state) {
 693     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
 694         if (ctrl.conn->local.isIPv6()) {
 695             debugs(9, 5, "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
 696             mb.appendf("EPSV 2%s", Ftp::crlf);
 697             state = SENT_EPSV_2;
 698             break;
 699         }
 700     // else fall through to skip EPSV 2
 701
 702     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
 703         if (ctrl.conn->local.isIPv4()) {
 704             debugs(9, 5, "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
 705             mb.appendf("EPSV 1%s", Ftp::crlf);
 706             state = SENT_EPSV_1;
 707             break;
 708         } else if (Config.Ftp.epsv_all) {
 709             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 710             failed(ERR_FTP_FAILURE, 0);
 711             return false;
 712         }
 713     // else fall through to skip EPSV 1
 714
 715     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
 716         debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
 717         mb.appendf("PASV%s", Ftp::crlf);
 718         state = SENT_PASV;
 719         break;
 720
 721     default: {
 722         bool doEpsv = true;
 723         if (Config.accessList.ftp_epsv) {
 724             ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL);
 725             doEpsv = checklist.fastCheck().allowed();
 726         }
 727         if (!doEpsv) {
 728             debugs(9, 5, "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")");
 729             mb.appendf("PASV%s", Ftp::crlf);
 730             state = SENT_PASV;
 731         } else if (Config.Ftp.epsv_all) {
 732             debugs(9, 5, "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")");
 733             mb.appendf("EPSV ALL%s", Ftp::crlf);
 734             state = SENT_EPSV_ALL;
 735         } else {
 736             if (ctrl.conn->local.isIPv6()) {
 737                 debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2");
 738                 mb.appendf("EPSV 2%s", Ftp::crlf);
 739                 state = SENT_EPSV_2;
 740             }
 741             if (ctrl.conn->local.isIPv4()) {
 742                 debugs(9, 5, "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1");
 743                 mb.appendf("EPSV 1%s", Ftp::crlf);
 744                 state = SENT_EPSV_1;
 745             }
 746         }
 747         break;
 748     }
 749     }
 750
 751     if (ctrl.message)
 752         wordlistDestroy(&ctrl.message);
 753     ctrl.message = NULL; //No message to return to client.
 754     ctrl.offset = 0; //reset readed response, to make room read the next response
 755
 756     writeCommand(mb.content());
 757
 758     shortenReadTimeout = true;
 759     return true;
 760 }
 761
 762 void
 763 Ftp::Client::connectDataChannel()
 764 {
 765     if (!Comm::IsConnOpen(ctrl.conn)) {
 766         debugs(9, 5, "The control connection to the remote end is closed");
 767         return;
 768     }
 769
 770     safe_free(ctrl.last_command);
 771
 772     safe_free(ctrl.last_reply);
 773
 774     ctrl.last_command = xstrdup("Connect to server data port");
 775
 776     // Generate a new data channel descriptor to be opened.
 777     Comm::ConnectionPointer conn = new Comm::Connection;
 778     conn->setAddrs(ctrl.conn->local, data.host);
 779     conn->local.port(0);
 780     conn->remote.port(data.port);
 781     conn->tos = ctrl.conn->tos;
 782     conn->nfmark = ctrl.conn->nfmark;
 783     // Using non-local addresses in TPROXY mode requires appropriate socket option.
 784     conn->flags |= ctrl.conn->flags & COMM_TRANSPARENT;
 785
 786     debugs(9, 3, "connecting to " << conn->remote);
 787
 788     typedef CommCbMemFunT<Client, CommConnectCbParams> Dialer;
 789     data.opener = JobCallback(9, 3, Dialer, this, Ftp::Client::dataChannelConnected);
 790     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect);
 791     cs->setHost(data.host);
 792     AsyncJob::Start(cs);
 793 }
 794
 795 bool
 796 Ftp::Client::openListenSocket()
 797 {
 798     return false;
 799 }
 800
 801 /// creates a data channel Comm close callback
 802 AsyncCall::Pointer
 803 Ftp::Client::dataCloser()
 804 {
 805     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 806     return JobCallback(9, 5, Dialer, this, Ftp::Client::dataClosed);
 807 }
 808
 809 /// handler called by Comm when FTP data channel is closed unexpectedly
 810 void
 811 Ftp::Client::dataClosed(const CommCloseCbParams &)
 812 {
 813     debugs(9, 4, status());
 814     if (data.listenConn != NULL) {
 815         data.listenConn->close();
 816         data.listenConn = NULL;
 817         // NP clear() does the: data.fd = -1;
 818     }
 819     data.clear();
 820 }
 821
 822 void
 823 Ftp::Client::writeCommand(const char *buf)
 824 {
 825     char *ebuf;
 826     /* trace FTP protocol communications at level 2 */
 827     debugs(9, 2, "ftp<< " << buf);
 828
 829     if (Config.Ftp.telnet)
 830         ebuf = escapeIAC(buf);
 831     else
 832         ebuf = xstrdup(buf);
 833
 834     safe_free(ctrl.last_command);
 835
 836     safe_free(ctrl.last_reply);
 837
 838     ctrl.last_command = ebuf;
 839
 840     if (!Comm::IsConnOpen(ctrl.conn)) {
 841         debugs(9, 2, "cannot send to closing ctrl " << ctrl.conn);
 842         // TODO: assert(ctrl.closer != NULL);
 843         return;
 844     }
 845
 846     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 847     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this,
 848                                           Ftp::Client::writeCommandCallback);
 849     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
 850
 851     scheduleReadControlReply(0);
 852 }
 853
 854 void
 855 Ftp::Client::writeCommandCallback(const CommIoCbParams &io)
 856 {
 857
 858     debugs(9, 5, "wrote " << io.size << " bytes");
 859
 860     if (io.size > 0) {
 861         fd_bytes(io.fd, io.size, FD_WRITE);
 862         statCounter.server.all.kbytes_out += io.size;
 863         statCounter.server.ftp.kbytes_out += io.size;
 864     }
 865
 866     if (io.flag == Comm::ERR_CLOSING)
 867         return;
 868
 869     if (io.flag) {
 870         debugs(9, DBG_IMPORTANT, "FTP command write error: " << io.conn << ": " << xstrerr(io.xerrno));
 871         failed(ERR_WRITE_ERROR, io.xerrno);
 872         /* failed closes ctrl.conn and frees ftpState */
 873         return;
 874     }
 875 }
 876
 877 /// handler called by Comm when FTP control channel is closed unexpectedly
 878 void
 879 Ftp::Client::ctrlClosed(const CommCloseCbParams &)
 880 {
 881     debugs(9, 4, status());
 882     ctrl.clear();
 883     doneWithFwd = "ctrlClosed()"; // assume FwdState is monitoring too
 884     mustStop("Ftp::Client::ctrlClosed");
 885 }
 886
 887 void
 888 Ftp::Client::timeout(const CommTimeoutCbParams &io)
 889 {
 890     debugs(9, 4, io.conn << ": '" << entry->url() << "'" );
 891
 892     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 893         return;
 894
 895     failed(ERR_READ_TIMEOUT, 0);
 896     /* failed() closes ctrl.conn and frees ftpState */
 897 }
 898
 899 const Comm::ConnectionPointer &
 900 Ftp::Client::dataConnection() const
 901 {
 902     return data.conn;
 903 }
 904
 905 void
 906 Ftp::Client::maybeReadVirginBody()
 907 {
 908     // too late to read
 909     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
 910         return;
 911
 912     if (data.read_pending)
 913         return;
 914
 915     initReadBuf();
 916
 917     const int read_sz = replyBodySpace(*data.readBuf, 0);
 918
 919     debugs(9, 9, "FTP may read up to " << read_sz << " bytes");
 920
 921     if (read_sz < 2) // see http.cc
 922         return;
 923
 924     data.read_pending = true;
 925
 926     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 927     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
 928                                       TimeoutDialer, this, Ftp::Client::timeout);
 929     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 930
 931     debugs(9,5,"queueing read on FD " << data.conn->fd);
 932
 933     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 934     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
 935                           JobCallback(9, 5, Dialer, this, Ftp::Client::dataRead));
 936 }
 937
 938 void
 939 Ftp::Client::dataRead(const CommIoCbParams &io)
 940 {
 941     int j;
 942     int bin;
 943
 944     data.read_pending = false;
 945
 946     debugs(9, 3, "FD " << io.fd << " Read " << io.size << " bytes");
 947
 948     if (io.size > 0) {
 949         statCounter.server.all.kbytes_in += io.size;
 950         statCounter.server.ftp.kbytes_in += io.size;
 951     }
 952
 953     if (io.flag == Comm::ERR_CLOSING)
 954         return;
 955
 956     assert(io.fd == data.conn->fd);
 957
 958     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 959         abortOnData("entry aborted during dataRead");
 960         return;
 961     }
 962
 963     if (io.flag == Comm::OK && io.size > 0) {
 964         debugs(9, 5, "appended " << io.size << " bytes to readBuf");
 965         data.readBuf->appended(io.size);
 966 #if USE_DELAY_POOLS
 967         DelayId delayId = entry->mem_obj->mostBytesAllowed();
 968         delayId.bytesIn(io.size);
 969 #endif
 970         ++ IOStats.Ftp.reads;
 971
 972         for (j = io.size - 1, bin = 0; j; ++bin)
 973             j >>= 1;
 974
 975         ++ IOStats.Ftp.read_hist[bin];
 976     }
 977
 978     if (io.flag != Comm::OK) {
 979         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 980                "FTP data read error: " << xstrerr(io.xerrno));
 981
 982         if (ignoreErrno(io.xerrno)) {
 983             maybeReadVirginBody();
 984         } else {
 985             failed(ERR_READ_ERROR, 0);
 986             /* failed closes ctrl.conn and frees ftpState */
 987             return;
 988         }
 989     } else if (io.size == 0) {
 990         debugs(9, 3, "Calling dataComplete() because io.size == 0");
 991         /*
 992          * DPW 2007-04-23
 993          * Dangerous curves ahead.  This call to dataComplete was
 994          * calling scheduleReadControlReply, handleControlReply,
 995          * and then ftpReadTransferDone.  If ftpReadTransferDone
 996          * gets unexpected status code, it closes down the control
 997          * socket and our FtpStateData object gets destroyed.   As
 998          * a workaround we no longer set the 'buffered_ok' flag in
 999          * the scheduleReadControlReply call.
1000          */
1001         dataComplete();
1002     }
1003
1004     processReplyBody();
1005 }
1006
1007 void
1008 Ftp::Client::dataComplete()
1009 {
1010     debugs(9, 3,status());
1011
1012     /* Connection closed; transfer done. */
1013
1014     /// Close data channel, if any, to conserve resources while we wait.
1015     data.close();
1016
1017     /* expect the "transfer complete" message on the control socket */
1018     /*
1019      * DPW 2007-04-23
1020      * Previously, this was the only place where we set the
1021      * 'buffered_ok' flag when calling scheduleReadControlReply().
1022      * It caused some problems if the FTP server returns an unexpected
1023      * status code after the data command.  FtpStateData was being
1024      * deleted in the middle of dataRead().
1025      */
1026     /* AYJ: 2011-01-13: Bug 2581.
1027      * 226 status is possibly waiting in the ctrl buffer.
1028      * The connection will hang if we DONT send buffered_ok.
1029      * This happens on all transfers which can be completely sent by the
1030      * server before the 150 started status message is read in by Squid.
1031      * ie all transfers of about one packet hang.
1032      */
1033     scheduleReadControlReply(1);
1034 }
1035
1036 void
1037 Ftp::Client::abortAll(const char *reason)
1038 {
1039     debugs(9, 3, "aborting transaction for " << reason <<
1040            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
1041     mustStop(reason);
1042 }
1043
1044 /**
1045  * Cancel the timeout on the Control socket and establish one
1046  * on the data socket
1047  */
1048 void
1049 Ftp::Client::switchTimeoutToDataChannel()
1050 {
1051     commUnsetConnTimeout(ctrl.conn);
1052
1053     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
1054     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this,
1055                                      Ftp::Client::timeout);
1056     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1057 }
1058
1059 void
1060 Ftp::Client::sentRequestBody(const CommIoCbParams &io)
1061 {
1062     if (io.size > 0)
1063         statCounter.server.ftp.kbytes_out += io.size;
1064     ::Client::sentRequestBody(io);
1065 }
1066
1067 /**
1068  * called after we wrote the last byte of the request body
1069  */
1070 void
1071 Ftp::Client::doneSendingRequestBody()
1072 {
1073     ::Client::doneSendingRequestBody();
1074     debugs(9, 3, status());
1075     dataComplete();
1076     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
1077      * if transfer type is 'stream' call dataComplete()
1078      * otherwise leave open. (reschedule control channel read?)
1079      */
1080 }
1081
1082 /// Parses FTP server control response into ctrl structure fields,
1083 /// setting bytesUsed and returning true on success.
1084 bool
1085 Ftp::Client::parseControlReply(size_t &bytesUsed)
1086 {
1087     char *s;
1088     char *sbuf;
1089     char *end;
1090     int usable;
1091     int complete = 0;
1092     wordlist *head = NULL;
1093     wordlist *list;
1094     wordlist **tail = &head;
1095     size_t linelen;
1096     debugs(9, 3, status());
1097     /*
1098      * We need a NULL-terminated buffer for scanning, ick
1099      */
1100     const size_t len = ctrl.offset;
1101     sbuf = (char *)xmalloc(len + 1);
1102     xstrncpy(sbuf, ctrl.buf, len + 1);
1103     end = sbuf + len - 1;
1104
1105     while (*end != '\r' && *end != '\n' && end > sbuf)
1106         --end;
1107
1108     usable = end - sbuf;
1109
1110     debugs(9, 3, "usable = " << usable);
1111
1112     if (usable == 0) {
1113         debugs(9, 3, "didn't find end of line");
1114         safe_free(sbuf);
1115         return false;
1116     }
1117
1118     debugs(9, 3, len << " bytes to play with");
1119     ++end;
1120     s = sbuf;
1121     s += strspn(s, crlf);
1122
1123     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1124         if (complete)
1125             break;
1126
1127         debugs(9, 5, "s = {" << s << "}");
1128
1129         linelen = strcspn(s, crlf) + 1;
1130
1131         if (linelen < 2)
1132             break;
1133
1134         if (linelen > 3)
1135             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1136
1137         list = new wordlist();
1138
1139         list->key = (char *)xmalloc(linelen);
1140
1141         xstrncpy(list->key, s, linelen);
1142
1143         /* trace the FTP communication chat at level 2 */
1144         debugs(9, 2, "ftp>> " << list->key);
1145
1146         if (complete) {
1147             // use list->key for last_reply because s contains the new line
1148             ctrl.last_reply = xstrdup(list->key + 4);
1149             ctrl.replycode = atoi(list->key);
1150         }
1151
1152         *tail = list;
1153
1154         tail = &list->next;
1155     }
1156
1157     bytesUsed = static_cast<size_t>(s - sbuf);
1158     safe_free(sbuf);
1159
1160     if (!complete) {
1161         wordlistDestroy(&head);
1162         return false;
1163     }
1164
1165     ctrl.message = head;
1166     assert(ctrl.replycode >= 0);
1167     assert(ctrl.last_reply);
1168     assert(ctrl.message);
1169     return true;
1170 }
1171
1172 }; // namespace Ftp
1173