src/clients/FtpClient.cc

   1 /*
   2  * Copyright (C) 1996-2018 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "client_side.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm/ConnOpener.h"
  16 #include "comm/Read.h"
  17 #include "comm/TcpAcceptor.h"
  18 #include "comm/Write.h"
  19 #include "errorpage.h"
  20 #include "fd.h"
  21 #include "ftp/Parsing.h"
  22 #include "http/Stream.h"
  23 #include "ip/tools.h"
  24 #include "SquidConfig.h"
  25 #include "SquidString.h"
  26 #include "StatCounters.h"
  27 #include "tools.h"
  28 #include "wordlist.h"
  29
  30 #include <set>
  31
  32 namespace Ftp
  33 {
  34
  35 const char *const crlf = "\r\n";
  36
  37 static char *
  38 escapeIAC(const char *buf)
  39 {
  40     int n;
  41     char *ret;
  42     unsigned const char *p;
  43     unsigned char *r;
  44
  45     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
  46         if (*p == 255)
  47             ++n;
  48
  49     ret = (char *)xmalloc(n);
  50
  51     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
  52         *r = *p;
  53         ++r;
  54
  55         if (*p == 255) {
  56             *r = 255;
  57             ++r;
  58         }
  59     }
  60
  61     *r = '\0';
  62     ++r;
  63     assert((r - (unsigned char *)ret) == n );
  64     return ret;
  65 }
  66
  67 /* Ftp::Channel */
  68
  69 /// configures the channel with a descriptor and registers a close handler
  70 void
  71 Ftp::Channel::opened(const Comm::ConnectionPointer &newConn,
  72                      const AsyncCall::Pointer &aCloser)
  73 {
  74     assert(!Comm::IsConnOpen(conn));
  75     assert(closer == NULL);
  76
  77     assert(Comm::IsConnOpen(newConn));
  78     assert(aCloser != NULL);
  79
  80     conn = newConn;
  81     closer = aCloser;
  82     comm_add_close_handler(conn->fd, closer);
  83 }
  84
  85 /// planned close: removes the close handler and calls comm_close
  86 void
  87 Ftp::Channel::close()
  88 {
  89     // channels with active listeners will be closed when the listener handler dies.
  90     if (Comm::IsConnOpen(conn)) {
  91         comm_remove_close_handler(conn->fd, closer);
  92         conn->close(); // we do not expect to be called back
  93     }
  94     clear();
  95 }
  96
  97 void
  98 Ftp::Channel::forget()
  99 {
 100     if (Comm::IsConnOpen(conn)) {
 101         commUnsetConnTimeout(conn);
 102         comm_remove_close_handler(conn->fd, closer);
 103     }
 104     clear();
 105 }
 106
 107 void
 108 Ftp::Channel::clear()
 109 {
 110     conn = NULL;
 111     closer = NULL;
 112 }
 113
 114 /* Ftp::CtrlChannel */
 115
 116 Ftp::CtrlChannel::CtrlChannel():
 117     buf(NULL),
 118     size(0),
 119     offset(0),
 120     message(NULL),
 121     last_command(NULL),
 122     last_reply(NULL),
 123     replycode(0)
 124 {
 125     buf = static_cast<char*>(memAllocBuf(4096, &size));
 126 }
 127
 128 Ftp::CtrlChannel::~CtrlChannel()
 129 {
 130     memFreeBuf(size, buf);
 131     if (message)
 132         wordlistDestroy(&message);
 133     safe_free(last_command);
 134     safe_free(last_reply);
 135 }
 136
 137 /* Ftp::DataChannel */
 138
 139 Ftp::DataChannel::DataChannel():
 140     readBuf(NULL),
 141     host(NULL),
 142     port(0),
 143     read_pending(false)
 144 {
 145 }
 146
 147 Ftp::DataChannel::~DataChannel()
 148 {
 149     delete readBuf;
 150 }
 151
 152 void
 153 Ftp::DataChannel::addr(const Ip::Address &import)
 154 {
 155     static char addrBuf[MAX_IPSTRLEN];
 156     import.toStr(addrBuf, sizeof(addrBuf));
 157     xfree(host);
 158     host = xstrdup(addrBuf);
 159     port = import.port();
 160 }
 161
 162 /* Ftp::Client */
 163
 164 Ftp::Client::Client(FwdState *fwdState):
 165     AsyncJob("Ftp::Client"),
 166     ::Client(fwdState),
 167      ctrl(),
 168      data(),
 169      state(BEGIN),
 170      old_request(NULL),
 171      old_reply(NULL),
 172      shortenReadTimeout(false)
 173 {
 174     ++statCounter.server.all.requests;
 175     ++statCounter.server.ftp.requests;
 176
 177     ctrl.last_command = xstrdup("Connect to server");
 178
 179     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 180     const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this,
 181                                       Ftp::Client::ctrlClosed);
 182     ctrl.opened(fwdState->serverConnection(), closer);
 183 }
 184
 185 Ftp::Client::~Client()
 186 {
 187     if (data.opener != NULL) {
 188         data.opener->cancel("Ftp::Client destructed");
 189         data.opener = NULL;
 190     }
 191     data.close();
 192
 193     safe_free(old_request);
 194     safe_free(old_reply);
 195     fwd = NULL; // refcounted
 196 }
 197
 198 void
 199 Ftp::Client::start()
 200 {
 201     scheduleReadControlReply(0);
 202 }
 203
 204 void
 205 Ftp::Client::initReadBuf()
 206 {
 207     if (data.readBuf == NULL) {
 208         data.readBuf = new MemBuf;
 209         data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 210     }
 211 }
 212
 213 /**
 214  * Close the FTP server connection(s). Used by serverComplete().
 215  */
 216 void
 217 Ftp::Client::closeServer()
 218 {
 219     if (Comm::IsConnOpen(ctrl.conn)) {
 220         debugs(9, 3, "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
 221         fwd->unregister(ctrl.conn);
 222         ctrl.close();
 223     }
 224
 225     if (Comm::IsConnOpen(data.conn)) {
 226         debugs(9, 3, "closing FTP data FD " << data.conn->fd << ", this " << this);
 227         data.close();
 228     }
 229
 230     debugs(9, 3, "FTP ctrl and data connections closed. this " << this);
 231 }
 232
 233 /**
 234  * Did we close all FTP server connection(s)?
 235  *
 236  \retval true  Both server control and data channels are closed. And not waiting for a new data connection to open.
 237  \retval false Either control channel or data is still active.
 238  */
 239 bool
 240 Ftp::Client::doneWithServer() const
 241 {
 242     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
 243 }
 244
 245 void
 246 Ftp::Client::failed(err_type error, int xerrno, ErrorState *err)
 247 {
 248     debugs(9, 3, "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
 249
 250     const char *command, *reply;
 251     ErrorState *ftperr;
 252
 253     if (err) {
 254         debugs(9, 6, "error=" << err->type << ", code=" << xerrno <<
 255                ", status=" << err->httpStatus);
 256         error = err->type;
 257         ftperr = err;
 258     } else {
 259         Http::StatusCode httpStatus = failedHttpStatus(error);
 260         ftperr = new ErrorState(error, httpStatus, fwd->request);
 261     }
 262
 263     ftperr->xerrno = xerrno;
 264
 265     ftperr->ftp.server_msg = ctrl.message;
 266     ctrl.message = NULL;
 267
 268     if (old_request)
 269         command = old_request;
 270     else
 271         command = ctrl.last_command;
 272
 273     if (command && strncmp(command, "PASS", 4) == 0)
 274         command = "PASS <yourpassword>";
 275
 276     if (old_reply)
 277         reply = old_reply;
 278     else
 279         reply = ctrl.last_reply;
 280
 281     if (command)
 282         ftperr->ftp.request = xstrdup(command);
 283
 284     if (reply)
 285         ftperr->ftp.reply = xstrdup(reply);
 286
 287     if (!err) {
 288         fwd->request->detailError(error, xerrno);
 289         fwd->fail(ftperr);
 290         closeServer(); // we failed, so no serverComplete()
 291     }
 292 }
 293
 294 Http::StatusCode
 295 Ftp::Client::failedHttpStatus(err_type &error)
 296 {
 297     if (error == ERR_NONE)
 298         error = ERR_FTP_FAILURE;
 299     return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout :
 300            Http::scBadGateway;
 301 }
 302
 303 /**
 304  * DPW 2007-04-23
 305  * Looks like there are no longer anymore callers that set
 306  * buffered_ok=1.  Perhaps it can be removed at some point.
 307  */
 308 void
 309 Ftp::Client::scheduleReadControlReply(int buffered_ok)
 310 {
 311     debugs(9, 3, ctrl.conn);
 312
 313     if (buffered_ok && ctrl.offset > 0) {
 314         /* We've already read some reply data */
 315         handleControlReply();
 316     } else {
 317
 318         if (!Comm::IsConnOpen(ctrl.conn)) {
 319             debugs(9, 3, "cannot read without ctrl " << ctrl.conn);
 320             return;
 321         }
 322         /*
 323          * Cancel the timeout on the Data socket (if any) and
 324          * establish one on the control socket.
 325          */
 326         if (Comm::IsConnOpen(data.conn)) {
 327             commUnsetConnTimeout(data.conn);
 328         }
 329
 330         const time_t tout = shortenReadTimeout ?
 331                             min(Config.Timeout.connect, Config.Timeout.read):
 332                             Config.Timeout.read;
 333         shortenReadTimeout = false; // we only need to do this once, after PASV
 334
 335         typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 336         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, Ftp::Client::timeout);
 337         commSetConnTimeout(ctrl.conn, tout, timeoutCall);
 338
 339         typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 340         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, Ftp::Client::readControlReply);
 341         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
 342     }
 343 }
 344
 345 void
 346 Ftp::Client::readControlReply(const CommIoCbParams &io)
 347 {
 348     debugs(9, 3, "FD " << io.fd << ", Read " << io.size << " bytes");
 349
 350     if (io.size > 0) {
 351         statCounter.server.all.kbytes_in += io.size;
 352         statCounter.server.ftp.kbytes_in += io.size;
 353     }
 354
 355     if (io.flag == Comm::ERR_CLOSING)
 356         return;
 357
 358     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 359         if (abortOnData("entry aborted during control reply read"))
 360             return;
 361     }
 362
 363     assert(ctrl.offset < ctrl.size);
 364
 365     if (io.flag == Comm::OK && io.size > 0) {
 366         fd_bytes(io.fd, io.size, FD_READ);
 367     }
 368
 369     if (io.flag != Comm::OK) {
 370         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 371                "FTP control reply read error: " << xstrerr(io.xerrno));
 372
 373         if (ignoreErrno(io.xerrno)) {
 374             scheduleReadControlReply(0);
 375         } else {
 376             failed(ERR_READ_ERROR, io.xerrno);
 377             /* failed closes ctrl.conn and frees ftpState */
 378         }
 379         return;
 380     }
 381
 382     if (io.size == 0) {
 383         if (entry->store_status == STORE_PENDING) {
 384             failed(ERR_FTP_FAILURE, 0);
 385             /* failed closes ctrl.conn and frees ftpState */
 386             return;
 387         }
 388
 389         /* XXX this may end up having to be serverComplete() .. */
 390         abortAll("zero control reply read");
 391         return;
 392     }
 393
 394     unsigned int len =io.size + ctrl.offset;
 395     ctrl.offset = len;
 396     assert(len <= ctrl.size);
 397     if (Comm::IsConnOpen(ctrl.conn))
 398         commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply
 399     handleControlReply();
 400 }
 401
 402 void
 403 Ftp::Client::handleControlReply()
 404 {
 405     debugs(9, 3, status());
 406
 407     size_t bytes_used = 0;
 408     wordlistDestroy(&ctrl.message);
 409
 410     if (!parseControlReply(bytes_used)) {
 411         /* didn't get complete reply yet */
 412
 413         if (ctrl.offset == ctrl.size) {
 414             ctrl.buf = static_cast<char*>(memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size));
 415         }
 416
 417         scheduleReadControlReply(0);
 418         return;
 419     }
 420
 421     assert(ctrl.message); // the entire FTP server response, line by line
 422     assert(ctrl.replycode >= 0); // FTP status code (from the last line)
 423     assert(ctrl.last_reply); // FTP reason (from the last line)
 424
 425     if (ctrl.offset == bytes_used) {
 426         /* used it all up */
 427         ctrl.offset = 0;
 428     } else {
 429         /* Got some data past the complete reply */
 430         assert(bytes_used < ctrl.offset);
 431         ctrl.offset -= bytes_used;
 432         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
 433     }
 434
 435     debugs(9, 3, "state=" << state << ", code=" << ctrl.replycode);
 436 }
 437
 438 bool
 439 Ftp::Client::handlePasvReply(Ip::Address &srvAddr)
 440 {
 441     int code = ctrl.replycode;
 442     char *buf;
 443     debugs(9, 3, status());
 444
 445     if (!Comm::IsConnOpen(ctrl.conn)) {
 446         debugs(9, 5, "The control connection to the remote end is closed");
 447         return false;
 448     }
 449
 450     if (code != 227) {
 451         debugs(9, 2, "PASV not supported by remote end");
 452         return false;
 453     }
 454
 455     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
 456     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 457     debugs(9, 5, "scanning: " << ctrl.last_reply);
 458
 459     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789");
 460
 461     const char *forceIp = Config.Ftp.sanitycheck ?
 462                           fd_table[ctrl.conn->fd].ipaddr : NULL;
 463     if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) {
 464         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
 465                ctrl.conn->remote << ": " << ctrl.last_reply);
 466         return false;
 467     }
 468
 469     data.addr(srvAddr);
 470
 471     return true;
 472 }
 473
 474 bool
 475 Ftp::Client::handleEpsvReply(Ip::Address &remoteAddr)
 476 {
 477     int code = ctrl.replycode;
 478     char *buf;
 479     debugs(9, 3, status());
 480
 481     if (!Comm::IsConnOpen(ctrl.conn)) {
 482         debugs(9, 5, "The control connection to the remote end is closed");
 483         return false;
 484     }
 485
 486     if (code != 229 && code != 522) {
 487         if (code == 200) {
 488             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
 489             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
 490              * Its okay to re-send EPSV 1/2 but nothing else. */
 491             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV");
 492         } else {
 493             debugs(9, 2, "EPSV not supported by remote end");
 494         }
 495         return sendPassive();
 496     }
 497
 498     if (code == 522) {
 499         /* Peer responded with a list of supported methods:
 500          *   522 Network protocol not supported, use (1)
 501          *   522 Network protocol not supported, use (1,2)
 502          *   522 Network protocol not supported, use (2)
 503          * TODO: Handle the (1,2) case which may happen after EPSV ALL. Close
 504          * data + control without self-destructing and re-open from scratch.
 505          */
 506         debugs(9, 5, "scanning: " << ctrl.last_reply);
 507         buf = ctrl.last_reply;
 508         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
 509             ++buf;
 510         if (buf != NULL && *buf == '\n')
 511             ++buf;
 512
 513         if (buf == NULL || *buf == '\0') {
 514             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 515             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
 516             return sendPassive();
 517         } else if (strcmp(buf, "(1)") == 0) {
 518             state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
 519             return sendPassive();
 520         } else if (strcmp(buf, "(2)") == 0) {
 521             if (Ip::EnableIpv6) {
 522                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
 523                 if (state == SENT_EPSV_2) {
 524                     return sendEprt();
 525                 } else {
 526                     /* or try the next Passive mode down the chain. */
 527                     return sendPassive();
 528                 }
 529             } else {
 530                 /* Server only accept EPSV in IPv6 traffic. */
 531                 state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
 532                 return sendPassive();
 533             }
 534         } else {
 535             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 536             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
 537             return sendPassive();
 538         }
 539         /* coverity[unreachable] */
 540         /* safeguard against possible future bugs in above conditions */
 541         failed(ERR_FTP_FAILURE, 0);
 542         return false;
 543     }
 544
 545     /*  229 Entering Extended Passive Mode (|||port|) */
 546     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 547     debugs(9, 5, "scanning: " << ctrl.last_reply);
 548
 549     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "(");
 550
 551     char h1, h2, h3, h4;
 552     unsigned short port;
 553     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
 554
 555     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
 556         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
 557                ctrl.conn->remote << ": " <<
 558                ctrl.last_reply);
 559
 560         return sendPassive();
 561     }
 562
 563     if (0 == port) {
 564         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 565                ctrl.conn->remote << ": " <<
 566                ctrl.last_reply);
 567
 568         return sendPassive();
 569     }
 570
 571     if (Config.Ftp.sanitycheck) {
 572         if (port < 1024) {
 573             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 574                    ctrl.conn->remote << ": " <<
 575                    ctrl.last_reply);
 576
 577             return sendPassive();
 578         }
 579     }
 580
 581     remoteAddr = ctrl.conn->remote;
 582     remoteAddr.port(port);
 583     data.addr(remoteAddr);
 584     return true;
 585 }
 586
 587 // FTP clients do not support EPRT and PORT commands yet.
 588 // The Ftp::Client::sendEprt() will fail because of the unimplemented
 589 // openListenSocket() or sendPort() methods
 590 bool
 591 Ftp::Client::sendEprt()
 592 {
 593     if (!Config.Ftp.eprt) {
 594         /* Disabled. Switch immediately to attempting old PORT command. */
 595         debugs(9, 3, "EPRT disabled by local administrator");
 596         return sendPort();
 597     }
 598
 599     debugs(9, 3, status());
 600
 601     if (!openListenSocket()) {
 602         failed(ERR_FTP_FAILURE, 0);
 603         return false;
 604     }
 605
 606     debugs(9, 3, "Listening for FTP data connection with FD " << data.conn);
 607     if (!Comm::IsConnOpen(data.conn)) {
 608         // TODO: Set error message.
 609         failed(ERR_FTP_FAILURE, 0);
 610         return false;
 611     }
 612
 613     static MemBuf mb;
 614     mb.reset();
 615     char buf[MAX_IPSTRLEN];
 616     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
 617     /* Which can be used by EITHER protocol. */
 618     debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port());
 619     mb.appendf("EPRT |%d|%s|%d|%s",
 620                ( data.conn->local.isIPv6() ? 2 : 1 ),
 621                data.conn->local.toStr(buf,MAX_IPSTRLEN),
 622                comm_local_port(data.conn->fd), Ftp::crlf );
 623
 624     state = SENT_EPRT;
 625     writeCommand(mb.content());
 626     return true;
 627 }
 628
 629 bool
 630 Ftp::Client::sendPort()
 631 {
 632     failed(ERR_FTP_FAILURE, 0);
 633     return false;
 634 }
 635
 636 bool
 637 Ftp::Client::sendPassive()
 638 {
 639     debugs(9, 3, status());
 640
 641     /** \par
 642       * Checks for EPSV ALL special conditions:
 643       * If enabled to be sent, squid MUST NOT request any other connect methods.
 644       * If 'ALL' is sent and fails the entire FTP Session fails.
 645       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
 646     if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) {
 647         // We are here because the last "EPSV 1" failed, but because of epsv_all
 648         // no other method allowed.
 649         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 650         failed(ERR_FTP_FAILURE, 0);
 651         return false;
 652     }
 653
 654     /// Closes any old FTP-Data connection which may exist. */
 655     data.close();
 656
 657     /** \par
 658       * Checks for previous EPSV/PASV failures on this server/session.
 659       * Diverts to EPRT immediately if they are not working. */
 660     if (!Config.Ftp.passive || state == SENT_PASV) {
 661         sendEprt();
 662         return true;
 663     }
 664
 665     static MemBuf mb;
 666     mb.reset();
 667     /** \par
 668       * Send EPSV (ALL,2,1) or PASV on the control channel.
 669       *
 670       *  - EPSV ALL  is used if enabled.
 671       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
 672       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
 673       *  - PASV      is used if EPSV 1 fails.
 674       */
 675     switch (state) {
 676     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
 677         if (ctrl.conn->local.isIPv6()) {
 678             debugs(9, 5, "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
 679             mb.appendf("EPSV 2%s", Ftp::crlf);
 680             state = SENT_EPSV_2;
 681             break;
 682         }
 683     // else fall through to skip EPSV 2
 684
 685     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
 686         if (ctrl.conn->local.isIPv4()) {
 687             debugs(9, 5, "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
 688             mb.appendf("EPSV 1%s", Ftp::crlf);
 689             state = SENT_EPSV_1;
 690             break;
 691         } else if (Config.Ftp.epsv_all) {
 692             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 693             failed(ERR_FTP_FAILURE, 0);
 694             return false;
 695         }
 696     // else fall through to skip EPSV 1
 697
 698     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
 699         debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
 700         mb.appendf("PASV%s", Ftp::crlf);
 701         state = SENT_PASV;
 702         break;
 703
 704     default: {
 705         bool doEpsv = true;
 706         if (Config.accessList.ftp_epsv) {
 707             ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL);
 708             doEpsv = checklist.fastCheck().allowed();
 709         }
 710         if (!doEpsv) {
 711             debugs(9, 5, "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")");
 712             mb.appendf("PASV%s", Ftp::crlf);
 713             state = SENT_PASV;
 714         } else if (Config.Ftp.epsv_all) {
 715             debugs(9, 5, "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")");
 716             mb.appendf("EPSV ALL%s", Ftp::crlf);
 717             state = SENT_EPSV_ALL;
 718         } else {
 719             if (ctrl.conn->local.isIPv6()) {
 720                 debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2");
 721                 mb.appendf("EPSV 2%s", Ftp::crlf);
 722                 state = SENT_EPSV_2;
 723             }
 724             if (ctrl.conn->local.isIPv4()) {
 725                 debugs(9, 5, "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1");
 726                 mb.appendf("EPSV 1%s", Ftp::crlf);
 727                 state = SENT_EPSV_1;
 728             }
 729         }
 730         break;
 731     }
 732     }
 733
 734     if (ctrl.message)
 735         wordlistDestroy(&ctrl.message);
 736     ctrl.message = NULL; //No message to return to client.
 737     ctrl.offset = 0; //reset readed response, to make room read the next response
 738
 739     writeCommand(mb.content());
 740
 741     shortenReadTimeout = true;
 742     return true;
 743 }
 744
 745 void
 746 Ftp::Client::connectDataChannel()
 747 {
 748     if (!Comm::IsConnOpen(ctrl.conn)) {
 749         debugs(9, 5, "The control connection to the remote end is closed");
 750         return;
 751     }
 752
 753     safe_free(ctrl.last_command);
 754
 755     safe_free(ctrl.last_reply);
 756
 757     ctrl.last_command = xstrdup("Connect to server data port");
 758
 759     // Generate a new data channel descriptor to be opened.
 760     Comm::ConnectionPointer conn = new Comm::Connection;
 761     conn->setAddrs(ctrl.conn->local, data.host);
 762     conn->local.port(0);
 763     conn->remote.port(data.port);
 764     conn->tos = ctrl.conn->tos;
 765     conn->nfmark = ctrl.conn->nfmark;
 766
 767     debugs(9, 3, "connecting to " << conn->remote);
 768
 769     typedef CommCbMemFunT<Client, CommConnectCbParams> Dialer;
 770     data.opener = JobCallback(9, 3, Dialer, this, Ftp::Client::dataChannelConnected);
 771     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect);
 772     cs->setHost(data.host);
 773     AsyncJob::Start(cs);
 774 }
 775
 776 bool
 777 Ftp::Client::openListenSocket()
 778 {
 779     return false;
 780 }
 781
 782 /// creates a data channel Comm close callback
 783 AsyncCall::Pointer
 784 Ftp::Client::dataCloser()
 785 {
 786     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 787     return JobCallback(9, 5, Dialer, this, Ftp::Client::dataClosed);
 788 }
 789
 790 /// handler called by Comm when FTP data channel is closed unexpectedly
 791 void
 792 Ftp::Client::dataClosed(const CommCloseCbParams &)
 793 {
 794     debugs(9, 4, status());
 795     if (data.listenConn != NULL) {
 796         data.listenConn->close();
 797         data.listenConn = NULL;
 798         // NP clear() does the: data.fd = -1;
 799     }
 800     data.clear();
 801 }
 802
 803 void
 804 Ftp::Client::writeCommand(const char *buf)
 805 {
 806     char *ebuf;
 807     /* trace FTP protocol communications at level 2 */
 808     debugs(9, 2, "ftp<< " << buf);
 809
 810     if (Config.Ftp.telnet)
 811         ebuf = escapeIAC(buf);
 812     else
 813         ebuf = xstrdup(buf);
 814
 815     safe_free(ctrl.last_command);
 816
 817     safe_free(ctrl.last_reply);
 818
 819     ctrl.last_command = ebuf;
 820
 821     if (!Comm::IsConnOpen(ctrl.conn)) {
 822         debugs(9, 2, "cannot send to closing ctrl " << ctrl.conn);
 823         // TODO: assert(ctrl.closer != NULL);
 824         return;
 825     }
 826
 827     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 828     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this,
 829                                           Ftp::Client::writeCommandCallback);
 830     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
 831
 832     scheduleReadControlReply(0);
 833 }
 834
 835 void
 836 Ftp::Client::writeCommandCallback(const CommIoCbParams &io)
 837 {
 838
 839     debugs(9, 5, "wrote " << io.size << " bytes");
 840
 841     if (io.size > 0) {
 842         fd_bytes(io.fd, io.size, FD_WRITE);
 843         statCounter.server.all.kbytes_out += io.size;
 844         statCounter.server.ftp.kbytes_out += io.size;
 845     }
 846
 847     if (io.flag == Comm::ERR_CLOSING)
 848         return;
 849
 850     if (io.flag) {
 851         debugs(9, DBG_IMPORTANT, "FTP command write error: " << io.conn << ": " << xstrerr(io.xerrno));
 852         failed(ERR_WRITE_ERROR, io.xerrno);
 853         /* failed closes ctrl.conn and frees ftpState */
 854         return;
 855     }
 856 }
 857
 858 /// handler called by Comm when FTP control channel is closed unexpectedly
 859 void
 860 Ftp::Client::ctrlClosed(const CommCloseCbParams &)
 861 {
 862     debugs(9, 4, status());
 863     ctrl.clear();
 864     doneWithFwd = "ctrlClosed()"; // assume FwdState is monitoring too
 865     mustStop("Ftp::Client::ctrlClosed");
 866 }
 867
 868 void
 869 Ftp::Client::timeout(const CommTimeoutCbParams &io)
 870 {
 871     debugs(9, 4, io.conn << ": '" << entry->url() << "'" );
 872
 873     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 874         return;
 875
 876     failed(ERR_READ_TIMEOUT, 0);
 877     /* failed() closes ctrl.conn and frees ftpState */
 878 }
 879
 880 const Comm::ConnectionPointer &
 881 Ftp::Client::dataConnection() const
 882 {
 883     return data.conn;
 884 }
 885
 886 void
 887 Ftp::Client::maybeReadVirginBody()
 888 {
 889     // too late to read
 890     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
 891         return;
 892
 893     if (data.read_pending)
 894         return;
 895
 896     initReadBuf();
 897
 898     const int read_sz = replyBodySpace(*data.readBuf, 0);
 899
 900     debugs(9, 9, "FTP may read up to " << read_sz << " bytes");
 901
 902     if (read_sz < 2) // see http.cc
 903         return;
 904
 905     data.read_pending = true;
 906
 907     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 908     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
 909                                       TimeoutDialer, this, Ftp::Client::timeout);
 910     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 911
 912     debugs(9,5,"queueing read on FD " << data.conn->fd);
 913
 914     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 915     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
 916                           JobCallback(9, 5, Dialer, this, Ftp::Client::dataRead));
 917 }
 918
 919 void
 920 Ftp::Client::dataRead(const CommIoCbParams &io)
 921 {
 922     int j;
 923     int bin;
 924
 925     data.read_pending = false;
 926
 927     debugs(9, 3, "FD " << io.fd << " Read " << io.size << " bytes");
 928
 929     if (io.size > 0) {
 930         statCounter.server.all.kbytes_in += io.size;
 931         statCounter.server.ftp.kbytes_in += io.size;
 932     }
 933
 934     if (io.flag == Comm::ERR_CLOSING)
 935         return;
 936
 937     assert(io.fd == data.conn->fd);
 938
 939     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 940         abortOnData("entry aborted during dataRead");
 941         return;
 942     }
 943
 944     if (io.flag == Comm::OK && io.size > 0) {
 945         debugs(9, 5, "appended " << io.size << " bytes to readBuf");
 946         data.readBuf->appended(io.size);
 947 #if USE_DELAY_POOLS
 948         DelayId delayId = entry->mem_obj->mostBytesAllowed();
 949         delayId.bytesIn(io.size);
 950 #endif
 951         ++ IOStats.Ftp.reads;
 952
 953         for (j = io.size - 1, bin = 0; j; ++bin)
 954             j >>= 1;
 955
 956         ++ IOStats.Ftp.read_hist[bin];
 957     }
 958
 959     if (io.flag != Comm::OK) {
 960         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 961                "FTP data read error: " << xstrerr(io.xerrno));
 962
 963         if (ignoreErrno(io.xerrno)) {
 964             maybeReadVirginBody();
 965         } else {
 966             failed(ERR_READ_ERROR, 0);
 967             /* failed closes ctrl.conn and frees ftpState */
 968             return;
 969         }
 970     } else if (io.size == 0) {
 971         debugs(9, 3, "Calling dataComplete() because io.size == 0");
 972         /*
 973          * DPW 2007-04-23
 974          * Dangerous curves ahead.  This call to dataComplete was
 975          * calling scheduleReadControlReply, handleControlReply,
 976          * and then ftpReadTransferDone.  If ftpReadTransferDone
 977          * gets unexpected status code, it closes down the control
 978          * socket and our FtpStateData object gets destroyed.   As
 979          * a workaround we no longer set the 'buffered_ok' flag in
 980          * the scheduleReadControlReply call.
 981          */
 982         dataComplete();
 983     }
 984
 985     processReplyBody();
 986 }
 987
 988 void
 989 Ftp::Client::dataComplete()
 990 {
 991     debugs(9, 3,status());
 992
 993     /* Connection closed; transfer done. */
 994
 995     /// Close data channel, if any, to conserve resources while we wait.
 996     data.close();
 997
 998     /* expect the "transfer complete" message on the control socket */
 999     /*
1000      * DPW 2007-04-23
1001      * Previously, this was the only place where we set the
1002      * 'buffered_ok' flag when calling scheduleReadControlReply().
1003      * It caused some problems if the FTP server returns an unexpected
1004      * status code after the data command.  FtpStateData was being
1005      * deleted in the middle of dataRead().
1006      */
1007     /* AYJ: 2011-01-13: Bug 2581.
1008      * 226 status is possibly waiting in the ctrl buffer.
1009      * The connection will hang if we DONT send buffered_ok.
1010      * This happens on all transfers which can be completly sent by the
1011      * server before the 150 started status message is read in by Squid.
1012      * ie all transfers of about one packet hang.
1013      */
1014     scheduleReadControlReply(1);
1015 }
1016
1017 void
1018 Ftp::Client::abortAll(const char *reason)
1019 {
1020     debugs(9, 3, "aborting transaction for " << reason <<
1021            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
1022     mustStop(reason);
1023 }
1024
1025 /**
1026  * Cancel the timeout on the Control socket and establish one
1027  * on the data socket
1028  */
1029 void
1030 Ftp::Client::switchTimeoutToDataChannel()
1031 {
1032     commUnsetConnTimeout(ctrl.conn);
1033
1034     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
1035     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this,
1036                                      Ftp::Client::timeout);
1037     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1038 }
1039
1040 void
1041 Ftp::Client::sentRequestBody(const CommIoCbParams &io)
1042 {
1043     if (io.size > 0)
1044         statCounter.server.ftp.kbytes_out += io.size;
1045     ::Client::sentRequestBody(io);
1046 }
1047
1048 /**
1049  * called after we wrote the last byte of the request body
1050  */
1051 void
1052 Ftp::Client::doneSendingRequestBody()
1053 {
1054     ::Client::doneSendingRequestBody();
1055     debugs(9, 3, status());
1056     dataComplete();
1057     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
1058      * if transfer type is 'stream' call dataComplete()
1059      * otherwise leave open. (reschedule control channel read?)
1060      */
1061 }
1062
1063 /// Parses FTP server control response into ctrl structure fields,
1064 /// setting bytesUsed and returning true on success.
1065 bool
1066 Ftp::Client::parseControlReply(size_t &bytesUsed)
1067 {
1068     char *s;
1069     char *sbuf;
1070     char *end;
1071     int usable;
1072     int complete = 0;
1073     wordlist *head = NULL;
1074     wordlist *list;
1075     wordlist **tail = &head;
1076     size_t linelen;
1077     debugs(9, 3, status());
1078     /*
1079      * We need a NULL-terminated buffer for scanning, ick
1080      */
1081     const size_t len = ctrl.offset;
1082     sbuf = (char *)xmalloc(len + 1);
1083     xstrncpy(sbuf, ctrl.buf, len + 1);
1084     end = sbuf + len - 1;
1085
1086     while (*end != '\r' && *end != '\n' && end > sbuf)
1087         --end;
1088
1089     usable = end - sbuf;
1090
1091     debugs(9, 3, "usable = " << usable);
1092
1093     if (usable == 0) {
1094         debugs(9, 3, "didn't find end of line");
1095         safe_free(sbuf);
1096         return false;
1097     }
1098
1099     debugs(9, 3, len << " bytes to play with");
1100     ++end;
1101     s = sbuf;
1102     s += strspn(s, crlf);
1103
1104     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1105         if (complete)
1106             break;
1107
1108         debugs(9, 5, "s = {" << s << "}");
1109
1110         linelen = strcspn(s, crlf) + 1;
1111
1112         if (linelen < 2)
1113             break;
1114
1115         if (linelen > 3)
1116             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1117
1118         list = new wordlist();
1119
1120         list->key = (char *)xmalloc(linelen);
1121
1122         xstrncpy(list->key, s, linelen);
1123
1124         /* trace the FTP communication chat at level 2 */
1125         debugs(9, 2, "ftp>> " << list->key);
1126
1127         if (complete) {
1128             // use list->key for last_reply because s contains the new line
1129             ctrl.last_reply = xstrdup(list->key + 4);
1130             ctrl.replycode = atoi(list->key);
1131         }
1132
1133         *tail = list;
1134
1135         tail = &list->next;
1136     }
1137
1138     bytesUsed = static_cast<size_t>(s - sbuf);
1139     safe_free(sbuf);
1140
1141     if (!complete) {
1142         wordlistDestroy(&head);
1143         return false;
1144     }
1145
1146     ctrl.message = head;
1147     assert(ctrl.replycode >= 0);
1148     assert(ctrl.last_reply);
1149     assert(ctrl.message);
1150     return true;
1151 }
1152
1153 }; // namespace Ftp
1154