src/clients/FtpClient.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "client_side.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm/ConnOpener.h"
  16 #include "comm/Read.h"
  17 #include "comm/TcpAcceptor.h"
  18 #include "comm/Write.h"
  19 #include "errorpage.h"
  20 #include "fd.h"
  21 #include "ftp/Parsing.h"
  22 #include "http/Stream.h"
  23 #include "ip/tools.h"
  24 #include "SquidConfig.h"
  25 #include "SquidString.h"
  26 #include "StatCounters.h"
  27 #include "tools.h"
  28 #include "wordlist.h"
  29
  30 #include <set>
  31
  32 namespace Ftp
  33 {
  34
  35 const char *const crlf = "\r\n";
  36
  37 static char *
  38 escapeIAC(const char *buf)
  39 {
  40     int n;
  41     char *ret;
  42     unsigned const char *p;
  43     unsigned char *r;
  44
  45     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
  46         if (*p == 255)
  47             ++n;
  48
  49     ret = (char *)xmalloc(n);
  50
  51     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
  52         *r = *p;
  53         ++r;
  54
  55         if (*p == 255) {
  56             *r = 255;
  57             ++r;
  58         }
  59     }
  60
  61     *r = '\0';
  62     ++r;
  63     assert((r - (unsigned char *)ret) == n );
  64     return ret;
  65 }
  66
  67 /* Ftp::Channel */
  68
  69 /// configures the channel with a descriptor and registers a close handler
  70 void
  71 Ftp::Channel::opened(const Comm::ConnectionPointer &newConn,
  72                      const AsyncCall::Pointer &aCloser)
  73 {
  74     assert(!Comm::IsConnOpen(conn));
  75     assert(closer == NULL);
  76
  77     assert(Comm::IsConnOpen(newConn));
  78     assert(aCloser != NULL);
  79
  80     conn = newConn;
  81     closer = aCloser;
  82     comm_add_close_handler(conn->fd, closer);
  83 }
  84
  85 /// planned close: removes the close handler and calls comm_close
  86 void
  87 Ftp::Channel::close()
  88 {
  89     // channels with active listeners will be closed when the listener handler dies.
  90     if (Comm::IsConnOpen(conn)) {
  91         comm_remove_close_handler(conn->fd, closer);
  92         conn->close(); // we do not expect to be called back
  93     }
  94     clear();
  95 }
  96
  97 void
  98 Ftp::Channel::forget()
  99 {
 100     if (Comm::IsConnOpen(conn)) {
 101         commUnsetConnTimeout(conn);
 102         comm_remove_close_handler(conn->fd, closer);
 103     }
 104     clear();
 105 }
 106
 107 void
 108 Ftp::Channel::clear()
 109 {
 110     conn = NULL;
 111     closer = NULL;
 112 }
 113
 114 /* Ftp::CtrlChannel */
 115
 116 Ftp::CtrlChannel::CtrlChannel():
 117     buf(NULL),
 118     size(0),
 119     offset(0),
 120     message(NULL),
 121     last_command(NULL),
 122     last_reply(NULL),
 123     replycode(0)
 124 {
 125     buf = static_cast<char*>(memAllocBuf(4096, &size));
 126 }
 127
 128 Ftp::CtrlChannel::~CtrlChannel()
 129 {
 130     memFreeBuf(size, buf);
 131     if (message)
 132         wordlistDestroy(&message);
 133     safe_free(last_command);
 134     safe_free(last_reply);
 135 }
 136
 137 /* Ftp::DataChannel */
 138
 139 Ftp::DataChannel::DataChannel():
 140     readBuf(NULL),
 141     host(NULL),
 142     port(0),
 143     read_pending(false)
 144 {
 145 }
 146
 147 Ftp::DataChannel::~DataChannel()
 148 {
 149     delete readBuf;
 150 }
 151
 152 void
 153 Ftp::DataChannel::addr(const Ip::Address &import)
 154 {
 155     static char addrBuf[MAX_IPSTRLEN];
 156     import.toStr(addrBuf, sizeof(addrBuf));
 157     xfree(host);
 158     host = xstrdup(addrBuf);
 159     port = import.port();
 160 }
 161
 162 /* Ftp::Client */
 163
 164 Ftp::Client::Client(FwdState *fwdState):
 165     AsyncJob("Ftp::Client"),
 166     ::Client(fwdState),
 167      ctrl(),
 168      data(),
 169      state(BEGIN),
 170      old_request(NULL),
 171      old_reply(NULL),
 172      shortenReadTimeout(false)
 173 {
 174     ++statCounter.server.all.requests;
 175     ++statCounter.server.ftp.requests;
 176
 177     ctrl.last_command = xstrdup("Connect to server");
 178
 179     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 180     const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this,
 181                                       Ftp::Client::ctrlClosed);
 182     ctrl.opened(fwdState->serverConnection(), closer);
 183 }
 184
 185 Ftp::Client::~Client()
 186 {
 187     if (data.opener != NULL) {
 188         data.opener->cancel("Ftp::Client destructed");
 189         data.opener = NULL;
 190     }
 191     data.close();
 192
 193     safe_free(old_request);
 194     safe_free(old_reply);
 195     fwd = NULL; // refcounted
 196 }
 197
 198 void
 199 Ftp::Client::start()
 200 {
 201     scheduleReadControlReply(0);
 202 }
 203
 204 void
 205 Ftp::Client::initReadBuf()
 206 {
 207     if (data.readBuf == NULL) {
 208         data.readBuf = new MemBuf;
 209         data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 210     }
 211 }
 212
 213 /**
 214  * Close the FTP server connection(s). Used by serverComplete().
 215  */
 216 void
 217 Ftp::Client::closeServer()
 218 {
 219     if (Comm::IsConnOpen(ctrl.conn)) {
 220         debugs(9, 3, "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
 221         fwd->unregister(ctrl.conn);
 222         ctrl.close();
 223     }
 224
 225     if (Comm::IsConnOpen(data.conn)) {
 226         debugs(9, 3, "closing FTP data FD " << data.conn->fd << ", this " << this);
 227         data.close();
 228     }
 229
 230     debugs(9, 3, "FTP ctrl and data connections closed. this " << this);
 231 }
 232
 233 /**
 234  * Did we close all FTP server connection(s)?
 235  *
 236  \retval true  Both server control and data channels are closed. And not waiting for a new data connection to open.
 237  \retval false Either control channel or data is still active.
 238  */
 239 bool
 240 Ftp::Client::doneWithServer() const
 241 {
 242     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
 243 }
 244
 245 void
 246 Ftp::Client::failed(err_type error, int xerrno, ErrorState *err)
 247 {
 248     debugs(9, 3, "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
 249
 250     const char *command, *reply;
 251     ErrorState *ftperr;
 252
 253     if (err) {
 254         debugs(9, 6, "error=" << err->type << ", code=" << xerrno <<
 255                ", status=" << err->httpStatus);
 256         error = err->type;
 257         ftperr = err;
 258     } else {
 259         Http::StatusCode httpStatus = failedHttpStatus(error);
 260         ftperr = new ErrorState(error, httpStatus, fwd->request);
 261     }
 262
 263     ftperr->xerrno = xerrno;
 264
 265     ftperr->ftp.server_msg = ctrl.message;
 266     ctrl.message = NULL;
 267
 268     if (old_request)
 269         command = old_request;
 270     else
 271         command = ctrl.last_command;
 272
 273     if (command && strncmp(command, "PASS", 4) == 0)
 274         command = "PASS <yourpassword>";
 275
 276     if (old_reply)
 277         reply = old_reply;
 278     else
 279         reply = ctrl.last_reply;
 280
 281     if (command)
 282         ftperr->ftp.request = xstrdup(command);
 283
 284     if (reply)
 285         ftperr->ftp.reply = xstrdup(reply);
 286
 287     if (!err) {
 288         fwd->request->detailError(error, xerrno);
 289         fwd->fail(ftperr);
 290         closeServer(); // we failed, so no serverComplete()
 291     }
 292 }
 293
 294 Http::StatusCode
 295 Ftp::Client::failedHttpStatus(err_type &error)
 296 {
 297     if (error == ERR_NONE)
 298         error = ERR_FTP_FAILURE;
 299     return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout :
 300            Http::scBadGateway;
 301 }
 302
 303 /**
 304  * DPW 2007-04-23
 305  * Looks like there are no longer anymore callers that set
 306  * buffered_ok=1.  Perhaps it can be removed at some point.
 307  */
 308 void
 309 Ftp::Client::scheduleReadControlReply(int buffered_ok)
 310 {
 311     debugs(9, 3, ctrl.conn);
 312
 313     if (buffered_ok && ctrl.offset > 0) {
 314         /* We've already read some reply data */
 315         handleControlReply();
 316     } else {
 317
 318         if (!Comm::IsConnOpen(ctrl.conn)) {
 319             debugs(9, 3, "cannot read without ctrl " << ctrl.conn);
 320             return;
 321         }
 322         /*
 323          * Cancel the timeout on the Data socket (if any) and
 324          * establish one on the control socket.
 325          */
 326         if (Comm::IsConnOpen(data.conn)) {
 327             commUnsetConnTimeout(data.conn);
 328         }
 329
 330         const time_t tout = shortenReadTimeout ?
 331                             min(Config.Timeout.connect, Config.Timeout.read):
 332                             Config.Timeout.read;
 333         shortenReadTimeout = false; // we only need to do this once, after PASV
 334
 335         typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 336         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, Ftp::Client::timeout);
 337         commSetConnTimeout(ctrl.conn, tout, timeoutCall);
 338
 339         typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 340         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, Ftp::Client::readControlReply);
 341         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
 342     }
 343 }
 344
 345 void
 346 Ftp::Client::readControlReply(const CommIoCbParams &io)
 347 {
 348     debugs(9, 3, "FD " << io.fd << ", Read " << io.size << " bytes");
 349
 350     if (io.size > 0) {
 351         statCounter.server.all.kbytes_in += io.size;
 352         statCounter.server.ftp.kbytes_in += io.size;
 353     }
 354
 355     if (io.flag == Comm::ERR_CLOSING)
 356         return;
 357
 358     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 359         if (abortOnData("entry aborted during control reply read"))
 360             return;
 361     }
 362
 363     assert(ctrl.offset < ctrl.size);
 364
 365     if (io.flag == Comm::OK && io.size > 0) {
 366         fd_bytes(io.fd, io.size, FD_READ);
 367     }
 368
 369     if (io.flag != Comm::OK) {
 370         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 371                "FTP control reply read error: " << xstrerr(io.xerrno));
 372
 373         if (ignoreErrno(io.xerrno)) {
 374             scheduleReadControlReply(0);
 375         } else {
 376             failed(ERR_READ_ERROR, io.xerrno);
 377             /* failed closes ctrl.conn and frees ftpState */
 378         }
 379         return;
 380     }
 381
 382     if (io.size == 0) {
 383         if (entry->store_status == STORE_PENDING) {
 384             failed(ERR_FTP_FAILURE, 0);
 385             /* failed closes ctrl.conn and frees ftpState */
 386             return;
 387         }
 388
 389         /* XXX this may end up having to be serverComplete() .. */
 390         abortAll("zero control reply read");
 391         return;
 392     }
 393
 394     unsigned int len =io.size + ctrl.offset;
 395     ctrl.offset = len;
 396     assert(len <= ctrl.size);
 397     if (Comm::IsConnOpen(ctrl.conn))
 398         commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply
 399     handleControlReply();
 400 }
 401
 402 void
 403 Ftp::Client::handleControlReply()
 404 {
 405     debugs(9, 3, status());
 406
 407     size_t bytes_used = 0;
 408     wordlistDestroy(&ctrl.message);
 409
 410     if (!parseControlReply(bytes_used)) {
 411         /* didn't get complete reply yet */
 412
 413         if (ctrl.offset == ctrl.size) {
 414             ctrl.buf = static_cast<char*>(memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size));
 415         }
 416
 417         scheduleReadControlReply(0);
 418         return;
 419     }
 420
 421     assert(ctrl.message); // the entire FTP server response, line by line
 422     assert(ctrl.replycode >= 0); // FTP status code (from the last line)
 423     assert(ctrl.last_reply); // FTP reason (from the last line)
 424
 425     if (ctrl.offset == bytes_used) {
 426         /* used it all up */
 427         ctrl.offset = 0;
 428     } else {
 429         /* Got some data past the complete reply */
 430         assert(bytes_used < ctrl.offset);
 431         ctrl.offset -= bytes_used;
 432         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
 433     }
 434
 435     debugs(9, 3, "state=" << state << ", code=" << ctrl.replycode);
 436 }
 437
 438 bool
 439 Ftp::Client::handlePasvReply(Ip::Address &srvAddr)
 440 {
 441     int code = ctrl.replycode;
 442     char *buf;
 443     debugs(9, 3, status());
 444
 445     if (code != 227) {
 446         debugs(9, 2, "PASV not supported by remote end");
 447         return false;
 448     }
 449
 450     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
 451     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 452     debugs(9, 5, "scanning: " << ctrl.last_reply);
 453
 454     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789");
 455
 456     const char *forceIp = Config.Ftp.sanitycheck ?
 457                           fd_table[ctrl.conn->fd].ipaddr : NULL;
 458     if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) {
 459         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
 460                ctrl.conn->remote << ": " << ctrl.last_reply);
 461         return false;
 462     }
 463
 464     data.addr(srvAddr);
 465
 466     return true;
 467 }
 468
 469 bool
 470 Ftp::Client::handleEpsvReply(Ip::Address &remoteAddr)
 471 {
 472     int code = ctrl.replycode;
 473     char *buf;
 474     debugs(9, 3, status());
 475
 476     if (code != 229 && code != 522) {
 477         if (code == 200) {
 478             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
 479             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
 480              * Its okay to re-send EPSV 1/2 but nothing else. */
 481             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV");
 482         } else {
 483             debugs(9, 2, "EPSV not supported by remote end");
 484         }
 485         return sendPassive();
 486     }
 487
 488     if (code == 522) {
 489         /* Peer responded with a list of supported methods:
 490          *   522 Network protocol not supported, use (1)
 491          *   522 Network protocol not supported, use (1,2)
 492          *   522 Network protocol not supported, use (2)
 493          * TODO: Handle the (1,2) case which may happen after EPSV ALL. Close
 494          * data + control without self-destructing and re-open from scratch.
 495          */
 496         debugs(9, 5, "scanning: " << ctrl.last_reply);
 497         buf = ctrl.last_reply;
 498         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
 499             ++buf;
 500         if (buf != NULL && *buf == '\n')
 501             ++buf;
 502
 503         if (buf == NULL || *buf == '\0') {
 504             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 505             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
 506             return sendPassive();
 507         } else if (strcmp(buf, "(1)") == 0) {
 508             state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
 509             return sendPassive();
 510         } else if (strcmp(buf, "(2)") == 0) {
 511             if (Ip::EnableIpv6) {
 512                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
 513                 if (state == SENT_EPSV_2) {
 514                     return sendEprt();
 515                 } else {
 516                     /* or try the next Passive mode down the chain. */
 517                     return sendPassive();
 518                 }
 519             } else {
 520                 /* Server only accept EPSV in IPv6 traffic. */
 521                 state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
 522                 return sendPassive();
 523             }
 524         } else {
 525             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 526             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
 527             return sendPassive();
 528         }
 529         /* coverity[unreachable] */
 530         /* safeguard against possible future bugs in above conditions */
 531         failed(ERR_FTP_FAILURE, 0);
 532         return false;
 533     }
 534
 535     /*  229 Entering Extended Passive Mode (|||port|) */
 536     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 537     debugs(9, 5, "scanning: " << ctrl.last_reply);
 538
 539     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "(");
 540
 541     char h1, h2, h3, h4;
 542     unsigned short port;
 543     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
 544
 545     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
 546         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
 547                ctrl.conn->remote << ": " <<
 548                ctrl.last_reply);
 549
 550         return sendPassive();
 551     }
 552
 553     if (0 == port) {
 554         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 555                ctrl.conn->remote << ": " <<
 556                ctrl.last_reply);
 557
 558         return sendPassive();
 559     }
 560
 561     if (Config.Ftp.sanitycheck) {
 562         if (port < 1024) {
 563             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 564                    ctrl.conn->remote << ": " <<
 565                    ctrl.last_reply);
 566
 567             return sendPassive();
 568         }
 569     }
 570
 571     remoteAddr = ctrl.conn->remote;
 572     remoteAddr.port(port);
 573     data.addr(remoteAddr);
 574     return true;
 575 }
 576
 577 // FTP clients do not support EPRT and PORT commands yet.
 578 // The Ftp::Client::sendEprt() will fail because of the unimplemented
 579 // openListenSocket() or sendPort() methods
 580 bool
 581 Ftp::Client::sendEprt()
 582 {
 583     if (!Config.Ftp.eprt) {
 584         /* Disabled. Switch immediately to attempting old PORT command. */
 585         debugs(9, 3, "EPRT disabled by local administrator");
 586         return sendPort();
 587     }
 588
 589     debugs(9, 3, status());
 590
 591     if (!openListenSocket()) {
 592         failed(ERR_FTP_FAILURE, 0);
 593         return false;
 594     }
 595
 596     debugs(9, 3, "Listening for FTP data connection with FD " << data.conn);
 597     if (!Comm::IsConnOpen(data.conn)) {
 598         // TODO: Set error message.
 599         failed(ERR_FTP_FAILURE, 0);
 600         return false;
 601     }
 602
 603     static MemBuf mb;
 604     mb.reset();
 605     char buf[MAX_IPSTRLEN];
 606     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
 607     /* Which can be used by EITHER protocol. */
 608     debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port());
 609     mb.appendf("EPRT |%d|%s|%d|%s",
 610                ( data.conn->local.isIPv6() ? 2 : 1 ),
 611                data.conn->local.toStr(buf,MAX_IPSTRLEN),
 612                comm_local_port(data.conn->fd), Ftp::crlf );
 613
 614     state = SENT_EPRT;
 615     writeCommand(mb.content());
 616     return true;
 617 }
 618
 619 bool
 620 Ftp::Client::sendPort()
 621 {
 622     failed(ERR_FTP_FAILURE, 0);
 623     return false;
 624 }
 625
 626 bool
 627 Ftp::Client::sendPassive()
 628 {
 629     debugs(9, 3, status());
 630
 631     /** \par
 632       * Checks for EPSV ALL special conditions:
 633       * If enabled to be sent, squid MUST NOT request any other connect methods.
 634       * If 'ALL' is sent and fails the entire FTP Session fails.
 635       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
 636     if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) {
 637         // We are here because the last "EPSV 1" failed, but because of epsv_all
 638         // no other method allowed.
 639         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 640         failed(ERR_FTP_FAILURE, 0);
 641         return false;
 642     }
 643
 644     /// Closes any old FTP-Data connection which may exist. */
 645     data.close();
 646
 647     /** \par
 648       * Checks for previous EPSV/PASV failures on this server/session.
 649       * Diverts to EPRT immediately if they are not working. */
 650     if (!Config.Ftp.passive || state == SENT_PASV) {
 651         sendEprt();
 652         return true;
 653     }
 654
 655     static MemBuf mb;
 656     mb.reset();
 657     /** \par
 658       * Send EPSV (ALL,2,1) or PASV on the control channel.
 659       *
 660       *  - EPSV ALL  is used if enabled.
 661       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
 662       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
 663       *  - PASV      is used if EPSV 1 fails.
 664       */
 665     switch (state) {
 666     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
 667         if (ctrl.conn->local.isIPv6()) {
 668             debugs(9, 5, "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
 669             mb.appendf("EPSV 2%s", Ftp::crlf);
 670             state = SENT_EPSV_2;
 671             break;
 672         }
 673     // else fall through to skip EPSV 2
 674
 675     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
 676         if (ctrl.conn->local.isIPv4()) {
 677             debugs(9, 5, "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
 678             mb.appendf("EPSV 1%s", Ftp::crlf);
 679             state = SENT_EPSV_1;
 680             break;
 681         } else if (Config.Ftp.epsv_all) {
 682             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 683             failed(ERR_FTP_FAILURE, 0);
 684             return false;
 685         }
 686     // else fall through to skip EPSV 1
 687
 688     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
 689         debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
 690         mb.appendf("PASV%s", Ftp::crlf);
 691         state = SENT_PASV;
 692         break;
 693
 694     default: {
 695         bool doEpsv = true;
 696         if (Config.accessList.ftp_epsv) {
 697             ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL);
 698             doEpsv = (checklist.fastCheck() == ACCESS_ALLOWED);
 699         }
 700         if (!doEpsv) {
 701             debugs(9, 5, "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")");
 702             mb.appendf("PASV%s", Ftp::crlf);
 703             state = SENT_PASV;
 704         } else if (Config.Ftp.epsv_all) {
 705             debugs(9, 5, "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")");
 706             mb.appendf("EPSV ALL%s", Ftp::crlf);
 707             state = SENT_EPSV_ALL;
 708         } else {
 709             if (ctrl.conn->local.isIPv6()) {
 710                 debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2");
 711                 mb.appendf("EPSV 2%s", Ftp::crlf);
 712                 state = SENT_EPSV_2;
 713             }
 714             if (ctrl.conn->local.isIPv4()) {
 715                 debugs(9, 5, "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1");
 716                 mb.appendf("EPSV 1%s", Ftp::crlf);
 717                 state = SENT_EPSV_1;
 718             }
 719         }
 720         break;
 721     }
 722     }
 723
 724     if (ctrl.message)
 725         wordlistDestroy(&ctrl.message);
 726     ctrl.message = NULL; //No message to return to client.
 727     ctrl.offset = 0; //reset readed response, to make room read the next response
 728
 729     writeCommand(mb.content());
 730
 731     shortenReadTimeout = true;
 732     return true;
 733 }
 734
 735 void
 736 Ftp::Client::connectDataChannel()
 737 {
 738     safe_free(ctrl.last_command);
 739
 740     safe_free(ctrl.last_reply);
 741
 742     ctrl.last_command = xstrdup("Connect to server data port");
 743
 744     // Generate a new data channel descriptor to be opened.
 745     Comm::ConnectionPointer conn = new Comm::Connection;
 746     conn->setAddrs(ctrl.conn->local, data.host);
 747     conn->local.port(0);
 748     conn->remote.port(data.port);
 749     conn->tos = ctrl.conn->tos;
 750     conn->nfmark = ctrl.conn->nfmark;
 751
 752     debugs(9, 3, "connecting to " << conn->remote);
 753
 754     typedef CommCbMemFunT<Client, CommConnectCbParams> Dialer;
 755     data.opener = JobCallback(9, 3, Dialer, this, Ftp::Client::dataChannelConnected);
 756     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect);
 757     cs->setHost(data.host);
 758     AsyncJob::Start(cs);
 759 }
 760
 761 bool
 762 Ftp::Client::openListenSocket()
 763 {
 764     return false;
 765 }
 766
 767 /// creates a data channel Comm close callback
 768 AsyncCall::Pointer
 769 Ftp::Client::dataCloser()
 770 {
 771     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 772     return JobCallback(9, 5, Dialer, this, Ftp::Client::dataClosed);
 773 }
 774
 775 /// handler called by Comm when FTP data channel is closed unexpectedly
 776 void
 777 Ftp::Client::dataClosed(const CommCloseCbParams &)
 778 {
 779     debugs(9, 4, status());
 780     if (data.listenConn != NULL) {
 781         data.listenConn->close();
 782         data.listenConn = NULL;
 783         // NP clear() does the: data.fd = -1;
 784     }
 785     data.clear();
 786 }
 787
 788 void
 789 Ftp::Client::writeCommand(const char *buf)
 790 {
 791     char *ebuf;
 792     /* trace FTP protocol communications at level 2 */
 793     debugs(9, 2, "ftp<< " << buf);
 794
 795     if (Config.Ftp.telnet)
 796         ebuf = escapeIAC(buf);
 797     else
 798         ebuf = xstrdup(buf);
 799
 800     safe_free(ctrl.last_command);
 801
 802     safe_free(ctrl.last_reply);
 803
 804     ctrl.last_command = ebuf;
 805
 806     if (!Comm::IsConnOpen(ctrl.conn)) {
 807         debugs(9, 2, "cannot send to closing ctrl " << ctrl.conn);
 808         // TODO: assert(ctrl.closer != NULL);
 809         return;
 810     }
 811
 812     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 813     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this,
 814                                           Ftp::Client::writeCommandCallback);
 815     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
 816
 817     scheduleReadControlReply(0);
 818 }
 819
 820 void
 821 Ftp::Client::writeCommandCallback(const CommIoCbParams &io)
 822 {
 823
 824     debugs(9, 5, "wrote " << io.size << " bytes");
 825
 826     if (io.size > 0) {
 827         fd_bytes(io.fd, io.size, FD_WRITE);
 828         statCounter.server.all.kbytes_out += io.size;
 829         statCounter.server.ftp.kbytes_out += io.size;
 830     }
 831
 832     if (io.flag == Comm::ERR_CLOSING)
 833         return;
 834
 835     if (io.flag) {
 836         debugs(9, DBG_IMPORTANT, "FTP command write error: " << io.conn << ": " << xstrerr(io.xerrno));
 837         failed(ERR_WRITE_ERROR, io.xerrno);
 838         /* failed closes ctrl.conn and frees ftpState */
 839         return;
 840     }
 841 }
 842
 843 /// handler called by Comm when FTP control channel is closed unexpectedly
 844 void
 845 Ftp::Client::ctrlClosed(const CommCloseCbParams &)
 846 {
 847     debugs(9, 4, status());
 848     ctrl.clear();
 849     doneWithFwd = "ctrlClosed()"; // assume FwdState is monitoring too
 850     mustStop("Ftp::Client::ctrlClosed");
 851 }
 852
 853 void
 854 Ftp::Client::timeout(const CommTimeoutCbParams &io)
 855 {
 856     debugs(9, 4, io.conn << ": '" << entry->url() << "'" );
 857
 858     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 859         return;
 860
 861     failed(ERR_READ_TIMEOUT, 0);
 862     /* failed() closes ctrl.conn and frees ftpState */
 863 }
 864
 865 const Comm::ConnectionPointer &
 866 Ftp::Client::dataConnection() const
 867 {
 868     return data.conn;
 869 }
 870
 871 void
 872 Ftp::Client::maybeReadVirginBody()
 873 {
 874     // too late to read
 875     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
 876         return;
 877
 878     if (data.read_pending)
 879         return;
 880
 881     initReadBuf();
 882
 883     const int read_sz = replyBodySpace(*data.readBuf, 0);
 884
 885     debugs(9, 9, "FTP may read up to " << read_sz << " bytes");
 886
 887     if (read_sz < 2) // see http.cc
 888         return;
 889
 890     data.read_pending = true;
 891
 892     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 893     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
 894                                       TimeoutDialer, this, Ftp::Client::timeout);
 895     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 896
 897     debugs(9,5,"queueing read on FD " << data.conn->fd);
 898
 899     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 900     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
 901                           JobCallback(9, 5, Dialer, this, Ftp::Client::dataRead));
 902 }
 903
 904 void
 905 Ftp::Client::dataRead(const CommIoCbParams &io)
 906 {
 907     int j;
 908     int bin;
 909
 910     data.read_pending = false;
 911
 912     debugs(9, 3, "FD " << io.fd << " Read " << io.size << " bytes");
 913
 914     if (io.size > 0) {
 915         statCounter.server.all.kbytes_in += io.size;
 916         statCounter.server.ftp.kbytes_in += io.size;
 917     }
 918
 919     if (io.flag == Comm::ERR_CLOSING)
 920         return;
 921
 922     assert(io.fd == data.conn->fd);
 923
 924     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 925         abortOnData("entry aborted during dataRead");
 926         return;
 927     }
 928
 929     if (io.flag == Comm::OK && io.size > 0) {
 930         debugs(9, 5, "appended " << io.size << " bytes to readBuf");
 931         data.readBuf->appended(io.size);
 932 #if USE_DELAY_POOLS
 933         DelayId delayId = entry->mem_obj->mostBytesAllowed();
 934         delayId.bytesIn(io.size);
 935 #endif
 936         ++ IOStats.Ftp.reads;
 937
 938         for (j = io.size - 1, bin = 0; j; ++bin)
 939             j >>= 1;
 940
 941         ++ IOStats.Ftp.read_hist[bin];
 942     }
 943
 944     if (io.flag != Comm::OK) {
 945         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 946                "FTP data read error: " << xstrerr(io.xerrno));
 947
 948         if (ignoreErrno(io.xerrno)) {
 949             maybeReadVirginBody();
 950         } else {
 951             failed(ERR_READ_ERROR, 0);
 952             /* failed closes ctrl.conn and frees ftpState */
 953             return;
 954         }
 955     } else if (io.size == 0) {
 956         debugs(9, 3, "Calling dataComplete() because io.size == 0");
 957         /*
 958          * DPW 2007-04-23
 959          * Dangerous curves ahead.  This call to dataComplete was
 960          * calling scheduleReadControlReply, handleControlReply,
 961          * and then ftpReadTransferDone.  If ftpReadTransferDone
 962          * gets unexpected status code, it closes down the control
 963          * socket and our FtpStateData object gets destroyed.   As
 964          * a workaround we no longer set the 'buffered_ok' flag in
 965          * the scheduleReadControlReply call.
 966          */
 967         dataComplete();
 968     }
 969
 970     processReplyBody();
 971 }
 972
 973 void
 974 Ftp::Client::dataComplete()
 975 {
 976     debugs(9, 3,status());
 977
 978     /* Connection closed; transfer done. */
 979
 980     /// Close data channel, if any, to conserve resources while we wait.
 981     data.close();
 982
 983     /* expect the "transfer complete" message on the control socket */
 984     /*
 985      * DPW 2007-04-23
 986      * Previously, this was the only place where we set the
 987      * 'buffered_ok' flag when calling scheduleReadControlReply().
 988      * It caused some problems if the FTP server returns an unexpected
 989      * status code after the data command.  FtpStateData was being
 990      * deleted in the middle of dataRead().
 991      */
 992     /* AYJ: 2011-01-13: Bug 2581.
 993      * 226 status is possibly waiting in the ctrl buffer.
 994      * The connection will hang if we DONT send buffered_ok.
 995      * This happens on all transfers which can be completly sent by the
 996      * server before the 150 started status message is read in by Squid.
 997      * ie all transfers of about one packet hang.
 998      */
 999     scheduleReadControlReply(1);
1000 }
1001
1002 void
1003 Ftp::Client::abortAll(const char *reason)
1004 {
1005     debugs(9, 3, "aborting transaction for " << reason <<
1006            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
1007     mustStop(reason);
1008 }
1009
1010 /**
1011  * Cancel the timeout on the Control socket and establish one
1012  * on the data socket
1013  */
1014 void
1015 Ftp::Client::switchTimeoutToDataChannel()
1016 {
1017     commUnsetConnTimeout(ctrl.conn);
1018
1019     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
1020     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this,
1021                                      Ftp::Client::timeout);
1022     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1023 }
1024
1025 void
1026 Ftp::Client::sentRequestBody(const CommIoCbParams &io)
1027 {
1028     if (io.size > 0)
1029         statCounter.server.ftp.kbytes_out += io.size;
1030     ::Client::sentRequestBody(io);
1031 }
1032
1033 /**
1034  * called after we wrote the last byte of the request body
1035  */
1036 void
1037 Ftp::Client::doneSendingRequestBody()
1038 {
1039     ::Client::doneSendingRequestBody();
1040     debugs(9, 3, status());
1041     dataComplete();
1042     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
1043      * if transfer type is 'stream' call dataComplete()
1044      * otherwise leave open. (reschedule control channel read?)
1045      */
1046 }
1047
1048 /// Parses FTP server control response into ctrl structure fields,
1049 /// setting bytesUsed and returning true on success.
1050 bool
1051 Ftp::Client::parseControlReply(size_t &bytesUsed)
1052 {
1053     char *s;
1054     char *sbuf;
1055     char *end;
1056     int usable;
1057     int complete = 0;
1058     wordlist *head = NULL;
1059     wordlist *list;
1060     wordlist **tail = &head;
1061     size_t linelen;
1062     debugs(9, 3, status());
1063     /*
1064      * We need a NULL-terminated buffer for scanning, ick
1065      */
1066     const size_t len = ctrl.offset;
1067     sbuf = (char *)xmalloc(len + 1);
1068     xstrncpy(sbuf, ctrl.buf, len + 1);
1069     end = sbuf + len - 1;
1070
1071     while (*end != '\r' && *end != '\n' && end > sbuf)
1072         --end;
1073
1074     usable = end - sbuf;
1075
1076     debugs(9, 3, "usable = " << usable);
1077
1078     if (usable == 0) {
1079         debugs(9, 3, "didn't find end of line");
1080         safe_free(sbuf);
1081         return false;
1082     }
1083
1084     debugs(9, 3, len << " bytes to play with");
1085     ++end;
1086     s = sbuf;
1087     s += strspn(s, crlf);
1088
1089     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1090         if (complete)
1091             break;
1092
1093         debugs(9, 5, "s = {" << s << "}");
1094
1095         linelen = strcspn(s, crlf) + 1;
1096
1097         if (linelen < 2)
1098             break;
1099
1100         if (linelen > 3)
1101             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1102
1103         list = new wordlist();
1104
1105         list->key = (char *)xmalloc(linelen);
1106
1107         xstrncpy(list->key, s, linelen);
1108
1109         /* trace the FTP communication chat at level 2 */
1110         debugs(9, 2, "ftp>> " << list->key);
1111
1112         if (complete) {
1113             // use list->key for last_reply because s contains the new line
1114             ctrl.last_reply = xstrdup(list->key + 4);
1115             ctrl.replycode = atoi(list->key);
1116         }
1117
1118         *tail = list;
1119
1120         tail = &list->next;
1121     }
1122
1123     bytesUsed = static_cast<size_t>(s - sbuf);
1124     safe_free(sbuf);
1125
1126     if (!complete) {
1127         wordlistDestroy(&head);
1128         return false;
1129     }
1130
1131     ctrl.message = head;
1132     assert(ctrl.replycode >= 0);
1133     assert(ctrl.last_reply);
1134     assert(ctrl.message);
1135     return true;
1136 }
1137
1138 }; // namespace Ftp
1139