src/clients/FtpClient.cc

   1 /*
   2  * Copyright (C) 1996-2016 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "client_side.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm/ConnOpener.h"
  16 #include "comm/Read.h"
  17 #include "comm/TcpAcceptor.h"
  18 #include "comm/Write.h"
  19 #include "errorpage.h"
  20 #include "fd.h"
  21 #include "ftp/Parsing.h"
  22 #include "ip/tools.h"
  23 #include "SquidConfig.h"
  24 #include "SquidString.h"
  25 #include "StatCounters.h"
  26 #include "tools.h"
  27 #include "wordlist.h"
  28
  29 #include <set>
  30
  31 namespace Ftp
  32 {
  33
  34 const char *const crlf = "\r\n";
  35
  36 static char *
  37 escapeIAC(const char *buf)
  38 {
  39     int n;
  40     char *ret;
  41     unsigned const char *p;
  42     unsigned char *r;
  43
  44     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
  45         if (*p == 255)
  46             ++n;
  47
  48     ret = (char *)xmalloc(n);
  49
  50     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
  51         *r = *p;
  52         ++r;
  53
  54         if (*p == 255) {
  55             *r = 255;
  56             ++r;
  57         }
  58     }
  59
  60     *r = '\0';
  61     ++r;
  62     assert((r - (unsigned char *)ret) == n );
  63     return ret;
  64 }
  65
  66 /* Ftp::Channel */
  67
  68 /// configures the channel with a descriptor and registers a close handler
  69 void
  70 Ftp::Channel::opened(const Comm::ConnectionPointer &newConn,
  71                      const AsyncCall::Pointer &aCloser)
  72 {
  73     assert(!Comm::IsConnOpen(conn));
  74     assert(closer == NULL);
  75
  76     assert(Comm::IsConnOpen(newConn));
  77     assert(aCloser != NULL);
  78
  79     conn = newConn;
  80     closer = aCloser;
  81     comm_add_close_handler(conn->fd, closer);
  82 }
  83
  84 /// planned close: removes the close handler and calls comm_close
  85 void
  86 Ftp::Channel::close()
  87 {
  88     // channels with active listeners will be closed when the listener handler dies.
  89     if (Comm::IsConnOpen(conn)) {
  90         comm_remove_close_handler(conn->fd, closer);
  91         conn->close(); // we do not expect to be called back
  92     }
  93     clear();
  94 }
  95
  96 void
  97 Ftp::Channel::forget()
  98 {
  99     if (Comm::IsConnOpen(conn)) {
 100         commUnsetConnTimeout(conn);
 101         comm_remove_close_handler(conn->fd, closer);
 102     }
 103     clear();
 104 }
 105
 106 void
 107 Ftp::Channel::clear()
 108 {
 109     conn = NULL;
 110     closer = NULL;
 111 }
 112
 113 /* Ftp::CtrlChannel */
 114
 115 Ftp::CtrlChannel::CtrlChannel():
 116     buf(NULL),
 117     size(0),
 118     offset(0),
 119     message(NULL),
 120     last_command(NULL),
 121     last_reply(NULL),
 122     replycode(0)
 123 {
 124     buf = static_cast<char*>(memAllocBuf(4096, &size));
 125 }
 126
 127 Ftp::CtrlChannel::~CtrlChannel()
 128 {
 129     memFreeBuf(size, buf);
 130     if (message)
 131         wordlistDestroy(&message);
 132     safe_free(last_command);
 133     safe_free(last_reply);
 134 }
 135
 136 /* Ftp::DataChannel */
 137
 138 Ftp::DataChannel::DataChannel():
 139     readBuf(NULL),
 140     host(NULL),
 141     port(0),
 142     read_pending(false)
 143 {
 144 }
 145
 146 Ftp::DataChannel::~DataChannel()
 147 {
 148     delete readBuf;
 149 }
 150
 151 void
 152 Ftp::DataChannel::addr(const Ip::Address &import)
 153 {
 154     static char addrBuf[MAX_IPSTRLEN];
 155     import.toStr(addrBuf, sizeof(addrBuf));
 156     xfree(host);
 157     host = xstrdup(addrBuf);
 158     port = import.port();
 159 }
 160
 161 /* Ftp::Client */
 162
 163 Ftp::Client::Client(FwdState *fwdState):
 164     AsyncJob("Ftp::Client"),
 165     ::Client(fwdState),
 166      ctrl(),
 167      data(),
 168      state(BEGIN),
 169      old_request(NULL),
 170      old_reply(NULL),
 171      shortenReadTimeout(false)
 172 {
 173     ++statCounter.server.all.requests;
 174     ++statCounter.server.ftp.requests;
 175
 176     ctrl.last_command = xstrdup("Connect to server");
 177
 178     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 179     const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this,
 180                                       Ftp::Client::ctrlClosed);
 181     ctrl.opened(fwdState->serverConnection(), closer);
 182 }
 183
 184 Ftp::Client::~Client()
 185 {
 186     if (data.opener != NULL) {
 187         data.opener->cancel("Ftp::Client destructed");
 188         data.opener = NULL;
 189     }
 190     data.close();
 191
 192     safe_free(old_request);
 193     safe_free(old_reply);
 194     fwd = NULL; // refcounted
 195 }
 196
 197 void
 198 Ftp::Client::start()
 199 {
 200     scheduleReadControlReply(0);
 201 }
 202
 203 void
 204 Ftp::Client::initReadBuf()
 205 {
 206     if (data.readBuf == NULL) {
 207         data.readBuf = new MemBuf;
 208         data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 209     }
 210 }
 211
 212 /**
 213  * Close the FTP server connection(s). Used by serverComplete().
 214  */
 215 void
 216 Ftp::Client::closeServer()
 217 {
 218     if (Comm::IsConnOpen(ctrl.conn)) {
 219         debugs(9, 3, "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
 220         fwd->unregister(ctrl.conn);
 221         ctrl.close();
 222     }
 223
 224     if (Comm::IsConnOpen(data.conn)) {
 225         debugs(9, 3, "closing FTP data FD " << data.conn->fd << ", this " << this);
 226         data.close();
 227     }
 228
 229     debugs(9, 3, "FTP ctrl and data connections closed. this " << this);
 230 }
 231
 232 /**
 233  * Did we close all FTP server connection(s)?
 234  *
 235  \retval true  Both server control and data channels are closed. And not waiting for a new data connection to open.
 236  \retval false Either control channel or data is still active.
 237  */
 238 bool
 239 Ftp::Client::doneWithServer() const
 240 {
 241     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
 242 }
 243
 244 void
 245 Ftp::Client::failed(err_type error, int xerrno, ErrorState *err)
 246 {
 247     debugs(9, 3, "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
 248
 249     const char *command, *reply;
 250     ErrorState *ftperr;
 251
 252     if (err) {
 253         debugs(9, 6, "error=" << err->type << ", code=" << xerrno <<
 254                ", status=" << err->httpStatus);
 255         error = err->type;
 256         ftperr = err;
 257     } else {
 258         Http::StatusCode httpStatus = failedHttpStatus(error);
 259         ftperr = new ErrorState(error, httpStatus, fwd->request);
 260     }
 261
 262     ftperr->xerrno = xerrno;
 263
 264     ftperr->ftp.server_msg = ctrl.message;
 265     ctrl.message = NULL;
 266
 267     if (old_request)
 268         command = old_request;
 269     else
 270         command = ctrl.last_command;
 271
 272     if (command && strncmp(command, "PASS", 4) == 0)
 273         command = "PASS <yourpassword>";
 274
 275     if (old_reply)
 276         reply = old_reply;
 277     else
 278         reply = ctrl.last_reply;
 279
 280     if (command)
 281         ftperr->ftp.request = xstrdup(command);
 282
 283     if (reply)
 284         ftperr->ftp.reply = xstrdup(reply);
 285
 286     if (!err) {
 287         fwd->request->detailError(error, xerrno);
 288         fwd->fail(ftperr);
 289         closeServer(); // we failed, so no serverComplete()
 290     }
 291 }
 292
 293 Http::StatusCode
 294 Ftp::Client::failedHttpStatus(err_type &error)
 295 {
 296     if (error == ERR_NONE)
 297         error = ERR_FTP_FAILURE;
 298     return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout :
 299            Http::scBadGateway;
 300 }
 301
 302 /**
 303  * DPW 2007-04-23
 304  * Looks like there are no longer anymore callers that set
 305  * buffered_ok=1.  Perhaps it can be removed at some point.
 306  */
 307 void
 308 Ftp::Client::scheduleReadControlReply(int buffered_ok)
 309 {
 310     debugs(9, 3, ctrl.conn);
 311
 312     if (buffered_ok && ctrl.offset > 0) {
 313         /* We've already read some reply data */
 314         handleControlReply();
 315     } else {
 316         /*
 317          * Cancel the timeout on the Data socket (if any) and
 318          * establish one on the control socket.
 319          */
 320         if (Comm::IsConnOpen(data.conn)) {
 321             commUnsetConnTimeout(data.conn);
 322         }
 323
 324         const time_t tout = shortenReadTimeout ?
 325                             min(Config.Timeout.connect, Config.Timeout.read):
 326                             Config.Timeout.read;
 327         shortenReadTimeout = false; // we only need to do this once, after PASV
 328
 329         typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 330         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, Ftp::Client::timeout);
 331         commSetConnTimeout(ctrl.conn, tout, timeoutCall);
 332
 333         typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 334         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, Ftp::Client::readControlReply);
 335         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
 336     }
 337 }
 338
 339 void
 340 Ftp::Client::readControlReply(const CommIoCbParams &io)
 341 {
 342     debugs(9, 3, "FD " << io.fd << ", Read " << io.size << " bytes");
 343
 344     if (io.size > 0) {
 345         statCounter.server.all.kbytes_in += io.size;
 346         statCounter.server.ftp.kbytes_in += io.size;
 347     }
 348
 349     if (io.flag == Comm::ERR_CLOSING)
 350         return;
 351
 352     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 353         abortTransaction("entry aborted during control reply read");
 354         return;
 355     }
 356
 357     assert(ctrl.offset < ctrl.size);
 358
 359     if (io.flag == Comm::OK && io.size > 0) {
 360         fd_bytes(io.fd, io.size, FD_READ);
 361     }
 362
 363     if (io.flag != Comm::OK) {
 364         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 365                "FTP control reply read error: " << xstrerr(io.xerrno));
 366
 367         if (ignoreErrno(io.xerrno)) {
 368             scheduleReadControlReply(0);
 369         } else {
 370             failed(ERR_READ_ERROR, io.xerrno);
 371             /* failed closes ctrl.conn and frees ftpState */
 372         }
 373         return;
 374     }
 375
 376     if (io.size == 0) {
 377         if (entry->store_status == STORE_PENDING) {
 378             failed(ERR_FTP_FAILURE, 0);
 379             /* failed closes ctrl.conn and frees ftpState */
 380             return;
 381         }
 382
 383         /* XXX this may end up having to be serverComplete() .. */
 384         abortTransaction("zero control reply read");
 385         return;
 386     }
 387
 388     unsigned int len =io.size + ctrl.offset;
 389     ctrl.offset = len;
 390     assert(len <= ctrl.size);
 391     if (Comm::IsConnOpen(ctrl.conn))
 392         commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply
 393     handleControlReply();
 394 }
 395
 396 void
 397 Ftp::Client::handleControlReply()
 398 {
 399     debugs(9, 3, status());
 400
 401     size_t bytes_used = 0;
 402     wordlistDestroy(&ctrl.message);
 403
 404     if (!parseControlReply(bytes_used)) {
 405         /* didn't get complete reply yet */
 406
 407         if (ctrl.offset == ctrl.size) {
 408             ctrl.buf = static_cast<char*>(memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size));
 409         }
 410
 411         scheduleReadControlReply(0);
 412         return;
 413     }
 414
 415     assert(ctrl.message); // the entire FTP server response, line by line
 416     assert(ctrl.replycode >= 0); // FTP status code (from the last line)
 417     assert(ctrl.last_reply); // FTP reason (from the last line)
 418
 419     if (ctrl.offset == bytes_used) {
 420         /* used it all up */
 421         ctrl.offset = 0;
 422     } else {
 423         /* Got some data past the complete reply */
 424         assert(bytes_used < ctrl.offset);
 425         ctrl.offset -= bytes_used;
 426         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
 427     }
 428
 429     debugs(9, 3, "state=" << state << ", code=" << ctrl.replycode);
 430 }
 431
 432 bool
 433 Ftp::Client::handlePasvReply(Ip::Address &srvAddr)
 434 {
 435     int code = ctrl.replycode;
 436     char *buf;
 437     debugs(9, 3, status());
 438
 439     if (code != 227) {
 440         debugs(9, 2, "PASV not supported by remote end");
 441         return false;
 442     }
 443
 444     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
 445     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 446     debugs(9, 5, "scanning: " << ctrl.last_reply);
 447
 448     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789");
 449
 450     const char *forceIp = Config.Ftp.sanitycheck ?
 451                           fd_table[ctrl.conn->fd].ipaddr : NULL;
 452     if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) {
 453         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
 454                ctrl.conn->remote << ": " << ctrl.last_reply);
 455         return false;
 456     }
 457
 458     data.addr(srvAddr);
 459
 460     return true;
 461 }
 462
 463 bool
 464 Ftp::Client::handleEpsvReply(Ip::Address &remoteAddr)
 465 {
 466     int code = ctrl.replycode;
 467     char *buf;
 468     debugs(9, 3, status());
 469
 470     if (code != 229 && code != 522) {
 471         if (code == 200) {
 472             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
 473             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
 474              * Its okay to re-send EPSV 1/2 but nothing else. */
 475             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV");
 476         } else {
 477             debugs(9, 2, "EPSV not supported by remote end");
 478         }
 479         return sendPassive();
 480     }
 481
 482     if (code == 522) {
 483         /* Peer responded with a list of supported methods:
 484          *   522 Network protocol not supported, use (1)
 485          *   522 Network protocol not supported, use (1,2)
 486          *   522 Network protocol not supported, use (2)
 487          * TODO: Handle the (1,2) case which may happen after EPSV ALL. Close
 488          * data + control without self-destructing and re-open from scratch.
 489          */
 490         debugs(9, 5, "scanning: " << ctrl.last_reply);
 491         buf = ctrl.last_reply;
 492         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
 493             ++buf;
 494         if (buf != NULL && *buf == '\n')
 495             ++buf;
 496
 497         if (buf == NULL || *buf == '\0') {
 498             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 499             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
 500             return sendPassive();
 501         } else if (strcmp(buf, "(1)") == 0) {
 502             state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
 503             return sendPassive();
 504         } else if (strcmp(buf, "(2)") == 0) {
 505             if (Ip::EnableIpv6) {
 506                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
 507                 if (state == SENT_EPSV_2) {
 508                     return sendEprt();
 509                 } else {
 510                     /* or try the next Passive mode down the chain. */
 511                     return sendPassive();
 512                 }
 513             } else {
 514                 /* Server only accept EPSV in IPv6 traffic. */
 515                 state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
 516                 return sendPassive();
 517             }
 518         } else {
 519             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 520             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
 521             return sendPassive();
 522         }
 523         /* coverity[unreachable] */
 524         /* safeguard against possible future bugs in above conditions */
 525         failed(ERR_FTP_FAILURE, 0);
 526         return false;
 527     }
 528
 529     /*  229 Entering Extended Passive Mode (|||port|) */
 530     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 531     debugs(9, 5, "scanning: " << ctrl.last_reply);
 532
 533     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "(");
 534
 535     char h1, h2, h3, h4;
 536     unsigned short port;
 537     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
 538
 539     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
 540         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
 541                ctrl.conn->remote << ": " <<
 542                ctrl.last_reply);
 543
 544         return sendPassive();
 545     }
 546
 547     if (0 == port) {
 548         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 549                ctrl.conn->remote << ": " <<
 550                ctrl.last_reply);
 551
 552         return sendPassive();
 553     }
 554
 555     if (Config.Ftp.sanitycheck) {
 556         if (port < 1024) {
 557             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 558                    ctrl.conn->remote << ": " <<
 559                    ctrl.last_reply);
 560
 561             return sendPassive();
 562         }
 563     }
 564
 565     remoteAddr = ctrl.conn->remote;
 566     remoteAddr.port(port);
 567     data.addr(remoteAddr);
 568     return true;
 569 }
 570
 571 // FTP clients do not support EPRT and PORT commands yet.
 572 // The Ftp::Client::sendEprt() will fail because of the unimplemented
 573 // openListenSocket() or sendPort() methods
 574 bool
 575 Ftp::Client::sendEprt()
 576 {
 577     if (!Config.Ftp.eprt) {
 578         /* Disabled. Switch immediately to attempting old PORT command. */
 579         debugs(9, 3, "EPRT disabled by local administrator");
 580         return sendPort();
 581     }
 582
 583     debugs(9, 3, status());
 584
 585     if (!openListenSocket()) {
 586         failed(ERR_FTP_FAILURE, 0);
 587         return false;
 588     }
 589
 590     debugs(9, 3, "Listening for FTP data connection with FD " << data.conn);
 591     if (!Comm::IsConnOpen(data.conn)) {
 592         // TODO: Set error message.
 593         failed(ERR_FTP_FAILURE, 0);
 594         return false;
 595     }
 596
 597     static MemBuf mb;
 598     mb.reset();
 599     char buf[MAX_IPSTRLEN];
 600     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
 601     /* Which can be used by EITHER protocol. */
 602     debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port());
 603     mb.appendf("EPRT |%d|%s|%d|%s",
 604                ( data.conn->local.isIPv6() ? 2 : 1 ),
 605                data.conn->local.toStr(buf,MAX_IPSTRLEN),
 606                comm_local_port(data.conn->fd), Ftp::crlf );
 607
 608     state = SENT_EPRT;
 609     writeCommand(mb.content());
 610     return true;
 611 }
 612
 613 bool
 614 Ftp::Client::sendPort()
 615 {
 616     failed(ERR_FTP_FAILURE, 0);
 617     return false;
 618 }
 619
 620 bool
 621 Ftp::Client::sendPassive()
 622 {
 623     debugs(9, 3, status());
 624
 625     /** \par
 626       * Checks for EPSV ALL special conditions:
 627       * If enabled to be sent, squid MUST NOT request any other connect methods.
 628       * If 'ALL' is sent and fails the entire FTP Session fails.
 629       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
 630     if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) {
 631         // We are here because the last "EPSV 1" failed, but because of epsv_all
 632         // no other method allowed.
 633         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 634         failed(ERR_FTP_FAILURE, 0);
 635         return false;
 636     }
 637
 638     /// Closes any old FTP-Data connection which may exist. */
 639     data.close();
 640
 641     /** \par
 642       * Checks for previous EPSV/PASV failures on this server/session.
 643       * Diverts to EPRT immediately if they are not working. */
 644     if (!Config.Ftp.passive || state == SENT_PASV) {
 645         sendEprt();
 646         return true;
 647     }
 648
 649     static MemBuf mb;
 650     mb.reset();
 651     /** \par
 652       * Send EPSV (ALL,2,1) or PASV on the control channel.
 653       *
 654       *  - EPSV ALL  is used if enabled.
 655       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
 656       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
 657       *  - PASV      is used if EPSV 1 fails.
 658       */
 659     switch (state) {
 660     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
 661         if (ctrl.conn->local.isIPv6()) {
 662             debugs(9, 5, "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
 663             mb.appendf("EPSV 2%s", Ftp::crlf);
 664             state = SENT_EPSV_2;
 665             break;
 666         }
 667     // else fall through to skip EPSV 2
 668
 669     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
 670         if (ctrl.conn->local.isIPv4()) {
 671             debugs(9, 5, "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
 672             mb.appendf("EPSV 1%s", Ftp::crlf);
 673             state = SENT_EPSV_1;
 674             break;
 675         } else if (Config.Ftp.epsv_all) {
 676             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 677             failed(ERR_FTP_FAILURE, 0);
 678             return false;
 679         }
 680     // else fall through to skip EPSV 1
 681
 682     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
 683         debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
 684         mb.appendf("PASV%s", Ftp::crlf);
 685         state = SENT_PASV;
 686         break;
 687
 688     default: {
 689         bool doEpsv = true;
 690         if (Config.accessList.ftp_epsv) {
 691             ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL);
 692             doEpsv = (checklist.fastCheck() == ACCESS_ALLOWED);
 693         }
 694         if (!doEpsv) {
 695             debugs(9, 5, "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")");
 696             mb.appendf("PASV%s", Ftp::crlf);
 697             state = SENT_PASV;
 698         } else if (Config.Ftp.epsv_all) {
 699             debugs(9, 5, "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")");
 700             mb.appendf("EPSV ALL%s", Ftp::crlf);
 701             state = SENT_EPSV_ALL;
 702         } else {
 703             if (ctrl.conn->local.isIPv6()) {
 704                 debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2");
 705                 mb.appendf("EPSV 2%s", Ftp::crlf);
 706                 state = SENT_EPSV_2;
 707             }
 708             if (ctrl.conn->local.isIPv4()) {
 709                 debugs(9, 5, "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1");
 710                 mb.appendf("EPSV 1%s", Ftp::crlf);
 711                 state = SENT_EPSV_1;
 712             }
 713         }
 714         break;
 715     }
 716     }
 717
 718     if (ctrl.message)
 719         wordlistDestroy(&ctrl.message);
 720     ctrl.message = NULL; //No message to return to client.
 721     ctrl.offset = 0; //reset readed response, to make room read the next response
 722
 723     writeCommand(mb.content());
 724
 725     shortenReadTimeout = true;
 726     return true;
 727 }
 728
 729 void
 730 Ftp::Client::connectDataChannel()
 731 {
 732     safe_free(ctrl.last_command);
 733
 734     safe_free(ctrl.last_reply);
 735
 736     ctrl.last_command = xstrdup("Connect to server data port");
 737
 738     // Generate a new data channel descriptor to be opened.
 739     Comm::ConnectionPointer conn = new Comm::Connection;
 740     conn->setAddrs(ctrl.conn->local, data.host);
 741     conn->local.port(0);
 742     conn->remote.port(data.port);
 743     conn->tos = ctrl.conn->tos;
 744     conn->nfmark = ctrl.conn->nfmark;
 745
 746     debugs(9, 3, "connecting to " << conn->remote);
 747
 748     typedef CommCbMemFunT<Client, CommConnectCbParams> Dialer;
 749     data.opener = JobCallback(9, 3, Dialer, this, Ftp::Client::dataChannelConnected);
 750     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect);
 751     cs->setHost(data.host);
 752     AsyncJob::Start(cs);
 753 }
 754
 755 bool
 756 Ftp::Client::openListenSocket()
 757 {
 758     return false;
 759 }
 760
 761 /// creates a data channel Comm close callback
 762 AsyncCall::Pointer
 763 Ftp::Client::dataCloser()
 764 {
 765     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 766     return JobCallback(9, 5, Dialer, this, Ftp::Client::dataClosed);
 767 }
 768
 769 /// handler called by Comm when FTP data channel is closed unexpectedly
 770 void
 771 Ftp::Client::dataClosed(const CommCloseCbParams &)
 772 {
 773     debugs(9, 4, status());
 774     if (data.listenConn != NULL) {
 775         data.listenConn->close();
 776         data.listenConn = NULL;
 777         // NP clear() does the: data.fd = -1;
 778     }
 779     data.clear();
 780 }
 781
 782 void
 783 Ftp::Client::writeCommand(const char *buf)
 784 {
 785     char *ebuf;
 786     /* trace FTP protocol communications at level 2 */
 787     debugs(9, 2, "ftp<< " << buf);
 788
 789     if (Config.Ftp.telnet)
 790         ebuf = escapeIAC(buf);
 791     else
 792         ebuf = xstrdup(buf);
 793
 794     safe_free(ctrl.last_command);
 795
 796     safe_free(ctrl.last_reply);
 797
 798     ctrl.last_command = ebuf;
 799
 800     if (!Comm::IsConnOpen(ctrl.conn)) {
 801         debugs(9, 2, "cannot send to closing ctrl " << ctrl.conn);
 802         // TODO: assert(ctrl.closer != NULL);
 803         return;
 804     }
 805
 806     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 807     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this,
 808                                           Ftp::Client::writeCommandCallback);
 809     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
 810
 811     scheduleReadControlReply(0);
 812 }
 813
 814 void
 815 Ftp::Client::writeCommandCallback(const CommIoCbParams &io)
 816 {
 817
 818     debugs(9, 5, "wrote " << io.size << " bytes");
 819
 820     if (io.size > 0) {
 821         fd_bytes(io.fd, io.size, FD_WRITE);
 822         statCounter.server.all.kbytes_out += io.size;
 823         statCounter.server.ftp.kbytes_out += io.size;
 824     }
 825
 826     if (io.flag == Comm::ERR_CLOSING)
 827         return;
 828
 829     if (io.flag) {
 830         debugs(9, DBG_IMPORTANT, "FTP command write error: " << io.conn << ": " << xstrerr(io.xerrno));
 831         failed(ERR_WRITE_ERROR, io.xerrno);
 832         /* failed closes ctrl.conn and frees ftpState */
 833         return;
 834     }
 835 }
 836
 837 /// handler called by Comm when FTP control channel is closed unexpectedly
 838 void
 839 Ftp::Client::ctrlClosed(const CommCloseCbParams &)
 840 {
 841     debugs(9, 4, status());
 842     ctrl.clear();
 843     mustStop("Ftp::Client::ctrlClosed");
 844 }
 845
 846 void
 847 Ftp::Client::timeout(const CommTimeoutCbParams &io)
 848 {
 849     debugs(9, 4, io.conn << ": '" << entry->url() << "'" );
 850
 851     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 852         return;
 853
 854     failed(ERR_READ_TIMEOUT, 0);
 855     /* failed() closes ctrl.conn and frees ftpState */
 856 }
 857
 858 const Comm::ConnectionPointer &
 859 Ftp::Client::dataConnection() const
 860 {
 861     return data.conn;
 862 }
 863
 864 void
 865 Ftp::Client::maybeReadVirginBody()
 866 {
 867     // too late to read
 868     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
 869         return;
 870
 871     if (data.read_pending)
 872         return;
 873
 874     initReadBuf();
 875
 876     const int read_sz = replyBodySpace(*data.readBuf, 0);
 877
 878     debugs(9, 9, "FTP may read up to " << read_sz << " bytes");
 879
 880     if (read_sz < 2) // see http.cc
 881         return;
 882
 883     data.read_pending = true;
 884
 885     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 886     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
 887                                       TimeoutDialer, this, Ftp::Client::timeout);
 888     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 889
 890     debugs(9,5,"queueing read on FD " << data.conn->fd);
 891
 892     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 893     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
 894                           JobCallback(9, 5, Dialer, this, Ftp::Client::dataRead));
 895 }
 896
 897 void
 898 Ftp::Client::dataRead(const CommIoCbParams &io)
 899 {
 900     int j;
 901     int bin;
 902
 903     data.read_pending = false;
 904
 905     debugs(9, 3, "FD " << io.fd << " Read " << io.size << " bytes");
 906
 907     if (io.size > 0) {
 908         statCounter.server.all.kbytes_in += io.size;
 909         statCounter.server.ftp.kbytes_in += io.size;
 910     }
 911
 912     if (io.flag == Comm::ERR_CLOSING)
 913         return;
 914
 915     assert(io.fd == data.conn->fd);
 916
 917     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 918         abortTransaction("entry aborted during dataRead");
 919         return;
 920     }
 921
 922     if (io.flag == Comm::OK && io.size > 0) {
 923         debugs(9, 5, "appended " << io.size << " bytes to readBuf");
 924         data.readBuf->appended(io.size);
 925 #if USE_DELAY_POOLS
 926         DelayId delayId = entry->mem_obj->mostBytesAllowed();
 927         delayId.bytesIn(io.size);
 928 #endif
 929         ++ IOStats.Ftp.reads;
 930
 931         for (j = io.size - 1, bin = 0; j; ++bin)
 932             j >>= 1;
 933
 934         ++ IOStats.Ftp.read_hist[bin];
 935     }
 936
 937     if (io.flag != Comm::OK) {
 938         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 939                "FTP data read error: " << xstrerr(io.xerrno));
 940
 941         if (ignoreErrno(io.xerrno)) {
 942             maybeReadVirginBody();
 943         } else {
 944             failed(ERR_READ_ERROR, 0);
 945             /* failed closes ctrl.conn and frees ftpState */
 946             return;
 947         }
 948     } else if (io.size == 0) {
 949         debugs(9, 3, "Calling dataComplete() because io.size == 0");
 950         /*
 951          * DPW 2007-04-23
 952          * Dangerous curves ahead.  This call to dataComplete was
 953          * calling scheduleReadControlReply, handleControlReply,
 954          * and then ftpReadTransferDone.  If ftpReadTransferDone
 955          * gets unexpected status code, it closes down the control
 956          * socket and our FtpStateData object gets destroyed.   As
 957          * a workaround we no longer set the 'buffered_ok' flag in
 958          * the scheduleReadControlReply call.
 959          */
 960         dataComplete();
 961     }
 962
 963     processReplyBody();
 964 }
 965
 966 void
 967 Ftp::Client::dataComplete()
 968 {
 969     debugs(9, 3,status());
 970
 971     /* Connection closed; transfer done. */
 972
 973     /// Close data channel, if any, to conserve resources while we wait.
 974     data.close();
 975
 976     /* expect the "transfer complete" message on the control socket */
 977     /*
 978      * DPW 2007-04-23
 979      * Previously, this was the only place where we set the
 980      * 'buffered_ok' flag when calling scheduleReadControlReply().
 981      * It caused some problems if the FTP server returns an unexpected
 982      * status code after the data command.  FtpStateData was being
 983      * deleted in the middle of dataRead().
 984      */
 985     /* AYJ: 2011-01-13: Bug 2581.
 986      * 226 status is possibly waiting in the ctrl buffer.
 987      * The connection will hang if we DONT send buffered_ok.
 988      * This happens on all transfers which can be completly sent by the
 989      * server before the 150 started status message is read in by Squid.
 990      * ie all transfers of about one packet hang.
 991      */
 992     scheduleReadControlReply(1);
 993 }
 994
 995 /**
 996  * Quickly abort the transaction
 997  *
 998  \todo destruction should be sufficient as the destructor should cleanup,
 999  * including canceling close handlers
1000  */
1001 void
1002 Ftp::Client::abortTransaction(const char *reason)
1003 {
1004     debugs(9, 3, "aborting transaction for " << reason <<
1005            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
1006     if (Comm::IsConnOpen(ctrl.conn)) {
1007         ctrl.conn->close();
1008         return;
1009     }
1010
1011     fwd->handleUnregisteredServerEnd();
1012     mustStop("Ftp::Client::abortTransaction");
1013 }
1014
1015 /**
1016  * Cancel the timeout on the Control socket and establish one
1017  * on the data socket
1018  */
1019 void
1020 Ftp::Client::switchTimeoutToDataChannel()
1021 {
1022     commUnsetConnTimeout(ctrl.conn);
1023
1024     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
1025     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this,
1026                                      Ftp::Client::timeout);
1027     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1028 }
1029
1030 void
1031 Ftp::Client::sentRequestBody(const CommIoCbParams &io)
1032 {
1033     if (io.size > 0)
1034         statCounter.server.ftp.kbytes_out += io.size;
1035     ::Client::sentRequestBody(io);
1036 }
1037
1038 /**
1039  * called after we wrote the last byte of the request body
1040  */
1041 void
1042 Ftp::Client::doneSendingRequestBody()
1043 {
1044     ::Client::doneSendingRequestBody();
1045     debugs(9, 3, status());
1046     dataComplete();
1047     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
1048      * if transfer type is 'stream' call dataComplete()
1049      * otherwise leave open. (reschedule control channel read?)
1050      */
1051 }
1052
1053 /// Parses FTP server control response into ctrl structure fields,
1054 /// setting bytesUsed and returning true on success.
1055 bool
1056 Ftp::Client::parseControlReply(size_t &bytesUsed)
1057 {
1058     char *s;
1059     char *sbuf;
1060     char *end;
1061     int usable;
1062     int complete = 0;
1063     wordlist *head = NULL;
1064     wordlist *list;
1065     wordlist **tail = &head;
1066     size_t linelen;
1067     debugs(9, 3, status());
1068     /*
1069      * We need a NULL-terminated buffer for scanning, ick
1070      */
1071     const size_t len = ctrl.offset;
1072     sbuf = (char *)xmalloc(len + 1);
1073     xstrncpy(sbuf, ctrl.buf, len + 1);
1074     end = sbuf + len - 1;
1075
1076     while (*end != '\r' && *end != '\n' && end > sbuf)
1077         --end;
1078
1079     usable = end - sbuf;
1080
1081     debugs(9, 3, "usable = " << usable);
1082
1083     if (usable == 0) {
1084         debugs(9, 3, "didn't find end of line");
1085         safe_free(sbuf);
1086         return false;
1087     }
1088
1089     debugs(9, 3, len << " bytes to play with");
1090     ++end;
1091     s = sbuf;
1092     s += strspn(s, crlf);
1093
1094     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1095         if (complete)
1096             break;
1097
1098         debugs(9, 5, "s = {" << s << "}");
1099
1100         linelen = strcspn(s, crlf) + 1;
1101
1102         if (linelen < 2)
1103             break;
1104
1105         if (linelen > 3)
1106             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1107
1108         list = new wordlist();
1109
1110         list->key = (char *)xmalloc(linelen);
1111
1112         xstrncpy(list->key, s, linelen);
1113
1114         /* trace the FTP communication chat at level 2 */
1115         debugs(9, 2, "ftp>> " << list->key);
1116
1117         if (complete) {
1118             // use list->key for last_reply because s contains the new line
1119             ctrl.last_reply = xstrdup(list->key + 4);
1120             ctrl.replycode = atoi(list->key);
1121         }
1122
1123         *tail = list;
1124
1125         tail = &list->next;
1126     }
1127
1128     bytesUsed = static_cast<size_t>(s - sbuf);
1129     safe_free(sbuf);
1130
1131     if (!complete) {
1132         wordlistDestroy(&head);
1133         return false;
1134     }
1135
1136     ctrl.message = head;
1137     assert(ctrl.replycode >= 0);
1138     assert(ctrl.last_reply);
1139     assert(ctrl.message);
1140     return true;
1141 }
1142
1143 }; // namespace Ftp
1144