src/clients/FtpClient.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "client_side.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm/ConnOpener.h"
  16 #include "comm/Read.h"
  17 #include "comm/TcpAcceptor.h"
  18 #include "comm/Write.h"
  19 #include "errorpage.h"
  20 #include "fd.h"
  21 #include "ftp/Parsing.h"
  22 #include "ip/tools.h"
  23 #include "Mem.h"
  24 #include "SquidConfig.h"
  25 #include "SquidString.h"
  26 #include "StatCounters.h"
  27 #include "tools.h"
  28 #include "wordlist.h"
  29
  30 #include <set>
  31
  32 namespace Ftp
  33 {
  34
  35 const char *const crlf = "\r\n";
  36
  37 static char *
  38 escapeIAC(const char *buf)
  39 {
  40     int n;
  41     char *ret;
  42     unsigned const char *p;
  43     unsigned char *r;
  44
  45     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
  46         if (*p == 255)
  47             ++n;
  48
  49     ret = (char *)xmalloc(n);
  50
  51     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
  52         *r = *p;
  53         ++r;
  54
  55         if (*p == 255) {
  56             *r = 255;
  57             ++r;
  58         }
  59     }
  60
  61     *r = '\0';
  62     ++r;
  63     assert((r - (unsigned char *)ret) == n );
  64     return ret;
  65 }
  66
  67 /* Ftp::Channel */
  68
  69 /// configures the channel with a descriptor and registers a close handler
  70 void
  71 Ftp::Channel::opened(const Comm::ConnectionPointer &newConn,
  72                      const AsyncCall::Pointer &aCloser)
  73 {
  74     assert(!Comm::IsConnOpen(conn));
  75     assert(closer == NULL);
  76
  77     assert(Comm::IsConnOpen(newConn));
  78     assert(aCloser != NULL);
  79
  80     conn = newConn;
  81     closer = aCloser;
  82     comm_add_close_handler(conn->fd, closer);
  83 }
  84
  85 /// planned close: removes the close handler and calls comm_close
  86 void
  87 Ftp::Channel::close()
  88 {
  89     // channels with active listeners will be closed when the listener handler dies.
  90     if (Comm::IsConnOpen(conn)) {
  91         comm_remove_close_handler(conn->fd, closer);
  92         conn->close(); // we do not expect to be called back
  93     }
  94     clear();
  95 }
  96
  97 void
  98 Ftp::Channel::forget()
  99 {
 100     if (Comm::IsConnOpen(conn)) {
 101         commUnsetConnTimeout(conn);
 102         comm_remove_close_handler(conn->fd, closer);
 103     }
 104     clear();
 105 }
 106
 107 void
 108 Ftp::Channel::clear()
 109 {
 110     conn = NULL;
 111     closer = NULL;
 112 }
 113
 114 /* Ftp::CtrlChannel */
 115
 116 Ftp::CtrlChannel::CtrlChannel():
 117         buf(NULL),
 118         size(0),
 119         offset(0),
 120         message(NULL),
 121         last_command(NULL),
 122         last_reply(NULL),
 123         replycode(0)
 124 {
 125     buf = static_cast<char*>(memAllocBuf(4096, &size));
 126 }
 127
 128 Ftp::CtrlChannel::~CtrlChannel()
 129 {
 130     memFreeBuf(size, buf);
 131     if (message)
 132         wordlistDestroy(&message);
 133     safe_free(last_command);
 134     safe_free(last_reply);
 135 }
 136
 137 /* Ftp::DataChannel */
 138
 139 Ftp::DataChannel::DataChannel():
 140         readBuf(NULL),
 141         host(NULL),
 142         port(0),
 143         read_pending(false)
 144 {
 145 }
 146
 147 Ftp::DataChannel::~DataChannel()
 148 {
 149     delete readBuf;
 150 }
 151
 152 void
 153 Ftp::DataChannel::addr(const Ip::Address &import)
 154 {
 155     static char addrBuf[MAX_IPSTRLEN];
 156     import.toStr(addrBuf, sizeof(addrBuf));
 157     xfree(host);
 158     host = xstrdup(addrBuf);
 159     port = import.port();
 160 }
 161
 162 /* Ftp::Client */
 163
 164 Ftp::Client::Client(FwdState *fwdState):
 165         AsyncJob("Ftp::Client"),
 166         ::ServerStateData(fwdState),
 167         ctrl(),
 168         data(),
 169         state(BEGIN),
 170         old_request(NULL),
 171         old_reply(NULL),
 172         shortenReadTimeout(false)
 173 {
 174     ++statCounter.server.all.requests;
 175     ++statCounter.server.ftp.requests;
 176
 177     ctrl.last_command = xstrdup("Connect to server");
 178
 179     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 180     const AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this,
 181                                       Ftp::Client::ctrlClosed);
 182     ctrl.opened(fwdState->serverConnection(), closer);
 183 }
 184
 185 Ftp::Client::~Client()
 186 {
 187     if (data.opener != NULL) {
 188         data.opener->cancel("Ftp::Client destructed");
 189         data.opener = NULL;
 190     }
 191     data.close();
 192
 193     safe_free(old_request);
 194     safe_free(old_reply);
 195     fwd = NULL; // refcounted
 196 }
 197
 198 void
 199 Ftp::Client::start()
 200 {
 201     scheduleReadControlReply(0);
 202 }
 203
 204 void
 205 Ftp::Client::initReadBuf()
 206 {
 207     if (data.readBuf == NULL) {
 208         data.readBuf = new MemBuf;
 209         data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
 210     }
 211 }
 212
 213 /**
 214  * Close the FTP server connection(s). Used by serverComplete().
 215  */
 216 void
 217 Ftp::Client::closeServer()
 218 {
 219     if (Comm::IsConnOpen(ctrl.conn)) {
 220         debugs(9, 3, "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
 221         fwd->unregister(ctrl.conn);
 222         ctrl.close();
 223     }
 224
 225     if (Comm::IsConnOpen(data.conn)) {
 226         debugs(9, 3, "closing FTP data FD " << data.conn->fd << ", this " << this);
 227         data.close();
 228     }
 229
 230     debugs(9, 3, "FTP ctrl and data connections closed. this " << this);
 231 }
 232
 233 /**
 234  * Did we close all FTP server connection(s)?
 235  *
 236  \retval true  Both server control and data channels are closed. And not waiting for a new data connection to open.
 237  \retval false Either control channel or data is still active.
 238  */
 239 bool
 240 Ftp::Client::doneWithServer() const
 241 {
 242     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
 243 }
 244
 245 void
 246 Ftp::Client::failed(err_type error, int xerrno)
 247 {
 248     debugs(9, 3, "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
 249
 250     const char *command, *reply;
 251     const Http::StatusCode httpStatus = failedHttpStatus(error);
 252     ErrorState *const ftperr = new ErrorState(error, httpStatus, fwd->request);
 253     ftperr->xerrno = xerrno;
 254
 255     ftperr->ftp.server_msg = ctrl.message;
 256     ctrl.message = NULL;
 257
 258     if (old_request)
 259         command = old_request;
 260     else
 261         command = ctrl.last_command;
 262
 263     if (command && strncmp(command, "PASS", 4) == 0)
 264         command = "PASS <yourpassword>";
 265
 266     if (old_reply)
 267         reply = old_reply;
 268     else
 269         reply = ctrl.last_reply;
 270
 271     if (command)
 272         ftperr->ftp.request = xstrdup(command);
 273
 274     if (reply)
 275         ftperr->ftp.reply = xstrdup(reply);
 276
 277     fwd->request->detailError(error, xerrno);
 278     fwd->fail(ftperr);
 279
 280     closeServer(); // we failed, so no serverComplete()
 281 }
 282
 283 Http::StatusCode
 284 Ftp::Client::failedHttpStatus(err_type &error)
 285 {
 286     if (error == ERR_NONE)
 287         error = ERR_FTP_FAILURE;
 288     return error == ERR_READ_TIMEOUT ? Http::scGatewayTimeout :
 289            Http::scBadGateway;
 290 }
 291
 292 /**
 293  * DPW 2007-04-23
 294  * Looks like there are no longer anymore callers that set
 295  * buffered_ok=1.  Perhaps it can be removed at some point.
 296  */
 297 void
 298 Ftp::Client::scheduleReadControlReply(int buffered_ok)
 299 {
 300     debugs(9, 3, ctrl.conn);
 301
 302     if (buffered_ok && ctrl.offset > 0) {
 303         /* We've already read some reply data */
 304         handleControlReply();
 305     } else {
 306         /*
 307          * Cancel the timeout on the Data socket (if any) and
 308          * establish one on the control socket.
 309          */
 310         if (Comm::IsConnOpen(data.conn)) {
 311             commUnsetConnTimeout(data.conn);
 312         }
 313
 314         const time_t tout = shortenReadTimeout ?
 315                             min(Config.Timeout.connect, Config.Timeout.read):
 316                             Config.Timeout.read;
 317         shortenReadTimeout = false; // we only need to do this once, after PASV
 318
 319         typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 320         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, Ftp::Client::timeout);
 321         commSetConnTimeout(ctrl.conn, tout, timeoutCall);
 322
 323         typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 324         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, Ftp::Client::readControlReply);
 325         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
 326     }
 327 }
 328
 329 void
 330 Ftp::Client::readControlReply(const CommIoCbParams &io)
 331 {
 332     debugs(9, 3, "FD " << io.fd << ", Read " << io.size << " bytes");
 333
 334     if (io.size > 0) {
 335         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
 336         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
 337     }
 338
 339     if (io.flag == Comm::ERR_CLOSING)
 340         return;
 341
 342     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 343         abortTransaction("entry aborted during control reply read");
 344         return;
 345     }
 346
 347     assert(ctrl.offset < ctrl.size);
 348
 349     if (io.flag == Comm::OK && io.size > 0) {
 350         fd_bytes(io.fd, io.size, FD_READ);
 351     }
 352
 353     if (io.flag != Comm::OK) {
 354         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 355                "FTP control reply read error: " << xstrerr(io.xerrno));
 356
 357         if (ignoreErrno(io.xerrno)) {
 358             scheduleReadControlReply(0);
 359         } else {
 360             failed(ERR_READ_ERROR, io.xerrno);
 361             /* failed closes ctrl.conn and frees ftpState */
 362         }
 363         return;
 364     }
 365
 366     if (io.size == 0) {
 367         if (entry->store_status == STORE_PENDING) {
 368             failed(ERR_FTP_FAILURE, 0);
 369             /* failed closes ctrl.conn and frees ftpState */
 370             return;
 371         }
 372
 373         /* XXX this may end up having to be serverComplete() .. */
 374         abortTransaction("zero control reply read");
 375         return;
 376     }
 377
 378     unsigned int len =io.size + ctrl.offset;
 379     ctrl.offset = len;
 380     assert(len <= ctrl.size);
 381     if (Comm::IsConnOpen(ctrl.conn))
 382         commUnsetConnTimeout(ctrl.conn); // we are done waiting for ctrl reply
 383     handleControlReply();
 384 }
 385
 386 void
 387 Ftp::Client::handleControlReply()
 388 {
 389     debugs(9, 3, status());
 390
 391     size_t bytes_used = 0;
 392     wordlistDestroy(&ctrl.message);
 393
 394     if (!parseControlReply(bytes_used)) {
 395         /* didn't get complete reply yet */
 396
 397         if (ctrl.offset == ctrl.size) {
 398             ctrl.buf = static_cast<char*>(memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size));
 399         }
 400
 401         scheduleReadControlReply(0);
 402         return;
 403     }
 404
 405     assert(ctrl.message); // the entire FTP server response, line by line
 406     assert(ctrl.replycode >= 0); // FTP status code (from the last line)
 407     assert(ctrl.last_reply); // FTP reason (from the last line)
 408
 409     if (ctrl.offset == bytes_used) {
 410         /* used it all up */
 411         ctrl.offset = 0;
 412     } else {
 413         /* Got some data past the complete reply */
 414         assert(bytes_used < ctrl.offset);
 415         ctrl.offset -= bytes_used;
 416         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
 417     }
 418
 419     debugs(9, 3, "state=" << state << ", code=" << ctrl.replycode);
 420 }
 421
 422 bool
 423 Ftp::Client::handlePasvReply(Ip::Address &srvAddr)
 424 {
 425     int code = ctrl.replycode;
 426     char *buf;
 427     debugs(9, 3, status());
 428
 429     if (code != 227) {
 430         debugs(9, 2, "PASV not supported by remote end");
 431         return false;
 432     }
 433
 434     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
 435     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 436     debugs(9, 5, "scanning: " << ctrl.last_reply);
 437
 438     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "0123456789");
 439
 440     const char *forceIp = Config.Ftp.sanitycheck ?
 441                           fd_table[ctrl.conn->fd].ipaddr : NULL;
 442     if (!Ftp::ParseIpPort(buf, forceIp, srvAddr)) {
 443         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
 444                ctrl.conn->remote << ": " << ctrl.last_reply);
 445         return false;
 446     }
 447
 448     data.addr(srvAddr);
 449
 450     return true;
 451 }
 452
 453 bool
 454 Ftp::Client::handleEpsvReply(Ip::Address &remoteAddr)
 455 {
 456     int code = ctrl.replycode;
 457     char *buf;
 458     debugs(9, 3, status());
 459
 460     if (code != 229 && code != 522) {
 461         if (code == 200) {
 462             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
 463             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
 464              * Its okay to re-send EPSV 1/2 but nothing else. */
 465             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". Wrong accept code for EPSV");
 466         } else {
 467             debugs(9, 2, "EPSV not supported by remote end");
 468         }
 469         return sendPassive();
 470     }
 471
 472     if (code == 522) {
 473         /* Peer responded with a list of supported methods:
 474          *   522 Network protocol not supported, use (1)
 475          *   522 Network protocol not supported, use (1,2)
 476          *   522 Network protocol not supported, use (2)
 477          * TODO: Handle the (1,2) case which may happen after EPSV ALL. Close
 478          * data + control without self-destructing and re-open from scratch.
 479          */
 480         debugs(9, 5, "scanning: " << ctrl.last_reply);
 481         buf = ctrl.last_reply;
 482         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
 483             ++buf;
 484         if (buf != NULL && *buf == '\n')
 485             ++buf;
 486
 487         if (buf == NULL || *buf == '\0') {
 488             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 489             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
 490             return sendPassive();
 491         } else if (strcmp(buf, "(1)") == 0) {
 492             state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
 493             return sendPassive();
 494         } else if (strcmp(buf, "(2)") == 0) {
 495             if (Ip::EnableIpv6) {
 496                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
 497                 if (state == SENT_EPSV_2) {
 498                     return sendEprt();
 499                 } else {
 500                     /* or try the next Passive mode down the chain. */
 501                     return sendPassive();
 502                 }
 503             } else {
 504                 /* Server only accept EPSV in IPv6 traffic. */
 505                 state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
 506                 return sendPassive();
 507             }
 508         } else {
 509             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
 510             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
 511             return sendPassive();
 512         }
 513         failed(ERR_FTP_FAILURE, 0);
 514         return false;
 515     }
 516
 517     /*  229 Entering Extended Passive Mode (|||port|) */
 518     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
 519     debugs(9, 5, "scanning: " << ctrl.last_reply);
 520
 521     buf = ctrl.last_reply + strcspn(ctrl.last_reply, "(");
 522
 523     char h1, h2, h3, h4;
 524     unsigned short port;
 525     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
 526
 527     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
 528         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
 529                ctrl.conn->remote << ": " <<
 530                ctrl.last_reply);
 531
 532         return sendPassive();
 533     }
 534
 535     if (0 == port) {
 536         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 537                ctrl.conn->remote << ": " <<
 538                ctrl.last_reply);
 539
 540         return sendPassive();
 541     }
 542
 543     if (Config.Ftp.sanitycheck) {
 544         if (port < 1024) {
 545             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
 546                    ctrl.conn->remote << ": " <<
 547                    ctrl.last_reply);
 548
 549             return sendPassive();
 550         }
 551     }
 552
 553     remoteAddr = ctrl.conn->remote;
 554     remoteAddr.port(port);
 555     data.addr(remoteAddr);
 556     return true;
 557 }
 558
 559 // FTP clients do not support EPRT and PORT commands yet.
 560 // The Ftp::Client::sendEprt() will fail because of the unimplemented
 561 // openListenSocket() or sendPort() methods
 562 bool
 563 Ftp::Client::sendEprt()
 564 {
 565     if (!Config.Ftp.eprt) {
 566         /* Disabled. Switch immediately to attempting old PORT command. */
 567         debugs(9, 3, "EPRT disabled by local administrator");
 568         return sendPort();
 569     }
 570
 571     debugs(9, 3, status());
 572
 573     if (!openListenSocket()) {
 574         failed(ERR_FTP_FAILURE, 0);
 575         return false;
 576     }
 577
 578     debugs(9, 3, "Listening for FTP data connection with FD " << data.conn);
 579     if (!Comm::IsConnOpen(data.conn)) {
 580         // TODO: Set error message.
 581         failed(ERR_FTP_FAILURE, 0);
 582         return false;
 583     }
 584
 585     static MemBuf mb;
 586     mb.reset();
 587     char buf[MAX_IPSTRLEN];
 588     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
 589     /* Which can be used by EITHER protocol. */
 590     debugs(9, 3, "Listening for FTP data connection on port" << comm_local_port(data.conn->fd) << " or port?" << data.conn->local.port());
 591     mb.Printf("EPRT |%d|%s|%d|%s",
 592               ( data.conn->local.isIPv6() ? 2 : 1 ),
 593               data.conn->local.toStr(buf,MAX_IPSTRLEN),
 594               comm_local_port(data.conn->fd), Ftp::crlf );
 595
 596     state = SENT_EPRT;
 597     writeCommand(mb.content());
 598     return true;
 599 }
 600
 601 bool
 602 Ftp::Client::sendPort()
 603 {
 604     failed(ERR_FTP_FAILURE, 0);
 605     return false;
 606 }
 607
 608 bool
 609 Ftp::Client::sendPassive()
 610 {
 611     debugs(9, 3, status());
 612
 613     /** \par
 614       * Checks for EPSV ALL special conditions:
 615       * If enabled to be sent, squid MUST NOT request any other connect methods.
 616       * If 'ALL' is sent and fails the entire FTP Session fails.
 617       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
 618     if (Config.Ftp.epsv_all && state == SENT_EPSV_1 ) {
 619         // We are here because the last "EPSV 1" failed, but because of epsv_all
 620         // no other method allowed.
 621         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 622         failed(ERR_FTP_FAILURE, 0);
 623         return false;
 624     }
 625
 626     /// Closes any old FTP-Data connection which may exist. */
 627     data.close();
 628
 629     /** \par
 630       * Checks for previous EPSV/PASV failures on this server/session.
 631       * Diverts to EPRT immediately if they are not working. */
 632     if (!Config.Ftp.passive || state == SENT_PASV) {
 633         sendEprt();
 634         return true;
 635     }
 636
 637     static MemBuf mb;
 638     mb.reset();
 639     /** \par
 640       * Send EPSV (ALL,2,1) or PASV on the control channel.
 641       *
 642       *  - EPSV ALL  is used if enabled.
 643       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
 644       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
 645       *  - PASV      is used if EPSV 1 fails.
 646       */
 647     switch (state) {
 648     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
 649         if (ctrl.conn->local.isIPv6()) {
 650             debugs(9, 5, "FTP Channel is IPv6 (" << ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
 651             mb.Printf("EPSV 2%s", Ftp::crlf);
 652             state = SENT_EPSV_2;
 653             break;
 654         }
 655         // else fall through to skip EPSV 2
 656
 657     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
 658         if (ctrl.conn->local.isIPv4()) {
 659             debugs(9, 5, "FTP Channel is IPv4 (" << ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
 660             mb.Printf("EPSV 1%s", Ftp::crlf);
 661             state = SENT_EPSV_1;
 662             break;
 663         } else if (Config.Ftp.epsv_all) {
 664             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
 665             failed(ERR_FTP_FAILURE, 0);
 666             return false;
 667         }
 668         // else fall through to skip EPSV 1
 669
 670     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
 671         debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
 672         mb.Printf("PASV%s", Ftp::crlf);
 673         state = SENT_PASV;
 674         break;
 675
 676     default: {
 677         bool doEpsv = true;
 678         if (Config.accessList.ftp_epsv) {
 679             ACLFilledChecklist checklist(Config.accessList.ftp_epsv, fwd->request, NULL);
 680             doEpsv = (checklist.fastCheck() == ACCESS_ALLOWED);
 681         }
 682         if (!doEpsv) {
 683             debugs(9, 5, "EPSV support manually disabled. Sending PASV for FTP Channel (" << ctrl.conn->remote <<")");
 684             mb.Printf("PASV%s", Ftp::crlf);
 685             state = SENT_PASV;
 686         } else if (Config.Ftp.epsv_all) {
 687             debugs(9, 5, "EPSV ALL manually enabled. Attempting with FTP Channel (" << ctrl.conn->remote <<")");
 688             mb.Printf("EPSV ALL%s", Ftp::crlf);
 689             state = SENT_EPSV_ALL;
 690         } else {
 691             if (ctrl.conn->local.isIPv6()) {
 692                 debugs(9, 5, "FTP Channel (" << ctrl.conn->remote << "). Sending default EPSV 2");
 693                 mb.Printf("EPSV 2%s", Ftp::crlf);
 694                 state = SENT_EPSV_2;
 695             }
 696             if (ctrl.conn->local.isIPv4()) {
 697                 debugs(9, 5, "Channel (" << ctrl.conn->remote <<"). Sending default EPSV 1");
 698                 mb.Printf("EPSV 1%s", Ftp::crlf);
 699                 state = SENT_EPSV_1;
 700             }
 701         }
 702         break;
 703     }
 704     }
 705
 706     if (ctrl.message)
 707         wordlistDestroy(&ctrl.message);
 708     ctrl.message = NULL; //No message to return to client.
 709     ctrl.offset = 0; //reset readed response, to make room read the next response
 710
 711     writeCommand(mb.content());
 712
 713     shortenReadTimeout = true;
 714     return true;
 715 }
 716
 717 void
 718 Ftp::Client::connectDataChannel()
 719 {
 720     safe_free(ctrl.last_command);
 721
 722     safe_free(ctrl.last_reply);
 723
 724     ctrl.last_command = xstrdup("Connect to server data port");
 725
 726     // Generate a new data channel descriptor to be opened.
 727     Comm::ConnectionPointer conn = new Comm::Connection;
 728     conn->setAddrs(ctrl.conn->local, data.host);
 729     conn->local.port(0);
 730     conn->remote.port(data.port);
 731     conn->tos = ctrl.conn->tos;
 732     conn->nfmark = ctrl.conn->nfmark;
 733
 734     debugs(9, 3, "connecting to " << conn->remote);
 735
 736     typedef CommCbMemFunT<Client, CommConnectCbParams> Dialer;
 737     data.opener = JobCallback(9, 3, Dialer, this, Ftp::Client::dataChannelConnected);
 738     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, data.opener, Config.Timeout.connect);
 739     cs->setHost(data.host);
 740     AsyncJob::Start(cs);
 741 }
 742
 743 bool
 744 Ftp::Client::openListenSocket()
 745 {
 746     return false;
 747 }
 748
 749 /// creates a data channel Comm close callback
 750 AsyncCall::Pointer
 751 Ftp::Client::dataCloser()
 752 {
 753     typedef CommCbMemFunT<Client, CommCloseCbParams> Dialer;
 754     return JobCallback(9, 5, Dialer, this, Ftp::Client::dataClosed);
 755 }
 756
 757 /// handler called by Comm when FTP data channel is closed unexpectedly
 758 void
 759 Ftp::Client::dataClosed(const CommCloseCbParams &io)
 760 {
 761     debugs(9, 4, status());
 762     if (data.listenConn != NULL) {
 763         data.listenConn->close();
 764         data.listenConn = NULL;
 765         // NP clear() does the: data.fd = -1;
 766     }
 767     data.clear();
 768 }
 769
 770 void
 771 Ftp::Client::writeCommand(const char *buf)
 772 {
 773     char *ebuf;
 774     /* trace FTP protocol communications at level 2 */
 775     debugs(9, 2, "ftp<< " << buf);
 776
 777     if (Config.Ftp.telnet)
 778         ebuf = escapeIAC(buf);
 779     else
 780         ebuf = xstrdup(buf);
 781
 782     safe_free(ctrl.last_command);
 783
 784     safe_free(ctrl.last_reply);
 785
 786     ctrl.last_command = ebuf;
 787
 788     if (!Comm::IsConnOpen(ctrl.conn)) {
 789         debugs(9, 2, "cannot send to closing ctrl " << ctrl.conn);
 790         // TODO: assert(ctrl.closer != NULL);
 791         return;
 792     }
 793
 794     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 795     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this,
 796                                           Ftp::Client::writeCommandCallback);
 797     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
 798
 799     scheduleReadControlReply(0);
 800 }
 801
 802 void
 803 Ftp::Client::writeCommandCallback(const CommIoCbParams &io)
 804 {
 805
 806     debugs(9, 5, "wrote " << io.size << " bytes");
 807
 808     if (io.size > 0) {
 809         fd_bytes(io.fd, io.size, FD_WRITE);
 810         kb_incr(&(statCounter.server.all.kbytes_out), io.size);
 811         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
 812     }
 813
 814     if (io.flag == Comm::ERR_CLOSING)
 815         return;
 816
 817     if (io.flag) {
 818         debugs(9, DBG_IMPORTANT, "FTP command write error: " << io.conn << ": " << xstrerr(io.xerrno));
 819         failed(ERR_WRITE_ERROR, io.xerrno);
 820         /* failed closes ctrl.conn and frees ftpState */
 821         return;
 822     }
 823 }
 824
 825 /// handler called by Comm when FTP control channel is closed unexpectedly
 826 void
 827 Ftp::Client::ctrlClosed(const CommCloseCbParams &io)
 828 {
 829     debugs(9, 4, status());
 830     ctrl.clear();
 831     mustStop("Ftp::Client::ctrlClosed");
 832 }
 833
 834 void
 835 Ftp::Client::timeout(const CommTimeoutCbParams &io)
 836 {
 837     debugs(9, 4, io.conn << ": '" << entry->url() << "'" );
 838
 839     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 840         return;
 841
 842     failed(ERR_READ_TIMEOUT, 0);
 843     /* failed() closes ctrl.conn and frees ftpState */
 844 }
 845
 846 const Comm::ConnectionPointer &
 847 Ftp::Client::dataConnection() const
 848 {
 849     return data.conn;
 850 }
 851
 852 void
 853 Ftp::Client::maybeReadVirginBody()
 854 {
 855     // too late to read
 856     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
 857         return;
 858
 859     if (data.read_pending)
 860         return;
 861
 862     initReadBuf();
 863
 864     const int read_sz = replyBodySpace(*data.readBuf, 0);
 865
 866     debugs(9, 9, "FTP may read up to " << read_sz << " bytes");
 867
 868     if (read_sz < 2) // see http.cc
 869         return;
 870
 871     data.read_pending = true;
 872
 873     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
 874     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
 875                                       TimeoutDialer, this, Ftp::Client::timeout);
 876     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 877
 878     debugs(9,5,"queueing read on FD " << data.conn->fd);
 879
 880     typedef CommCbMemFunT<Client, CommIoCbParams> Dialer;
 881     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
 882                           JobCallback(9, 5, Dialer, this, Ftp::Client::dataRead));
 883 }
 884
 885 void
 886 Ftp::Client::dataRead(const CommIoCbParams &io)
 887 {
 888     int j;
 889     int bin;
 890
 891     data.read_pending = false;
 892
 893     debugs(9, 3, "FD " << io.fd << " Read " << io.size << " bytes");
 894
 895     if (io.size > 0) {
 896         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
 897         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
 898     }
 899
 900     if (io.flag == Comm::ERR_CLOSING)
 901         return;
 902
 903     assert(io.fd == data.conn->fd);
 904
 905     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 906         abortTransaction("entry aborted during dataRead");
 907         return;
 908     }
 909
 910     if (io.flag == Comm::OK && io.size > 0) {
 911         debugs(9, 5, "appended " << io.size << " bytes to readBuf");
 912         data.readBuf->appended(io.size);
 913 #if USE_DELAY_POOLS
 914         DelayId delayId = entry->mem_obj->mostBytesAllowed();
 915         delayId.bytesIn(io.size);
 916 #endif
 917         ++ IOStats.Ftp.reads;
 918
 919         for (j = io.size - 1, bin = 0; j; ++bin)
 920             j >>= 1;
 921
 922         ++ IOStats.Ftp.read_hist[bin];
 923     }
 924
 925     if (io.flag != Comm::OK) {
 926         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
 927                "FTP data read error: " << xstrerr(io.xerrno));
 928
 929         if (ignoreErrno(io.xerrno)) {
 930             maybeReadVirginBody();
 931         } else {
 932             failed(ERR_READ_ERROR, 0);
 933             /* failed closes ctrl.conn and frees ftpState */
 934             return;
 935         }
 936     } else if (io.size == 0) {
 937         debugs(9, 3, "Calling dataComplete() because io.size == 0");
 938         /*
 939          * DPW 2007-04-23
 940          * Dangerous curves ahead.  This call to dataComplete was
 941          * calling scheduleReadControlReply, handleControlReply,
 942          * and then ftpReadTransferDone.  If ftpReadTransferDone
 943          * gets unexpected status code, it closes down the control
 944          * socket and our FtpStateData object gets destroyed.   As
 945          * a workaround we no longer set the 'buffered_ok' flag in
 946          * the scheduleReadControlReply call.
 947          */
 948         dataComplete();
 949     }
 950
 951     processReplyBody();
 952 }
 953
 954 void
 955 Ftp::Client::dataComplete()
 956 {
 957     debugs(9, 3,status());
 958
 959     /* Connection closed; transfer done. */
 960
 961     /// Close data channel, if any, to conserve resources while we wait.
 962     data.close();
 963
 964     /* expect the "transfer complete" message on the control socket */
 965     /*
 966      * DPW 2007-04-23
 967      * Previously, this was the only place where we set the
 968      * 'buffered_ok' flag when calling scheduleReadControlReply().
 969      * It caused some problems if the FTP server returns an unexpected
 970      * status code after the data command.  FtpStateData was being
 971      * deleted in the middle of dataRead().
 972      */
 973     /* AYJ: 2011-01-13: Bug 2581.
 974      * 226 status is possibly waiting in the ctrl buffer.
 975      * The connection will hang if we DONT send buffered_ok.
 976      * This happens on all transfers which can be completly sent by the
 977      * server before the 150 started status message is read in by Squid.
 978      * ie all transfers of about one packet hang.
 979      */
 980     scheduleReadControlReply(1);
 981 }
 982
 983 /**
 984  * Quickly abort the transaction
 985  *
 986  \todo destruction should be sufficient as the destructor should cleanup,
 987  * including canceling close handlers
 988  */
 989 void
 990 Ftp::Client::abortTransaction(const char *reason)
 991 {
 992     debugs(9, 3, "aborting transaction for " << reason <<
 993            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
 994     if (Comm::IsConnOpen(ctrl.conn)) {
 995         ctrl.conn->close();
 996         return;
 997     }
 998
 999     fwd->handleUnregisteredServerEnd();
1000     mustStop("Ftp::Client::abortTransaction");
1001 }
1002
1003 /**
1004  * Cancel the timeout on the Control socket and establish one
1005  * on the data socket
1006  */
1007 void
1008 Ftp::Client::switchTimeoutToDataChannel()
1009 {
1010     commUnsetConnTimeout(ctrl.conn);
1011
1012     typedef CommCbMemFunT<Client, CommTimeoutCbParams> TimeoutDialer;
1013     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this,
1014                                      Ftp::Client::timeout);
1015     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1016 }
1017
1018 void
1019 Ftp::Client::sentRequestBody(const CommIoCbParams &io)
1020 {
1021     if (io.size > 0)
1022         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
1023     ::ServerStateData::sentRequestBody(io);
1024 }
1025
1026 /**
1027  * called after we wrote the last byte of the request body
1028  */
1029 void
1030 Ftp::Client::doneSendingRequestBody()
1031 {
1032     ::ServerStateData::doneSendingRequestBody();
1033     debugs(9, 3, status());
1034     dataComplete();
1035     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
1036      * if transfer type is 'stream' call dataComplete()
1037      * otherwise leave open. (reschedule control channel read?)
1038      */
1039 }
1040
1041 /// Parses FTP server control response into ctrl structure fields,
1042 /// setting bytesUsed and returning true on success.
1043 bool
1044 Ftp::Client::parseControlReply(size_t &bytesUsed)
1045 {
1046     char *s;
1047     char *sbuf;
1048     char *end;
1049     int usable;
1050     int complete = 0;
1051     wordlist *head = NULL;
1052     wordlist *list;
1053     wordlist **tail = &head;
1054     size_t linelen;
1055     debugs(9, 3, status());
1056     /*
1057      * We need a NULL-terminated buffer for scanning, ick
1058      */
1059     const size_t len = ctrl.offset;
1060     sbuf = (char *)xmalloc(len + 1);
1061     xstrncpy(sbuf, ctrl.buf, len + 1);
1062     end = sbuf + len - 1;
1063
1064     while (*end != '\r' && *end != '\n' && end > sbuf)
1065         --end;
1066
1067     usable = end - sbuf;
1068
1069     debugs(9, 3, "usable = " << usable);
1070
1071     if (usable == 0) {
1072         debugs(9, 3, "didn't find end of line");
1073         safe_free(sbuf);
1074         return false;
1075     }
1076
1077     debugs(9, 3, len << " bytes to play with");
1078     ++end;
1079     s = sbuf;
1080     s += strspn(s, crlf);
1081
1082     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1083         if (complete)
1084             break;
1085
1086         debugs(9, 5, "s = {" << s << "}");
1087
1088         linelen = strcspn(s, crlf) + 1;
1089
1090         if (linelen < 2)
1091             break;
1092
1093         if (linelen > 3)
1094             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1095
1096         list = new wordlist();
1097
1098         list->key = (char *)xmalloc(linelen);
1099
1100         xstrncpy(list->key, s, linelen);
1101
1102         /* trace the FTP communication chat at level 2 */
1103         debugs(9, 2, "ftp>> " << list->key);
1104
1105         if (complete) {
1106             // use list->key for last_reply because s contains the new line
1107             ctrl.last_reply = xstrdup(list->key + 4);
1108             ctrl.replycode = atoi(list->key);
1109         }
1110
1111         *tail = list;
1112
1113         tail = &list->next;
1114     }
1115
1116     bytesUsed = static_cast<size_t>(s - sbuf);
1117     safe_free(sbuf);
1118
1119     if (!complete) {
1120         wordlistDestroy(&head);
1121         return false;
1122     }
1123
1124     ctrl.message = head;
1125     assert(ctrl.replycode >= 0);
1126     assert(ctrl.last_reply);
1127     assert(ctrl.message);
1128     return true;
1129 }
1130
1131 }; // namespace Ftp