src/ftp.cc

   1 /*
   2  * DEBUG: section 09    File Transfer Protocol (FTP)
   3  * AUTHOR: Harvest Derived
   4  *
   5  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   6  * ----------------------------------------------------------
   7  *
   8  *  Squid is the result of efforts by numerous individuals from
   9  *  the Internet community; see the CONTRIBUTORS file for full
  10  *  details.   Many organizations have provided support for Squid's
  11  *  development; see the SPONSORS file for full details.  Squid is
  12  *  Copyrighted (C) 2001 by the Regents of the University of
  13  *  California; see the COPYRIGHT file for full details.  Squid
  14  *  incorporates software developed and/or copyrighted by other
  15  *  sources; see the CREDITS file for full details.
  16  *
  17  *  This program is free software; you can redistribute it and/or modify
  18  *  it under the terms of the GNU General Public License as published by
  19  *  the Free Software Foundation; either version 2 of the License, or
  20  *  (at your option) any later version.
  21  *
  22  *  This program is distributed in the hope that it will be useful,
  23  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  24  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  25  *  GNU General Public License for more details.
  26  *
  27  *  You should have received a copy of the GNU General Public License
  28  *  along with this program; if not, write to the Free Software
  29  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  30  *
  31  */
  32
  33 #include "squid.h"
  34 #include "comm.h"
  35 #include "comm/ConnOpener.h"
  36 #include "comm/TcpAcceptor.h"
  37 #include "comm/Write.h"
  38 #include "CommCalls.h"
  39 #include "compat/strtoll.h"
  40 #include "errorpage.h"
  41 #include "fd.h"
  42 #include "fde.h"
  43 #include "forward.h"
  44 #include "html_quote.h"
  45 #include "HttpHdrContRange.h"
  46 #include "HttpHeader.h"
  47 #include "HttpHeaderRange.h"
  48 #include "HttpReply.h"
  49 #include "HttpRequest.h"
  50 #include "ip/tools.h"
  51 #include "Mem.h"
  52 #include "MemBuf.h"
  53 #include "mime.h"
  54 #include "rfc1738.h"
  55 #include "Server.h"
  56 #include "SquidConfig.h"
  57 #include "SquidString.h"
  58 #include "SquidTime.h"
  59 #include "StatCounters.h"
  60 #include "Store.h"
  61 #include "tools.h"
  62 #include "URL.h"
  63 #include "URLScheme.h"
  64 #include "wordlist.h"
  65
  66 #if USE_DELAY_POOLS
  67 #include "DelayPools.h"
  68 #include "MemObject.h"
  69 #endif
  70
  71 #if HAVE_ERRNO_H
  72 #include <errno.h>
  73 #endif
  74
  75 /**
  76  \defgroup ServerProtocolFTPInternal Server-Side FTP Internals
  77  \ingroup ServerProtocolFTPAPI
  78  */
  79
  80 /// \ingroup ServerProtocolFTPInternal
  81 static const char *const crlf = "\r\n";
  82
  83 #define CTRL_BUFLEN 1024
  84 /// \ingroup ServerProtocolFTPInternal
  85 static char cbuf[CTRL_BUFLEN];
  86
  87 /// \ingroup ServerProtocolFTPInternal
  88 typedef enum {
  89     BEGIN,
  90     SENT_USER,
  91     SENT_PASS,
  92     SENT_TYPE,
  93     SENT_MDTM,
  94     SENT_SIZE,
  95     SENT_EPRT,
  96     SENT_PORT,
  97     SENT_EPSV_ALL,
  98     SENT_EPSV_1,
  99     SENT_EPSV_2,
 100     SENT_PASV,
 101     SENT_CWD,
 102     SENT_LIST,
 103     SENT_NLST,
 104     SENT_REST,
 105     SENT_RETR,
 106     SENT_STOR,
 107     SENT_QUIT,
 108     READING_DATA,
 109     WRITING_DATA,
 110     SENT_MKDIR
 111 } ftp_state_t;
 112
 113 /// \ingroup ServerProtocolFTPInternal
 114 struct _ftp_flags {
 115
 116     /* passive mode */
 117     bool pasv_supported;  ///< PASV command is allowed
 118     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
 119     bool pasv_only;
 120     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
 121
 122     /* authentication */
 123     bool authenticated;         ///< authentication success
 124     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
 125     bool tried_auth_nopass;     ///< auth tried username with no password already.
 126
 127     /* other */
 128     bool isdir;
 129     bool skip_whitespace;
 130     bool rest_supported;
 131     bool http_header_sent;
 132     bool tried_nlst;
 133     bool need_base_href;
 134     bool dir_slash;
 135     bool root_dir;
 136     bool no_dotdot;
 137     bool binary;
 138     bool try_slash_hack;
 139     bool put;
 140     bool put_mkdir;
 141     bool listformat_unknown;
 142     bool listing;
 143     bool completed_forwarding;
 144 };
 145
 146 class FtpStateData;
 147
 148 /// \ingroup ServerProtocolFTPInternal
 149 typedef void (FTPSM) (FtpStateData *);
 150
 151 /// common code for FTP control and data channels
 152 /// does not own the channel descriptor, which is managed by FtpStateData
 153 class FtpChannel
 154 {
 155 public:
 156     FtpChannel() {};
 157
 158     /// called after the socket is opened, sets up close handler
 159     void opened(const Comm::ConnectionPointer &conn, const AsyncCall::Pointer &aCloser);
 160
 161     /** Handles all operations needed to properly close the active channel FD.
 162      * clearing the close handler, clearing the listen socket properly, and calling comm_close
 163      */
 164     void close();
 165
 166     void clear(); ///< just drops conn and close handler. does not close active connections.
 167
 168     Comm::ConnectionPointer conn; ///< channel descriptor
 169
 170     /** A temporary handle to the connection being listened on.
 171      * Closing this will also close the waiting Data channel acceptor.
 172      * If a data connection has already been accepted but is still waiting in the event queue
 173      * the callback will still happen and needs to be handled (usually dropped).
 174      */
 175     Comm::ConnectionPointer listenConn;
 176
 177     AsyncCall::Pointer opener; ///< Comm opener handler callback.
 178 private:
 179     AsyncCall::Pointer closer; ///< Comm close handler callback
 180 };
 181
 182 /// \ingroup ServerProtocolFTPInternal
 183 class FtpStateData : public ServerStateData
 184 {
 185
 186 public:
 187     void *operator new (size_t);
 188     void operator delete (void *);
 189     void *toCbdata() { return this; }
 190
 191     FtpStateData(FwdState *, const Comm::ConnectionPointer &conn);
 192     ~FtpStateData();
 193     char user[MAX_URL];
 194     char password[MAX_URL];
 195     int password_url;
 196     char *reply_hdr;
 197     int reply_hdr_state;
 198     String clean_url;
 199     String title_url;
 200     String base_href;
 201     int conn_att;
 202     int login_att;
 203     ftp_state_t state;
 204     time_t mdtm;
 205     int64_t theSize;
 206     wordlist *pathcomps;
 207     char *filepath;
 208     char *dirpath;
 209     int64_t restart_offset;
 210     char *proxy_host;
 211     size_t list_width;
 212     String cwd_message;
 213     char *old_request;
 214     char *old_reply;
 215     char *old_filepath;
 216     char typecode;
 217     MemBuf listing;             ///< FTP directory listing in HTML format.
 218
 219     // \todo: optimize ctrl and data structs member order, to minimize size
 220     /// FTP control channel info; the channel is opened once per transaction
 221     struct CtrlChannel: public FtpChannel {
 222         char *buf;
 223         size_t size;
 224         size_t offset;
 225         wordlist *message;
 226         char *last_command;
 227         char *last_reply;
 228         int replycode;
 229     } ctrl;
 230
 231     /// FTP data channel info; the channel may be opened/closed a few times
 232     struct DataChannel: public FtpChannel {
 233         MemBuf *readBuf;
 234         char *host;
 235         unsigned short port;
 236         bool read_pending;
 237     } data;
 238
 239     struct _ftp_flags flags;
 240
 241 private:
 242     CBDATA_CLASS(FtpStateData);
 243
 244 public:
 245     // these should all be private
 246     virtual void start();
 247     void loginParser(const char *, int escaped);
 248     int restartable();
 249     void appendSuccessHeader();
 250     void hackShortcut(FTPSM * nextState);
 251     void failed(err_type, int xerrno);
 252     void failedErrorMessage(err_type, int xerrno);
 253     void unhack();
 254     void scheduleReadControlReply(int);
 255     void handleControlReply();
 256     void readStor();
 257     void parseListing();
 258     MemBuf *htmlifyListEntry(const char *line);
 259     void completedListing(void);
 260     void dataComplete();
 261     void dataRead(const CommIoCbParams &io);
 262
 263     /// ignore timeout on CTRL channel. set read timeout on DATA channel.
 264     void switchTimeoutToDataChannel();
 265     /// create a data channel acceptor and start listening.
 266     void listenForDataChannel(const Comm::ConnectionPointer &conn, const char *note);
 267
 268     int checkAuth(const HttpHeader * req_hdr);
 269     void checkUrlpath();
 270     void buildTitleUrl();
 271     void writeReplyBody(const char *, size_t len);
 272     void printfReplyBody(const char *fmt, ...);
 273     virtual const Comm::ConnectionPointer & dataConnection() const;
 274     virtual void maybeReadVirginBody();
 275     virtual void closeServer();
 276     virtual void completeForwarding();
 277     virtual void abortTransaction(const char *reason);
 278     void processHeadResponse();
 279     void processReplyBody();
 280     void writeCommand(const char *buf);
 281     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 282     int64_t getCurrentOffset() const { return currentOffset; }
 283
 284     static CNCB ftpPasvCallback;
 285     static PF ftpDataWrite;
 286     void ftpTimeout(const CommTimeoutCbParams &io);
 287     void ctrlClosed(const CommCloseCbParams &io);
 288     void dataClosed(const CommCloseCbParams &io);
 289     void ftpReadControlReply(const CommIoCbParams &io);
 290     void ftpWriteCommandCallback(const CommIoCbParams &io);
 291     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 292
 293     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 294     const char *ftpRealm(void);
 295     void loginFailed(void);
 296     static wordlist *ftpParseControlReply(char *, size_t, int *, size_t *);
 297
 298     // sending of the request body to the server
 299     virtual void sentRequestBody(const CommIoCbParams&);
 300     virtual void doneSendingRequestBody();
 301
 302     virtual void haveParsedReplyHeaders();
 303
 304     virtual bool doneWithServer() const;
 305     virtual bool haveControlChannel(const char *caller_name) const;
 306     AsyncCall::Pointer dataCloser(); /// creates a Comm close callback
 307     AsyncCall::Pointer dataOpener(); /// creates a Comm connect callback
 308
 309 private:
 310     // BodyConsumer for HTTP: consume request body.
 311     virtual void handleRequestBodyProducerAborted();
 312 };
 313
 314 CBDATA_CLASS_INIT(FtpStateData);
 315
 316 void *
 317 FtpStateData::operator new (size_t)
 318 {
 319     CBDATA_INIT_TYPE(FtpStateData);
 320     FtpStateData *result = cbdataAlloc(FtpStateData);
 321     return result;
 322 }
 323
 324 void
 325 FtpStateData::operator delete (void *address)
 326 {
 327     FtpStateData *t = static_cast<FtpStateData *>(address);
 328     cbdataFree(t);
 329 }
 330
 331 /// \ingroup ServerProtocolFTPInternal
 332 typedef struct {
 333     char type;
 334     int64_t size;
 335     char *date;
 336     char *name;
 337     char *showname;
 338     char *link;
 339 } ftpListParts;
 340
 341 /// \ingroup ServerProtocolFTPInternal
 342 #define FTP_LOGIN_ESCAPED       1
 343
 344 /// \ingroup ServerProtocolFTPInternal
 345 #define FTP_LOGIN_NOT_ESCAPED   0
 346
 347 /*
 348  * State machine functions
 349  * send == state transition
 350  * read == wait for response, and select next state transition
 351  * other == Transition logic
 352  */
 353 static FTPSM ftpReadWelcome;
 354 static FTPSM ftpSendUser;
 355 static FTPSM ftpReadUser;
 356 static FTPSM ftpSendPass;
 357 static FTPSM ftpReadPass;
 358 static FTPSM ftpSendType;
 359 static FTPSM ftpReadType;
 360 static FTPSM ftpSendMdtm;
 361 static FTPSM ftpReadMdtm;
 362 static FTPSM ftpSendSize;
 363 static FTPSM ftpReadSize;
 364 static FTPSM ftpSendEPRT;
 365 static FTPSM ftpReadEPRT;
 366 static FTPSM ftpSendPORT;
 367 static FTPSM ftpReadPORT;
 368 static FTPSM ftpSendPassive;
 369 static FTPSM ftpReadEPSV;
 370 static FTPSM ftpReadPasv;
 371 static FTPSM ftpTraverseDirectory;
 372 static FTPSM ftpListDir;
 373 static FTPSM ftpGetFile;
 374 static FTPSM ftpSendCwd;
 375 static FTPSM ftpReadCwd;
 376 static FTPSM ftpRestOrList;
 377 static FTPSM ftpSendList;
 378 static FTPSM ftpSendNlst;
 379 static FTPSM ftpReadList;
 380 static FTPSM ftpSendRest;
 381 static FTPSM ftpReadRest;
 382 static FTPSM ftpSendRetr;
 383 static FTPSM ftpReadRetr;
 384 static FTPSM ftpReadTransferDone;
 385 static FTPSM ftpSendStor;
 386 static FTPSM ftpReadStor;
 387 static FTPSM ftpWriteTransferDone;
 388 static FTPSM ftpSendReply;
 389 static FTPSM ftpSendMkdir;
 390 static FTPSM ftpReadMkdir;
 391 static FTPSM ftpFail;
 392 static FTPSM ftpSendQuit;
 393 static FTPSM ftpReadQuit;
 394
 395 /************************************************
 396 ** Debugs Levels used here                     **
 397 *************************************************
 398 0       CRITICAL Events
 399 1       IMPORTANT Events
 400         Protocol and Transmission failures.
 401 2       FTP Protocol Chatter
 402 3       Logic Flows
 403 4       Data Parsing Flows
 404 5       Data Dumps
 405 7       ??
 406 ************************************************/
 407
 408 /************************************************
 409 ** State Machine Description (excluding hacks) **
 410 *************************************************
 411 From                    To
 412 ---------------------------------------
 413 Welcome                 User
 414 User                    Pass
 415 Pass                    Type
 416 Type                    TraverseDirectory / GetFile
 417 TraverseDirectory       Cwd / GetFile / ListDir
 418 Cwd                     TraverseDirectory / Mkdir
 419 GetFile                 Mdtm
 420 Mdtm                    Size
 421 Size                    Epsv
 422 ListDir                 Epsv
 423 Epsv                    FileOrList
 424 FileOrList              Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 425 Rest                    Retr
 426 Retr / Nlst / List      DataRead* (on datachannel)
 427 DataRead*               ReadTransferDone
 428 ReadTransferDone        DataTransferDone
 429 Stor                    DataWrite* (on datachannel)
 430 DataWrite*              RequestPutBody** (from client)
 431 RequestPutBody**        DataWrite* / WriteTransferDone
 432 WriteTransferDone       DataTransferDone
 433 DataTransferDone        Quit
 434 Quit                    -
 435 ************************************************/
 436
 437 /// \ingroup ServerProtocolFTPInternal
 438 FTPSM *FTP_SM_FUNCS[] = {
 439     ftpReadWelcome,             /* BEGIN */
 440     ftpReadUser,                /* SENT_USER */
 441     ftpReadPass,                /* SENT_PASS */
 442     ftpReadType,                /* SENT_TYPE */
 443     ftpReadMdtm,                /* SENT_MDTM */
 444     ftpReadSize,                /* SENT_SIZE */
 445     ftpReadEPRT,                /* SENT_EPRT */
 446     ftpReadPORT,                /* SENT_PORT */
 447     ftpReadEPSV,                /* SENT_EPSV_ALL */
 448     ftpReadEPSV,                /* SENT_EPSV_1 */
 449     ftpReadEPSV,                /* SENT_EPSV_2 */
 450     ftpReadPasv,                /* SENT_PASV */
 451     ftpReadCwd,         /* SENT_CWD */
 452     ftpReadList,                /* SENT_LIST */
 453     ftpReadList,                /* SENT_NLST */
 454     ftpReadRest,                /* SENT_REST */
 455     ftpReadRetr,                /* SENT_RETR */
 456     ftpReadStor,                /* SENT_STOR */
 457     ftpReadQuit,                /* SENT_QUIT */
 458     ftpReadTransferDone,        /* READING_DATA (RETR,LIST,NLST) */
 459     ftpWriteTransferDone,       /* WRITING_DATA (STOR) */
 460     ftpReadMkdir                /* SENT_MKDIR */
 461 };
 462
 463 /// handler called by Comm when FTP control channel is closed unexpectedly
 464 void
 465 FtpStateData::ctrlClosed(const CommCloseCbParams &io)
 466 {
 467     debugs(9, 4, HERE);
 468     ctrl.clear();
 469     mustStop("FtpStateData::ctrlClosed");
 470 }
 471
 472 /// handler called by Comm when FTP data channel is closed unexpectedly
 473 void
 474 FtpStateData::dataClosed(const CommCloseCbParams &io)
 475 {
 476     debugs(9, 4, HERE);
 477     if (data.listenConn != NULL) {
 478         data.listenConn->close();
 479         data.listenConn = NULL;
 480         // NP clear() does the: data.fd = -1;
 481     }
 482     data.clear();
 483     failed(ERR_FTP_FAILURE, 0);
 484     /* failed closes ctrl.conn and frees ftpState */
 485
 486     /* NP: failure recovery may be possible when its only a data.conn failure.
 487      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 488      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 489      */
 490 }
 491
 492 FtpStateData::FtpStateData(FwdState *theFwdState, const Comm::ConnectionPointer &conn) : AsyncJob("FtpStateData"), ServerStateData(theFwdState)
 493 {
 494     const char *url = entry->url();
 495     debugs(9, 3, HERE << "'" << url << "'" );
 496     ++ statCounter.server.all.requests;
 497     ++ statCounter.server.ftp.requests;
 498     theSize = -1;
 499     mdtm = -1;
 500
 501     if (Config.Ftp.passive && !flags.pasv_failed)
 502         flags.pasv_supported = 1;
 503
 504     flags.rest_supported = 1;
 505
 506     typedef CommCbMemFunT<FtpStateData, CommCloseCbParams> Dialer;
 507     AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this, FtpStateData::ctrlClosed);
 508     ctrl.opened(conn, closer);
 509
 510     if (request->method == Http::METHOD_PUT)
 511         flags.put = 1;
 512 }
 513
 514 FtpStateData::~FtpStateData()
 515 {
 516     debugs(9, 3, HERE << entry->url()  );
 517
 518     if (reply_hdr) {
 519         memFree(reply_hdr, MEM_8K_BUF);
 520         reply_hdr = NULL;
 521     }
 522
 523     if (data.opener != NULL) {
 524         data.opener->cancel("FtpStateData destructed");
 525         data.opener = NULL;
 526     }
 527     data.close();
 528
 529     if (Comm::IsConnOpen(ctrl.conn)) {
 530         debugs(9, DBG_IMPORTANT, HERE << "Internal bug: FtpStateData left " <<
 531                "open control channel " << ctrl.conn);
 532     }
 533
 534     if (ctrl.buf) {
 535         memFreeBuf(ctrl.size, ctrl.buf);
 536         ctrl.buf = NULL;
 537     }
 538
 539     if (data.readBuf) {
 540         if (!data.readBuf->isNull())
 541             data.readBuf->clean();
 542
 543         delete data.readBuf;
 544     }
 545
 546     if (pathcomps)
 547         wordlistDestroy(&pathcomps);
 548
 549     if (ctrl.message)
 550         wordlistDestroy(&ctrl.message);
 551
 552     cwd_message.clean();
 553
 554     safe_free(ctrl.last_reply);
 555
 556     safe_free(ctrl.last_command);
 557
 558     safe_free(old_request);
 559
 560     safe_free(old_reply);
 561
 562     safe_free(old_filepath);
 563
 564     title_url.clean();
 565
 566     base_href.clean();
 567
 568     safe_free(filepath);
 569
 570     safe_free(dirpath);
 571
 572     safe_free(data.host);
 573
 574     fwd = NULL; // refcounted
 575 }
 576
 577 /**
 578  * Parse a possible login username:password pair.
 579  * Produces filled member variables user, password, password_url if anything found.
 580  */
 581 void
 582 FtpStateData::loginParser(const char *login, int escaped)
 583 {
 584     const char *u = NULL; // end of the username sub-string
 585     int len;              // length of the current sub-string to handle.
 586
 587     int total_len = strlen(login);
 588
 589     debugs(9, 4, HERE << ": login='" << login << "', escaped=" << escaped);
 590     debugs(9, 9, HERE << ": IN : login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 591
 592     if ((u = strchr(login, ':'))) {
 593
 594         /* if there was a username part */
 595         if (u > login) {
 596             len = u - login;
 597             ++u; // jump off the delimiter.
 598             if (len > MAX_URL)
 599                 len = MAX_URL-1;
 600             xstrncpy(user, login, len +1);
 601             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<"), escaped=" << escaped);
 602             if (escaped)
 603                 rfc1738_unescape(user);
 604             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<") unescaped.");
 605         }
 606
 607         /* if there was a password part */
 608         len = login + total_len - u;
 609         if ( len > 0) {
 610             if (len > MAX_URL)
 611                 len = MAX_URL -1;
 612             xstrncpy(password, u, len +1);
 613             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<"), escaped=" << escaped);
 614             if (escaped) {
 615                 rfc1738_unescape(password);
 616                 password_url = 1;
 617             }
 618             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<") unescaped.");
 619         }
 620     } else if (login[0]) {
 621         /* no password, just username */
 622         if (total_len > MAX_URL)
 623             total_len = MAX_URL -1;
 624         xstrncpy(user, login, total_len +1);
 625         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<"), escaped=" << escaped);
 626         if (escaped)
 627             rfc1738_unescape(user);
 628         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<") unescaped.");
 629     }
 630
 631     debugs(9, 9, HERE << ": OUT: login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 632 }
 633
 634 /**
 635  * Cancel the timeout on the Control socket and establish one
 636  * on the data socket
 637  */
 638 void
 639 FtpStateData::switchTimeoutToDataChannel()
 640 {
 641     commUnsetConnTimeout(ctrl.conn);
 642
 643     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
 644     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, FtpStateData::ftpTimeout);
 645     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 646 }
 647
 648 void
 649 FtpStateData::listenForDataChannel(const Comm::ConnectionPointer &conn, const char *note)
 650 {
 651     assert(!Comm::IsConnOpen(data.conn));
 652
 653     typedef CommCbMemFunT<FtpStateData, CommAcceptCbParams> AcceptDialer;
 654     typedef AsyncCallT<AcceptDialer> AcceptCall;
 655     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, FtpStateData::ftpAcceptDataConnection));
 656     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 657
 658     /* open the conn if its not already open */
 659     if (!Comm::IsConnOpen(conn)) {
 660         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 661         if (!Comm::IsConnOpen(conn)) {
 662             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 663             return;
 664         }
 665         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 666     }
 667
 668     assert(Comm::IsConnOpen(conn));
 669     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 670
 671     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 672     data.opened(conn, dataCloser());
 673     switchTimeoutToDataChannel();
 674 }
 675
 676 void
 677 FtpStateData::ftpTimeout(const CommTimeoutCbParams &io)
 678 {
 679     debugs(9, 4, HERE << io.conn << ": '" << entry->url() << "'" );
 680
 681     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 682         return;
 683
 684     if (SENT_PASV == state) {
 685         /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */
 686         flags.pasv_supported = false;
 687         debugs(9, DBG_IMPORTANT, "ftpTimeout: timeout in SENT_PASV state" );
 688
 689         // cancel the data connection setup.
 690         if (data.opener != NULL) {
 691             data.opener->cancel("timeout");
 692             data.opener = NULL;
 693         }
 694         data.close();
 695     }
 696
 697     failed(ERR_READ_TIMEOUT, 0);
 698     /* failed() closes ctrl.conn and frees ftpState */
 699 }
 700
 701 /// \ingroup ServerProtocolFTPInternal
 702 static const char *Month[] = {
 703     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 704     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 705 };
 706
 707 /// \ingroup ServerProtocolFTPInternal
 708 static int
 709 is_month(const char *buf)
 710 {
 711     int i;
 712
 713     for (i = 0; i < 12; ++i)
 714         if (!strcasecmp(buf, Month[i]))
 715             return 1;
 716
 717     return 0;
 718 }
 719
 720 /// \ingroup ServerProtocolFTPInternal
 721 static void
 722 ftpListPartsFree(ftpListParts ** parts)
 723 {
 724     safe_free((*parts)->date);
 725     safe_free((*parts)->name);
 726     safe_free((*parts)->showname);
 727     safe_free((*parts)->link);
 728     safe_free(*parts);
 729 }
 730
 731 /// \ingroup ServerProtocolFTPInternal
 732 #define MAX_TOKENS 64
 733
 734 /// \ingroup ServerProtocolFTPInternal
 735 static ftpListParts *
 736 ftpListParseParts(const char *buf, struct _ftp_flags flags)
 737 {
 738     ftpListParts *p = NULL;
 739     char *t = NULL;
 740     const char *ct = NULL;
 741     char *tokens[MAX_TOKENS];
 742     int i;
 743     int n_tokens;
 744     static char tbuf[128];
 745     char *xbuf = NULL;
 746     static int scan_ftp_initialized = 0;
 747     static regex_t scan_ftp_integer;
 748     static regex_t scan_ftp_time;
 749     static regex_t scan_ftp_dostime;
 750     static regex_t scan_ftp_dosdate;
 751
 752     if (!scan_ftp_initialized) {
 753         scan_ftp_initialized = 1;
 754         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 755         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 756         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 757         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 758     }
 759
 760     if (buf == NULL)
 761         return NULL;
 762
 763     if (*buf == '\0')
 764         return NULL;
 765
 766     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 767
 768     n_tokens = 0;
 769
 770     memset(tokens, 0, sizeof(tokens));
 771
 772     xbuf = xstrdup(buf);
 773
 774     if (flags.tried_nlst) {
 775         /* Machine readable format, one name per line */
 776         p->name = xbuf;
 777         p->type = '\0';
 778         return p;
 779     }
 780
 781     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 782         tokens[n_tokens] = xstrdup(t);
 783         ++n_tokens;
 784     }
 785
 786     xfree(xbuf);
 787
 788     /* locate the Month field */
 789     for (i = 3; i < n_tokens - 2; ++i) {
 790         char *size = tokens[i - 1];
 791         char *month = tokens[i];
 792         char *day = tokens[i + 1];
 793         char *year = tokens[i + 2];
 794
 795         if (!is_month(month))
 796             continue;
 797
 798         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 799             continue;
 800
 801         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 802             continue;
 803
 804         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0)     /* Yr | hh:mm */
 805             continue;
 806
 807         snprintf(tbuf, 128, "%s %2s %5s",
 808                  month, day, year);
 809
 810         if (!strstr(buf, tbuf))
 811             snprintf(tbuf, 128, "%s %2s %-5s",
 812                      month, day, year);
 813
 814         char const *copyFrom = NULL;
 815
 816         if ((copyFrom = strstr(buf, tbuf))) {
 817             p->type = *tokens[0];
 818             p->size = strtoll(size, NULL, 10);
 819             p->date = xstrdup(tbuf);
 820
 821             if (flags.skip_whitespace) {
 822                 copyFrom += strlen(tbuf);
 823
 824                 while (strchr(w_space, *copyFrom))
 825                     ++copyFrom;
 826             } else {
 827                 /* XXX assumes a single space between date and filename
 828                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 829                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 830                 copyFrom += strlen(tbuf) + 1;
 831             }
 832
 833             p->name = xstrdup(copyFrom);
 834
 835             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 836                 *t = '\0';
 837                 p->link = xstrdup(t + 4);
 838             }
 839
 840             goto found;
 841         }
 842
 843         break;
 844     }
 845
 846     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 847     if (n_tokens > 3 &&
 848             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 849             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 850         if (!strcasecmp(tokens[2], "<dir>")) {
 851             p->type = 'd';
 852         } else {
 853             p->type = '-';
 854             p->size = strtoll(tokens[2], NULL, 10);
 855         }
 856
 857         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 858         p->date = xstrdup(tbuf);
 859
 860         if (p->type == 'd') {
 861             /* Directory.. name begins with first printable after <dir> */
 862             ct = strstr(buf, tokens[2]);
 863             ct += strlen(tokens[2]);
 864
 865             while (xisspace(*ct))
 866                 ++ct;
 867
 868             if (!*ct)
 869                 ct = NULL;
 870         } else {
 871             /* A file. Name begins after size, with a space in between */
 872             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 873             ct = strstr(buf, tbuf);
 874
 875             if (ct) {
 876                 ct += strlen(tokens[2]) + 2;
 877             }
 878         }
 879
 880         p->name = xstrdup(ct ? ct : tokens[3]);
 881         goto found;
 882     }
 883
 884     /* Try EPLF format; carson@lehman.com */
 885     if (buf[0] == '+') {
 886         ct = buf + 1;
 887         p->type = 0;
 888
 889         while (ct && *ct) {
 890             time_t tm;
 891             int l = strcspn(ct, ",");
 892             char *tmp;
 893
 894             if (l < 1)
 895                 goto blank;
 896
 897             switch (*ct) {
 898
 899             case '\t':
 900                 p->name = xstrndup(ct + 1, l + 1);
 901                 break;
 902
 903             case 's':
 904                 p->size = atoi(ct + 1);
 905                 break;
 906
 907             case 'm':
 908                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 909
 910                 if (tmp != ct + 1)
 911                     break;      /* not a valid integer */
 912
 913                 p->date = xstrdup(ctime(&tm));
 914
 915                 *(strstr(p->date, "\n")) = '\0';
 916
 917                 break;
 918
 919             case '/':
 920                 p->type = 'd';
 921
 922                 break;
 923
 924             case 'r':
 925                 p->type = '-';
 926
 927                 break;
 928
 929             case 'i':
 930                 break;
 931
 932             default:
 933                 break;
 934             }
 935
 936 blank:
 937             ct = strstr(ct, ",");
 938
 939             if (ct) {
 940                 ++ct;
 941             }
 942         }
 943
 944         if (p->type == 0) {
 945             p->type = '-';
 946         }
 947
 948         if (p->name)
 949             goto found;
 950         else
 951             safe_free(p->date);
 952     }
 953
 954 found:
 955
 956     for (i = 0; i < n_tokens; ++i)
 957         xfree(tokens[i]);
 958
 959     if (!p->name)
 960         ftpListPartsFree(&p);   /* cleanup */
 961
 962     return p;
 963 }
 964
 965 MemBuf *
 966 FtpStateData::htmlifyListEntry(const char *line)
 967 {
 968     char icon[2048];
 969     char href[2048 + 40];
 970     char text[ 2048];
 971     char size[ 2048];
 972     char chdir[ 2048 + 40];
 973     char view[ 2048 + 40];
 974     char download[ 2048 + 40];
 975     char link[ 2048 + 40];
 976     MemBuf *html;
 977     char prefix[2048];
 978     ftpListParts *parts;
 979     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 980
 981     debugs(9, 7, HERE << " line ={" << line << "}");
 982
 983     if (strlen(line) > 1024) {
 984         html = new MemBuf();
 985         html->init();
 986         html->Printf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 987         return html;
 988     }
 989
 990     if (flags.dir_slash && dirpath && typecode != 'D')
 991         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 992     else
 993         prefix[0] = '\0';
 994
 995     if ((parts = ftpListParseParts(line, flags)) == NULL) {
 996         const char *p;
 997
 998         html = new MemBuf();
 999         html->init();
1000         html->Printf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
1001
1002         for (p = line; *p && xisspace(*p); ++p);
1003         if (*p && !xisspace(*p))
1004             flags.listformat_unknown = 1;
1005
1006         return html;
1007     }
1008
1009     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
1010         ftpListPartsFree(&parts);
1011         return NULL;
1012     }
1013
1014     parts->size += 1023;
1015     parts->size >>= 10;
1016     parts->showname = xstrdup(parts->name);
1017
1018     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
1019     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
1020
1021     xstrncpy(text, parts->showname, 2048);
1022
1023     switch (parts->type) {
1024
1025     case 'd':
1026         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1027                  mimeGetIconURL("internal-dir"),
1028                  "[DIR]");
1029         strcat(href, "/");      /* margin is allocated above */
1030         break;
1031
1032     case 'l':
1033         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1034                  mimeGetIconURL("internal-link"),
1035                  "[LINK]");
1036         /* sometimes there is an 'l' flag, but no "->" link */
1037
1038         if (parts->link) {
1039             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
1040             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
1041                      *link2 != '/' ? prefix : "", link2,
1042                      html_quote(parts->link));
1043             safe_free(link2);
1044         }
1045
1046         break;
1047
1048     case '\0':
1049         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1050                  mimeGetIconURL(parts->name),
1051                  "[UNKNOWN]");
1052         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
1053                  "alt=\"[DIR]\"></a>",
1054                  rfc1738_escape_part(parts->name),
1055                  mimeGetIconURL("internal-dir"));
1056         break;
1057
1058     case '-':
1059
1060     default:
1061         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1062                  mimeGetIconURL(parts->name),
1063                  "[FILE]");
1064         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
1065         break;
1066     }
1067
1068     if (parts->type != 'd') {
1069         if (mimeGetViewOption(parts->name)) {
1070             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
1071                      "alt=\"[VIEW]\"></a>",
1072                      prefix, href, mimeGetIconURL("internal-view"));
1073         }
1074
1075         if (mimeGetDownloadOption(parts->name)) {
1076             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
1077                      "alt=\"[DOWNLOAD]\"></a>",
1078                      prefix, href, mimeGetIconURL("internal-download"));
1079         }
1080     }
1081
1082     /* construct the table row from parts. */
1083     html = new MemBuf();
1084     html->init();
1085     html->Printf("<tr class=\"entry\">"
1086                  "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
1087                  "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
1088                  "<td class=\"date\">%s</td>"
1089                  "<td class=\"size\">%s</td>"
1090                  "<td class=\"actions\">%s%s%s%s</td>"
1091                  "</tr>\n",
1092                  prefix, href, icon,
1093                  prefix, href, html_quote(text),
1094                  parts->date,
1095                  size,
1096                  chdir, view, download, link);
1097
1098     ftpListPartsFree(&parts);
1099     return html;
1100 }
1101
1102 void
1103 FtpStateData::parseListing()
1104 {
1105     char *buf = data.readBuf->content();
1106     char *sbuf;                 /* NULL-terminated copy of termedBuf */
1107     char *end;
1108     char *line;
1109     char *s;
1110     MemBuf *t;
1111     size_t linelen;
1112     size_t usable;
1113     size_t len = data.readBuf->contentSize();
1114
1115     if (!len) {
1116         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
1117         return;
1118     }
1119
1120     /*
1121      * We need a NULL-terminated buffer for scanning, ick
1122      */
1123     sbuf = (char *)xmalloc(len + 1);
1124     xstrncpy(sbuf, buf, len + 1);
1125     end = sbuf + len - 1;
1126
1127     while (*end != '\r' && *end != '\n' && end > sbuf)
1128         --end;
1129
1130     usable = end - sbuf;
1131
1132     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
1133
1134     if (usable == 0) {
1135         if (buf[0] == '\0' && len == 1) {
1136             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
1137             data.readBuf->consume(len);
1138         } else {
1139             debugs(9, 3, HERE << "didn't find end for " << entry->url());
1140             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
1141         }
1142         xfree(sbuf);
1143         return;
1144     }
1145
1146     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
1147
1148     line = (char *)memAllocate(MEM_4K_BUF);
1149     ++end;
1150     s = sbuf;
1151     s += strspn(s, crlf);
1152
1153     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1154         debugs(9, 7, HERE << "s = {" << s << "}");
1155         linelen = strcspn(s, crlf) + 1;
1156
1157         if (linelen < 2)
1158             break;
1159
1160         if (linelen > 4096)
1161             linelen = 4096;
1162
1163         xstrncpy(line, s, linelen);
1164
1165         debugs(9, 7, HERE << "{" << line << "}");
1166
1167         if (!strncmp(line, "total", 5))
1168             continue;
1169
1170         t = htmlifyListEntry(line);
1171
1172         if ( t != NULL) {
1173             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
1174             listing.append(t->content(), t->contentSize());
1175 //leak?            delete t;
1176         }
1177     }
1178
1179     debugs(9, 7, HERE << "Done.");
1180     data.readBuf->consume(usable);
1181     memFree(line, MEM_4K_BUF);
1182     xfree(sbuf);
1183 }
1184
1185 const Comm::ConnectionPointer &
1186 FtpStateData::dataConnection() const
1187 {
1188     return data.conn;
1189 }
1190
1191 void
1192 FtpStateData::dataComplete()
1193 {
1194     debugs(9, 3,HERE);
1195
1196     /* Connection closed; transfer done. */
1197
1198     /// Close data channel, if any, to conserve resources while we wait.
1199     data.close();
1200
1201     /* expect the "transfer complete" message on the control socket */
1202     /*
1203      * DPW 2007-04-23
1204      * Previously, this was the only place where we set the
1205      * 'buffered_ok' flag when calling scheduleReadControlReply().
1206      * It caused some problems if the FTP server returns an unexpected
1207      * status code after the data command.  FtpStateData was being
1208      * deleted in the middle of dataRead().
1209      */
1210     /* AYJ: 2011-01-13: Bug 2581.
1211      * 226 status is possibly waiting in the ctrl buffer.
1212      * The connection will hang if we DONT send buffered_ok.
1213      * This happens on all transfers which can be completly sent by the
1214      * server before the 150 started status message is read in by Squid.
1215      * ie all transfers of about one packet hang.
1216      */
1217     scheduleReadControlReply(1);
1218 }
1219
1220 void
1221 FtpStateData::maybeReadVirginBody()
1222 {
1223     // too late to read
1224     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
1225         return;
1226
1227     if (data.read_pending)
1228         return;
1229
1230     const int read_sz = replyBodySpace(*data.readBuf, 0);
1231
1232     debugs(11,9, HERE << "FTP may read up to " << read_sz << " bytes");
1233
1234     if (read_sz < 2)    // see http.cc
1235         return;
1236
1237     data.read_pending = true;
1238
1239     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1240     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
1241                                       TimeoutDialer, this, FtpStateData::ftpTimeout);
1242     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1243
1244     debugs(9,5,HERE << "queueing read on FD " << data.conn->fd);
1245
1246     typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1247     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
1248                           JobCallback(9, 5, Dialer, this, FtpStateData::dataRead));
1249 }
1250
1251 void
1252 FtpStateData::dataRead(const CommIoCbParams &io)
1253 {
1254     int j;
1255     int bin;
1256
1257     data.read_pending = false;
1258
1259     debugs(9, 3, HERE << "ftpDataRead: FD " << io.fd << " Read " << io.size << " bytes");
1260
1261     if (io.size > 0) {
1262         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
1263         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
1264     }
1265
1266     if (io.flag == COMM_ERR_CLOSING)
1267         return;
1268
1269     assert(io.fd == data.conn->fd);
1270
1271     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1272         abortTransaction("entry aborted during dataRead");
1273         return;
1274     }
1275
1276     if (io.flag == COMM_OK && io.size > 0) {
1277         debugs(9,5,HERE << "appended " << io.size << " bytes to readBuf");
1278         data.readBuf->appended(io.size);
1279 #if USE_DELAY_POOLS
1280         DelayId delayId = entry->mem_obj->mostBytesAllowed();
1281         delayId.bytesIn(io.size);
1282 #endif
1283         ++ IOStats.Ftp.reads;
1284
1285         for (j = io.size - 1, bin = 0; j; ++bin)
1286             j >>= 1;
1287
1288         ++ IOStats.Ftp.read_hist[bin];
1289     }
1290
1291     if (io.flag != COMM_OK) {
1292         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
1293                "ftpDataRead: read error: " << xstrerr(io.xerrno));
1294
1295         if (ignoreErrno(io.xerrno)) {
1296             typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1297             AsyncCall::Pointer timeoutCall = JobCallback(9, 5,
1298                                              TimeoutDialer, this, FtpStateData::ftpTimeout);
1299             commSetConnTimeout(io.conn, Config.Timeout.read, timeoutCall);
1300
1301             maybeReadVirginBody();
1302         } else {
1303             failed(ERR_READ_ERROR, 0);
1304             /* failed closes ctrl.conn and frees ftpState */
1305             return;
1306         }
1307     } else if (io.size == 0) {
1308         debugs(9,3, HERE << "Calling dataComplete() because io.size == 0");
1309         /*
1310          * DPW 2007-04-23
1311          * Dangerous curves ahead.  This call to dataComplete was
1312          * calling scheduleReadControlReply, handleControlReply,
1313          * and then ftpReadTransferDone.  If ftpReadTransferDone
1314          * gets unexpected status code, it closes down the control
1315          * socket and our FtpStateData object gets destroyed.   As
1316          * a workaround we no longer set the 'buffered_ok' flag in
1317          * the scheduleReadControlReply call.
1318          */
1319         dataComplete();
1320     }
1321
1322     processReplyBody();
1323 }
1324
1325 void
1326 FtpStateData::processReplyBody()
1327 {
1328     debugs(9, 3, HERE << "FtpStateData::processReplyBody starting.");
1329
1330     if (request->method == Http::METHOD_HEAD && (flags.isdir || theSize != -1)) {
1331         serverComplete();
1332         return;
1333     }
1334
1335     /* Directory listings are special. They write ther own headers via the error objects */
1336     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
1337         appendSuccessHeader();
1338
1339     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1340         /*
1341          * probably was aborted because content length exceeds one
1342          * of the maximum size limits.
1343          */
1344         abortTransaction("entry aborted after calling appendSuccessHeader()");
1345         return;
1346     }
1347
1348 #if USE_ADAPTATION
1349
1350     if (adaptationAccessCheckPending) {
1351         debugs(9,3, HERE << "returning from FtpStateData::processReplyBody due to adaptationAccessCheckPending");
1352         return;
1353     }
1354
1355 #endif
1356
1357     if (flags.isdir) {
1358         if (!flags.listing) {
1359             flags.listing = 1;
1360             listing.reset();
1361         }
1362         parseListing();
1363         maybeReadVirginBody();
1364         return;
1365     } else if (const int csize = data.readBuf->contentSize()) {
1366         writeReplyBody(data.readBuf->content(), csize);
1367         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1368         data.readBuf->consume(csize);
1369     }
1370
1371     entry->flush();
1372
1373     maybeReadVirginBody();
1374 }
1375
1376 /**
1377  * Locates the FTP user:password login.
1378  *
1379  * Highest to lowest priority:
1380  *  - Checks URL (ftp://user:pass@domain)
1381  *  - Authorization: Basic header
1382  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1383  *
1384  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1385  *
1386  * TODO: we might be able to do something about locating username from other sources:
1387  *       ie, external ACL user=* tag or ident lookup
1388  *
1389  \retval 1      if we have everything needed to complete this request.
1390  \retval 0      if something is missing.
1391  */
1392 int
1393 FtpStateData::checkAuth(const HttpHeader * req_hdr)
1394 {
1395     /* default username */
1396     xstrncpy(user, "anonymous", MAX_URL);
1397
1398 #if HAVE_AUTH_MODULE_BASIC
1399     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1400     const char *auth;
1401     if ( (auth = req_hdr->getAuth(HDR_AUTHORIZATION, "Basic")) ) {
1402         flags.authenticated = 1;
1403         loginParser(auth, FTP_LOGIN_NOT_ESCAPED);
1404     }
1405     /* we fail with authorization-required error later IFF the FTP server requests it */
1406 #endif
1407
1408     /* Test URL login syntax. Overrides any headers received. */
1409     loginParser(request->login, FTP_LOGIN_ESCAPED);
1410
1411     /* name is missing. thats fatal. */
1412     if (!user[0])
1413         fatal("FTP login parsing destroyed username info");
1414
1415     /* name + password == success */
1416     if (password[0])
1417         return 1;
1418
1419     /* Setup default FTP password settings */
1420     /* this has to be done last so that we can have a no-password case above. */
1421     if (!password[0]) {
1422         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1423             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1424             flags.tried_auth_anonymous=1;
1425             return 1;
1426         } else if (!flags.tried_auth_nopass) {
1427             xstrncpy(password, null_string, MAX_URL);
1428             flags.tried_auth_nopass=1;
1429             return 1;
1430         }
1431     }
1432
1433     return 0;                   /* different username */
1434 }
1435
1436 static String str_type_eq;
1437 void
1438 FtpStateData::checkUrlpath()
1439 {
1440     int l;
1441     size_t t;
1442
1443     if (str_type_eq.undefined()) //hack. String doesn't support global-static
1444         str_type_eq="type=";
1445
1446     if ((t = request->urlpath.rfind(';')) != String::npos) {
1447         if (request->urlpath.substr(t+1,t+1+str_type_eq.size())==str_type_eq) {
1448             typecode = (char)xtoupper(request->urlpath[t+str_type_eq.size()+1]);
1449             request->urlpath.cut(t);
1450         }
1451     }
1452
1453     l = request->urlpath.size();
1454     /* check for null path */
1455
1456     if (!l) {
1457         flags.isdir = 1;
1458         flags.root_dir = 1;
1459         flags.need_base_href = 1;       /* Work around broken browsers */
1460     } else if (!request->urlpath.cmp("/%2f/")) {
1461         /* UNIX root directory */
1462         flags.isdir = 1;
1463         flags.root_dir = 1;
1464     } else if ((l >= 1) && (request->urlpath[l - 1] == '/')) {
1465         /* Directory URL, ending in / */
1466         flags.isdir = 1;
1467
1468         if (l == 1)
1469             flags.root_dir = 1;
1470     } else {
1471         flags.dir_slash = 1;
1472     }
1473 }
1474
1475 void
1476 FtpStateData::buildTitleUrl()
1477 {
1478     title_url = "ftp://";
1479
1480     if (strcmp(user, "anonymous")) {
1481         title_url.append(user);
1482         title_url.append("@");
1483     }
1484
1485     title_url.append(request->GetHost());
1486
1487     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1488         title_url.append(":");
1489         title_url.append(xitoa(request->port));
1490     }
1491
1492     title_url.append (request->urlpath);
1493
1494     base_href = "ftp://";
1495
1496     if (strcmp(user, "anonymous") != 0) {
1497         base_href.append(rfc1738_escape_part(user));
1498
1499         if (password_url) {
1500             base_href.append (":");
1501             base_href.append(rfc1738_escape_part(password));
1502         }
1503
1504         base_href.append("@");
1505     }
1506
1507     base_href.append(request->GetHost());
1508
1509     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1510         base_href.append(":");
1511         base_href.append(xitoa(request->port));
1512     }
1513
1514     base_href.append(request->urlpath);
1515     base_href.append("/");
1516 }
1517
1518 /// \ingroup ServerProtocolFTPAPI
1519 void
1520 ftpStart(FwdState * fwd)
1521 {
1522     AsyncJob::Start(new FtpStateData(fwd, fwd->serverConnection()));
1523 }
1524
1525 void
1526 FtpStateData::start()
1527 {
1528     if (!checkAuth(&request->header)) {
1529         /* create appropriate reply */
1530         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1531         entry->replaceHttpReply(reply);
1532         serverComplete();
1533         return;
1534     }
1535
1536     checkUrlpath();
1537     buildTitleUrl();
1538     debugs(9, 5, HERE << "FD " << ctrl.conn->fd << " : host=" << request->GetHost() <<
1539            ", path=" << request->urlpath << ", user=" << user << ", passwd=" << password);
1540
1541     state = BEGIN;
1542     ctrl.last_command = xstrdup("Connect to server");
1543     ctrl.buf = (char *)memAllocBuf(4096, &ctrl.size);
1544     ctrl.offset = 0;
1545     data.readBuf = new MemBuf;
1546     data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
1547     scheduleReadControlReply(0);
1548 }
1549
1550 /* ====================================================================== */
1551
1552 /// \ingroup ServerProtocolFTPInternal
1553 static char *
1554 escapeIAC(const char *buf)
1555 {
1556     int n;
1557     char *ret;
1558     unsigned const char *p;
1559     unsigned char *r;
1560
1561     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
1562         if (*p == 255)
1563             ++n;
1564
1565     ret = (char *)xmalloc(n);
1566
1567     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
1568         *r = *p;
1569         ++r;
1570
1571         if (*p == 255) {
1572             *r = 255;
1573             ++r;
1574         }
1575     }
1576
1577     *r = '\0';
1578     ++r;
1579     assert((r - (unsigned char *)ret) == n );
1580     return ret;
1581 }
1582
1583 void
1584 FtpStateData::writeCommand(const char *buf)
1585 {
1586     char *ebuf;
1587     /* trace FTP protocol communications at level 2 */
1588     debugs(9, 2, "ftp<< " << buf);
1589
1590     if (Config.Ftp.telnet)
1591         ebuf = escapeIAC(buf);
1592     else
1593         ebuf = xstrdup(buf);
1594
1595     safe_free(ctrl.last_command);
1596
1597     safe_free(ctrl.last_reply);
1598
1599     ctrl.last_command = ebuf;
1600
1601     if (!Comm::IsConnOpen(ctrl.conn)) {
1602         debugs(9, 2, HERE << "cannot send to closing ctrl " << ctrl.conn);
1603         // TODO: assert(ctrl.closer != NULL);
1604         return;
1605     }
1606
1607     typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1608     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this, FtpStateData::ftpWriteCommandCallback);
1609     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
1610
1611     scheduleReadControlReply(0);
1612 }
1613
1614 void
1615 FtpStateData::ftpWriteCommandCallback(const CommIoCbParams &io)
1616 {
1617
1618     debugs(9, 5, "ftpWriteCommandCallback: wrote " << io.size << " bytes");
1619
1620     if (io.size > 0) {
1621         fd_bytes(io.fd, io.size, FD_WRITE);
1622         kb_incr(&(statCounter.server.all.kbytes_out), io.size);
1623         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
1624     }
1625
1626     if (io.flag == COMM_ERR_CLOSING)
1627         return;
1628
1629     if (io.flag) {
1630         debugs(9, DBG_IMPORTANT, "ftpWriteCommandCallback: " << io.conn << ": " << xstrerr(io.xerrno));
1631         failed(ERR_WRITE_ERROR, io.xerrno);
1632         /* failed closes ctrl.conn and frees ftpState */
1633         return;
1634     }
1635 }
1636
1637 wordlist *
1638 FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, size_t *used)
1639 {
1640     char *s;
1641     char *sbuf;
1642     char *end;
1643     int usable;
1644     int complete = 0;
1645     wordlist *head = NULL;
1646     wordlist *list;
1647     wordlist **tail = &head;
1648     size_t offset;
1649     size_t linelen;
1650     int code = -1;
1651     debugs(9, 3, HERE);
1652     /*
1653      * We need a NULL-terminated buffer for scanning, ick
1654      */
1655     sbuf = (char *)xmalloc(len + 1);
1656     xstrncpy(sbuf, buf, len + 1);
1657     end = sbuf + len - 1;
1658
1659     while (*end != '\r' && *end != '\n' && end > sbuf)
1660         --end;
1661
1662     usable = end - sbuf;
1663
1664     debugs(9, 3, HERE << "usable = " << usable);
1665
1666     if (usable == 0) {
1667         debugs(9, 3, HERE << "didn't find end of line");
1668         safe_free(sbuf);
1669         return NULL;
1670     }
1671
1672     debugs(9, 3, HERE << len << " bytes to play with");
1673     ++end;
1674     s = sbuf;
1675     s += strspn(s, crlf);
1676
1677     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1678         if (complete)
1679             break;
1680
1681         debugs(9, 5, HERE << "s = {" << s << "}");
1682
1683         linelen = strcspn(s, crlf) + 1;
1684
1685         if (linelen < 2)
1686             break;
1687
1688         if (linelen > 3)
1689             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1690
1691         if (complete)
1692             code = atoi(s);
1693
1694         offset = 0;
1695
1696         if (linelen > 3)
1697             if (*s >= '0' && *s <= '9' && (*(s + 3) == '-' || *(s + 3) == ' '))
1698                 offset = 4;
1699
1700         list = new wordlist();
1701
1702         list->key = (char *)xmalloc(linelen - offset);
1703
1704         xstrncpy(list->key, s + offset, linelen - offset);
1705
1706         /* trace the FTP communication chat at level 2 */
1707         debugs(9, 2, "ftp>> " << code << " " << list->key);
1708
1709         *tail = list;
1710
1711         tail = &list->next;
1712     }
1713
1714     *used = (size_t) (s - sbuf);
1715     safe_free(sbuf);
1716
1717     if (!complete)
1718         wordlistDestroy(&head);
1719
1720     if (codep)
1721         *codep = code;
1722
1723     return head;
1724 }
1725
1726 /**
1727  * DPW 2007-04-23
1728  * Looks like there are no longer anymore callers that set
1729  * buffered_ok=1.  Perhaps it can be removed at some point.
1730  */
1731 void
1732 FtpStateData::scheduleReadControlReply(int buffered_ok)
1733 {
1734     debugs(9, 3, HERE << ctrl.conn);
1735
1736     if (buffered_ok && ctrl.offset > 0) {
1737         /* We've already read some reply data */
1738         handleControlReply();
1739     } else {
1740         /*
1741          * Cancel the timeout on the Data socket (if any) and
1742          * establish one on the control socket.
1743          */
1744         if (Comm::IsConnOpen(data.conn)) {
1745             commUnsetConnTimeout(data.conn);
1746         }
1747
1748         typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1749         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, FtpStateData::ftpTimeout);
1750         commSetConnTimeout(ctrl.conn, Config.Timeout.read, timeoutCall);
1751
1752         typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1753         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, FtpStateData::ftpReadControlReply);
1754         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
1755     }
1756 }
1757
1758 void FtpStateData::ftpReadControlReply(const CommIoCbParams &io)
1759 {
1760     debugs(9, 3, "ftpReadControlReply: FD " << io.fd << ", Read " << io.size << " bytes");
1761
1762     if (io.size > 0) {
1763         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
1764         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
1765     }
1766
1767     if (io.flag == COMM_ERR_CLOSING)
1768         return;
1769
1770     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1771         abortTransaction("entry aborted during control reply read");
1772         return;
1773     }
1774
1775     assert(ctrl.offset < ctrl.size);
1776
1777     if (io.flag == COMM_OK && io.size > 0) {
1778         fd_bytes(io.fd, io.size, FD_READ);
1779     }
1780
1781     if (io.flag != COMM_OK) {
1782         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
1783                "ftpReadControlReply: read error: " << xstrerr(io.xerrno));
1784
1785         if (ignoreErrno(io.xerrno)) {
1786             scheduleReadControlReply(0);
1787         } else {
1788             failed(ERR_READ_ERROR, io.xerrno);
1789             /* failed closes ctrl.conn and frees ftpState */
1790         }
1791         return;
1792     }
1793
1794     if (io.size == 0) {
1795         if (entry->store_status == STORE_PENDING) {
1796             failed(ERR_FTP_FAILURE, 0);
1797             /* failed closes ctrl.conn and frees ftpState */
1798             return;
1799         }
1800
1801         /* XXX this may end up having to be serverComplete() .. */
1802         abortTransaction("zero control reply read");
1803         return;
1804     }
1805
1806     unsigned int len =io.size + ctrl.offset;
1807     ctrl.offset = len;
1808     assert(len <= ctrl.size);
1809     handleControlReply();
1810 }
1811
1812 void
1813 FtpStateData::handleControlReply()
1814 {
1815     wordlist **W;
1816     size_t bytes_used = 0;
1817     wordlistDestroy(&ctrl.message);
1818     ctrl.message = ftpParseControlReply(ctrl.buf,
1819                                         ctrl.offset, &ctrl.replycode, &bytes_used);
1820
1821     if (ctrl.message == NULL) {
1822         /* didn't get complete reply yet */
1823
1824         if (ctrl.offset == ctrl.size) {
1825             ctrl.buf = (char *)memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size);
1826         }
1827
1828         scheduleReadControlReply(0);
1829         return;
1830     } else if (ctrl.offset == bytes_used) {
1831         /* used it all up */
1832         ctrl.offset = 0;
1833     } else {
1834         /* Got some data past the complete reply */
1835         assert(bytes_used < ctrl.offset);
1836         ctrl.offset -= bytes_used;
1837         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
1838     }
1839
1840     /* Move the last line of the reply message to ctrl.last_reply */
1841     for (W = &ctrl.message; (*W)->next; W = &(*W)->next);
1842     safe_free(ctrl.last_reply);
1843
1844     ctrl.last_reply = xstrdup((*W)->key);
1845
1846     wordlistDestroy(W);
1847
1848     /* Copy the rest of the message to cwd_message to be printed in
1849      * error messages
1850      */
1851     if (ctrl.message) {
1852         for (wordlist *w = ctrl.message; w; w = w->next) {
1853             cwd_message.append('\n');
1854             cwd_message.append(w->key);
1855         }
1856     }
1857
1858     debugs(9, 3, HERE << "state=" << state << ", code=" << ctrl.replycode);
1859
1860     FTP_SM_FUNCS[state] (this);
1861 }
1862
1863 /* ====================================================================== */
1864
1865 /// \ingroup ServerProtocolFTPInternal
1866 static void
1867 ftpReadWelcome(FtpStateData * ftpState)
1868 {
1869     int code = ftpState->ctrl.replycode;
1870     debugs(9, 3, HERE);
1871
1872     if (ftpState->flags.pasv_only)
1873         ++ ftpState->login_att;
1874
1875     if (code == 220) {
1876         if (ftpState->ctrl.message) {
1877             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1878                 ftpState->flags.skip_whitespace = 1;
1879         }
1880
1881         ftpSendUser(ftpState);
1882     } else if (code == 120) {
1883         if (NULL != ftpState->ctrl.message)
1884             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1885
1886         return;
1887     } else {
1888         ftpFail(ftpState);
1889     }
1890 }
1891
1892 /**
1893  * Translate FTP login failure into HTTP error
1894  * this is an attmpt to get the 407 message to show up outside Squid.
1895  * its NOT a general failure. But a correct FTP response type.
1896  */
1897 void
1898 FtpStateData::loginFailed()
1899 {
1900     ErrorState *err = NULL;
1901     const char *command, *reply;
1902
1903     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1904         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1905             // 421/426 - Service Overload - retry permitted.
1906             err = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
1907         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1908             // 43x - Invalid or Credential Error - retry challenge required.
1909             err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1910         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1911             // 53x - Credentials Missing - retry challenge required
1912             if (password_url) // but they were in the URI! major fail.
1913                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
1914             else
1915                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1916         }
1917     }
1918
1919     // any other problems are general falures.
1920     if (!err) {
1921         ftpFail(this);
1922         return;
1923     }
1924
1925     err->ftp.server_msg = ctrl.message;
1926
1927     ctrl.message = NULL;
1928
1929     if (old_request)
1930         command = old_request;
1931     else
1932         command = ctrl.last_command;
1933
1934     if (command && strncmp(command, "PASS", 4) == 0)
1935         command = "PASS <yourpassword>";
1936
1937     if (old_reply)
1938         reply = old_reply;
1939     else
1940         reply = ctrl.last_reply;
1941
1942     if (command)
1943         err->ftp.request = xstrdup(command);
1944
1945     if (reply)
1946         err->ftp.reply = xstrdup(reply);
1947
1948     HttpReply *newrep = err->BuildHttpReply();
1949     delete err;
1950
1951 #if HAVE_AUTH_MODULE_BASIC
1952     /* add Authenticate header */
1953     newrep->header.putAuth("Basic", ftpRealm());
1954 #endif
1955
1956     // add it to the store entry for response....
1957     entry->replaceHttpReply(newrep);
1958     serverComplete();
1959 }
1960
1961 const char *
1962 FtpStateData::ftpRealm()
1963 {
1964     static char realm[8192];
1965
1966     /* This request is not fully authenticated */
1967     if (!request) {
1968         snprintf(realm, 8192, "FTP %s unknown", user);
1969     } else if (request->port == 21) {
1970         snprintf(realm, 8192, "FTP %s %s", user, request->GetHost());
1971     } else {
1972         snprintf(realm, 8192, "FTP %s %s port %d", user, request->GetHost(), request->port);
1973     }
1974     return realm;
1975 }
1976
1977 /// \ingroup ServerProtocolFTPInternal
1978 static void
1979 ftpSendUser(FtpStateData * ftpState)
1980 {
1981     /* check the server control channel is still available */
1982     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1983         return;
1984
1985     if (ftpState->proxy_host != NULL)
1986         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n",
1987                  ftpState->user,
1988                  ftpState->request->GetHost());
1989     else
1990         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1991
1992     ftpState->writeCommand(cbuf);
1993
1994     ftpState->state = SENT_USER;
1995 }
1996
1997 /// \ingroup ServerProtocolFTPInternal
1998 static void
1999 ftpReadUser(FtpStateData * ftpState)
2000 {
2001     int code = ftpState->ctrl.replycode;
2002     debugs(9, 3, HERE);
2003
2004     if (code == 230) {
2005         ftpReadPass(ftpState);
2006     } else if (code == 331) {
2007         ftpSendPass(ftpState);
2008     } else {
2009         ftpState->loginFailed();
2010     }
2011 }
2012
2013 /// \ingroup ServerProtocolFTPInternal
2014 static void
2015 ftpSendPass(FtpStateData * ftpState)
2016 {
2017     /* check the server control channel is still available */
2018     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
2019         return;
2020
2021     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
2022     ftpState->writeCommand(cbuf);
2023     ftpState->state = SENT_PASS;
2024 }
2025
2026 /// \ingroup ServerProtocolFTPInternal
2027 static void
2028 ftpReadPass(FtpStateData * ftpState)
2029 {
2030     int code = ftpState->ctrl.replycode;
2031     debugs(9, 3, HERE << "code=" << code);
2032
2033     if (code == 230) {
2034         ftpSendType(ftpState);
2035     } else {
2036         ftpState->loginFailed();
2037     }
2038 }
2039
2040 /// \ingroup ServerProtocolFTPInternal
2041 static void
2042 ftpSendType(FtpStateData * ftpState)
2043 {
2044     const char *t;
2045     const char *filename;
2046     char mode;
2047
2048     /* check the server control channel is still available */
2049     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
2050         return;
2051
2052     /*
2053      * Ref section 3.2.2 of RFC 1738
2054      */
2055     mode = ftpState->typecode;
2056
2057     switch (mode) {
2058
2059     case 'D':
2060         mode = 'A';
2061         break;
2062
2063     case 'A':
2064
2065     case 'I':
2066         break;
2067
2068     default:
2069
2070         if (ftpState->flags.isdir) {
2071             mode = 'A';
2072         } else {
2073             t = ftpState->request->urlpath.rpos('/');
2074             filename = t ? t + 1 : ftpState->request->urlpath.termedBuf();
2075             mode = mimeGetTransferMode(filename);
2076         }
2077
2078         break;
2079     }
2080
2081     if (mode == 'I')
2082         ftpState->flags.binary = 1;
2083     else
2084         ftpState->flags.binary = 0;
2085
2086     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
2087
2088     ftpState->writeCommand(cbuf);
2089
2090     ftpState->state = SENT_TYPE;
2091 }
2092
2093 /// \ingroup ServerProtocolFTPInternal
2094 static void
2095 ftpReadType(FtpStateData * ftpState)
2096 {
2097     int code = ftpState->ctrl.replycode;
2098     char *path;
2099     char *d, *p;
2100     debugs(9, 3, HERE << "code=" << code);
2101
2102     if (code == 200) {
2103         p = path = xstrdup(ftpState->request->urlpath.termedBuf());
2104
2105         if (*p == '/')
2106             ++p;
2107
2108         while (*p) {
2109             d = p;
2110             p += strcspn(p, "/");
2111
2112             if (*p) {
2113                 *p = '\0';
2114                 ++p;
2115             }
2116
2117             rfc1738_unescape(d);
2118
2119             if (*d)
2120                 wordlistAdd(&ftpState->pathcomps, d);
2121         }
2122
2123         xfree(path);
2124
2125         if (ftpState->pathcomps)
2126             ftpTraverseDirectory(ftpState);
2127         else
2128             ftpListDir(ftpState);
2129     } else {
2130         ftpFail(ftpState);
2131     }
2132 }
2133
2134 /// \ingroup ServerProtocolFTPInternal
2135 static void
2136 ftpTraverseDirectory(FtpStateData * ftpState)
2137 {
2138     wordlist *w;
2139     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
2140
2141     safe_free(ftpState->dirpath);
2142     ftpState->dirpath = ftpState->filepath;
2143     ftpState->filepath = NULL;
2144
2145     /* Done? */
2146
2147     if (ftpState->pathcomps == NULL) {
2148         debugs(9, 3, HERE << "the final component was a directory");
2149         ftpListDir(ftpState);
2150         return;
2151     }
2152
2153     /* Go to next path component */
2154     w = ftpState->pathcomps;
2155
2156     ftpState->filepath = w->key;
2157
2158     ftpState->pathcomps = w->next;
2159
2160     delete w;
2161
2162     /* Check if we are to CWD or RETR */
2163     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
2164         ftpSendCwd(ftpState);
2165     } else {
2166         debugs(9, 3, HERE << "final component is probably a file");
2167         ftpGetFile(ftpState);
2168         return;
2169     }
2170 }
2171
2172 /// \ingroup ServerProtocolFTPInternal
2173 static void
2174 ftpSendCwd(FtpStateData * ftpState)
2175 {
2176     char *path = NULL;
2177
2178     /* check the server control channel is still available */
2179     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
2180         return;
2181
2182     debugs(9, 3, HERE);
2183
2184     path = ftpState->filepath;
2185
2186     if (!strcmp(path, "..") || !strcmp(path, "/")) {
2187         ftpState->flags.no_dotdot = 1;
2188     } else {
2189         ftpState->flags.no_dotdot = 0;
2190     }
2191
2192     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
2193
2194     ftpState->writeCommand(cbuf);
2195
2196     ftpState->state = SENT_CWD;
2197 }
2198
2199 /// \ingroup ServerProtocolFTPInternal
2200 static void
2201 ftpReadCwd(FtpStateData * ftpState)
2202 {
2203     int code = ftpState->ctrl.replycode;
2204     debugs(9, 3, HERE);
2205
2206     if (code >= 200 && code < 300) {
2207         /* CWD OK */
2208         ftpState->unhack();
2209
2210         /* Reset cwd_message to only include the last message */
2211         ftpState->cwd_message.reset("");
2212         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
2213             ftpState->cwd_message.append(' ');
2214             ftpState->cwd_message.append(w->key);
2215         }
2216         ftpState->ctrl.message = NULL;
2217
2218         /* Continue to traverse the path */
2219         ftpTraverseDirectory(ftpState);
2220     } else {
2221         /* CWD FAILED */
2222
2223         if (!ftpState->flags.put)
2224             ftpFail(ftpState);
2225         else
2226             ftpSendMkdir(ftpState);
2227     }
2228 }
2229
2230 /// \ingroup ServerProtocolFTPInternal
2231 static void
2232 ftpSendMkdir(FtpStateData * ftpState)
2233 {
2234     char *path = NULL;
2235
2236     /* check the server control channel is still available */
2237     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
2238         return;
2239
2240     path = ftpState->filepath;
2241     debugs(9, 3, HERE << "with path=" << path);
2242     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
2243     ftpState->writeCommand(cbuf);
2244     ftpState->state = SENT_MKDIR;
2245 }
2246
2247 /// \ingroup ServerProtocolFTPInternal
2248 static void
2249 ftpReadMkdir(FtpStateData * ftpState)
2250 {
2251     char *path = ftpState->filepath;
2252     int code = ftpState->ctrl.replycode;
2253
2254     debugs(9, 3, HERE << "path " << path << ", code " << code);
2255
2256     if (code == 257) {          /* success */
2257         ftpSendCwd(ftpState);
2258     } else if (code == 550) {   /* dir exists */
2259
2260         if (ftpState->flags.put_mkdir) {
2261             ftpState->flags.put_mkdir = 1;
2262             ftpSendCwd(ftpState);
2263         } else
2264             ftpSendReply(ftpState);
2265     } else
2266         ftpSendReply(ftpState);
2267 }
2268
2269 /// \ingroup ServerProtocolFTPInternal
2270 static void
2271 ftpGetFile(FtpStateData * ftpState)
2272 {
2273     assert(*ftpState->filepath != '\0');
2274     ftpState->flags.isdir = 0;
2275     ftpSendMdtm(ftpState);
2276 }
2277
2278 /// \ingroup ServerProtocolFTPInternal
2279 static void
2280 ftpListDir(FtpStateData * ftpState)
2281 {
2282     if (ftpState->flags.dir_slash) {
2283         debugs(9, 3, HERE << "Directory path did not end in /");
2284         ftpState->title_url.append("/");
2285         ftpState->flags.isdir = 1;
2286     }
2287
2288     ftpSendPassive(ftpState);
2289 }
2290
2291 /// \ingroup ServerProtocolFTPInternal
2292 static void
2293 ftpSendMdtm(FtpStateData * ftpState)
2294 {
2295     /* check the server control channel is still available */
2296     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
2297         return;
2298
2299     assert(*ftpState->filepath != '\0');
2300     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
2301     ftpState->writeCommand(cbuf);
2302     ftpState->state = SENT_MDTM;
2303 }
2304
2305 /// \ingroup ServerProtocolFTPInternal
2306 static void
2307 ftpReadMdtm(FtpStateData * ftpState)
2308 {
2309     int code = ftpState->ctrl.replycode;
2310     debugs(9, 3, HERE);
2311
2312     if (code == 213) {
2313         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
2314         ftpState->unhack();
2315     } else if (code < 0) {
2316         ftpFail(ftpState);
2317         return;
2318     }
2319
2320     ftpSendSize(ftpState);
2321 }
2322
2323 /// \ingroup ServerProtocolFTPInternal
2324 static void
2325 ftpSendSize(FtpStateData * ftpState)
2326 {
2327     /* check the server control channel is still available */
2328     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
2329         return;
2330
2331     /* Only send SIZE for binary transfers. The returned size
2332      * is useless on ASCII transfers */
2333
2334     if (ftpState->flags.binary) {
2335         assert(ftpState->filepath != NULL);
2336         assert(*ftpState->filepath != '\0');
2337         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
2338         ftpState->writeCommand(cbuf);
2339         ftpState->state = SENT_SIZE;
2340     } else
2341         /* Skip to next state no non-binary transfers */
2342         ftpSendPassive(ftpState);
2343 }
2344
2345 /// \ingroup ServerProtocolFTPInternal
2346 static void
2347 ftpReadSize(FtpStateData * ftpState)
2348 {
2349     int code = ftpState->ctrl.replycode;
2350     debugs(9, 3, HERE);
2351
2352     if (code == 213) {
2353         ftpState->unhack();
2354         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
2355
2356         if (ftpState->theSize == 0) {
2357             debugs(9, 2, "SIZE reported " <<
2358                    ftpState->ctrl.last_reply << " on " <<
2359                    ftpState->title_url);
2360             ftpState->theSize = -1;
2361         }
2362     } else if (code < 0) {
2363         ftpFail(ftpState);
2364         return;
2365     }
2366
2367     ftpSendPassive(ftpState);
2368 }
2369
2370 /**
2371  \ingroup ServerProtocolFTPInternal
2372  */
2373 static void
2374 ftpReadEPSV(FtpStateData* ftpState)
2375 {
2376     int code = ftpState->ctrl.replycode;
2377     Ip::Address ipa_remote;
2378     char *buf;
2379     debugs(9, 3, HERE);
2380
2381     if (code != 229 && code != 522) {
2382         if (code == 200) {
2383             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
2384             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
2385              * Its okay to re-send EPSV 1/2 but nothing else. */
2386             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ftpState->ctrl.conn->remote << ". Wrong accept code for EPSV");
2387         } else {
2388             debugs(9, 2, "EPSV not supported by remote end");
2389             ftpState->state = SENT_EPSV_1; /* simulate having failed EPSV 1 (last EPSV to try before shifting to PASV) */
2390         }
2391         ftpSendPassive(ftpState);
2392         return;
2393     }
2394
2395     if (code == 522) {
2396         /* server response with list of supported methods   */
2397         /*   522 Network protocol not supported, use (1)    */
2398         /*   522 Network protocol not supported, use (1,2)  */
2399         /*   522 Network protocol not supported, use (2)  */
2400         /* TODO: handle the (1,2) case. We might get it back after EPSV ALL
2401          * which means close data + control without self-destructing and re-open from scratch. */
2402         debugs(9, 5, HERE << "scanning: " << ftpState->ctrl.last_reply);
2403         buf = ftpState->ctrl.last_reply;
2404         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
2405             ++buf;
2406         if (buf != NULL && *buf == '\n')
2407             ++buf;
2408
2409         if (buf == NULL || *buf == '\0') {
2410             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
2411             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ftpState->ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
2412             ftpSendPassive(ftpState);
2413         } else if (strcmp(buf, "(1)") == 0) {
2414             ftpState->state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
2415             ftpSendPassive(ftpState);
2416         } else if (strcmp(buf, "(2)") == 0) {
2417             if (Ip::EnableIpv6) {
2418                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
2419                 if (ftpState->state == SENT_EPSV_2) {
2420                     ftpSendEPRT(ftpState);
2421                 } else {
2422                     /* or try the next Passive mode down the chain. */
2423                     ftpSendPassive(ftpState);
2424                 }
2425             } else {
2426                 /* Server only accept EPSV in IPv6 traffic. */
2427                 ftpState->state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
2428                 ftpSendPassive(ftpState);
2429             }
2430         } else {
2431             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
2432             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ftpState->ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
2433             ftpSendPassive(ftpState);
2434         }
2435         return;
2436     }
2437
2438     /*  229 Entering Extended Passive Mode (|||port|) */
2439     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
2440     debugs(9, 5, "scanning: " << ftpState->ctrl.last_reply);
2441
2442     buf = ftpState->ctrl.last_reply + strcspn(ftpState->ctrl.last_reply, "(");
2443
2444     char h1, h2, h3, h4;
2445     unsigned short port;
2446     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
2447
2448     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
2449         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
2450                ftpState->ctrl.conn->remote << ": " <<
2451                ftpState->ctrl.last_reply);
2452
2453         ftpSendPassive(ftpState);
2454         return;
2455     }
2456
2457     if (0 == port) {
2458         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
2459                ftpState->ctrl.conn->remote << ": " <<
2460                ftpState->ctrl.last_reply);
2461
2462         ftpSendPassive(ftpState);
2463         return;
2464     }
2465
2466     if (Config.Ftp.sanitycheck) {
2467         if (port < 1024) {
2468             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
2469                    ftpState->ctrl.conn->remote << ": " <<
2470                    ftpState->ctrl.last_reply);
2471
2472             ftpSendPassive(ftpState);
2473             return;
2474         }
2475     }
2476
2477     ftpState->data.port = port;
2478
2479     safe_free(ftpState->data.host);
2480     ftpState->data.host = xstrdup(fd_table[ftpState->ctrl.conn->fd].ipaddr);
2481
2482     safe_free(ftpState->ctrl.last_command);
2483
2484     safe_free(ftpState->ctrl.last_reply);
2485
2486     ftpState->ctrl.last_command = xstrdup("Connect to server data port");
2487
2488     // Generate a new data channel descriptor to be opened.
2489     Comm::ConnectionPointer conn = new Comm::Connection;
2490     conn->local = ftpState->ctrl.conn->local;
2491     conn->local.SetPort(0);
2492     conn->remote = ftpState->ctrl.conn->remote;
2493     conn->remote.SetPort(port);
2494
2495     debugs(9, 3, HERE << "connecting to " << conn->remote);
2496
2497     ftpState->data.opener = commCbCall(9,3, "FtpStateData::ftpPasvCallback", CommConnectCbPtrFun(FtpStateData::ftpPasvCallback, ftpState));
2498     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, ftpState->data.opener, Config.Timeout.connect);
2499     cs->setHost(ftpState->data.host);
2500     AsyncJob::Start(cs);
2501 }
2502
2503 /** \ingroup ServerProtocolFTPInternal
2504  *
2505  * Send Passive connection request.
2506  * Default method is to use modern EPSV request.
2507  * The failover mechanism should check for previous state and re-call with alternates on failure.
2508  */
2509 static void
2510 ftpSendPassive(FtpStateData * ftpState)
2511 {
2512     /** Checks the server control channel is still available before running. */
2513     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
2514         return;
2515
2516     debugs(9, 3, HERE);
2517
2518     /** \par
2519       * Checks for EPSV ALL special conditions:
2520       * If enabled to be sent, squid MUST NOT request any other connect methods.
2521       * If 'ALL' is sent and fails the entire FTP Session fails.
2522       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
2523     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent && ftpState->state == SENT_EPSV_1 ) {
2524         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
2525         ftpFail(ftpState);
2526         return;
2527     }
2528
2529     /** \par
2530       * Checks for 'HEAD' method request and passes off for special handling by FtpStateData::processHeadResponse(). */
2531     if (ftpState->request->method == Http::METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
2532         ftpState->processHeadResponse(); // may call serverComplete
2533         return;
2534     }
2535
2536     /// Closes any old FTP-Data connection which may exist. */
2537     ftpState->data.close();
2538
2539     /** \par
2540       * Checks for previous EPSV/PASV failures on this server/session.
2541       * Diverts to EPRT immediately if they are not working. */
2542     if (!ftpState->flags.pasv_supported) {
2543         ftpSendEPRT(ftpState);
2544         return;
2545     }
2546
2547     /** \par
2548       * Send EPSV (ALL,2,1) or PASV on the control channel.
2549       *
2550       *  - EPSV ALL  is used if enabled.
2551       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
2552       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
2553       *  - PASV      is used if EPSV 1 fails.
2554       */
2555     switch (ftpState->state) {
2556     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
2557         ftpState->flags.epsv_all_sent = true;
2558         if (ftpState->ctrl.conn->local.IsIPv6()) {
2559             debugs(9, 5, HERE << "FTP Channel is IPv6 (" << ftpState->ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
2560             snprintf(cbuf, CTRL_BUFLEN, "EPSV 2\r\n");
2561             ftpState->state = SENT_EPSV_2;
2562             break;
2563         }
2564         // else fall through to skip EPSV 2
2565
2566     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
2567         if (ftpState->ctrl.conn->local.IsIPv4()) {
2568             debugs(9, 5, HERE << "FTP Channel is IPv4 (" << ftpState->ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
2569             snprintf(cbuf, CTRL_BUFLEN, "EPSV 1\r\n");
2570             ftpState->state = SENT_EPSV_1;
2571             break;
2572         } else if (ftpState->flags.epsv_all_sent) {
2573             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
2574             ftpFail(ftpState);
2575             return;
2576         }
2577         // else fall through to skip EPSV 1
2578
2579     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
2580         debugs(9, 5, HERE << "FTP Channel (" << ftpState->ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
2581         snprintf(cbuf, CTRL_BUFLEN, "PASV\r\n");
2582         ftpState->state = SENT_PASV;
2583         break;
2584
2585     default:
2586         if (!Config.Ftp.epsv) {
2587             debugs(9, 5, HERE << "EPSV support manually disabled. Sending PASV for FTP Channel (" << ftpState->ctrl.conn->remote <<")");
2588             snprintf(cbuf, CTRL_BUFLEN, "PASV\r\n");
2589             ftpState->state = SENT_PASV;
2590         } else if (Config.Ftp.epsv_all) {
2591             debugs(9, 5, HERE << "EPSV ALL manually enabled. Attempting with FTP Channel (" << ftpState->ctrl.conn->remote <<")");
2592             snprintf(cbuf, CTRL_BUFLEN, "EPSV ALL\r\n");
2593             ftpState->state = SENT_EPSV_ALL;
2594             /* block other non-EPSV connections being attempted */
2595             ftpState->flags.epsv_all_sent = true;
2596         } else {
2597             if (ftpState->ctrl.conn->local.IsIPv6()) {
2598                 debugs(9, 5, HERE << "FTP Channel (" << ftpState->ctrl.conn->remote << "). Sending default EPSV 2");
2599                 snprintf(cbuf, CTRL_BUFLEN, "EPSV 2\r\n");
2600                 ftpState->state = SENT_EPSV_2;
2601             }
2602             if (ftpState->ctrl.conn->local.IsIPv4()) {
2603                 debugs(9, 5, HERE << "Channel (" << ftpState->ctrl.conn->remote <<"). Sending default EPSV 1");
2604                 snprintf(cbuf, CTRL_BUFLEN, "EPSV 1\r\n");
2605                 ftpState->state = SENT_EPSV_1;
2606             }
2607         }
2608         break;
2609     }
2610
2611     ftpState->writeCommand(cbuf);
2612
2613     /*
2614      * ugly hack for ftp servers like ftp.netscape.com that sometimes
2615      * dont acknowledge PASV commands. Use connect timeout to be faster then read timeout (minutes).
2616      */
2617     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
2618     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
2619                                       TimeoutDialer, ftpState, FtpStateData::ftpTimeout);
2620     commSetConnTimeout(ftpState->ctrl.conn, Config.Timeout.connect, timeoutCall);
2621 }
2622
2623 void
2624 FtpStateData::processHeadResponse()
2625 {
2626     debugs(9, 5, HERE << "handling HEAD response");
2627     ftpSendQuit(this);
2628     appendSuccessHeader();
2629
2630     /*
2631      * On rare occasions I'm seeing the entry get aborted after
2632      * ftpReadControlReply() and before here, probably when
2633      * trying to write to the client.
2634      */
2635     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
2636         abortTransaction("entry aborted while processing HEAD");
2637         return;
2638     }
2639
2640 #if USE_ADAPTATION
2641     if (adaptationAccessCheckPending) {
2642         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
2643         return;
2644     }
2645 #endif
2646
2647     // processReplyBody calls serverComplete() since there is no body
2648     processReplyBody();
2649 }
2650
2651 /// \ingroup ServerProtocolFTPInternal
2652 static void
2653 ftpReadPasv(FtpStateData * ftpState)
2654 {
2655     int code = ftpState->ctrl.replycode;
2656     int h1, h2, h3, h4;
2657     int p1, p2;
2658     int n;
2659     unsigned short port;
2660     Ip::Address ipa_remote;
2661     char *buf;
2662     LOCAL_ARRAY(char, ipaddr, 1024);
2663     debugs(9, 3, HERE);
2664
2665     if (code != 227) {
2666         debugs(9, 2, "PASV not supported by remote end");
2667         ftpSendEPRT(ftpState);
2668         return;
2669     }
2670
2671     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
2672     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
2673     debugs(9, 5, HERE << "scanning: " << ftpState->ctrl.last_reply);
2674
2675     buf = ftpState->ctrl.last_reply + strcspn(ftpState->ctrl.last_reply, "0123456789");
2676
2677     n = sscanf(buf, "%d,%d,%d,%d,%d,%d", &h1, &h2, &h3, &h4, &p1, &p2);
2678
2679     if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) {
2680         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2681                ftpState->ctrl.conn->remote << ": " <<
2682                ftpState->ctrl.last_reply);
2683
2684         ftpSendEPRT(ftpState);
2685         return;
2686     }
2687
2688     snprintf(ipaddr, 1024, "%d.%d.%d.%d", h1, h2, h3, h4);
2689
2690     ipa_remote = ipaddr;
2691
2692     if ( ipa_remote.IsAnyAddr() ) {
2693         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2694                ftpState->ctrl.conn->remote << ": " <<
2695                ftpState->ctrl.last_reply);
2696
2697         ftpSendEPRT(ftpState);
2698         return;
2699     }
2700
2701     port = ((p1 << 8) + p2);
2702
2703     if (0 == port) {
2704         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2705                ftpState->ctrl.conn->remote << ": " <<
2706                ftpState->ctrl.last_reply);
2707
2708         ftpSendEPRT(ftpState);
2709         return;
2710     }
2711
2712     if (Config.Ftp.sanitycheck) {
2713         if (port < 1024) {
2714             debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2715                    ftpState->ctrl.conn->remote << ": " <<
2716                    ftpState->ctrl.last_reply);
2717
2718             ftpSendEPRT(ftpState);
2719             return;
2720         }
2721     }
2722
2723     ftpState->data.port = port;
2724
2725     safe_free(ftpState->data.host);
2726     if (Config.Ftp.sanitycheck)
2727         ftpState->data.host = xstrdup(fd_table[ftpState->ctrl.conn->fd].ipaddr);
2728     else
2729         ftpState->data.host = xstrdup(ipaddr);
2730
2731     safe_free(ftpState->ctrl.last_command);
2732
2733     safe_free(ftpState->ctrl.last_reply);
2734
2735     ftpState->ctrl.last_command = xstrdup("Connect to server data port");
2736
2737     Comm::ConnectionPointer conn = new Comm::Connection;
2738     conn->local = ftpState->ctrl.conn->local;
2739     conn->local.SetPort(0);
2740     conn->remote = ipaddr;
2741     conn->remote.SetPort(port);
2742
2743     debugs(9, 3, HERE << "connecting to " << conn->remote);
2744
2745     ftpState->data.opener = commCbCall(9,3, "FtpStateData::ftpPasvCallback", CommConnectCbPtrFun(FtpStateData::ftpPasvCallback, ftpState));
2746     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, ftpState->data.opener, Config.Timeout.connect);
2747     cs->setHost(ftpState->data.host);
2748     AsyncJob::Start(cs);
2749 }
2750
2751 void
2752 FtpStateData::ftpPasvCallback(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data)
2753 {
2754     FtpStateData *ftpState = (FtpStateData *)data;
2755     debugs(9, 3, HERE);
2756     ftpState->data.opener = NULL;
2757
2758     if (status != COMM_OK) {
2759         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
2760
2761         // ABORT on timeouts. server may be waiting on a broken TCP link.
2762         if (status == COMM_TIMEOUT)
2763             ftpState->writeCommand("ABOR");
2764
2765         // try another connection attempt with some other method
2766         ftpSendPassive(ftpState);
2767         return;
2768     }
2769
2770     ftpState->data.opened(conn, ftpState->dataCloser());
2771     ftpRestOrList(ftpState);
2772 }
2773
2774 /// \ingroup ServerProtocolFTPInternal
2775 static void
2776 ftpOpenListenSocket(FtpStateData * ftpState, int fallback)
2777 {
2778     /// Close old data channels, if any. We may open a new one below.
2779     if (ftpState->data.conn != NULL) {
2780         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
2781             // NP: in fact it points to the control channel. just clear it.
2782             ftpState->data.clear();
2783         else
2784             ftpState->data.close();
2785     }
2786     safe_free(ftpState->data.host);
2787
2788     /*
2789      * Set up a listen socket on the same local address as the
2790      * control connection.
2791      */
2792     Comm::ConnectionPointer temp = new Comm::Connection;
2793     temp->local = ftpState->ctrl.conn->local;
2794
2795     /*
2796      * REUSEADDR is needed in fallback mode, since the same port is
2797      * used for both control and data.
2798      */
2799     if (fallback) {
2800         int on = 1;
2801         setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
2802         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
2803         temp->flags |= COMM_REUSEADDR;
2804     } else {
2805         /* if not running in fallback mode a new port needs to be retrieved */
2806         temp->local.SetPort(0);
2807     }
2808
2809     ftpState->listenForDataChannel(temp, ftpState->entry->url());
2810 }
2811
2812 /// \ingroup ServerProtocolFTPInternal
2813 static void
2814 ftpSendPORT(FtpStateData * ftpState)
2815 {
2816     /* check the server control channel is still available */
2817     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
2818         return;
2819
2820     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
2821         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
2822         return;
2823     }
2824
2825     debugs(9, 3, HERE);
2826     ftpState->flags.pasv_supported = 0;
2827     ftpOpenListenSocket(ftpState, 0);
2828
2829     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
2830         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.IsIPv4() ) {
2831             /* non-IPv4 CANNOT send PORT command.                       */
2832             /* we got here by attempting and failing an EPRT            */
2833             /* using the same reply code should simulate a PORT failure */
2834             ftpReadPORT(ftpState);
2835             return;
2836         }
2837
2838         /* XXX Need to set error message */
2839         ftpFail(ftpState);
2840         return;
2841     }
2842
2843     // pull out the internal IP address bytes to send in PORT command...
2844     // source them from the listen_conn->local
2845
2846     struct addrinfo *AI = NULL;
2847     ftpState->data.listenConn->local.GetAddrInfo(AI, AF_INET);
2848     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
2849     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
2850     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
2851              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
2852              portptr[0], portptr[1]);
2853     ftpState->writeCommand(cbuf);
2854     ftpState->state = SENT_PORT;
2855
2856     ftpState->data.listenConn->local.FreeAddrInfo(AI);
2857 }
2858
2859 /// \ingroup ServerProtocolFTPInternal
2860 static void
2861 ftpReadPORT(FtpStateData * ftpState)
2862 {
2863     int code = ftpState->ctrl.replycode;
2864     debugs(9, 3, HERE);
2865
2866     if (code != 200) {
2867         /* Fall back on using the same port as the control connection */
2868         debugs(9, 3, "PORT not supported by remote end");
2869         ftpOpenListenSocket(ftpState, 1);
2870     }
2871
2872     ftpRestOrList(ftpState);
2873 }
2874
2875 /// \ingroup ServerProtocolFTPInternal
2876 static void
2877 ftpSendEPRT(FtpStateData * ftpState)
2878 {
2879     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
2880         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
2881         return;
2882     }
2883
2884     if (!Config.Ftp.eprt) {
2885         /* Disabled. Switch immediately to attempting old PORT command. */
2886         debugs(9, 3, "EPRT disabled by local administrator");
2887         ftpSendPORT(ftpState);
2888         return;
2889     }
2890
2891     debugs(9, 3, HERE);
2892     ftpState->flags.pasv_supported = 0;
2893
2894     ftpOpenListenSocket(ftpState, 0);
2895     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
2896     if (!Comm::IsConnOpen(ftpState->data.conn)) {
2897         /* XXX Need to set error message */
2898         ftpFail(ftpState);
2899         return;
2900     }
2901
2902     char buf[MAX_IPSTRLEN];
2903
2904     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
2905     /* Which can be used by EITHER protocol. */
2906     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
2907              ( ftpState->data.listenConn->local.IsIPv6() ? 2 : 1 ),
2908              ftpState->data.listenConn->local.NtoA(buf,MAX_IPSTRLEN),
2909              ftpState->data.listenConn->local.GetPort() );
2910
2911     ftpState->writeCommand(cbuf);
2912     ftpState->state = SENT_EPRT;
2913 }
2914
2915 static void
2916 ftpReadEPRT(FtpStateData * ftpState)
2917 {
2918     int code = ftpState->ctrl.replycode;
2919     debugs(9, 3, HERE);
2920
2921     if (code != 200) {
2922         /* Failover to attempting old PORT command. */
2923         debugs(9, 3, "EPRT not supported by remote end");
2924         ftpSendPORT(ftpState);
2925         return;
2926     }
2927
2928     ftpRestOrList(ftpState);
2929 }
2930
2931 /**
2932  \ingroup ServerProtocolFTPInternal
2933  \par
2934  * "read" handler to accept FTP data connections.
2935  *
2936  \param io    comm accept(2) callback parameters
2937  */
2938 void
2939 FtpStateData::ftpAcceptDataConnection(const CommAcceptCbParams &io)
2940 {
2941     debugs(9, 3, HERE);
2942
2943     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
2944         abortTransaction("entry aborted when accepting data conn");
2945         data.listenConn->close();
2946         data.listenConn = NULL;
2947         return;
2948     }
2949
2950     if (io.flag != COMM_OK) {
2951         data.listenConn->close();
2952         data.listenConn = NULL;
2953         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
2954         /** \todo Need to send error message on control channel*/
2955         ftpFail(this);
2956         return;
2957     }
2958
2959     /* data listening conn is no longer even open. abort. */
2960     if (!Comm::IsConnOpen(data.listenConn)) {
2961         data.listenConn = NULL; // ensure that it's cleared and not just closed.
2962         return;
2963     }
2964
2965     /* data listening conn is no longer even open. abort. */
2966     if (!Comm::IsConnOpen(data.conn)) {
2967         data.clear(); // ensure that it's cleared and not just closed.
2968         return;
2969     }
2970
2971     /** \par
2972      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
2973      * made by the remote client which is connected to the FTP control socket.
2974      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
2975      * This prevents third-party hacks, but also third-party load balancing handshakes.
2976      */
2977     if (Config.Ftp.sanitycheck) {
2978         // accept if either our data or ctrl connection is talking to this remote peer.
2979         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
2980             debugs(9, DBG_IMPORTANT,
2981                    "FTP data connection from unexpected server (" <<
2982                    io.conn->remote << "), expecting " <<
2983                    data.conn->remote << " or " << ctrl.conn->remote);
2984
2985             /* close the bad sources connection down ASAP. */
2986             io.conn->close();
2987
2988             /* drop the bad connection (io) by ignoring the attempt. */
2989             return;
2990         }
2991     }
2992
2993     /** On COMM_OK start using the accepted data socket and discard the temporary listen socket. */
2994     data.close();
2995     data.opened(io.conn, dataCloser());
2996     static char ntoapeer[MAX_IPSTRLEN];
2997     io.conn->remote.NtoA(ntoapeer,sizeof(ntoapeer));
2998     data.host = xstrdup(ntoapeer);
2999
3000     debugs(9, 3, HERE << "Connected data socket on " <<
3001            io.conn << ". FD table says: " <<
3002            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
3003            "data-peer= " << fd_table[data.conn->fd].ipaddr);
3004
3005     assert(haveControlChannel("ftpAcceptDataConnection"));
3006     assert(ctrl.message == NULL);
3007
3008     // Ctrl channel operations will determine what happens to this data connection
3009 }
3010
3011 /// \ingroup ServerProtocolFTPInternal
3012 static void
3013 ftpRestOrList(FtpStateData * ftpState)
3014 {
3015     debugs(9, 3, HERE);
3016
3017     if (ftpState->typecode == 'D') {
3018         ftpState->flags.isdir = 1;
3019
3020         if (ftpState->flags.put) {
3021             ftpSendMkdir(ftpState);     /* PUT name;type=d */
3022         } else {
3023             ftpSendNlst(ftpState);      /* GET name;type=d  sec 3.2.2 of RFC 1738 */
3024         }
3025     } else if (ftpState->flags.put) {
3026         ftpSendStor(ftpState);
3027     } else if (ftpState->flags.isdir)
3028         ftpSendList(ftpState);
3029     else if (ftpState->restartable())
3030         ftpSendRest(ftpState);
3031     else
3032         ftpSendRetr(ftpState);
3033 }
3034
3035 /// \ingroup ServerProtocolFTPInternal
3036 static void
3037 ftpSendStor(FtpStateData * ftpState)
3038 {
3039     /* check the server control channel is still available */
3040     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
3041         return;
3042
3043     debugs(9, 3, HERE);
3044
3045     if (ftpState->filepath != NULL) {
3046         /* Plain file upload */
3047         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
3048         ftpState->writeCommand(cbuf);
3049         ftpState->state = SENT_STOR;
3050     } else if (ftpState->request->header.getInt64(HDR_CONTENT_LENGTH) > 0) {
3051         /* File upload without a filename. use STOU to generate one */
3052         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
3053         ftpState->writeCommand(cbuf);
3054         ftpState->state = SENT_STOR;
3055     } else {
3056         /* No file to transfer. Only create directories if needed */
3057         ftpSendReply(ftpState);
3058     }
3059 }
3060
3061 /// \ingroup ServerProtocolFTPInternal
3062 /// \deprecated use ftpState->readStor() instead.
3063 static void
3064 ftpReadStor(FtpStateData * ftpState)
3065 {
3066     ftpState->readStor();
3067 }
3068
3069 void FtpStateData::readStor()
3070 {
3071     int code = ctrl.replycode;
3072     debugs(9, 3, HERE);
3073
3074     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
3075         if (!startRequestBodyFlow()) { // register to receive body data
3076             ftpFail(this);
3077             return;
3078         }
3079
3080         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
3081         debugs(9, 3, HERE << "starting data transfer");
3082         switchTimeoutToDataChannel();
3083         sendMoreRequestBody();
3084         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
3085         state = WRITING_DATA;
3086         debugs(9, 3, HERE << "writing data channel");
3087     } else if (code == 150) {
3088         /* When client code is 150 with no data channel, Accept data channel. */
3089         debugs(9, 3, "ftpReadStor: accepting data channel");
3090         listenForDataChannel(data.conn, data.host);
3091     } else {
3092         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
3093         ftpFail(this);
3094     }
3095 }
3096
3097 /// \ingroup ServerProtocolFTPInternal
3098 static void
3099 ftpSendRest(FtpStateData * ftpState)
3100 {
3101     /* check the server control channel is still available */
3102     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
3103         return;
3104
3105     debugs(9, 3, HERE);
3106
3107     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
3108     ftpState->writeCommand(cbuf);
3109     ftpState->state = SENT_REST;
3110 }
3111
3112 int
3113 FtpStateData::restartable()
3114 {
3115     if (restart_offset > 0)
3116         return 1;
3117
3118     if (!request->range)
3119         return 0;
3120
3121     if (!flags.binary)
3122         return 0;
3123
3124     if (theSize <= 0)
3125         return 0;
3126
3127     int64_t desired_offset = request->range->lowestOffset(theSize);
3128
3129     if (desired_offset <= 0)
3130         return 0;
3131
3132     if (desired_offset >= theSize)
3133         return 0;
3134
3135     restart_offset = desired_offset;
3136     return 1;
3137 }
3138
3139 /// \ingroup ServerProtocolFTPInternal
3140 static void
3141 ftpReadRest(FtpStateData * ftpState)
3142 {
3143     int code = ftpState->ctrl.replycode;
3144     debugs(9, 3, HERE);
3145     assert(ftpState->restart_offset > 0);
3146
3147     if (code == 350) {
3148         ftpState->setCurrentOffset(ftpState->restart_offset);
3149         ftpSendRetr(ftpState);
3150     } else if (code > 0) {
3151         debugs(9, 3, HERE << "REST not supported");
3152         ftpState->flags.rest_supported = 0;
3153         ftpSendRetr(ftpState);
3154     } else {
3155         ftpFail(ftpState);
3156     }
3157 }
3158
3159 /// \ingroup ServerProtocolFTPInternal
3160 static void
3161 ftpSendList(FtpStateData * ftpState)
3162 {
3163     /* check the server control channel is still available */
3164     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
3165         return;
3166
3167     debugs(9, 3, HERE);
3168
3169     if (ftpState->filepath) {
3170         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
3171     } else {
3172         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
3173     }
3174
3175     ftpState->writeCommand(cbuf);
3176     ftpState->state = SENT_LIST;
3177 }
3178
3179 /// \ingroup ServerProtocolFTPInternal
3180 static void
3181 ftpSendNlst(FtpStateData * ftpState)
3182 {
3183     /* check the server control channel is still available */
3184     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
3185         return;
3186
3187     debugs(9, 3, HERE);
3188
3189     ftpState->flags.tried_nlst = 1;
3190
3191     if (ftpState->filepath) {
3192         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
3193     } else {
3194         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
3195     }
3196
3197     ftpState->writeCommand(cbuf);
3198     ftpState->state = SENT_NLST;
3199 }
3200
3201 /// \ingroup ServerProtocolFTPInternal
3202 static void
3203 ftpReadList(FtpStateData * ftpState)
3204 {
3205     int code = ftpState->ctrl.replycode;
3206     debugs(9, 3, HERE);
3207
3208     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
3209         /* Begin data transfer */
3210         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3211         ftpState->switchTimeoutToDataChannel();
3212         ftpState->maybeReadVirginBody();
3213         ftpState->state = READING_DATA;
3214         return;
3215     } else if (code == 150) {
3216         /* Accept data channel */
3217         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3218         ftpState->listenForDataChannel(ftpState->data.conn, ftpState->data.host);
3219         return;
3220     } else if (!ftpState->flags.tried_nlst && code > 300) {
3221         ftpSendNlst(ftpState);
3222     } else {
3223         ftpFail(ftpState);
3224         return;
3225     }
3226 }
3227
3228 /// \ingroup ServerProtocolFTPInternal
3229 static void
3230 ftpSendRetr(FtpStateData * ftpState)
3231 {
3232     /* check the server control channel is still available */
3233     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
3234         return;
3235
3236     debugs(9, 3, HERE);
3237
3238     assert(ftpState->filepath != NULL);
3239     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
3240     ftpState->writeCommand(cbuf);
3241     ftpState->state = SENT_RETR;
3242 }
3243
3244 /// \ingroup ServerProtocolFTPInternal
3245 static void
3246 ftpReadRetr(FtpStateData * ftpState)
3247 {
3248     int code = ftpState->ctrl.replycode;
3249     debugs(9, 3, HERE);
3250
3251     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
3252         /* Begin data transfer */
3253         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3254         ftpState->switchTimeoutToDataChannel();
3255         ftpState->maybeReadVirginBody();
3256         ftpState->state = READING_DATA;
3257     } else if (code == 150) {
3258         /* Accept data channel */
3259         ftpState->listenForDataChannel(ftpState->data.conn, ftpState->data.host);
3260     } else if (code >= 300) {
3261         if (!ftpState->flags.try_slash_hack) {
3262             /* Try this as a directory missing trailing slash... */
3263             ftpState->hackShortcut(ftpSendCwd);
3264         } else {
3265             ftpFail(ftpState);
3266         }
3267     } else {
3268         ftpFail(ftpState);
3269     }
3270 }
3271
3272 /**
3273  * Generate the HTTP headers and template fluff around an FTP
3274  * directory listing display.
3275  */
3276 void
3277 FtpStateData::completedListing()
3278 {
3279     assert(entry);
3280     entry->lock();
3281     ErrorState ferr(ERR_DIR_LISTING, Http::scOkay, request);
3282     ferr.ftp.listing = &listing;
3283     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
3284     ferr.ftp.server_msg = ctrl.message;
3285     ctrl.message = NULL;
3286     entry->replaceHttpReply( ferr.BuildHttpReply() );
3287     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
3288     entry->flush();
3289     entry->unlock();
3290 }
3291
3292 /// \ingroup ServerProtocolFTPInternal
3293 static void
3294 ftpReadTransferDone(FtpStateData * ftpState)
3295 {
3296     int code = ftpState->ctrl.replycode;
3297     debugs(9, 3, HERE);
3298
3299     if (code == 226 || code == 250) {
3300         /* Connection closed; retrieval done. */
3301         if (ftpState->flags.listing) {
3302             ftpState->completedListing();
3303             /* QUIT operation handles sending the reply to client */
3304         }
3305         ftpSendQuit(ftpState);
3306     } else {                    /* != 226 */
3307         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
3308         ftpState->failed(ERR_FTP_FAILURE, 0);
3309         /* failed closes ctrl.conn and frees ftpState */
3310         return;
3311     }
3312 }
3313
3314 // premature end of the request body
3315 void
3316 FtpStateData::handleRequestBodyProducerAborted()
3317 {
3318     ServerStateData::handleRequestBodyProducerAborted();
3319     debugs(9, 3, HERE << "ftpState=" << this);
3320     failed(ERR_READ_ERROR, 0);
3321 }
3322
3323 /**
3324  * This will be called when the put write is completed
3325  */
3326 void
3327 FtpStateData::sentRequestBody(const CommIoCbParams &io)
3328 {
3329     if (io.size > 0)
3330         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
3331     ServerStateData::sentRequestBody(io);
3332 }
3333
3334 /// \ingroup ServerProtocolFTPInternal
3335 static void
3336 ftpWriteTransferDone(FtpStateData * ftpState)
3337 {
3338     int code = ftpState->ctrl.replycode;
3339     debugs(9, 3, HERE);
3340
3341     if (!(code == 226 || code == 250)) {
3342         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
3343         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
3344         return;
3345     }
3346
3347     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
3348     ftpSendReply(ftpState);
3349 }
3350
3351 /// \ingroup ServerProtocolFTPInternal
3352 static void
3353 ftpSendQuit(FtpStateData * ftpState)
3354 {
3355     /* check the server control channel is still available */
3356     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
3357         return;
3358
3359     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
3360     ftpState->writeCommand(cbuf);
3361     ftpState->state = SENT_QUIT;
3362 }
3363
3364 /**
3365  * \ingroup ServerProtocolFTPInternal
3366  *
3367  *  This completes a client FTP operation with success or other page
3368  *  generated and stored in the entry field by the code issuing QUIT.
3369  */
3370 static void
3371 ftpReadQuit(FtpStateData * ftpState)
3372 {
3373     ftpState->serverComplete();
3374 }
3375
3376 /// \ingroup ServerProtocolFTPInternal
3377 static void
3378 ftpTrySlashHack(FtpStateData * ftpState)
3379 {
3380     char *path;
3381     ftpState->flags.try_slash_hack = 1;
3382     /* Free old paths */
3383
3384     debugs(9, 3, HERE);
3385
3386     if (ftpState->pathcomps)
3387         wordlistDestroy(&ftpState->pathcomps);
3388
3389     safe_free(ftpState->filepath);
3390
3391     /* Build the new path (urlpath begins with /) */
3392     path = xstrdup(ftpState->request->urlpath.termedBuf());
3393
3394     rfc1738_unescape(path);
3395
3396     ftpState->filepath = path;
3397
3398     /* And off we go */
3399     ftpGetFile(ftpState);
3400 }
3401
3402 /**
3403  * Forget hack status. Next error is shown to the user
3404  */
3405 void
3406 FtpStateData::unhack()
3407 {
3408     debugs(9, 3, HERE);
3409
3410     if (old_request != NULL) {
3411         safe_free(old_request);
3412         safe_free(old_reply);
3413     }
3414 }
3415
3416 void
3417 FtpStateData::hackShortcut(FTPSM * nextState)
3418 {
3419     /* Clear some unwanted state */
3420     setCurrentOffset(0);
3421     restart_offset = 0;
3422     /* Save old error message & some state info */
3423
3424     debugs(9, 3, HERE);
3425
3426     if (old_request == NULL) {
3427         old_request = ctrl.last_command;
3428         ctrl.last_command = NULL;
3429         old_reply = ctrl.last_reply;
3430         ctrl.last_reply = NULL;
3431
3432         if (pathcomps == NULL && filepath != NULL)
3433             old_filepath = xstrdup(filepath);
3434     }
3435
3436     /* Jump to the "hack" state */
3437     nextState(this);
3438 }
3439
3440 /// \ingroup ServerProtocolFTPInternal
3441 static void
3442 ftpFail(FtpStateData *ftpState)
3443 {
3444     debugs(9, 6, HERE << "flags(" <<
3445            (ftpState->flags.isdir?"IS_DIR,":"") <<
3446            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
3447            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
3448            "slashhack=" << (ftpState->request->urlpath.caseCmp("/%2f", 4)==0? "T":"F") );
3449
3450     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
3451     if (!ftpState->flags.isdir &&       /* Not a directory */
3452             !ftpState->flags.try_slash_hack &&  /* Not in slash hack */
3453             ftpState->mdtm <= 0 && ftpState->theSize < 0 &&     /* Not known as a file */
3454             ftpState->request->urlpath.caseCmp("/%2f", 4) != 0) {       /* No slash encoded */
3455
3456         switch (ftpState->state) {
3457
3458         case SENT_CWD:
3459
3460         case SENT_RETR:
3461             /* Try the / hack */
3462             ftpState->hackShortcut(ftpTrySlashHack);
3463             return;
3464
3465         default:
3466             break;
3467         }
3468     }
3469
3470     ftpState->failed(ERR_NONE, 0);
3471     /* failed() closes ctrl.conn and frees this */
3472 }
3473
3474 void
3475 FtpStateData::failed(err_type error, int xerrno)
3476 {
3477     debugs(9,3,HERE << "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
3478     if (entry->isEmpty())
3479         failedErrorMessage(error, xerrno);
3480
3481     serverComplete();
3482 }
3483
3484 void
3485 FtpStateData::failedErrorMessage(err_type error, int xerrno)
3486 {
3487     ErrorState *ftperr = NULL;
3488     const char *command, *reply;
3489
3490     /* Translate FTP errors into HTTP errors */
3491     switch (error) {
3492
3493     case ERR_NONE:
3494
3495         switch (state) {
3496
3497         case SENT_USER:
3498
3499         case SENT_PASS:
3500
3501             if (ctrl.replycode > 500)
3502                 if (password_url)
3503                     ftperr = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
3504                 else
3505                     ftperr = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
3506
3507             else if (ctrl.replycode == 421)
3508                 ftperr = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
3509
3510             break;
3511
3512         case SENT_CWD:
3513
3514         case SENT_RETR:
3515             if (ctrl.replycode == 550)
3516                 ftperr = new ErrorState(ERR_FTP_NOT_FOUND, Http::scNotFound, fwd->request);
3517
3518             break;
3519
3520         default:
3521             break;
3522         }
3523
3524         break;
3525
3526     case ERR_READ_TIMEOUT:
3527         ftperr = new ErrorState(error, Http::scGateway_Timeout, fwd->request);
3528         break;
3529
3530     default:
3531         ftperr = new ErrorState(error, Http::scBadGateway, fwd->request);
3532         break;
3533     }
3534
3535     if (ftperr == NULL)
3536         ftperr = new ErrorState(ERR_FTP_FAILURE, Http::scBadGateway, fwd->request);
3537
3538     ftperr->xerrno = xerrno;
3539
3540     ftperr->ftp.server_msg = ctrl.message;
3541     ctrl.message = NULL;
3542
3543     if (old_request)
3544         command = old_request;
3545     else
3546         command = ctrl.last_command;
3547
3548     if (command && strncmp(command, "PASS", 4) == 0)
3549         command = "PASS <yourpassword>";
3550
3551     if (old_reply)
3552         reply = old_reply;
3553     else
3554         reply = ctrl.last_reply;
3555
3556     if (command)
3557         ftperr->ftp.request = xstrdup(command);
3558
3559     if (reply)
3560         ftperr->ftp.reply = xstrdup(reply);
3561
3562     entry->replaceHttpReply( ftperr->BuildHttpReply() );
3563     delete ftperr;
3564 }
3565
3566 /// \ingroup ServerProtocolFTPInternal
3567 static void
3568 ftpSendReply(FtpStateData * ftpState)
3569 {
3570     int code = ftpState->ctrl.replycode;
3571     Http::StatusCode http_code;
3572     err_type err_code = ERR_NONE;
3573
3574     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
3575
3576     if (cbdataReferenceValid(ftpState))
3577         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
3578
3579     if (code == 226 || code == 250) {
3580         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
3581         http_code = (ftpState->mdtm > 0) ? Http::scAccepted : Http::scCreated;
3582     } else if (code == 227) {
3583         err_code = ERR_FTP_PUT_CREATED;
3584         http_code = Http::scCreated;
3585     } else {
3586         err_code = ERR_FTP_PUT_ERROR;
3587         http_code = Http::scInternalServerError;
3588     }
3589
3590     ErrorState err(err_code, http_code, ftpState->request);
3591
3592     if (ftpState->old_request)
3593         err.ftp.request = xstrdup(ftpState->old_request);
3594     else
3595         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
3596
3597     if (ftpState->old_reply)
3598         err.ftp.reply = xstrdup(ftpState->old_reply);
3599     else if (ftpState->ctrl.last_reply)
3600         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
3601     else
3602         err.ftp.reply = xstrdup("");
3603
3604     // TODO: interpret as FTP-specific error code
3605     err.detailError(code);
3606
3607     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
3608
3609     ftpSendQuit(ftpState);
3610 }
3611
3612 void
3613 FtpStateData::appendSuccessHeader()
3614 {
3615     const char *mime_type = NULL;
3616     const char *mime_enc = NULL;
3617     String urlpath = request->urlpath;
3618     const char *filename = NULL;
3619     const char *t = NULL;
3620
3621     debugs(9, 3, HERE);
3622
3623     if (flags.http_header_sent)
3624         return;
3625
3626     HttpReply *reply = new HttpReply;
3627
3628     flags.http_header_sent = 1;
3629
3630     assert(entry->isEmpty());
3631
3632     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
3633
3634     entry->buffer();    /* released when done processing current data payload */
3635
3636     filename = (t = urlpath.rpos('/')) ? t + 1 : urlpath.termedBuf();
3637
3638     if (flags.isdir) {
3639         mime_type = "text/html";
3640     } else {
3641         switch (typecode) {
3642
3643         case 'I':
3644             mime_type = "application/octet-stream";
3645             mime_enc = mimeGetContentEncoding(filename);
3646             break;
3647
3648         case 'A':
3649             mime_type = "text/plain";
3650             break;
3651
3652         default:
3653             mime_type = mimeGetContentType(filename);
3654             mime_enc = mimeGetContentEncoding(filename);
3655             break;
3656         }
3657     }
3658
3659     /* set standard stuff */
3660
3661     if (0 == getCurrentOffset()) {
3662         /* Full reply */
3663         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
3664     } else if (theSize < getCurrentOffset()) {
3665         /*
3666          * DPW 2007-05-04
3667          * offset should not be larger than theSize.  We should
3668          * not be seeing this condition any more because we'll only
3669          * send REST if we know the theSize and if it is less than theSize.
3670          */
3671         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
3672                " current offset=" << getCurrentOffset() <<
3673                ", but theSize=" << theSize <<
3674                ".  assuming full content response");
3675         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
3676     } else {
3677         /* Partial reply */
3678         HttpHdrRangeSpec range_spec;
3679         range_spec.offset = getCurrentOffset();
3680         range_spec.length = theSize - getCurrentOffset();
3681         reply->setHeaders(Http::scPartialContent, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
3682         httpHeaderAddContRange(&reply->header, range_spec, theSize);
3683     }
3684
3685     /* additional info */
3686     if (mime_enc)
3687         reply->header.putStr(HDR_CONTENT_ENCODING, mime_enc);
3688
3689     setVirginReply(reply);
3690     adaptOrFinalizeReply();
3691 }
3692
3693 void
3694 FtpStateData::haveParsedReplyHeaders()
3695 {
3696     ServerStateData::haveParsedReplyHeaders();
3697
3698     StoreEntry *e = entry;
3699
3700     e->timestampsSet();
3701
3702     if (flags.authenticated) {
3703         /*
3704          * Authenticated requests can't be cached.
3705          */
3706         e->release();
3707     } else if (EBIT_TEST(e->flags, ENTRY_CACHABLE) && !getCurrentOffset()) {
3708         e->setPublicKey();
3709     } else {
3710         e->release();
3711     }
3712 }
3713
3714 HttpReply *
3715 FtpStateData::ftpAuthRequired(HttpRequest * request, const char *realm)
3716 {
3717     ErrorState err(ERR_CACHE_ACCESS_DENIED, Http::scUnauthorized, request);
3718     HttpReply *newrep = err.BuildHttpReply();
3719 #if HAVE_AUTH_MODULE_BASIC
3720     /* add Authenticate header */
3721     newrep->header.putAuth("Basic", realm);
3722 #endif
3723     return newrep;
3724 }
3725
3726 /**
3727  \ingroup ServerProtocolFTPAPI
3728  \todo Should be a URL class API call.
3729  *
3730  *  Construct an URI with leading / in PATH portion for use by CWD command
3731  *  possibly others. FTP encodes absolute paths as beginning with '/'
3732  *  after the initial URI path delimiter, which happens to be / itself.
3733  *  This makes FTP absolute URI appear as:  ftp:host:port//root/path
3734  *  To encompass older software which compacts multiple // to / in transit
3735  *  We use standard URI-encoding on the second / making it
3736  *  ftp:host:port/%2froot/path  AKA 'the FTP %2f hack'.
3737  */
3738 const char *
3739 ftpUrlWith2f(HttpRequest * request)
3740 {
3741     String newbuf = "%2f";
3742
3743     if (request->protocol != AnyP::PROTO_FTP)
3744         return NULL;
3745
3746     if ( request->urlpath[0]=='/' ) {
3747         newbuf.append(request->urlpath);
3748         request->urlpath.absorb(newbuf);
3749         safe_free(request->canonical);
3750     } else if ( !strncmp(request->urlpath.termedBuf(), "%2f", 3) ) {
3751         newbuf.append(request->urlpath.substr(1,request->urlpath.size()));
3752         request->urlpath.absorb(newbuf);
3753         safe_free(request->canonical);
3754     }
3755
3756     return urlCanonical(request);
3757 }
3758
3759 void
3760 FtpStateData::printfReplyBody(const char *fmt, ...)
3761 {
3762     va_list args;
3763     va_start (args, fmt);
3764     static char buf[4096];
3765     buf[0] = '\0';
3766     vsnprintf(buf, 4096, fmt, args);
3767     writeReplyBody(buf, strlen(buf));
3768     va_end(args);
3769 }
3770
3771 /**
3772  * Call this when there is data from the origin server
3773  * which should be sent to either StoreEntry, or to ICAP...
3774  */
3775 void
3776 FtpStateData::writeReplyBody(const char *dataToWrite, size_t dataLength)
3777 {
3778     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
3779     addVirginReplyBody(dataToWrite, dataLength);
3780 }
3781
3782 /**
3783  * called after we wrote the last byte of the request body
3784  */
3785 void
3786 FtpStateData::doneSendingRequestBody()
3787 {
3788     ServerStateData::doneSendingRequestBody();
3789     debugs(9,3, HERE);
3790     dataComplete();
3791     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
3792      * if transfer type is 'stream' call dataComplete()
3793      * otherwise leave open. (reschedule control channel read?)
3794      */
3795 }
3796
3797 /**
3798  * A hack to ensure we do not double-complete on the forward entry.
3799  *
3800  \todo FtpStateData logic should probably be rewritten to avoid
3801  *      double-completion or FwdState should be rewritten to allow it.
3802  */
3803 void
3804 FtpStateData::completeForwarding()
3805 {
3806     if (fwd == NULL || flags.completed_forwarding) {
3807         debugs(9, 3, HERE << "completeForwarding avoids " <<
3808                "double-complete on FD " << ctrl.conn->fd << ", Data FD " << data.conn->fd <<
3809                ", this " << this << ", fwd " << fwd);
3810         return;
3811     }
3812
3813     flags.completed_forwarding = true;
3814     ServerStateData::completeForwarding();
3815 }
3816
3817 /**
3818  * Close the FTP server connection(s). Used by serverComplete().
3819  */
3820 void
3821 FtpStateData::closeServer()
3822 {
3823     if (Comm::IsConnOpen(ctrl.conn)) {
3824         debugs(9,3, HERE << "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
3825         fwd->unregister(ctrl.conn);
3826         ctrl.close();
3827     }
3828
3829     if (Comm::IsConnOpen(data.conn)) {
3830         debugs(9,3, HERE << "closing FTP data FD " << data.conn->fd << ", this " << this);
3831         data.close();
3832     }
3833
3834     debugs(9,3, HERE << "FTP ctrl and data connections closed. this " << this);
3835 }
3836
3837 /**
3838  * Did we close all FTP server connection(s)?
3839  *
3840  \retval true   Both server control and data channels are closed. And not waiting for a new data connection to open.
3841  \retval false  Either control channel or data is still active.
3842  */
3843 bool
3844 FtpStateData::doneWithServer() const
3845 {
3846     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
3847 }
3848
3849 /**
3850  * Have we lost the FTP server control channel?
3851  *
3852  \retval true   The server control channel is available.
3853  \retval false  The server control channel is not available.
3854  */
3855 bool
3856 FtpStateData::haveControlChannel(const char *caller_name) const
3857 {
3858     if (doneWithServer())
3859         return false;
3860
3861     /* doneWithServer() only checks BOTH channels are closed. */
3862     if (!Comm::IsConnOpen(ctrl.conn)) {
3863         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
3864         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
3865         return false;
3866     }
3867
3868     return true;
3869 }
3870
3871 /**
3872  * Quickly abort the transaction
3873  *
3874  \todo destruction should be sufficient as the destructor should cleanup,
3875  *      including canceling close handlers
3876  */
3877 void
3878 FtpStateData::abortTransaction(const char *reason)
3879 {
3880     debugs(9, 3, HERE << "aborting transaction for " << reason <<
3881            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
3882     if (Comm::IsConnOpen(ctrl.conn)) {
3883         ctrl.conn->close();
3884         return;
3885     }
3886
3887     fwd->handleUnregisteredServerEnd();
3888     mustStop("FtpStateData::abortTransaction");
3889 }
3890
3891 /// creates a data channel Comm close callback
3892 AsyncCall::Pointer
3893 FtpStateData::dataCloser()
3894 {
3895     typedef CommCbMemFunT<FtpStateData, CommCloseCbParams> Dialer;
3896     return JobCallback(9, 5, Dialer, this, FtpStateData::dataClosed);
3897 }
3898
3899 /// configures the channel with a descriptor and registers a close handler
3900 void
3901 FtpChannel::opened(const Comm::ConnectionPointer &newConn, const AsyncCall::Pointer &aCloser)
3902 {
3903     assert(!Comm::IsConnOpen(conn));
3904     assert(closer == NULL);
3905
3906     assert(Comm::IsConnOpen(newConn));
3907     assert(aCloser != NULL);
3908
3909     conn = newConn;
3910     closer = aCloser;
3911     comm_add_close_handler(conn->fd, closer);
3912 }
3913
3914 /// planned close: removes the close handler and calls comm_close
3915 void
3916 FtpChannel::close()
3917 {
3918     // channels with active listeners will be closed when the listener handler dies.
3919     if (Comm::IsConnOpen(conn)) {
3920         comm_remove_close_handler(conn->fd, closer);
3921         conn->close(); // we do not expect to be called back
3922     }
3923     clear();
3924 }
3925
3926 void
3927 FtpChannel::clear()
3928 {
3929     conn = NULL;
3930     closer = NULL;
3931 }