src/ftp.cc

   1 /*
   2  * $Id$
   3  *
   4  * DEBUG: section 09    File Transfer Protocol (FTP)
   5  * AUTHOR: Harvest Derived
   6  *
   7  * SQUID Web Proxy Cache          http://www.squid-cache.org/
   8  * ----------------------------------------------------------
   9  *
  10  *  Squid is the result of efforts by numerous individuals from
  11  *  the Internet community; see the CONTRIBUTORS file for full
  12  *  details.   Many organizations have provided support for Squid's
  13  *  development; see the SPONSORS file for full details.  Squid is
  14  *  Copyrighted (C) 2001 by the Regents of the University of
  15  *  California; see the COPYRIGHT file for full details.  Squid
  16  *  incorporates software developed and/or copyrighted by other
  17  *  sources; see the CREDITS file for full details.
  18  *
  19  *  This program is free software; you can redistribute it and/or modify
  20  *  it under the terms of the GNU General Public License as published by
  21  *  the Free Software Foundation; either version 2 of the License, or
  22  *  (at your option) any later version.
  23  *
  24  *  This program is distributed in the hope that it will be useful,
  25  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  *  GNU General Public License for more details.
  28  *
  29  *  You should have received a copy of the GNU General Public License
  30  *  along with this program; if not, write to the Free Software
  31  *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, USA.
  32  *
  33  */
  34
  35 #include "squid.h"
  36 #include "comm.h"
  37 #include "comm/ConnOpener.h"
  38 #include "comm/TcpAcceptor.h"
  39 #include "comm/Write.h"
  40 #include "CommCalls.h"
  41 #include "compat/strtoll.h"
  42 #include "errorpage.h"
  43 #include "fde.h"
  44 #include "forward.h"
  45 #include "html_quote.h"
  46 #include "HttpHdrContRange.h"
  47 #include "HttpHeader.h"
  48 #include "HttpHeaderRange.h"
  49 #include "HttpReply.h"
  50 #include "HttpRequest.h"
  51 #include "ip/tools.h"
  52 #include "MemBuf.h"
  53 #include "protos.h"
  54 #include "rfc1738.h"
  55 #include "Server.h"
  56 #include "SquidString.h"
  57 #include "SquidTime.h"
  58 #include "StatCounters.h"
  59 #include "Store.h"
  60 #include "URLScheme.h"
  61 #include "wordlist.h"
  62
  63 #if USE_DELAY_POOLS
  64 #include "DelayPools.h"
  65 #include "MemObject.h"
  66 #endif
  67
  68 #if HAVE_ERRNO_H
  69 #include <errno.h>
  70 #endif
  71
  72 /**
  73  \defgroup ServerProtocolFTPInternal Server-Side FTP Internals
  74  \ingroup ServerProtocolFTPAPI
  75  */
  76
  77 /// \ingroup ServerProtocolFTPInternal
  78 static const char *const crlf = "\r\n";
  79
  80 #define CTRL_BUFLEN 1024
  81 /// \ingroup ServerProtocolFTPInternal
  82 static char cbuf[CTRL_BUFLEN];
  83
  84 /// \ingroup ServerProtocolFTPInternal
  85 typedef enum {
  86     BEGIN,
  87     SENT_USER,
  88     SENT_PASS,
  89     SENT_TYPE,
  90     SENT_MDTM,
  91     SENT_SIZE,
  92     SENT_EPRT,
  93     SENT_PORT,
  94     SENT_EPSV_ALL,
  95     SENT_EPSV_1,
  96     SENT_EPSV_2,
  97     SENT_PASV,
  98     SENT_CWD,
  99     SENT_LIST,
 100     SENT_NLST,
 101     SENT_REST,
 102     SENT_RETR,
 103     SENT_STOR,
 104     SENT_QUIT,
 105     READING_DATA,
 106     WRITING_DATA,
 107     SENT_MKDIR
 108 } ftp_state_t;
 109
 110 /// \ingroup ServerProtocolFTPInternal
 111 struct _ftp_flags {
 112
 113     /* passive mode */
 114     bool pasv_supported;  ///< PASV command is allowed
 115     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
 116     bool pasv_only;
 117     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
 118
 119     /* authentication */
 120     bool authenticated;         ///< authentication success
 121     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
 122     bool tried_auth_nopass;     ///< auth tried username with no password already.
 123
 124     /* other */
 125     bool isdir;
 126     bool skip_whitespace;
 127     bool rest_supported;
 128     bool http_header_sent;
 129     bool tried_nlst;
 130     bool need_base_href;
 131     bool dir_slash;
 132     bool root_dir;
 133     bool no_dotdot;
 134     bool binary;
 135     bool try_slash_hack;
 136     bool put;
 137     bool put_mkdir;
 138     bool listformat_unknown;
 139     bool listing;
 140     bool completed_forwarding;
 141 };
 142
 143 class FtpStateData;
 144
 145 /// \ingroup ServerProtocolFTPInternal
 146 typedef void (FTPSM) (FtpStateData *);
 147
 148 /// common code for FTP control and data channels
 149 /// does not own the channel descriptor, which is managed by FtpStateData
 150 class FtpChannel
 151 {
 152 public:
 153     FtpChannel() {};
 154
 155     /// called after the socket is opened, sets up close handler
 156     void opened(const Comm::ConnectionPointer &conn, const AsyncCall::Pointer &aCloser);
 157
 158     /** Handles all operations needed to properly close the active channel FD.
 159      * clearing the close handler, clearing the listen socket properly, and calling comm_close
 160      */
 161     void close();
 162
 163     void clear(); ///< just drops conn and close handler. does not close active connections.
 164
 165     Comm::ConnectionPointer conn; ///< channel descriptor
 166
 167     /** A temporary handle to the connection being listened on.
 168      * Closing this will also close the waiting Data channel acceptor.
 169      * If a data connection has already been accepted but is still waiting in the event queue
 170      * the callback will still happen and needs to be handled (usually dropped).
 171      */
 172     Comm::ConnectionPointer listenConn;
 173
 174     AsyncCall::Pointer opener; ///< Comm opener handler callback.
 175 private:
 176     AsyncCall::Pointer closer; ///< Comm close handler callback
 177 };
 178
 179 /// \ingroup ServerProtocolFTPInternal
 180 class FtpStateData : public ServerStateData
 181 {
 182
 183 public:
 184     void *operator new (size_t);
 185     void operator delete (void *);
 186     void *toCbdata() { return this; }
 187
 188     FtpStateData(FwdState *, const Comm::ConnectionPointer &conn);
 189     ~FtpStateData();
 190     char user[MAX_URL];
 191     char password[MAX_URL];
 192     int password_url;
 193     char *reply_hdr;
 194     int reply_hdr_state;
 195     String clean_url;
 196     String title_url;
 197     String base_href;
 198     int conn_att;
 199     int login_att;
 200     ftp_state_t state;
 201     time_t mdtm;
 202     int64_t theSize;
 203     wordlist *pathcomps;
 204     char *filepath;
 205     char *dirpath;
 206     int64_t restart_offset;
 207     char *proxy_host;
 208     size_t list_width;
 209     String cwd_message;
 210     char *old_request;
 211     char *old_reply;
 212     char *old_filepath;
 213     char typecode;
 214     MemBuf listing;             ///< FTP directory listing in HTML format.
 215
 216     // \todo: optimize ctrl and data structs member order, to minimize size
 217     /// FTP control channel info; the channel is opened once per transaction
 218     struct CtrlChannel: public FtpChannel {
 219         char *buf;
 220         size_t size;
 221         size_t offset;
 222         wordlist *message;
 223         char *last_command;
 224         char *last_reply;
 225         int replycode;
 226     } ctrl;
 227
 228     /// FTP data channel info; the channel may be opened/closed a few times
 229     struct DataChannel: public FtpChannel {
 230         MemBuf *readBuf;
 231         char *host;
 232         unsigned short port;
 233         bool read_pending;
 234     } data;
 235
 236     struct _ftp_flags flags;
 237
 238 private:
 239     CBDATA_CLASS(FtpStateData);
 240
 241 public:
 242     // these should all be private
 243     virtual void start();
 244     void loginParser(const char *, int escaped);
 245     int restartable();
 246     void appendSuccessHeader();
 247     void hackShortcut(FTPSM * nextState);
 248     void failed(err_type, int xerrno);
 249     void failedErrorMessage(err_type, int xerrno);
 250     void unhack();
 251     void scheduleReadControlReply(int);
 252     void handleControlReply();
 253     void readStor();
 254     void parseListing();
 255     MemBuf *htmlifyListEntry(const char *line);
 256     void completedListing(void);
 257     void dataComplete();
 258     void dataRead(const CommIoCbParams &io);
 259
 260     /// ignore timeout on CTRL channel. set read timeout on DATA channel.
 261     void switchTimeoutToDataChannel();
 262     /// create a data channel acceptor and start listening.
 263     void listenForDataChannel(const Comm::ConnectionPointer &conn, const char *note);
 264
 265     int checkAuth(const HttpHeader * req_hdr);
 266     void checkUrlpath();
 267     void buildTitleUrl();
 268     void writeReplyBody(const char *, size_t len);
 269     void printfReplyBody(const char *fmt, ...);
 270     virtual const Comm::ConnectionPointer & dataConnection() const;
 271     virtual void maybeReadVirginBody();
 272     virtual void closeServer();
 273     virtual void completeForwarding();
 274     virtual void abortTransaction(const char *reason);
 275     void processHeadResponse();
 276     void processReplyBody();
 277     void writeCommand(const char *buf);
 278     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 279     int64_t getCurrentOffset() const { return currentOffset; }
 280
 281     static CNCB ftpPasvCallback;
 282     static PF ftpDataWrite;
 283     void ftpTimeout(const CommTimeoutCbParams &io);
 284     void ctrlClosed(const CommCloseCbParams &io);
 285     void dataClosed(const CommCloseCbParams &io);
 286     void ftpReadControlReply(const CommIoCbParams &io);
 287     void ftpWriteCommandCallback(const CommIoCbParams &io);
 288     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 289
 290     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 291     const char *ftpRealm(void);
 292     void loginFailed(void);
 293     static wordlist *ftpParseControlReply(char *, size_t, int *, size_t *);
 294
 295     // sending of the request body to the server
 296     virtual void sentRequestBody(const CommIoCbParams&);
 297     virtual void doneSendingRequestBody();
 298
 299     virtual void haveParsedReplyHeaders();
 300
 301     virtual bool doneWithServer() const;
 302     virtual bool haveControlChannel(const char *caller_name) const;
 303     AsyncCall::Pointer dataCloser(); /// creates a Comm close callback
 304     AsyncCall::Pointer dataOpener(); /// creates a Comm connect callback
 305
 306 private:
 307     // BodyConsumer for HTTP: consume request body.
 308     virtual void handleRequestBodyProducerAborted();
 309 };
 310
 311 CBDATA_CLASS_INIT(FtpStateData);
 312
 313 void *
 314 FtpStateData::operator new (size_t)
 315 {
 316     CBDATA_INIT_TYPE(FtpStateData);
 317     FtpStateData *result = cbdataAlloc(FtpStateData);
 318     return result;
 319 }
 320
 321 void
 322 FtpStateData::operator delete (void *address)
 323 {
 324     FtpStateData *t = static_cast<FtpStateData *>(address);
 325     cbdataFree(t);
 326 }
 327
 328 /// \ingroup ServerProtocolFTPInternal
 329 typedef struct {
 330     char type;
 331     int64_t size;
 332     char *date;
 333     char *name;
 334     char *showname;
 335     char *link;
 336 } ftpListParts;
 337
 338 /// \ingroup ServerProtocolFTPInternal
 339 #define FTP_LOGIN_ESCAPED       1
 340
 341 /// \ingroup ServerProtocolFTPInternal
 342 #define FTP_LOGIN_NOT_ESCAPED   0
 343
 344 /*
 345  * State machine functions
 346  * send == state transition
 347  * read == wait for response, and select next state transition
 348  * other == Transition logic
 349  */
 350 static FTPSM ftpReadWelcome;
 351 static FTPSM ftpSendUser;
 352 static FTPSM ftpReadUser;
 353 static FTPSM ftpSendPass;
 354 static FTPSM ftpReadPass;
 355 static FTPSM ftpSendType;
 356 static FTPSM ftpReadType;
 357 static FTPSM ftpSendMdtm;
 358 static FTPSM ftpReadMdtm;
 359 static FTPSM ftpSendSize;
 360 static FTPSM ftpReadSize;
 361 static FTPSM ftpSendEPRT;
 362 static FTPSM ftpReadEPRT;
 363 static FTPSM ftpSendPORT;
 364 static FTPSM ftpReadPORT;
 365 static FTPSM ftpSendPassive;
 366 static FTPSM ftpReadEPSV;
 367 static FTPSM ftpReadPasv;
 368 static FTPSM ftpTraverseDirectory;
 369 static FTPSM ftpListDir;
 370 static FTPSM ftpGetFile;
 371 static FTPSM ftpSendCwd;
 372 static FTPSM ftpReadCwd;
 373 static FTPSM ftpRestOrList;
 374 static FTPSM ftpSendList;
 375 static FTPSM ftpSendNlst;
 376 static FTPSM ftpReadList;
 377 static FTPSM ftpSendRest;
 378 static FTPSM ftpReadRest;
 379 static FTPSM ftpSendRetr;
 380 static FTPSM ftpReadRetr;
 381 static FTPSM ftpReadTransferDone;
 382 static FTPSM ftpSendStor;
 383 static FTPSM ftpReadStor;
 384 static FTPSM ftpWriteTransferDone;
 385 static FTPSM ftpSendReply;
 386 static FTPSM ftpSendMkdir;
 387 static FTPSM ftpReadMkdir;
 388 static FTPSM ftpFail;
 389 static FTPSM ftpSendQuit;
 390 static FTPSM ftpReadQuit;
 391
 392 /************************************************
 393 ** Debugs Levels used here                     **
 394 *************************************************
 395 0       CRITICAL Events
 396 1       IMPORTANT Events
 397         Protocol and Transmission failures.
 398 2       FTP Protocol Chatter
 399 3       Logic Flows
 400 4       Data Parsing Flows
 401 5       Data Dumps
 402 7       ??
 403 ************************************************/
 404
 405 /************************************************
 406 ** State Machine Description (excluding hacks) **
 407 *************************************************
 408 From                    To
 409 ---------------------------------------
 410 Welcome                 User
 411 User                    Pass
 412 Pass                    Type
 413 Type                    TraverseDirectory / GetFile
 414 TraverseDirectory       Cwd / GetFile / ListDir
 415 Cwd                     TraverseDirectory / Mkdir
 416 GetFile                 Mdtm
 417 Mdtm                    Size
 418 Size                    Epsv
 419 ListDir                 Epsv
 420 Epsv                    FileOrList
 421 FileOrList              Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 422 Rest                    Retr
 423 Retr / Nlst / List      DataRead* (on datachannel)
 424 DataRead*               ReadTransferDone
 425 ReadTransferDone        DataTransferDone
 426 Stor                    DataWrite* (on datachannel)
 427 DataWrite*              RequestPutBody** (from client)
 428 RequestPutBody**        DataWrite* / WriteTransferDone
 429 WriteTransferDone       DataTransferDone
 430 DataTransferDone        Quit
 431 Quit                    -
 432 ************************************************/
 433
 434 /// \ingroup ServerProtocolFTPInternal
 435 FTPSM *FTP_SM_FUNCS[] = {
 436     ftpReadWelcome,             /* BEGIN */
 437     ftpReadUser,                /* SENT_USER */
 438     ftpReadPass,                /* SENT_PASS */
 439     ftpReadType,                /* SENT_TYPE */
 440     ftpReadMdtm,                /* SENT_MDTM */
 441     ftpReadSize,                /* SENT_SIZE */
 442     ftpReadEPRT,                /* SENT_EPRT */
 443     ftpReadPORT,                /* SENT_PORT */
 444     ftpReadEPSV,                /* SENT_EPSV_ALL */
 445     ftpReadEPSV,                /* SENT_EPSV_1 */
 446     ftpReadEPSV,                /* SENT_EPSV_2 */
 447     ftpReadPasv,                /* SENT_PASV */
 448     ftpReadCwd,         /* SENT_CWD */
 449     ftpReadList,                /* SENT_LIST */
 450     ftpReadList,                /* SENT_NLST */
 451     ftpReadRest,                /* SENT_REST */
 452     ftpReadRetr,                /* SENT_RETR */
 453     ftpReadStor,                /* SENT_STOR */
 454     ftpReadQuit,                /* SENT_QUIT */
 455     ftpReadTransferDone,        /* READING_DATA (RETR,LIST,NLST) */
 456     ftpWriteTransferDone,       /* WRITING_DATA (STOR) */
 457     ftpReadMkdir                /* SENT_MKDIR */
 458 };
 459
 460 /// handler called by Comm when FTP control channel is closed unexpectedly
 461 void
 462 FtpStateData::ctrlClosed(const CommCloseCbParams &io)
 463 {
 464     debugs(9, 4, HERE);
 465     ctrl.clear();
 466     mustStop("FtpStateData::ctrlClosed");
 467 }
 468
 469 /// handler called by Comm when FTP data channel is closed unexpectedly
 470 void
 471 FtpStateData::dataClosed(const CommCloseCbParams &io)
 472 {
 473     debugs(9, 4, HERE);
 474     if (data.listenConn != NULL) {
 475         data.listenConn->close();
 476         data.listenConn = NULL;
 477         // NP clear() does the: data.fd = -1;
 478     }
 479     data.clear();
 480     failed(ERR_FTP_FAILURE, 0);
 481     /* failed closes ctrl.conn and frees ftpState */
 482
 483     /* NP: failure recovery may be possible when its only a data.conn failure.
 484      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 485      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 486      */
 487 }
 488
 489 FtpStateData::FtpStateData(FwdState *theFwdState, const Comm::ConnectionPointer &conn) : AsyncJob("FtpStateData"), ServerStateData(theFwdState)
 490 {
 491     const char *url = entry->url();
 492     debugs(9, 3, HERE << "'" << url << "'" );
 493     ++ statCounter.server.all.requests;
 494     ++ statCounter.server.ftp.requests;
 495     theSize = -1;
 496     mdtm = -1;
 497
 498     if (Config.Ftp.passive && !flags.pasv_failed)
 499         flags.pasv_supported = 1;
 500
 501     flags.rest_supported = 1;
 502
 503     typedef CommCbMemFunT<FtpStateData, CommCloseCbParams> Dialer;
 504     AsyncCall::Pointer closer = JobCallback(9, 5, Dialer, this, FtpStateData::ctrlClosed);
 505     ctrl.opened(conn, closer);
 506
 507     if (request->method == METHOD_PUT)
 508         flags.put = 1;
 509 }
 510
 511 FtpStateData::~FtpStateData()
 512 {
 513     debugs(9, 3, HERE << entry->url()  );
 514
 515     if (reply_hdr) {
 516         memFree(reply_hdr, MEM_8K_BUF);
 517         reply_hdr = NULL;
 518     }
 519
 520     if (data.opener != NULL) {
 521         data.opener->cancel("FtpStateData destructed");
 522         data.opener = NULL;
 523     }
 524     data.close();
 525
 526     if (Comm::IsConnOpen(ctrl.conn)) {
 527         debugs(9, DBG_IMPORTANT, HERE << "Internal bug: FtpStateData left " <<
 528                "open control channel " << ctrl.conn);
 529     }
 530
 531     if (ctrl.buf) {
 532         memFreeBuf(ctrl.size, ctrl.buf);
 533         ctrl.buf = NULL;
 534     }
 535
 536     if (data.readBuf) {
 537         if (!data.readBuf->isNull())
 538             data.readBuf->clean();
 539
 540         delete data.readBuf;
 541     }
 542
 543     if (pathcomps)
 544         wordlistDestroy(&pathcomps);
 545
 546     if (ctrl.message)
 547         wordlistDestroy(&ctrl.message);
 548
 549     cwd_message.clean();
 550
 551     safe_free(ctrl.last_reply);
 552
 553     safe_free(ctrl.last_command);
 554
 555     safe_free(old_request);
 556
 557     safe_free(old_reply);
 558
 559     safe_free(old_filepath);
 560
 561     title_url.clean();
 562
 563     base_href.clean();
 564
 565     safe_free(filepath);
 566
 567     safe_free(dirpath);
 568
 569     safe_free(data.host);
 570
 571     fwd = NULL; // refcounted
 572 }
 573
 574 /**
 575  * Parse a possible login username:password pair.
 576  * Produces filled member variables user, password, password_url if anything found.
 577  */
 578 void
 579 FtpStateData::loginParser(const char *login, int escaped)
 580 {
 581     const char *u = NULL; // end of the username sub-string
 582     int len;              // length of the current sub-string to handle.
 583
 584     int total_len = strlen(login);
 585
 586     debugs(9, 4, HERE << ": login='" << login << "', escaped=" << escaped);
 587     debugs(9, 9, HERE << ": IN : login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 588
 589     if ((u = strchr(login, ':'))) {
 590
 591         /* if there was a username part */
 592         if (u > login) {
 593             len = u - login;
 594             ++u; // jump off the delimiter.
 595             if (len > MAX_URL)
 596                 len = MAX_URL-1;
 597             xstrncpy(user, login, len +1);
 598             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<"), escaped=" << escaped);
 599             if (escaped)
 600                 rfc1738_unescape(user);
 601             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<") unescaped.");
 602         }
 603
 604         /* if there was a password part */
 605         len = login + total_len - u;
 606         if ( len > 0) {
 607             if (len > MAX_URL)
 608                 len = MAX_URL -1;
 609             xstrncpy(password, u, len +1);
 610             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<"), escaped=" << escaped);
 611             if (escaped) {
 612                 rfc1738_unescape(password);
 613                 password_url = 1;
 614             }
 615             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<") unescaped.");
 616         }
 617     } else if (login[0]) {
 618         /* no password, just username */
 619         if (total_len > MAX_URL)
 620             total_len = MAX_URL -1;
 621         xstrncpy(user, login, total_len +1);
 622         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<"), escaped=" << escaped);
 623         if (escaped)
 624             rfc1738_unescape(user);
 625         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<") unescaped.");
 626     }
 627
 628     debugs(9, 9, HERE << ": OUT: login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 629 }
 630
 631 /**
 632  * Cancel the timeout on the Control socket and establish one
 633  * on the data socket
 634  */
 635 void
 636 FtpStateData::switchTimeoutToDataChannel()
 637 {
 638     commUnsetConnTimeout(ctrl.conn);
 639
 640     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
 641     AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, FtpStateData::ftpTimeout);
 642     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
 643 }
 644
 645 void
 646 FtpStateData::listenForDataChannel(const Comm::ConnectionPointer &conn, const char *note)
 647 {
 648     assert(!Comm::IsConnOpen(data.conn));
 649
 650     typedef CommCbMemFunT<FtpStateData, CommAcceptCbParams> AcceptDialer;
 651     typedef AsyncCallT<AcceptDialer> AcceptCall;
 652     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, FtpStateData::ftpAcceptDataConnection));
 653     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 654
 655     /* open the conn if its not already open */
 656     if (!Comm::IsConnOpen(conn)) {
 657         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 658         if (!Comm::IsConnOpen(conn)) {
 659             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 660             return;
 661         }
 662         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 663     }
 664
 665     assert(Comm::IsConnOpen(conn));
 666     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 667
 668     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 669     data.opened(conn, dataCloser());
 670     switchTimeoutToDataChannel();
 671 }
 672
 673 void
 674 FtpStateData::ftpTimeout(const CommTimeoutCbParams &io)
 675 {
 676     debugs(9, 4, HERE << io.conn << ": '" << entry->url() << "'" );
 677
 678     if (abortOnBadEntry("entry went bad while waiting for a timeout"))
 679         return;
 680
 681     if (SENT_PASV == state && io.conn->fd == data.conn->fd) {
 682         /* stupid ftp.netscape.com */
 683         flags.pasv_supported = false;
 684         debugs(9, DBG_IMPORTANT, "ftpTimeout: timeout in SENT_PASV state" );
 685     }
 686
 687     failed(ERR_READ_TIMEOUT, 0);
 688     /* failed() closes ctrl.conn and frees ftpState */
 689 }
 690
 691 #if DEAD_CODE // obsoleted by ERR_DIR_LISTING
 692 void
 693 FtpStateData::listingFinish()
 694 {
 695     // TODO: figure out what this means and how to show it ...
 696
 697     if (flags.listformat_unknown && !flags.tried_nlst) {
 698         printfReplyBody("<a href=\"%s/;type=d\">[As plain directory]</a>\n",
 699                         flags.dir_slash ? rfc1738_escape_part(old_filepath) : ".");
 700     } else if (typecode == 'D') {
 701         const char *path = flags.dir_slash ? filepath : ".";
 702         printfReplyBody("<a href=\"%s/\">[As extended directory]</a>\n", rfc1738_escape_part(path));
 703     }
 704 }
 705 #endif /* DEAD_CODE */
 706
 707 /// \ingroup ServerProtocolFTPInternal
 708 static const char *Month[] = {
 709     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 710     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 711 };
 712
 713 /// \ingroup ServerProtocolFTPInternal
 714 static int
 715 is_month(const char *buf)
 716 {
 717     int i;
 718
 719     for (i = 0; i < 12; ++i)
 720         if (!strcasecmp(buf, Month[i]))
 721             return 1;
 722
 723     return 0;
 724 }
 725
 726 /// \ingroup ServerProtocolFTPInternal
 727 static void
 728 ftpListPartsFree(ftpListParts ** parts)
 729 {
 730     safe_free((*parts)->date);
 731     safe_free((*parts)->name);
 732     safe_free((*parts)->showname);
 733     safe_free((*parts)->link);
 734     safe_free(*parts);
 735 }
 736
 737 /// \ingroup ServerProtocolFTPInternal
 738 #define MAX_TOKENS 64
 739
 740 /// \ingroup ServerProtocolFTPInternal
 741 static ftpListParts *
 742 ftpListParseParts(const char *buf, struct _ftp_flags flags)
 743 {
 744     ftpListParts *p = NULL;
 745     char *t = NULL;
 746     const char *ct = NULL;
 747     char *tokens[MAX_TOKENS];
 748     int i;
 749     int n_tokens;
 750     static char tbuf[128];
 751     char *xbuf = NULL;
 752     static int scan_ftp_initialized = 0;
 753     static regex_t scan_ftp_integer;
 754     static regex_t scan_ftp_time;
 755     static regex_t scan_ftp_dostime;
 756     static regex_t scan_ftp_dosdate;
 757
 758     if (!scan_ftp_initialized) {
 759         scan_ftp_initialized = 1;
 760         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 761         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 762         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 763         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 764     }
 765
 766     if (buf == NULL)
 767         return NULL;
 768
 769     if (*buf == '\0')
 770         return NULL;
 771
 772     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 773
 774     n_tokens = 0;
 775
 776     memset(tokens, 0, sizeof(tokens));
 777
 778     xbuf = xstrdup(buf);
 779
 780     if (flags.tried_nlst) {
 781         /* Machine readable format, one name per line */
 782         p->name = xbuf;
 783         p->type = '\0';
 784         return p;
 785     }
 786
 787     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 788         tokens[n_tokens] = xstrdup(t);
 789         ++n_tokens;
 790     }
 791
 792     xfree(xbuf);
 793
 794     /* locate the Month field */
 795     for (i = 3; i < n_tokens - 2; ++i) {
 796         char *size = tokens[i - 1];
 797         char *month = tokens[i];
 798         char *day = tokens[i + 1];
 799         char *year = tokens[i + 2];
 800
 801         if (!is_month(month))
 802             continue;
 803
 804         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 805             continue;
 806
 807         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 808             continue;
 809
 810         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0)     /* Yr | hh:mm */
 811             continue;
 812
 813         snprintf(tbuf, 128, "%s %2s %5s",
 814                  month, day, year);
 815
 816         if (!strstr(buf, tbuf))
 817             snprintf(tbuf, 128, "%s %2s %-5s",
 818                      month, day, year);
 819
 820         char const *copyFrom = NULL;
 821
 822         if ((copyFrom = strstr(buf, tbuf))) {
 823             p->type = *tokens[0];
 824             p->size = strtoll(size, NULL, 10);
 825             p->date = xstrdup(tbuf);
 826
 827             if (flags.skip_whitespace) {
 828                 copyFrom += strlen(tbuf);
 829
 830                 while (strchr(w_space, *copyFrom))
 831                     ++copyFrom;
 832             } else {
 833                 /* XXX assumes a single space between date and filename
 834                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 835                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 836                 copyFrom += strlen(tbuf) + 1;
 837             }
 838
 839             p->name = xstrdup(copyFrom);
 840
 841             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 842                 *t = '\0';
 843                 p->link = xstrdup(t + 4);
 844             }
 845
 846             goto found;
 847         }
 848
 849         break;
 850     }
 851
 852     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 853     if (n_tokens > 3 &&
 854             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 855             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 856         if (!strcasecmp(tokens[2], "<dir>")) {
 857             p->type = 'd';
 858         } else {
 859             p->type = '-';
 860             p->size = strtoll(tokens[2], NULL, 10);
 861         }
 862
 863         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 864         p->date = xstrdup(tbuf);
 865
 866         if (p->type == 'd') {
 867             /* Directory.. name begins with first printable after <dir> */
 868             ct = strstr(buf, tokens[2]);
 869             ct += strlen(tokens[2]);
 870
 871             while (xisspace(*ct))
 872                 ++ct;
 873
 874             if (!*ct)
 875                 ct = NULL;
 876         } else {
 877             /* A file. Name begins after size, with a space in between */
 878             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 879             ct = strstr(buf, tbuf);
 880
 881             if (ct) {
 882                 ct += strlen(tokens[2]) + 2;
 883             }
 884         }
 885
 886         p->name = xstrdup(ct ? ct : tokens[3]);
 887         goto found;
 888     }
 889
 890     /* Try EPLF format; carson@lehman.com */
 891     if (buf[0] == '+') {
 892         ct = buf + 1;
 893         p->type = 0;
 894
 895         while (ct && *ct) {
 896             time_t tm;
 897             int l = strcspn(ct, ",");
 898             char *tmp;
 899
 900             if (l < 1)
 901                 goto blank;
 902
 903             switch (*ct) {
 904
 905             case '\t':
 906                 p->name = xstrndup(ct + 1, l + 1);
 907                 break;
 908
 909             case 's':
 910                 p->size = atoi(ct + 1);
 911                 break;
 912
 913             case 'm':
 914                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 915
 916                 if (tmp != ct + 1)
 917                     break;      /* not a valid integer */
 918
 919                 p->date = xstrdup(ctime(&tm));
 920
 921                 *(strstr(p->date, "\n")) = '\0';
 922
 923                 break;
 924
 925             case '/':
 926                 p->type = 'd';
 927
 928                 break;
 929
 930             case 'r':
 931                 p->type = '-';
 932
 933                 break;
 934
 935             case 'i':
 936                 break;
 937
 938             default:
 939                 break;
 940             }
 941
 942 blank:
 943             ct = strstr(ct, ",");
 944
 945             if (ct) {
 946                 ++ct;
 947             }
 948         }
 949
 950         if (p->type == 0) {
 951             p->type = '-';
 952         }
 953
 954         if (p->name)
 955             goto found;
 956         else
 957             safe_free(p->date);
 958     }
 959
 960 found:
 961
 962     for (i = 0; i < n_tokens; ++i)
 963         xfree(tokens[i]);
 964
 965     if (!p->name)
 966         ftpListPartsFree(&p);   /* cleanup */
 967
 968     return p;
 969 }
 970
 971 MemBuf *
 972 FtpStateData::htmlifyListEntry(const char *line)
 973 {
 974     char icon[2048];
 975     char href[2048 + 40];
 976     char text[ 2048];
 977     char size[ 2048];
 978     char chdir[ 2048 + 40];
 979     char view[ 2048 + 40];
 980     char download[ 2048 + 40];
 981     char link[ 2048 + 40];
 982     MemBuf *html;
 983     char prefix[2048];
 984     ftpListParts *parts;
 985     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 986
 987     debugs(9, 7, HERE << " line ={" << line << "}");
 988
 989     if (strlen(line) > 1024) {
 990         html = new MemBuf();
 991         html->init();
 992         html->Printf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 993         return html;
 994     }
 995
 996     if (flags.dir_slash && dirpath && typecode != 'D')
 997         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 998     else
 999         prefix[0] = '\0';
1000
1001     if ((parts = ftpListParseParts(line, flags)) == NULL) {
1002         const char *p;
1003
1004         html = new MemBuf();
1005         html->init();
1006         html->Printf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
1007
1008         for (p = line; *p && xisspace(*p); ++p);
1009         if (*p && !xisspace(*p))
1010             flags.listformat_unknown = 1;
1011
1012         return html;
1013     }
1014
1015     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
1016         ftpListPartsFree(&parts);
1017         return NULL;
1018     }
1019
1020     parts->size += 1023;
1021     parts->size >>= 10;
1022     parts->showname = xstrdup(parts->name);
1023
1024     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
1025     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
1026
1027     xstrncpy(text, parts->showname, 2048);
1028
1029     switch (parts->type) {
1030
1031     case 'd':
1032         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1033                  mimeGetIconURL("internal-dir"),
1034                  "[DIR]");
1035         strcat(href, "/");      /* margin is allocated above */
1036         break;
1037
1038     case 'l':
1039         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1040                  mimeGetIconURL("internal-link"),
1041                  "[LINK]");
1042         /* sometimes there is an 'l' flag, but no "->" link */
1043
1044         if (parts->link) {
1045             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
1046             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
1047                      *link2 != '/' ? prefix : "", link2,
1048                      html_quote(parts->link));
1049             safe_free(link2);
1050         }
1051
1052         break;
1053
1054     case '\0':
1055         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1056                  mimeGetIconURL(parts->name),
1057                  "[UNKNOWN]");
1058         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
1059                  "alt=\"[DIR]\"></a>",
1060                  rfc1738_escape_part(parts->name),
1061                  mimeGetIconURL("internal-dir"));
1062         break;
1063
1064     case '-':
1065
1066     default:
1067         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
1068                  mimeGetIconURL(parts->name),
1069                  "[FILE]");
1070         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
1071         break;
1072     }
1073
1074     if (parts->type != 'd') {
1075         if (mimeGetViewOption(parts->name)) {
1076             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
1077                      "alt=\"[VIEW]\"></a>",
1078                      prefix, href, mimeGetIconURL("internal-view"));
1079         }
1080
1081         if (mimeGetDownloadOption(parts->name)) {
1082             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
1083                      "alt=\"[DOWNLOAD]\"></a>",
1084                      prefix, href, mimeGetIconURL("internal-download"));
1085         }
1086     }
1087
1088     /* construct the table row from parts. */
1089     html = new MemBuf();
1090     html->init();
1091     html->Printf("<tr class=\"entry\">"
1092                  "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
1093                  "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
1094                  "<td class=\"date\">%s</td>"
1095                  "<td class=\"size\">%s</td>"
1096                  "<td class=\"actions\">%s%s%s%s</td>"
1097                  "</tr>\n",
1098                  prefix, href, icon,
1099                  prefix, href, html_quote(text),
1100                  parts->date,
1101                  size,
1102                  chdir, view, download, link);
1103
1104     ftpListPartsFree(&parts);
1105     return html;
1106 }
1107
1108 void
1109 FtpStateData::parseListing()
1110 {
1111     char *buf = data.readBuf->content();
1112     char *sbuf;                 /* NULL-terminated copy of termedBuf */
1113     char *end;
1114     char *line;
1115     char *s;
1116     MemBuf *t;
1117     size_t linelen;
1118     size_t usable;
1119     size_t len = data.readBuf->contentSize();
1120
1121     if (!len) {
1122         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
1123         return;
1124     }
1125
1126     /*
1127      * We need a NULL-terminated buffer for scanning, ick
1128      */
1129     sbuf = (char *)xmalloc(len + 1);
1130     xstrncpy(sbuf, buf, len + 1);
1131     end = sbuf + len - 1;
1132
1133     while (*end != '\r' && *end != '\n' && end > sbuf)
1134         --end;
1135
1136     usable = end - sbuf;
1137
1138     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
1139
1140     if (usable == 0) {
1141         if (buf[0] == '\0' && len == 1) {
1142             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
1143             data.readBuf->consume(len);
1144         } else {
1145             debugs(9, 3, HERE << "didn't find end for " << entry->url());
1146             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
1147         }
1148         xfree(sbuf);
1149         return;
1150     }
1151
1152     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
1153
1154     line = (char *)memAllocate(MEM_4K_BUF);
1155     ++end;
1156     s = sbuf;
1157     s += strspn(s, crlf);
1158
1159     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1160         debugs(9, 7, HERE << "s = {" << s << "}");
1161         linelen = strcspn(s, crlf) + 1;
1162
1163         if (linelen < 2)
1164             break;
1165
1166         if (linelen > 4096)
1167             linelen = 4096;
1168
1169         xstrncpy(line, s, linelen);
1170
1171         debugs(9, 7, HERE << "{" << line << "}");
1172
1173         if (!strncmp(line, "total", 5))
1174             continue;
1175
1176         t = htmlifyListEntry(line);
1177
1178         if ( t != NULL) {
1179             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
1180             listing.append(t->content(), t->contentSize());
1181 //leak?            delete t;
1182         }
1183     }
1184
1185     debugs(9, 7, HERE << "Done.");
1186     data.readBuf->consume(usable);
1187     memFree(line, MEM_4K_BUF);
1188     xfree(sbuf);
1189 }
1190
1191 const Comm::ConnectionPointer &
1192 FtpStateData::dataConnection() const
1193 {
1194     return data.conn;
1195 }
1196
1197 void
1198 FtpStateData::dataComplete()
1199 {
1200     debugs(9, 3,HERE);
1201
1202     /* Connection closed; transfer done. */
1203
1204     /// Close data channel, if any, to conserve resources while we wait.
1205     data.close();
1206
1207     /* expect the "transfer complete" message on the control socket */
1208     /*
1209      * DPW 2007-04-23
1210      * Previously, this was the only place where we set the
1211      * 'buffered_ok' flag when calling scheduleReadControlReply().
1212      * It caused some problems if the FTP server returns an unexpected
1213      * status code after the data command.  FtpStateData was being
1214      * deleted in the middle of dataRead().
1215      */
1216     /* AYJ: 2011-01-13: Bug 2581.
1217      * 226 status is possibly waiting in the ctrl buffer.
1218      * The connection will hang if we DONT send buffered_ok.
1219      * This happens on all transfers which can be completly sent by the
1220      * server before the 150 started status message is read in by Squid.
1221      * ie all transfers of about one packet hang.
1222      */
1223     scheduleReadControlReply(1);
1224 }
1225
1226 void
1227 FtpStateData::maybeReadVirginBody()
1228 {
1229     // too late to read
1230     if (!Comm::IsConnOpen(data.conn) || fd_table[data.conn->fd].closing())
1231         return;
1232
1233     if (data.read_pending)
1234         return;
1235
1236     const int read_sz = replyBodySpace(*data.readBuf, 0);
1237
1238     debugs(11,9, HERE << "FTP may read up to " << read_sz << " bytes");
1239
1240     if (read_sz < 2)    // see http.cc
1241         return;
1242
1243     data.read_pending = true;
1244
1245     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1246     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
1247                                       TimeoutDialer, this, FtpStateData::ftpTimeout);
1248     commSetConnTimeout(data.conn, Config.Timeout.read, timeoutCall);
1249
1250     debugs(9,5,HERE << "queueing read on FD " << data.conn->fd);
1251
1252     typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1253     entry->delayAwareRead(data.conn, data.readBuf->space(), read_sz,
1254                           JobCallback(9, 5, Dialer, this, FtpStateData::dataRead));
1255 }
1256
1257 void
1258 FtpStateData::dataRead(const CommIoCbParams &io)
1259 {
1260     int j;
1261     int bin;
1262
1263     data.read_pending = false;
1264
1265     debugs(9, 3, HERE << "ftpDataRead: FD " << io.fd << " Read " << io.size << " bytes");
1266
1267     if (io.size > 0) {
1268         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
1269         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
1270     }
1271
1272     if (io.flag == COMM_ERR_CLOSING)
1273         return;
1274
1275     assert(io.fd == data.conn->fd);
1276
1277     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1278         abortTransaction("entry aborted during dataRead");
1279         return;
1280     }
1281
1282     if (io.flag == COMM_OK && io.size > 0) {
1283         debugs(9,5,HERE << "appended " << io.size << " bytes to readBuf");
1284         data.readBuf->appended(io.size);
1285 #if USE_DELAY_POOLS
1286         DelayId delayId = entry->mem_obj->mostBytesAllowed();
1287         delayId.bytesIn(io.size);
1288 #endif
1289         ++ IOStats.Ftp.reads;
1290
1291         for (j = io.size - 1, bin = 0; j; ++bin)
1292             j >>= 1;
1293
1294         ++ IOStats.Ftp.read_hist[bin];
1295     }
1296
1297     if (io.flag != COMM_OK) {
1298         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
1299                "ftpDataRead: read error: " << xstrerr(io.xerrno));
1300
1301         if (ignoreErrno(io.xerrno)) {
1302             typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1303             AsyncCall::Pointer timeoutCall = JobCallback(9, 5,
1304                                              TimeoutDialer, this, FtpStateData::ftpTimeout);
1305             commSetConnTimeout(io.conn, Config.Timeout.read, timeoutCall);
1306
1307             maybeReadVirginBody();
1308         } else {
1309             failed(ERR_READ_ERROR, 0);
1310             /* failed closes ctrl.conn and frees ftpState */
1311             return;
1312         }
1313     } else if (io.size == 0) {
1314         debugs(9,3, HERE << "Calling dataComplete() because io.size == 0");
1315         /*
1316          * DPW 2007-04-23
1317          * Dangerous curves ahead.  This call to dataComplete was
1318          * calling scheduleReadControlReply, handleControlReply,
1319          * and then ftpReadTransferDone.  If ftpReadTransferDone
1320          * gets unexpected status code, it closes down the control
1321          * socket and our FtpStateData object gets destroyed.   As
1322          * a workaround we no longer set the 'buffered_ok' flag in
1323          * the scheduleReadControlReply call.
1324          */
1325         dataComplete();
1326     }
1327
1328     processReplyBody();
1329 }
1330
1331 void
1332 FtpStateData::processReplyBody()
1333 {
1334     debugs(9, 3, HERE << "FtpStateData::processReplyBody starting.");
1335
1336     if (request->method == METHOD_HEAD && (flags.isdir || theSize != -1)) {
1337         serverComplete();
1338         return;
1339     }
1340
1341     /* Directory listings are special. They write ther own headers via the error objects */
1342     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
1343         appendSuccessHeader();
1344
1345     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1346         /*
1347          * probably was aborted because content length exceeds one
1348          * of the maximum size limits.
1349          */
1350         abortTransaction("entry aborted after calling appendSuccessHeader()");
1351         return;
1352     }
1353
1354 #if USE_ADAPTATION
1355
1356     if (adaptationAccessCheckPending) {
1357         debugs(9,3, HERE << "returning from FtpStateData::processReplyBody due to adaptationAccessCheckPending");
1358         return;
1359     }
1360
1361 #endif
1362
1363     if (flags.isdir) {
1364         if (!flags.listing) {
1365             flags.listing = 1;
1366             listing.reset();
1367         }
1368         parseListing();
1369         maybeReadVirginBody();
1370         return;
1371     } else if (const int csize = data.readBuf->contentSize()) {
1372         writeReplyBody(data.readBuf->content(), csize);
1373         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1374         data.readBuf->consume(csize);
1375     }
1376
1377     entry->flush();
1378
1379     maybeReadVirginBody();
1380 }
1381
1382 /**
1383  * Locates the FTP user:password login.
1384  *
1385  * Highest to lowest priority:
1386  *  - Checks URL (ftp://user:pass@domain)
1387  *  - Authorization: Basic header
1388  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1389  *
1390  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1391  *
1392  * TODO: we might be able to do something about locating username from other sources:
1393  *       ie, external ACL user=* tag or ident lookup
1394  *
1395  \retval 1      if we have everything needed to complete this request.
1396  \retval 0      if something is missing.
1397  */
1398 int
1399 FtpStateData::checkAuth(const HttpHeader * req_hdr)
1400 {
1401     /* default username */
1402     xstrncpy(user, "anonymous", MAX_URL);
1403
1404 #if HAVE_AUTH_MODULE_BASIC
1405     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1406     const char *auth;
1407     if ( (auth = req_hdr->getAuth(HDR_AUTHORIZATION, "Basic")) ) {
1408         flags.authenticated = 1;
1409         loginParser(auth, FTP_LOGIN_NOT_ESCAPED);
1410     }
1411     /* we fail with authorization-required error later IFF the FTP server requests it */
1412 #endif
1413
1414     /* Test URL login syntax. Overrides any headers received. */
1415     loginParser(request->login, FTP_LOGIN_ESCAPED);
1416
1417     /* name is missing. thats fatal. */
1418     if (!user[0])
1419         fatal("FTP login parsing destroyed username info");
1420
1421     /* name + password == success */
1422     if (password[0])
1423         return 1;
1424
1425     /* Setup default FTP password settings */
1426     /* this has to be done last so that we can have a no-password case above. */
1427     if (!password[0]) {
1428         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1429             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1430             flags.tried_auth_anonymous=1;
1431             return 1;
1432         } else if (!flags.tried_auth_nopass) {
1433             xstrncpy(password, null_string, MAX_URL);
1434             flags.tried_auth_nopass=1;
1435             return 1;
1436         }
1437     }
1438
1439     return 0;                   /* different username */
1440 }
1441
1442 static String str_type_eq;
1443 void
1444 FtpStateData::checkUrlpath()
1445 {
1446     int l;
1447     size_t t;
1448
1449     if (str_type_eq.undefined()) //hack. String doesn't support global-static
1450         str_type_eq="type=";
1451
1452     if ((t = request->urlpath.rfind(';')) != String::npos) {
1453         if (request->urlpath.substr(t+1,t+1+str_type_eq.size())==str_type_eq) {
1454             typecode = (char)xtoupper(request->urlpath[t+str_type_eq.size()+1]);
1455             request->urlpath.cut(t);
1456         }
1457     }
1458
1459     l = request->urlpath.size();
1460     /* check for null path */
1461
1462     if (!l) {
1463         flags.isdir = 1;
1464         flags.root_dir = 1;
1465         flags.need_base_href = 1;       /* Work around broken browsers */
1466     } else if (!request->urlpath.cmp("/%2f/")) {
1467         /* UNIX root directory */
1468         flags.isdir = 1;
1469         flags.root_dir = 1;
1470     } else if ((l >= 1) && (request->urlpath[l - 1] == '/')) {
1471         /* Directory URL, ending in / */
1472         flags.isdir = 1;
1473
1474         if (l == 1)
1475             flags.root_dir = 1;
1476     } else {
1477         flags.dir_slash = 1;
1478     }
1479 }
1480
1481 void
1482 FtpStateData::buildTitleUrl()
1483 {
1484     title_url = "ftp://";
1485
1486     if (strcmp(user, "anonymous")) {
1487         title_url.append(user);
1488         title_url.append("@");
1489     }
1490
1491     title_url.append(request->GetHost());
1492
1493     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1494         title_url.append(":");
1495         title_url.append(xitoa(request->port));
1496     }
1497
1498     title_url.append (request->urlpath);
1499
1500     base_href = "ftp://";
1501
1502     if (strcmp(user, "anonymous") != 0) {
1503         base_href.append(rfc1738_escape_part(user));
1504
1505         if (password_url) {
1506             base_href.append (":");
1507             base_href.append(rfc1738_escape_part(password));
1508         }
1509
1510         base_href.append("@");
1511     }
1512
1513     base_href.append(request->GetHost());
1514
1515     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1516         base_href.append(":");
1517         base_href.append(xitoa(request->port));
1518     }
1519
1520     base_href.append(request->urlpath);
1521     base_href.append("/");
1522 }
1523
1524 /// \ingroup ServerProtocolFTPAPI
1525 void
1526 ftpStart(FwdState * fwd)
1527 {
1528     AsyncJob::Start(new FtpStateData(fwd, fwd->serverConnection()));
1529 }
1530
1531 void
1532 FtpStateData::start()
1533 {
1534     if (!checkAuth(&request->header)) {
1535         /* create appropriate reply */
1536         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1537         entry->replaceHttpReply(reply);
1538         serverComplete();
1539         return;
1540     }
1541
1542     checkUrlpath();
1543     buildTitleUrl();
1544     debugs(9, 5, HERE << "FD " << ctrl.conn->fd << " : host=" << request->GetHost() <<
1545            ", path=" << request->urlpath << ", user=" << user << ", passwd=" << password);
1546
1547     state = BEGIN;
1548     ctrl.last_command = xstrdup("Connect to server");
1549     ctrl.buf = (char *)memAllocBuf(4096, &ctrl.size);
1550     ctrl.offset = 0;
1551     data.readBuf = new MemBuf;
1552     data.readBuf->init(4096, SQUID_TCP_SO_RCVBUF);
1553     scheduleReadControlReply(0);
1554 }
1555
1556 /* ====================================================================== */
1557
1558 /// \ingroup ServerProtocolFTPInternal
1559 static char *
1560 escapeIAC(const char *buf)
1561 {
1562     int n;
1563     char *ret;
1564     unsigned const char *p;
1565     unsigned char *r;
1566
1567     for (p = (unsigned const char *)buf, n = 1; *p; ++n, ++p)
1568         if (*p == 255)
1569             ++n;
1570
1571     ret = (char *)xmalloc(n);
1572
1573     for (p = (unsigned const char *)buf, r=(unsigned char *)ret; *p; ++p) {
1574         *r = *p;
1575         ++r;
1576
1577         if (*p == 255) {
1578             *r = 255;
1579             ++r;
1580         }
1581     }
1582
1583     *r = '\0';
1584     ++r;
1585     assert((r - (unsigned char *)ret) == n );
1586     return ret;
1587 }
1588
1589 void
1590 FtpStateData::writeCommand(const char *buf)
1591 {
1592     char *ebuf;
1593     /* trace FTP protocol communications at level 2 */
1594     debugs(9, 2, "ftp<< " << buf);
1595
1596     if (Config.Ftp.telnet)
1597         ebuf = escapeIAC(buf);
1598     else
1599         ebuf = xstrdup(buf);
1600
1601     safe_free(ctrl.last_command);
1602
1603     safe_free(ctrl.last_reply);
1604
1605     ctrl.last_command = ebuf;
1606
1607     if (!Comm::IsConnOpen(ctrl.conn)) {
1608         debugs(9, 2, HERE << "cannot send to closing ctrl " << ctrl.conn);
1609         // TODO: assert(ctrl.closer != NULL);
1610         return;
1611     }
1612
1613     typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1614     AsyncCall::Pointer call = JobCallback(9, 5, Dialer, this, FtpStateData::ftpWriteCommandCallback);
1615     Comm::Write(ctrl.conn, ctrl.last_command, strlen(ctrl.last_command), call, NULL);
1616
1617     scheduleReadControlReply(0);
1618 }
1619
1620 void
1621 FtpStateData::ftpWriteCommandCallback(const CommIoCbParams &io)
1622 {
1623
1624     debugs(9, 5, "ftpWriteCommandCallback: wrote " << io.size << " bytes");
1625
1626     if (io.size > 0) {
1627         fd_bytes(io.fd, io.size, FD_WRITE);
1628         kb_incr(&(statCounter.server.all.kbytes_out), io.size);
1629         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
1630     }
1631
1632     if (io.flag == COMM_ERR_CLOSING)
1633         return;
1634
1635     if (io.flag) {
1636         debugs(9, DBG_IMPORTANT, "ftpWriteCommandCallback: " << io.conn << ": " << xstrerr(io.xerrno));
1637         failed(ERR_WRITE_ERROR, io.xerrno);
1638         /* failed closes ctrl.conn and frees ftpState */
1639         return;
1640     }
1641 }
1642
1643 wordlist *
1644 FtpStateData::ftpParseControlReply(char *buf, size_t len, int *codep, size_t *used)
1645 {
1646     char *s;
1647     char *sbuf;
1648     char *end;
1649     int usable;
1650     int complete = 0;
1651     wordlist *head = NULL;
1652     wordlist *list;
1653     wordlist **tail = &head;
1654     size_t offset;
1655     size_t linelen;
1656     int code = -1;
1657     debugs(9, 3, HERE);
1658     /*
1659      * We need a NULL-terminated buffer for scanning, ick
1660      */
1661     sbuf = (char *)xmalloc(len + 1);
1662     xstrncpy(sbuf, buf, len + 1);
1663     end = sbuf + len - 1;
1664
1665     while (*end != '\r' && *end != '\n' && end > sbuf)
1666         --end;
1667
1668     usable = end - sbuf;
1669
1670     debugs(9, 3, HERE << "usable = " << usable);
1671
1672     if (usable == 0) {
1673         debugs(9, 3, HERE << "didn't find end of line");
1674         safe_free(sbuf);
1675         return NULL;
1676     }
1677
1678     debugs(9, 3, HERE << len << " bytes to play with");
1679     ++end;
1680     s = sbuf;
1681     s += strspn(s, crlf);
1682
1683     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
1684         if (complete)
1685             break;
1686
1687         debugs(9, 5, HERE << "s = {" << s << "}");
1688
1689         linelen = strcspn(s, crlf) + 1;
1690
1691         if (linelen < 2)
1692             break;
1693
1694         if (linelen > 3)
1695             complete = (*s >= '0' && *s <= '9' && *(s + 3) == ' ');
1696
1697         if (complete)
1698             code = atoi(s);
1699
1700         offset = 0;
1701
1702         if (linelen > 3)
1703             if (*s >= '0' && *s <= '9' && (*(s + 3) == '-' || *(s + 3) == ' '))
1704                 offset = 4;
1705
1706         list = new wordlist();
1707
1708         list->key = (char *)xmalloc(linelen - offset);
1709
1710         xstrncpy(list->key, s + offset, linelen - offset);
1711
1712         /* trace the FTP communication chat at level 2 */
1713         debugs(9, 2, "ftp>> " << code << " " << list->key);
1714
1715         *tail = list;
1716
1717         tail = &list->next;
1718     }
1719
1720     *used = (size_t) (s - sbuf);
1721     safe_free(sbuf);
1722
1723     if (!complete)
1724         wordlistDestroy(&head);
1725
1726     if (codep)
1727         *codep = code;
1728
1729     return head;
1730 }
1731
1732 /**
1733  * DPW 2007-04-23
1734  * Looks like there are no longer anymore callers that set
1735  * buffered_ok=1.  Perhaps it can be removed at some point.
1736  */
1737 void
1738 FtpStateData::scheduleReadControlReply(int buffered_ok)
1739 {
1740     debugs(9, 3, HERE << ctrl.conn);
1741
1742     if (buffered_ok && ctrl.offset > 0) {
1743         /* We've already read some reply data */
1744         handleControlReply();
1745     } else {
1746         /*
1747          * Cancel the timeout on the Data socket (if any) and
1748          * establish one on the control socket.
1749          */
1750         if (Comm::IsConnOpen(data.conn)) {
1751             commUnsetConnTimeout(data.conn);
1752         }
1753
1754         typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
1755         AsyncCall::Pointer timeoutCall = JobCallback(9, 5, TimeoutDialer, this, FtpStateData::ftpTimeout);
1756         commSetConnTimeout(ctrl.conn, Config.Timeout.read, timeoutCall);
1757
1758         typedef CommCbMemFunT<FtpStateData, CommIoCbParams> Dialer;
1759         AsyncCall::Pointer reader = JobCallback(9, 5, Dialer, this, FtpStateData::ftpReadControlReply);
1760         comm_read(ctrl.conn, ctrl.buf + ctrl.offset, ctrl.size - ctrl.offset, reader);
1761     }
1762 }
1763
1764 void FtpStateData::ftpReadControlReply(const CommIoCbParams &io)
1765 {
1766     debugs(9, 3, "ftpReadControlReply: FD " << io.fd << ", Read " << io.size << " bytes");
1767
1768     if (io.size > 0) {
1769         kb_incr(&(statCounter.server.all.kbytes_in), io.size);
1770         kb_incr(&(statCounter.server.ftp.kbytes_in), io.size);
1771     }
1772
1773     if (io.flag == COMM_ERR_CLOSING)
1774         return;
1775
1776     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1777         abortTransaction("entry aborted during control reply read");
1778         return;
1779     }
1780
1781     assert(ctrl.offset < ctrl.size);
1782
1783     if (io.flag == COMM_OK && io.size > 0) {
1784         fd_bytes(io.fd, io.size, FD_READ);
1785     }
1786
1787     if (io.flag != COMM_OK) {
1788         debugs(50, ignoreErrno(io.xerrno) ? 3 : DBG_IMPORTANT,
1789                "ftpReadControlReply: read error: " << xstrerr(io.xerrno));
1790
1791         if (ignoreErrno(io.xerrno)) {
1792             scheduleReadControlReply(0);
1793         } else {
1794             failed(ERR_READ_ERROR, io.xerrno);
1795             /* failed closes ctrl.conn and frees ftpState */
1796         }
1797         return;
1798     }
1799
1800     if (io.size == 0) {
1801         if (entry->store_status == STORE_PENDING) {
1802             failed(ERR_FTP_FAILURE, 0);
1803             /* failed closes ctrl.conn and frees ftpState */
1804             return;
1805         }
1806
1807         /* XXX this may end up having to be serverComplete() .. */
1808         abortTransaction("zero control reply read");
1809         return;
1810     }
1811
1812     unsigned int len =io.size + ctrl.offset;
1813     ctrl.offset = len;
1814     assert(len <= ctrl.size);
1815     handleControlReply();
1816 }
1817
1818 void
1819 FtpStateData::handleControlReply()
1820 {
1821     wordlist **W;
1822     size_t bytes_used = 0;
1823     wordlistDestroy(&ctrl.message);
1824     ctrl.message = ftpParseControlReply(ctrl.buf,
1825                                         ctrl.offset, &ctrl.replycode, &bytes_used);
1826
1827     if (ctrl.message == NULL) {
1828         /* didn't get complete reply yet */
1829
1830         if (ctrl.offset == ctrl.size) {
1831             ctrl.buf = (char *)memReallocBuf(ctrl.buf, ctrl.size << 1, &ctrl.size);
1832         }
1833
1834         scheduleReadControlReply(0);
1835         return;
1836     } else if (ctrl.offset == bytes_used) {
1837         /* used it all up */
1838         ctrl.offset = 0;
1839     } else {
1840         /* Got some data past the complete reply */
1841         assert(bytes_used < ctrl.offset);
1842         ctrl.offset -= bytes_used;
1843         memmove(ctrl.buf, ctrl.buf + bytes_used, ctrl.offset);
1844     }
1845
1846     /* Move the last line of the reply message to ctrl.last_reply */
1847     for (W = &ctrl.message; (*W)->next; W = &(*W)->next);
1848     safe_free(ctrl.last_reply);
1849
1850     ctrl.last_reply = xstrdup((*W)->key);
1851
1852     wordlistDestroy(W);
1853
1854     /* Copy the rest of the message to cwd_message to be printed in
1855      * error messages
1856      */
1857     if (ctrl.message) {
1858         for (wordlist *w = ctrl.message; w; w = w->next) {
1859             cwd_message.append('\n');
1860             cwd_message.append(w->key);
1861         }
1862     }
1863
1864     debugs(9, 3, HERE << "state=" << state << ", code=" << ctrl.replycode);
1865
1866     FTP_SM_FUNCS[state] (this);
1867 }
1868
1869 /* ====================================================================== */
1870
1871 /// \ingroup ServerProtocolFTPInternal
1872 static void
1873 ftpReadWelcome(FtpStateData * ftpState)
1874 {
1875     int code = ftpState->ctrl.replycode;
1876     debugs(9, 3, HERE);
1877
1878     if (ftpState->flags.pasv_only)
1879         ++ ftpState->login_att;
1880
1881     if (code == 220) {
1882         if (ftpState->ctrl.message) {
1883             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1884                 ftpState->flags.skip_whitespace = 1;
1885         }
1886
1887         ftpSendUser(ftpState);
1888     } else if (code == 120) {
1889         if (NULL != ftpState->ctrl.message)
1890             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1891
1892         return;
1893     } else {
1894         ftpFail(ftpState);
1895     }
1896 }
1897
1898 /**
1899  * Translate FTP login failure into HTTP error
1900  * this is an attmpt to get the 407 message to show up outside Squid.
1901  * its NOT a general failure. But a correct FTP response type.
1902  */
1903 void
1904 FtpStateData::loginFailed()
1905 {
1906     ErrorState *err = NULL;
1907     const char *command, *reply;
1908
1909     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1910         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1911             // 421/426 - Service Overload - retry permitted.
1912             err = new ErrorState(ERR_FTP_UNAVAILABLE, HTTP_SERVICE_UNAVAILABLE, fwd->request);
1913         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1914             // 43x - Invalid or Credential Error - retry challenge required.
1915             err = new ErrorState(ERR_FTP_FORBIDDEN, HTTP_UNAUTHORIZED, fwd->request);
1916         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1917             // 53x - Credentials Missing - retry challenge required
1918             if (password_url) // but they were in the URI! major fail.
1919                 err = new ErrorState(ERR_FTP_FORBIDDEN, HTTP_FORBIDDEN, fwd->request);
1920             else
1921                 err = new ErrorState(ERR_FTP_FORBIDDEN, HTTP_UNAUTHORIZED, fwd->request);
1922         }
1923     }
1924
1925     // any other problems are general falures.
1926     if (!err) {
1927         ftpFail(this);
1928         return;
1929     }
1930
1931     err->ftp.server_msg = ctrl.message;
1932
1933     ctrl.message = NULL;
1934
1935     if (old_request)
1936         command = old_request;
1937     else
1938         command = ctrl.last_command;
1939
1940     if (command && strncmp(command, "PASS", 4) == 0)
1941         command = "PASS <yourpassword>";
1942
1943     if (old_reply)
1944         reply = old_reply;
1945     else
1946         reply = ctrl.last_reply;
1947
1948     if (command)
1949         err->ftp.request = xstrdup(command);
1950
1951     if (reply)
1952         err->ftp.reply = xstrdup(reply);
1953
1954     HttpReply *newrep = err->BuildHttpReply();
1955     delete err;
1956
1957 #if HAVE_AUTH_MODULE_BASIC
1958     /* add Authenticate header */
1959     newrep->header.putAuth("Basic", ftpRealm());
1960 #endif
1961
1962     // add it to the store entry for response....
1963     entry->replaceHttpReply(newrep);
1964     serverComplete();
1965 }
1966
1967 const char *
1968 FtpStateData::ftpRealm()
1969 {
1970     static char realm[8192];
1971
1972     /* This request is not fully authenticated */
1973     if (!request) {
1974         snprintf(realm, 8192, "FTP %s unknown", user);
1975     } else if (request->port == 21) {
1976         snprintf(realm, 8192, "FTP %s %s", user, request->GetHost());
1977     } else {
1978         snprintf(realm, 8192, "FTP %s %s port %d", user, request->GetHost(), request->port);
1979     }
1980     return realm;
1981 }
1982
1983 /// \ingroup ServerProtocolFTPInternal
1984 static void
1985 ftpSendUser(FtpStateData * ftpState)
1986 {
1987     /* check the server control channel is still available */
1988     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1989         return;
1990
1991     if (ftpState->proxy_host != NULL)
1992         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n",
1993                  ftpState->user,
1994                  ftpState->request->GetHost());
1995     else
1996         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1997
1998     ftpState->writeCommand(cbuf);
1999
2000     ftpState->state = SENT_USER;
2001 }
2002
2003 /// \ingroup ServerProtocolFTPInternal
2004 static void
2005 ftpReadUser(FtpStateData * ftpState)
2006 {
2007     int code = ftpState->ctrl.replycode;
2008     debugs(9, 3, HERE);
2009
2010     if (code == 230) {
2011         ftpReadPass(ftpState);
2012     } else if (code == 331) {
2013         ftpSendPass(ftpState);
2014     } else {
2015         ftpState->loginFailed();
2016     }
2017 }
2018
2019 /// \ingroup ServerProtocolFTPInternal
2020 static void
2021 ftpSendPass(FtpStateData * ftpState)
2022 {
2023     /* check the server control channel is still available */
2024     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
2025         return;
2026
2027     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
2028     ftpState->writeCommand(cbuf);
2029     ftpState->state = SENT_PASS;
2030 }
2031
2032 /// \ingroup ServerProtocolFTPInternal
2033 static void
2034 ftpReadPass(FtpStateData * ftpState)
2035 {
2036     int code = ftpState->ctrl.replycode;
2037     debugs(9, 3, HERE << "code=" << code);
2038
2039     if (code == 230) {
2040         ftpSendType(ftpState);
2041     } else {
2042         ftpState->loginFailed();
2043     }
2044 }
2045
2046 /// \ingroup ServerProtocolFTPInternal
2047 static void
2048 ftpSendType(FtpStateData * ftpState)
2049 {
2050     const char *t;
2051     const char *filename;
2052     char mode;
2053
2054     /* check the server control channel is still available */
2055     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
2056         return;
2057
2058     /*
2059      * Ref section 3.2.2 of RFC 1738
2060      */
2061     mode = ftpState->typecode;
2062
2063     switch (mode) {
2064
2065     case 'D':
2066         mode = 'A';
2067         break;
2068
2069     case 'A':
2070
2071     case 'I':
2072         break;
2073
2074     default:
2075
2076         if (ftpState->flags.isdir) {
2077             mode = 'A';
2078         } else {
2079             t = ftpState->request->urlpath.rpos('/');
2080             filename = t ? t + 1 : ftpState->request->urlpath.termedBuf();
2081             mode = mimeGetTransferMode(filename);
2082         }
2083
2084         break;
2085     }
2086
2087     if (mode == 'I')
2088         ftpState->flags.binary = 1;
2089     else
2090         ftpState->flags.binary = 0;
2091
2092     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
2093
2094     ftpState->writeCommand(cbuf);
2095
2096     ftpState->state = SENT_TYPE;
2097 }
2098
2099 /// \ingroup ServerProtocolFTPInternal
2100 static void
2101 ftpReadType(FtpStateData * ftpState)
2102 {
2103     int code = ftpState->ctrl.replycode;
2104     char *path;
2105     char *d, *p;
2106     debugs(9, 3, HERE << "code=" << code);
2107
2108     if (code == 200) {
2109         p = path = xstrdup(ftpState->request->urlpath.termedBuf());
2110
2111         if (*p == '/')
2112             ++p;
2113
2114         while (*p) {
2115             d = p;
2116             p += strcspn(p, "/");
2117
2118             if (*p) {
2119                 *p = '\0';
2120                 ++p;
2121             }
2122
2123             rfc1738_unescape(d);
2124
2125             if (*d)
2126                 wordlistAdd(&ftpState->pathcomps, d);
2127         }
2128
2129         xfree(path);
2130
2131         if (ftpState->pathcomps)
2132             ftpTraverseDirectory(ftpState);
2133         else
2134             ftpListDir(ftpState);
2135     } else {
2136         ftpFail(ftpState);
2137     }
2138 }
2139
2140 /// \ingroup ServerProtocolFTPInternal
2141 static void
2142 ftpTraverseDirectory(FtpStateData * ftpState)
2143 {
2144     wordlist *w;
2145     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
2146
2147     safe_free(ftpState->dirpath);
2148     ftpState->dirpath = ftpState->filepath;
2149     ftpState->filepath = NULL;
2150
2151     /* Done? */
2152
2153     if (ftpState->pathcomps == NULL) {
2154         debugs(9, 3, HERE << "the final component was a directory");
2155         ftpListDir(ftpState);
2156         return;
2157     }
2158
2159     /* Go to next path component */
2160     w = ftpState->pathcomps;
2161
2162     ftpState->filepath = w->key;
2163
2164     ftpState->pathcomps = w->next;
2165
2166     delete w;
2167
2168     /* Check if we are to CWD or RETR */
2169     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
2170         ftpSendCwd(ftpState);
2171     } else {
2172         debugs(9, 3, HERE << "final component is probably a file");
2173         ftpGetFile(ftpState);
2174         return;
2175     }
2176 }
2177
2178 /// \ingroup ServerProtocolFTPInternal
2179 static void
2180 ftpSendCwd(FtpStateData * ftpState)
2181 {
2182     char *path = NULL;
2183
2184     /* check the server control channel is still available */
2185     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
2186         return;
2187
2188     debugs(9, 3, HERE);
2189
2190     path = ftpState->filepath;
2191
2192     if (!strcmp(path, "..") || !strcmp(path, "/")) {
2193         ftpState->flags.no_dotdot = 1;
2194     } else {
2195         ftpState->flags.no_dotdot = 0;
2196     }
2197
2198     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
2199
2200     ftpState->writeCommand(cbuf);
2201
2202     ftpState->state = SENT_CWD;
2203 }
2204
2205 /// \ingroup ServerProtocolFTPInternal
2206 static void
2207 ftpReadCwd(FtpStateData * ftpState)
2208 {
2209     int code = ftpState->ctrl.replycode;
2210     debugs(9, 3, HERE);
2211
2212     if (code >= 200 && code < 300) {
2213         /* CWD OK */
2214         ftpState->unhack();
2215
2216         /* Reset cwd_message to only include the last message */
2217         ftpState->cwd_message.reset("");
2218         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
2219             ftpState->cwd_message.append(' ');
2220             ftpState->cwd_message.append(w->key);
2221         }
2222         ftpState->ctrl.message = NULL;
2223
2224         /* Continue to traverse the path */
2225         ftpTraverseDirectory(ftpState);
2226     } else {
2227         /* CWD FAILED */
2228
2229         if (!ftpState->flags.put)
2230             ftpFail(ftpState);
2231         else
2232             ftpSendMkdir(ftpState);
2233     }
2234 }
2235
2236 /// \ingroup ServerProtocolFTPInternal
2237 static void
2238 ftpSendMkdir(FtpStateData * ftpState)
2239 {
2240     char *path = NULL;
2241
2242     /* check the server control channel is still available */
2243     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
2244         return;
2245
2246     path = ftpState->filepath;
2247     debugs(9, 3, HERE << "with path=" << path);
2248     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
2249     ftpState->writeCommand(cbuf);
2250     ftpState->state = SENT_MKDIR;
2251 }
2252
2253 /// \ingroup ServerProtocolFTPInternal
2254 static void
2255 ftpReadMkdir(FtpStateData * ftpState)
2256 {
2257     char *path = ftpState->filepath;
2258     int code = ftpState->ctrl.replycode;
2259
2260     debugs(9, 3, HERE << "path " << path << ", code " << code);
2261
2262     if (code == 257) {          /* success */
2263         ftpSendCwd(ftpState);
2264     } else if (code == 550) {   /* dir exists */
2265
2266         if (ftpState->flags.put_mkdir) {
2267             ftpState->flags.put_mkdir = 1;
2268             ftpSendCwd(ftpState);
2269         } else
2270             ftpSendReply(ftpState);
2271     } else
2272         ftpSendReply(ftpState);
2273 }
2274
2275 /// \ingroup ServerProtocolFTPInternal
2276 static void
2277 ftpGetFile(FtpStateData * ftpState)
2278 {
2279     assert(*ftpState->filepath != '\0');
2280     ftpState->flags.isdir = 0;
2281     ftpSendMdtm(ftpState);
2282 }
2283
2284 /// \ingroup ServerProtocolFTPInternal
2285 static void
2286 ftpListDir(FtpStateData * ftpState)
2287 {
2288     if (ftpState->flags.dir_slash) {
2289         debugs(9, 3, HERE << "Directory path did not end in /");
2290         ftpState->title_url.append("/");
2291         ftpState->flags.isdir = 1;
2292     }
2293
2294     ftpSendPassive(ftpState);
2295 }
2296
2297 /// \ingroup ServerProtocolFTPInternal
2298 static void
2299 ftpSendMdtm(FtpStateData * ftpState)
2300 {
2301     /* check the server control channel is still available */
2302     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
2303         return;
2304
2305     assert(*ftpState->filepath != '\0');
2306     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
2307     ftpState->writeCommand(cbuf);
2308     ftpState->state = SENT_MDTM;
2309 }
2310
2311 /// \ingroup ServerProtocolFTPInternal
2312 static void
2313 ftpReadMdtm(FtpStateData * ftpState)
2314 {
2315     int code = ftpState->ctrl.replycode;
2316     debugs(9, 3, HERE);
2317
2318     if (code == 213) {
2319         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
2320         ftpState->unhack();
2321     } else if (code < 0) {
2322         ftpFail(ftpState);
2323         return;
2324     }
2325
2326     ftpSendSize(ftpState);
2327 }
2328
2329 /// \ingroup ServerProtocolFTPInternal
2330 static void
2331 ftpSendSize(FtpStateData * ftpState)
2332 {
2333     /* check the server control channel is still available */
2334     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
2335         return;
2336
2337     /* Only send SIZE for binary transfers. The returned size
2338      * is useless on ASCII transfers */
2339
2340     if (ftpState->flags.binary) {
2341         assert(ftpState->filepath != NULL);
2342         assert(*ftpState->filepath != '\0');
2343         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
2344         ftpState->writeCommand(cbuf);
2345         ftpState->state = SENT_SIZE;
2346     } else
2347         /* Skip to next state no non-binary transfers */
2348         ftpSendPassive(ftpState);
2349 }
2350
2351 /// \ingroup ServerProtocolFTPInternal
2352 static void
2353 ftpReadSize(FtpStateData * ftpState)
2354 {
2355     int code = ftpState->ctrl.replycode;
2356     debugs(9, 3, HERE);
2357
2358     if (code == 213) {
2359         ftpState->unhack();
2360         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
2361
2362         if (ftpState->theSize == 0) {
2363             debugs(9, 2, "SIZE reported " <<
2364                    ftpState->ctrl.last_reply << " on " <<
2365                    ftpState->title_url);
2366             ftpState->theSize = -1;
2367         }
2368     } else if (code < 0) {
2369         ftpFail(ftpState);
2370         return;
2371     }
2372
2373     ftpSendPassive(ftpState);
2374 }
2375
2376 /**
2377  \ingroup ServerProtocolFTPInternal
2378  */
2379 static void
2380 ftpReadEPSV(FtpStateData* ftpState)
2381 {
2382     int code = ftpState->ctrl.replycode;
2383     Ip::Address ipa_remote;
2384     char *buf;
2385     debugs(9, 3, HERE);
2386
2387     if (code != 229 && code != 522) {
2388         if (code == 200) {
2389             /* handle broken servers (RFC 2428 says OK code for EPSV MUST be 229 not 200) */
2390             /* vsftpd for one send '200 EPSV ALL ok.' without even port info.
2391              * Its okay to re-send EPSV 1/2 but nothing else. */
2392             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ftpState->ctrl.conn->remote << ". Wrong accept code for EPSV");
2393         } else {
2394             debugs(9, 2, "EPSV not supported by remote end");
2395             ftpState->state = SENT_EPSV_1; /* simulate having failed EPSV 1 (last EPSV to try before shifting to PASV) */
2396         }
2397         ftpSendPassive(ftpState);
2398         return;
2399     }
2400
2401     if (code == 522) {
2402         /* server response with list of supported methods   */
2403         /*   522 Network protocol not supported, use (1)    */
2404         /*   522 Network protocol not supported, use (1,2)  */
2405         /*   522 Network protocol not supported, use (2)  */
2406         /* TODO: handle the (1,2) case. We might get it back after EPSV ALL
2407          * which means close data + control without self-destructing and re-open from scratch. */
2408         debugs(9, 5, HERE << "scanning: " << ftpState->ctrl.last_reply);
2409         buf = ftpState->ctrl.last_reply;
2410         while (buf != NULL && *buf != '\0' && *buf != '\n' && *buf != '(')
2411             ++buf;
2412         if (buf != NULL && *buf == '\n')
2413             ++buf;
2414
2415         if (buf == NULL || *buf == '\0') {
2416             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
2417             debugs(9, DBG_IMPORTANT, "Broken FTP Server at " << ftpState->ctrl.conn->remote << ". 522 error missing protocol negotiation hints");
2418             ftpSendPassive(ftpState);
2419         } else if (strcmp(buf, "(1)") == 0) {
2420             ftpState->state = SENT_EPSV_2; /* simulate having sent and failed EPSV 2 */
2421             ftpSendPassive(ftpState);
2422         } else if (strcmp(buf, "(2)") == 0) {
2423             if (Ip::EnableIpv6) {
2424                 /* If server only supports EPSV 2 and we have already tried that. Go straight to EPRT */
2425                 if (ftpState->state == SENT_EPSV_2) {
2426                     ftpSendEPRT(ftpState);
2427                 } else {
2428                     /* or try the next Passive mode down the chain. */
2429                     ftpSendPassive(ftpState);
2430                 }
2431             } else {
2432                 /* Server only accept EPSV in IPv6 traffic. */
2433                 ftpState->state = SENT_EPSV_1; /* simulate having sent and failed EPSV 1 */
2434                 ftpSendPassive(ftpState);
2435             }
2436         } else {
2437             /* handle broken server (RFC 2428 says MUST specify supported protocols in 522) */
2438             debugs(9, DBG_IMPORTANT, "WARNING: Server at " << ftpState->ctrl.conn->remote << " sent unknown protocol negotiation hint: " << buf);
2439             ftpSendPassive(ftpState);
2440         }
2441         return;
2442     }
2443
2444     /*  229 Entering Extended Passive Mode (|||port|) */
2445     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
2446     debugs(9, 5, "scanning: " << ftpState->ctrl.last_reply);
2447
2448     buf = ftpState->ctrl.last_reply + strcspn(ftpState->ctrl.last_reply, "(");
2449
2450     char h1, h2, h3, h4;
2451     unsigned short port;
2452     int n = sscanf(buf, "(%c%c%c%hu%c)", &h1, &h2, &h3, &port, &h4);
2453
2454     if (n < 4 || h1 != h2 || h1 != h3 || h1 != h4) {
2455         debugs(9, DBG_IMPORTANT, "Invalid EPSV reply from " <<
2456                ftpState->ctrl.conn->remote << ": " <<
2457                ftpState->ctrl.last_reply);
2458
2459         ftpSendPassive(ftpState);
2460         return;
2461     }
2462
2463     if (0 == port) {
2464         debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
2465                ftpState->ctrl.conn->remote << ": " <<
2466                ftpState->ctrl.last_reply);
2467
2468         ftpSendPassive(ftpState);
2469         return;
2470     }
2471
2472     if (Config.Ftp.sanitycheck) {
2473         if (port < 1024) {
2474             debugs(9, DBG_IMPORTANT, "Unsafe EPSV reply from " <<
2475                    ftpState->ctrl.conn->remote << ": " <<
2476                    ftpState->ctrl.last_reply);
2477
2478             ftpSendPassive(ftpState);
2479             return;
2480         }
2481     }
2482
2483     ftpState->data.port = port;
2484
2485     safe_free(ftpState->data.host);
2486     ftpState->data.host = xstrdup(fd_table[ftpState->ctrl.conn->fd].ipaddr);
2487
2488     safe_free(ftpState->ctrl.last_command);
2489
2490     safe_free(ftpState->ctrl.last_reply);
2491
2492     ftpState->ctrl.last_command = xstrdup("Connect to server data port");
2493
2494     // Generate a new data channel descriptor to be opened.
2495     Comm::ConnectionPointer conn = new Comm::Connection;
2496     conn->local = ftpState->ctrl.conn->local;
2497     conn->local.SetPort(0);
2498     conn->remote = ftpState->ctrl.conn->remote;
2499     conn->remote.SetPort(port);
2500
2501     debugs(9, 3, HERE << "connecting to " << conn->remote);
2502
2503     ftpState->data.opener = commCbCall(9,3, "FtpStateData::ftpPasvCallback", CommConnectCbPtrFun(FtpStateData::ftpPasvCallback, ftpState));
2504     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, ftpState->data.opener, Config.Timeout.connect);
2505     cs->setHost(ftpState->data.host);
2506     AsyncJob::Start(cs);
2507 }
2508
2509 /** \ingroup ServerProtocolFTPInternal
2510  *
2511  * Send Passive connection request.
2512  * Default method is to use modern EPSV request.
2513  * The failover mechanism should check for previous state and re-call with alternates on failure.
2514  */
2515 static void
2516 ftpSendPassive(FtpStateData * ftpState)
2517 {
2518     /** Checks the server control channel is still available before running. */
2519     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
2520         return;
2521
2522     debugs(9, 3, HERE);
2523
2524     /** \par
2525       * Checks for EPSV ALL special conditions:
2526       * If enabled to be sent, squid MUST NOT request any other connect methods.
2527       * If 'ALL' is sent and fails the entire FTP Session fails.
2528       * NP: By my reading exact EPSV protocols maybe attempted, but only EPSV method. */
2529     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent && ftpState->state == SENT_EPSV_1 ) {
2530         debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
2531         ftpFail(ftpState);
2532         return;
2533     }
2534
2535     /** \par
2536       * Checks for 'HEAD' method request and passes off for special handling by FtpStateData::processHeadResponse(). */
2537     if (ftpState->request->method == METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
2538         ftpState->processHeadResponse(); // may call serverComplete
2539         return;
2540     }
2541
2542     /// Closes any old FTP-Data connection which may exist. */
2543     ftpState->data.close();
2544
2545     /** \par
2546       * Checks for previous EPSV/PASV failures on this server/session.
2547       * Diverts to EPRT immediately if they are not working. */
2548     if (!ftpState->flags.pasv_supported) {
2549         ftpSendEPRT(ftpState);
2550         return;
2551     }
2552
2553     /** \par
2554       * Send EPSV (ALL,2,1) or PASV on the control channel.
2555       *
2556       *  - EPSV ALL  is used if enabled.
2557       *  - EPSV 2    is used if ALL is disabled and IPv6 is available and ctrl channel is IPv6.
2558       *  - EPSV 1    is used if EPSV 2 (IPv6) fails or is not available or ctrl channel is IPv4.
2559       *  - PASV      is used if EPSV 1 fails.
2560       */
2561     switch (ftpState->state) {
2562     case SENT_EPSV_ALL: /* EPSV ALL resulted in a bad response. Try ther EPSV methods. */
2563         ftpState->flags.epsv_all_sent = true;
2564         if (ftpState->ctrl.conn->local.IsIPv6()) {
2565             debugs(9, 5, HERE << "FTP Channel is IPv6 (" << ftpState->ctrl.conn->remote << ") attempting EPSV 2 after EPSV ALL has failed.");
2566             snprintf(cbuf, CTRL_BUFLEN, "EPSV 2\r\n");
2567             ftpState->state = SENT_EPSV_2;
2568             break;
2569         }
2570         // else fall through to skip EPSV 2
2571
2572     case SENT_EPSV_2: /* EPSV IPv6 failed. Try EPSV IPv4 */
2573         if (ftpState->ctrl.conn->local.IsIPv4()) {
2574             debugs(9, 5, HERE << "FTP Channel is IPv4 (" << ftpState->ctrl.conn->remote << ") attempting EPSV 1 after EPSV ALL has failed.");
2575             snprintf(cbuf, CTRL_BUFLEN, "EPSV 1\r\n");
2576             ftpState->state = SENT_EPSV_1;
2577             break;
2578         } else if (ftpState->flags.epsv_all_sent) {
2579             debugs(9, DBG_IMPORTANT, "FTP does not allow PASV method after 'EPSV ALL' has been sent.");
2580             ftpFail(ftpState);
2581             return;
2582         }
2583         // else fall through to skip EPSV 1
2584
2585     case SENT_EPSV_1: /* EPSV options exhausted. Try PASV now. */
2586         debugs(9, 5, HERE << "FTP Channel (" << ftpState->ctrl.conn->remote << ") rejects EPSV connection attempts. Trying PASV instead.");
2587         snprintf(cbuf, CTRL_BUFLEN, "PASV\r\n");
2588         ftpState->state = SENT_PASV;
2589         break;
2590
2591     default:
2592         if (!Config.Ftp.epsv) {
2593             debugs(9, 5, HERE << "EPSV support manually disabled. Sending PASV for FTP Channel (" << ftpState->ctrl.conn->remote <<")");
2594             snprintf(cbuf, CTRL_BUFLEN, "PASV\r\n");
2595             ftpState->state = SENT_PASV;
2596         } else if (Config.Ftp.epsv_all) {
2597             debugs(9, 5, HERE << "EPSV ALL manually enabled. Attempting with FTP Channel (" << ftpState->ctrl.conn->remote <<")");
2598             snprintf(cbuf, CTRL_BUFLEN, "EPSV ALL\r\n");
2599             ftpState->state = SENT_EPSV_ALL;
2600             /* block other non-EPSV connections being attempted */
2601             ftpState->flags.epsv_all_sent = true;
2602         } else {
2603             if (ftpState->ctrl.conn->local.IsIPv6()) {
2604                 debugs(9, 5, HERE << "FTP Channel (" << ftpState->ctrl.conn->remote << "). Sending default EPSV 2");
2605                 snprintf(cbuf, CTRL_BUFLEN, "EPSV 2\r\n");
2606                 ftpState->state = SENT_EPSV_2;
2607             }
2608             if (ftpState->ctrl.conn->local.IsIPv4()) {
2609                 debugs(9, 5, HERE << "Channel (" << ftpState->ctrl.conn->remote <<"). Sending default EPSV 1");
2610                 snprintf(cbuf, CTRL_BUFLEN, "EPSV 1\r\n");
2611                 ftpState->state = SENT_EPSV_1;
2612             }
2613         }
2614         break;
2615     }
2616
2617     ftpState->writeCommand(cbuf);
2618
2619     /*
2620      * ugly hack for ftp servers like ftp.netscape.com that sometimes
2621      * dont acknowledge PASV commands. Use connect timeout to be faster then read timeout (minutes).
2622      */
2623     typedef CommCbMemFunT<FtpStateData, CommTimeoutCbParams> TimeoutDialer;
2624     AsyncCall::Pointer timeoutCall =  JobCallback(9, 5,
2625                                       TimeoutDialer, ftpState, FtpStateData::ftpTimeout);
2626     commSetConnTimeout(ftpState->ctrl.conn, Config.Timeout.connect, timeoutCall);
2627 }
2628
2629 void
2630 FtpStateData::processHeadResponse()
2631 {
2632     debugs(9, 5, HERE << "handling HEAD response");
2633     ftpSendQuit(this);
2634     appendSuccessHeader();
2635
2636     /*
2637      * On rare occasions I'm seeing the entry get aborted after
2638      * ftpReadControlReply() and before here, probably when
2639      * trying to write to the client.
2640      */
2641     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
2642         abortTransaction("entry aborted while processing HEAD");
2643         return;
2644     }
2645
2646 #if USE_ADAPTATION
2647     if (adaptationAccessCheckPending) {
2648         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
2649         return;
2650     }
2651 #endif
2652
2653     // processReplyBody calls serverComplete() since there is no body
2654     processReplyBody();
2655 }
2656
2657 /// \ingroup ServerProtocolFTPInternal
2658 static void
2659 ftpReadPasv(FtpStateData * ftpState)
2660 {
2661     int code = ftpState->ctrl.replycode;
2662     int h1, h2, h3, h4;
2663     int p1, p2;
2664     int n;
2665     unsigned short port;
2666     Ip::Address ipa_remote;
2667     char *buf;
2668     LOCAL_ARRAY(char, ipaddr, 1024);
2669     debugs(9, 3, HERE);
2670
2671     if (code != 227) {
2672         debugs(9, 2, "PASV not supported by remote end");
2673         ftpSendEPRT(ftpState);
2674         return;
2675     }
2676
2677     /*  227 Entering Passive Mode (h1,h2,h3,h4,p1,p2).  */
2678     /*  ANSI sez [^0-9] is undefined, it breaks on Watcom cc */
2679     debugs(9, 5, HERE << "scanning: " << ftpState->ctrl.last_reply);
2680
2681     buf = ftpState->ctrl.last_reply + strcspn(ftpState->ctrl.last_reply, "0123456789");
2682
2683     n = sscanf(buf, "%d,%d,%d,%d,%d,%d", &h1, &h2, &h3, &h4, &p1, &p2);
2684
2685     if (n != 6 || p1 < 0 || p2 < 0 || p1 > 255 || p2 > 255) {
2686         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2687                ftpState->ctrl.conn->remote << ": " <<
2688                ftpState->ctrl.last_reply);
2689
2690         ftpSendEPRT(ftpState);
2691         return;
2692     }
2693
2694     snprintf(ipaddr, 1024, "%d.%d.%d.%d", h1, h2, h3, h4);
2695
2696     ipa_remote = ipaddr;
2697
2698     if ( ipa_remote.IsAnyAddr() ) {
2699         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2700                ftpState->ctrl.conn->remote << ": " <<
2701                ftpState->ctrl.last_reply);
2702
2703         ftpSendEPRT(ftpState);
2704         return;
2705     }
2706
2707     port = ((p1 << 8) + p2);
2708
2709     if (0 == port) {
2710         debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2711                ftpState->ctrl.conn->remote << ": " <<
2712                ftpState->ctrl.last_reply);
2713
2714         ftpSendEPRT(ftpState);
2715         return;
2716     }
2717
2718     if (Config.Ftp.sanitycheck) {
2719         if (port < 1024) {
2720             debugs(9, DBG_IMPORTANT, "Unsafe PASV reply from " <<
2721                    ftpState->ctrl.conn->remote << ": " <<
2722                    ftpState->ctrl.last_reply);
2723
2724             ftpSendEPRT(ftpState);
2725             return;
2726         }
2727     }
2728
2729     ftpState->data.port = port;
2730
2731     safe_free(ftpState->data.host);
2732     if (Config.Ftp.sanitycheck)
2733         ftpState->data.host = xstrdup(fd_table[ftpState->ctrl.conn->fd].ipaddr);
2734     else
2735         ftpState->data.host = xstrdup(ipaddr);
2736
2737     safe_free(ftpState->ctrl.last_command);
2738
2739     safe_free(ftpState->ctrl.last_reply);
2740
2741     ftpState->ctrl.last_command = xstrdup("Connect to server data port");
2742
2743     Comm::ConnectionPointer conn = new Comm::Connection;
2744     conn->local = ftpState->ctrl.conn->local;
2745     conn->local.SetPort(0);
2746     conn->remote = ipaddr;
2747     conn->remote.SetPort(port);
2748
2749     debugs(9, 3, HERE << "connecting to " << conn->remote);
2750
2751     ftpState->data.opener = commCbCall(9,3, "FtpStateData::ftpPasvCallback", CommConnectCbPtrFun(FtpStateData::ftpPasvCallback, ftpState));
2752     Comm::ConnOpener *cs = new Comm::ConnOpener(conn, ftpState->data.opener, Config.Timeout.connect);
2753     cs->setHost(ftpState->data.host);
2754     AsyncJob::Start(cs);
2755 }
2756
2757 void
2758 FtpStateData::ftpPasvCallback(const Comm::ConnectionPointer &conn, comm_err_t status, int xerrno, void *data)
2759 {
2760     FtpStateData *ftpState = (FtpStateData *)data;
2761     debugs(9, 3, HERE);
2762     ftpState->data.opener = NULL;
2763
2764     if (status != COMM_OK) {
2765         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
2766
2767         // ABORT on timeouts. server may be waiting on a broken TCP link.
2768         if (status == COMM_TIMEOUT)
2769             ftpState->writeCommand("ABOR");
2770
2771         // try another connection attempt with some other method
2772         ftpSendPassive(ftpState);
2773         return;
2774     }
2775
2776     ftpState->data.opened(conn, ftpState->dataCloser());
2777     ftpRestOrList(ftpState);
2778 }
2779
2780 /// \ingroup ServerProtocolFTPInternal
2781 static void
2782 ftpOpenListenSocket(FtpStateData * ftpState, int fallback)
2783 {
2784     /// Close old data channels, if any. We may open a new one below.
2785     if (ftpState->data.conn != NULL) {
2786         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
2787             // NP: in fact it points to the control channel. just clear it.
2788             ftpState->data.clear();
2789         else
2790             ftpState->data.close();
2791     }
2792     safe_free(ftpState->data.host);
2793
2794     /*
2795      * Set up a listen socket on the same local address as the
2796      * control connection.
2797      */
2798     Comm::ConnectionPointer temp = new Comm::Connection;
2799     temp->local = ftpState->ctrl.conn->local;
2800
2801     /*
2802      * REUSEADDR is needed in fallback mode, since the same port is
2803      * used for both control and data.
2804      */
2805     if (fallback) {
2806         int on = 1;
2807         setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
2808         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
2809         temp->flags |= COMM_REUSEADDR;
2810     } else {
2811         /* if not running in fallback mode a new port needs to be retrieved */
2812         temp->local.SetPort(0);
2813     }
2814
2815     ftpState->listenForDataChannel(temp, ftpState->entry->url());
2816 }
2817
2818 /// \ingroup ServerProtocolFTPInternal
2819 static void
2820 ftpSendPORT(FtpStateData * ftpState)
2821 {
2822     /* check the server control channel is still available */
2823     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
2824         return;
2825
2826     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
2827         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
2828         return;
2829     }
2830
2831     debugs(9, 3, HERE);
2832     ftpState->flags.pasv_supported = 0;
2833     ftpOpenListenSocket(ftpState, 0);
2834
2835     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
2836         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.IsIPv4() ) {
2837             /* non-IPv4 CANNOT send PORT command.                       */
2838             /* we got here by attempting and failing an EPRT            */
2839             /* using the same reply code should simulate a PORT failure */
2840             ftpReadPORT(ftpState);
2841             return;
2842         }
2843
2844         /* XXX Need to set error message */
2845         ftpFail(ftpState);
2846         return;
2847     }
2848
2849     // pull out the internal IP address bytes to send in PORT command...
2850     // source them from the listen_conn->local
2851
2852     struct addrinfo *AI = NULL;
2853     ftpState->data.listenConn->local.GetAddrInfo(AI, AF_INET);
2854     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
2855     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
2856     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
2857              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
2858              portptr[0], portptr[1]);
2859     ftpState->writeCommand(cbuf);
2860     ftpState->state = SENT_PORT;
2861
2862     ftpState->data.listenConn->local.FreeAddrInfo(AI);
2863 }
2864
2865 /// \ingroup ServerProtocolFTPInternal
2866 static void
2867 ftpReadPORT(FtpStateData * ftpState)
2868 {
2869     int code = ftpState->ctrl.replycode;
2870     debugs(9, 3, HERE);
2871
2872     if (code != 200) {
2873         /* Fall back on using the same port as the control connection */
2874         debugs(9, 3, "PORT not supported by remote end");
2875         ftpOpenListenSocket(ftpState, 1);
2876     }
2877
2878     ftpRestOrList(ftpState);
2879 }
2880
2881 /// \ingroup ServerProtocolFTPInternal
2882 static void
2883 ftpSendEPRT(FtpStateData * ftpState)
2884 {
2885     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
2886         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
2887         return;
2888     }
2889
2890     if (!Config.Ftp.eprt) {
2891         /* Disabled. Switch immediately to attempting old PORT command. */
2892         debugs(9, 3, "EPRT disabled by local administrator");
2893         ftpSendPORT(ftpState);
2894         return;
2895     }
2896
2897     debugs(9, 3, HERE);
2898     ftpState->flags.pasv_supported = 0;
2899
2900     ftpOpenListenSocket(ftpState, 0);
2901     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
2902     if (!Comm::IsConnOpen(ftpState->data.conn)) {
2903         /* XXX Need to set error message */
2904         ftpFail(ftpState);
2905         return;
2906     }
2907
2908     char buf[MAX_IPSTRLEN];
2909
2910     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
2911     /* Which can be used by EITHER protocol. */
2912     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
2913              ( ftpState->data.listenConn->local.IsIPv6() ? 2 : 1 ),
2914              ftpState->data.listenConn->local.NtoA(buf,MAX_IPSTRLEN),
2915              ftpState->data.listenConn->local.GetPort() );
2916
2917     ftpState->writeCommand(cbuf);
2918     ftpState->state = SENT_EPRT;
2919 }
2920
2921 static void
2922 ftpReadEPRT(FtpStateData * ftpState)
2923 {
2924     int code = ftpState->ctrl.replycode;
2925     debugs(9, 3, HERE);
2926
2927     if (code != 200) {
2928         /* Failover to attempting old PORT command. */
2929         debugs(9, 3, "EPRT not supported by remote end");
2930         ftpSendPORT(ftpState);
2931         return;
2932     }
2933
2934     ftpRestOrList(ftpState);
2935 }
2936
2937 /**
2938  \ingroup ServerProtocolFTPInternal
2939  \par
2940  * "read" handler to accept FTP data connections.
2941  *
2942  \param io    comm accept(2) callback parameters
2943  */
2944 void
2945 FtpStateData::ftpAcceptDataConnection(const CommAcceptCbParams &io)
2946 {
2947     debugs(9, 3, HERE);
2948
2949     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
2950         abortTransaction("entry aborted when accepting data conn");
2951         data.listenConn->close();
2952         data.listenConn = NULL;
2953         return;
2954     }
2955
2956     if (io.flag != COMM_OK) {
2957         data.listenConn->close();
2958         data.listenConn = NULL;
2959         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
2960         /** \todo Need to send error message on control channel*/
2961         ftpFail(this);
2962         return;
2963     }
2964
2965     /* data listening conn is no longer even open. abort. */
2966     if (!Comm::IsConnOpen(data.listenConn)) {
2967         data.listenConn = NULL; // ensure that it's cleared and not just closed.
2968         return;
2969     }
2970
2971     /* data listening conn is no longer even open. abort. */
2972     if (!Comm::IsConnOpen(data.conn)) {
2973         data.clear(); // ensure that it's cleared and not just closed.
2974         return;
2975     }
2976
2977     /** \par
2978      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
2979      * made by the remote client which is connected to the FTP control socket.
2980      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
2981      * This prevents third-party hacks, but also third-party load balancing handshakes.
2982      */
2983     if (Config.Ftp.sanitycheck) {
2984         // accept if either our data or ctrl connection is talking to this remote peer.
2985         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
2986             debugs(9, DBG_IMPORTANT,
2987                    "FTP data connection from unexpected server (" <<
2988                    io.conn->remote << "), expecting " <<
2989                    data.conn->remote << " or " << ctrl.conn->remote);
2990
2991             /* close the bad sources connection down ASAP. */
2992             io.conn->close();
2993
2994             /* drop the bad connection (io) by ignoring the attempt. */
2995             return;
2996         }
2997     }
2998
2999     /** On COMM_OK start using the accepted data socket and discard the temporary listen socket. */
3000     data.close();
3001     data.opened(io.conn, dataCloser());
3002     static char ntoapeer[MAX_IPSTRLEN];
3003     io.conn->remote.NtoA(ntoapeer,sizeof(ntoapeer));
3004     data.host = xstrdup(ntoapeer);
3005
3006     debugs(9, 3, HERE << "Connected data socket on " <<
3007            io.conn << ". FD table says: " <<
3008            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
3009            "data-peer= " << fd_table[data.conn->fd].ipaddr);
3010
3011     assert(haveControlChannel("ftpAcceptDataConnection"));
3012     assert(ctrl.message == NULL);
3013
3014     // Ctrl channel operations will determine what happens to this data connection
3015 }
3016
3017 /// \ingroup ServerProtocolFTPInternal
3018 static void
3019 ftpRestOrList(FtpStateData * ftpState)
3020 {
3021     debugs(9, 3, HERE);
3022
3023     if (ftpState->typecode == 'D') {
3024         ftpState->flags.isdir = 1;
3025
3026         if (ftpState->flags.put) {
3027             ftpSendMkdir(ftpState);     /* PUT name;type=d */
3028         } else {
3029             ftpSendNlst(ftpState);      /* GET name;type=d  sec 3.2.2 of RFC 1738 */
3030         }
3031     } else if (ftpState->flags.put) {
3032         ftpSendStor(ftpState);
3033     } else if (ftpState->flags.isdir)
3034         ftpSendList(ftpState);
3035     else if (ftpState->restartable())
3036         ftpSendRest(ftpState);
3037     else
3038         ftpSendRetr(ftpState);
3039 }
3040
3041 /// \ingroup ServerProtocolFTPInternal
3042 static void
3043 ftpSendStor(FtpStateData * ftpState)
3044 {
3045     /* check the server control channel is still available */
3046     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
3047         return;
3048
3049     debugs(9, 3, HERE);
3050
3051     if (ftpState->filepath != NULL) {
3052         /* Plain file upload */
3053         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
3054         ftpState->writeCommand(cbuf);
3055         ftpState->state = SENT_STOR;
3056     } else if (ftpState->request->header.getInt64(HDR_CONTENT_LENGTH) > 0) {
3057         /* File upload without a filename. use STOU to generate one */
3058         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
3059         ftpState->writeCommand(cbuf);
3060         ftpState->state = SENT_STOR;
3061     } else {
3062         /* No file to transfer. Only create directories if needed */
3063         ftpSendReply(ftpState);
3064     }
3065 }
3066
3067 /// \ingroup ServerProtocolFTPInternal
3068 /// \deprecated use ftpState->readStor() instead.
3069 static void
3070 ftpReadStor(FtpStateData * ftpState)
3071 {
3072     ftpState->readStor();
3073 }
3074
3075 void FtpStateData::readStor()
3076 {
3077     int code = ctrl.replycode;
3078     debugs(9, 3, HERE);
3079
3080     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
3081         if (!startRequestBodyFlow()) { // register to receive body data
3082             ftpFail(this);
3083             return;
3084         }
3085
3086         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
3087         debugs(9, 3, HERE << "starting data transfer");
3088         switchTimeoutToDataChannel();
3089         sendMoreRequestBody();
3090         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
3091         state = WRITING_DATA;
3092         debugs(9, 3, HERE << "writing data channel");
3093     } else if (code == 150) {
3094         /* When client code is 150 with no data channel, Accept data channel. */
3095         debugs(9, 3, "ftpReadStor: accepting data channel");
3096         listenForDataChannel(data.conn, data.host);
3097     } else {
3098         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
3099         ftpFail(this);
3100     }
3101 }
3102
3103 /// \ingroup ServerProtocolFTPInternal
3104 static void
3105 ftpSendRest(FtpStateData * ftpState)
3106 {
3107     /* check the server control channel is still available */
3108     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
3109         return;
3110
3111     debugs(9, 3, HERE);
3112
3113     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
3114     ftpState->writeCommand(cbuf);
3115     ftpState->state = SENT_REST;
3116 }
3117
3118 int
3119 FtpStateData::restartable()
3120 {
3121     if (restart_offset > 0)
3122         return 1;
3123
3124     if (!request->range)
3125         return 0;
3126
3127     if (!flags.binary)
3128         return 0;
3129
3130     if (theSize <= 0)
3131         return 0;
3132
3133     int64_t desired_offset = request->range->lowestOffset(theSize);
3134
3135     if (desired_offset <= 0)
3136         return 0;
3137
3138     if (desired_offset >= theSize)
3139         return 0;
3140
3141     restart_offset = desired_offset;
3142     return 1;
3143 }
3144
3145 /// \ingroup ServerProtocolFTPInternal
3146 static void
3147 ftpReadRest(FtpStateData * ftpState)
3148 {
3149     int code = ftpState->ctrl.replycode;
3150     debugs(9, 3, HERE);
3151     assert(ftpState->restart_offset > 0);
3152
3153     if (code == 350) {
3154         ftpState->setCurrentOffset(ftpState->restart_offset);
3155         ftpSendRetr(ftpState);
3156     } else if (code > 0) {
3157         debugs(9, 3, HERE << "REST not supported");
3158         ftpState->flags.rest_supported = 0;
3159         ftpSendRetr(ftpState);
3160     } else {
3161         ftpFail(ftpState);
3162     }
3163 }
3164
3165 /// \ingroup ServerProtocolFTPInternal
3166 static void
3167 ftpSendList(FtpStateData * ftpState)
3168 {
3169     /* check the server control channel is still available */
3170     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
3171         return;
3172
3173     debugs(9, 3, HERE);
3174
3175     if (ftpState->filepath) {
3176         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
3177     } else {
3178         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
3179     }
3180
3181     ftpState->writeCommand(cbuf);
3182     ftpState->state = SENT_LIST;
3183 }
3184
3185 /// \ingroup ServerProtocolFTPInternal
3186 static void
3187 ftpSendNlst(FtpStateData * ftpState)
3188 {
3189     /* check the server control channel is still available */
3190     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
3191         return;
3192
3193     debugs(9, 3, HERE);
3194
3195     ftpState->flags.tried_nlst = 1;
3196
3197     if (ftpState->filepath) {
3198         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
3199     } else {
3200         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
3201     }
3202
3203     ftpState->writeCommand(cbuf);
3204     ftpState->state = SENT_NLST;
3205 }
3206
3207 /// \ingroup ServerProtocolFTPInternal
3208 static void
3209 ftpReadList(FtpStateData * ftpState)
3210 {
3211     int code = ftpState->ctrl.replycode;
3212     debugs(9, 3, HERE);
3213
3214     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
3215         /* Begin data transfer */
3216         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3217         ftpState->switchTimeoutToDataChannel();
3218         ftpState->maybeReadVirginBody();
3219         ftpState->state = READING_DATA;
3220         return;
3221     } else if (code == 150) {
3222         /* Accept data channel */
3223         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3224         ftpState->listenForDataChannel(ftpState->data.conn, ftpState->data.host);
3225         return;
3226     } else if (!ftpState->flags.tried_nlst && code > 300) {
3227         ftpSendNlst(ftpState);
3228     } else {
3229         ftpFail(ftpState);
3230         return;
3231     }
3232 }
3233
3234 /// \ingroup ServerProtocolFTPInternal
3235 static void
3236 ftpSendRetr(FtpStateData * ftpState)
3237 {
3238     /* check the server control channel is still available */
3239     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
3240         return;
3241
3242     debugs(9, 3, HERE);
3243
3244     assert(ftpState->filepath != NULL);
3245     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
3246     ftpState->writeCommand(cbuf);
3247     ftpState->state = SENT_RETR;
3248 }
3249
3250 /// \ingroup ServerProtocolFTPInternal
3251 static void
3252 ftpReadRetr(FtpStateData * ftpState)
3253 {
3254     int code = ftpState->ctrl.replycode;
3255     debugs(9, 3, HERE);
3256
3257     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
3258         /* Begin data transfer */
3259         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
3260         ftpState->switchTimeoutToDataChannel();
3261         ftpState->maybeReadVirginBody();
3262         ftpState->state = READING_DATA;
3263     } else if (code == 150) {
3264         /* Accept data channel */
3265         ftpState->listenForDataChannel(ftpState->data.conn, ftpState->data.host);
3266     } else if (code >= 300) {
3267         if (!ftpState->flags.try_slash_hack) {
3268             /* Try this as a directory missing trailing slash... */
3269             ftpState->hackShortcut(ftpSendCwd);
3270         } else {
3271             ftpFail(ftpState);
3272         }
3273     } else {
3274         ftpFail(ftpState);
3275     }
3276 }
3277
3278 /**
3279  * Generate the HTTP headers and template fluff around an FTP
3280  * directory listing display.
3281  */
3282 void
3283 FtpStateData::completedListing()
3284 {
3285     assert(entry);
3286     entry->lock();
3287     ErrorState ferr(ERR_DIR_LISTING, HTTP_OK, request);
3288     ferr.ftp.listing = &listing;
3289     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
3290     ferr.ftp.server_msg = ctrl.message;
3291     ctrl.message = NULL;
3292     entry->replaceHttpReply( ferr.BuildHttpReply() );
3293     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
3294     entry->flush();
3295     entry->unlock();
3296 }
3297
3298 /// \ingroup ServerProtocolFTPInternal
3299 static void
3300 ftpReadTransferDone(FtpStateData * ftpState)
3301 {
3302     int code = ftpState->ctrl.replycode;
3303     debugs(9, 3, HERE);
3304
3305     if (code == 226 || code == 250) {
3306         /* Connection closed; retrieval done. */
3307         if (ftpState->flags.listing) {
3308             ftpState->completedListing();
3309             /* QUIT operation handles sending the reply to client */
3310         }
3311         ftpSendQuit(ftpState);
3312     } else {                    /* != 226 */
3313         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
3314         ftpState->failed(ERR_FTP_FAILURE, 0);
3315         /* failed closes ctrl.conn and frees ftpState */
3316         return;
3317     }
3318 }
3319
3320 // premature end of the request body
3321 void
3322 FtpStateData::handleRequestBodyProducerAborted()
3323 {
3324     ServerStateData::handleRequestBodyProducerAborted();
3325     debugs(9, 3, HERE << "ftpState=" << this);
3326     failed(ERR_READ_ERROR, 0);
3327 }
3328
3329 /**
3330  * This will be called when the put write is completed
3331  */
3332 void
3333 FtpStateData::sentRequestBody(const CommIoCbParams &io)
3334 {
3335     if (io.size > 0)
3336         kb_incr(&(statCounter.server.ftp.kbytes_out), io.size);
3337     ServerStateData::sentRequestBody(io);
3338 }
3339
3340 /// \ingroup ServerProtocolFTPInternal
3341 static void
3342 ftpWriteTransferDone(FtpStateData * ftpState)
3343 {
3344     int code = ftpState->ctrl.replycode;
3345     debugs(9, 3, HERE);
3346
3347     if (!(code == 226 || code == 250)) {
3348         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
3349         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
3350         return;
3351     }
3352
3353     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
3354     ftpSendReply(ftpState);
3355 }
3356
3357 /// \ingroup ServerProtocolFTPInternal
3358 static void
3359 ftpSendQuit(FtpStateData * ftpState)
3360 {
3361     /* check the server control channel is still available */
3362     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
3363         return;
3364
3365     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
3366     ftpState->writeCommand(cbuf);
3367     ftpState->state = SENT_QUIT;
3368 }
3369
3370 /**
3371  * \ingroup ServerProtocolFTPInternal
3372  *
3373  *  This completes a client FTP operation with success or other page
3374  *  generated and stored in the entry field by the code issuing QUIT.
3375  */
3376 static void
3377 ftpReadQuit(FtpStateData * ftpState)
3378 {
3379     ftpState->serverComplete();
3380 }
3381
3382 /// \ingroup ServerProtocolFTPInternal
3383 static void
3384 ftpTrySlashHack(FtpStateData * ftpState)
3385 {
3386     char *path;
3387     ftpState->flags.try_slash_hack = 1;
3388     /* Free old paths */
3389
3390     debugs(9, 3, HERE);
3391
3392     if (ftpState->pathcomps)
3393         wordlistDestroy(&ftpState->pathcomps);
3394
3395     safe_free(ftpState->filepath);
3396
3397     /* Build the new path (urlpath begins with /) */
3398     path = xstrdup(ftpState->request->urlpath.termedBuf());
3399
3400     rfc1738_unescape(path);
3401
3402     ftpState->filepath = path;
3403
3404     /* And off we go */
3405     ftpGetFile(ftpState);
3406 }
3407
3408 /**
3409  * Forget hack status. Next error is shown to the user
3410  */
3411 void
3412 FtpStateData::unhack()
3413 {
3414     debugs(9, 3, HERE);
3415
3416     if (old_request != NULL) {
3417         safe_free(old_request);
3418         safe_free(old_reply);
3419     }
3420 }
3421
3422 void
3423 FtpStateData::hackShortcut(FTPSM * nextState)
3424 {
3425     /* Clear some unwanted state */
3426     setCurrentOffset(0);
3427     restart_offset = 0;
3428     /* Save old error message & some state info */
3429
3430     debugs(9, 3, HERE);
3431
3432     if (old_request == NULL) {
3433         old_request = ctrl.last_command;
3434         ctrl.last_command = NULL;
3435         old_reply = ctrl.last_reply;
3436         ctrl.last_reply = NULL;
3437
3438         if (pathcomps == NULL && filepath != NULL)
3439             old_filepath = xstrdup(filepath);
3440     }
3441
3442     /* Jump to the "hack" state */
3443     nextState(this);
3444 }
3445
3446 /// \ingroup ServerProtocolFTPInternal
3447 static void
3448 ftpFail(FtpStateData *ftpState)
3449 {
3450     debugs(9, 6, HERE << "flags(" <<
3451            (ftpState->flags.isdir?"IS_DIR,":"") <<
3452            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
3453            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
3454            "slashhack=" << (ftpState->request->urlpath.caseCmp("/%2f", 4)==0? "T":"F") );
3455
3456     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
3457     if (!ftpState->flags.isdir &&       /* Not a directory */
3458             !ftpState->flags.try_slash_hack &&  /* Not in slash hack */
3459             ftpState->mdtm <= 0 && ftpState->theSize < 0 &&     /* Not known as a file */
3460             ftpState->request->urlpath.caseCmp("/%2f", 4) != 0) {       /* No slash encoded */
3461
3462         switch (ftpState->state) {
3463
3464         case SENT_CWD:
3465
3466         case SENT_RETR:
3467             /* Try the / hack */
3468             ftpState->hackShortcut(ftpTrySlashHack);
3469             return;
3470
3471         default:
3472             break;
3473         }
3474     }
3475
3476     ftpState->failed(ERR_NONE, 0);
3477     /* failed() closes ctrl.conn and frees this */
3478 }
3479
3480 void
3481 FtpStateData::failed(err_type error, int xerrno)
3482 {
3483     debugs(9,3,HERE << "entry-null=" << (entry?entry->isEmpty():0) << ", entry=" << entry);
3484     if (entry->isEmpty())
3485         failedErrorMessage(error, xerrno);
3486
3487     serverComplete();
3488 }
3489
3490 void
3491 FtpStateData::failedErrorMessage(err_type error, int xerrno)
3492 {
3493     ErrorState *ftperr = NULL;
3494     const char *command, *reply;
3495
3496     /* Translate FTP errors into HTTP errors */
3497     switch (error) {
3498
3499     case ERR_NONE:
3500
3501         switch (state) {
3502
3503         case SENT_USER:
3504
3505         case SENT_PASS:
3506
3507             if (ctrl.replycode > 500)
3508                 if (password_url)
3509                     ftperr = new ErrorState(ERR_FTP_FORBIDDEN, HTTP_FORBIDDEN, fwd->request);
3510                 else
3511                     ftperr = new ErrorState(ERR_FTP_FORBIDDEN, HTTP_UNAUTHORIZED, fwd->request);
3512
3513             else if (ctrl.replycode == 421)
3514                 ftperr = new ErrorState(ERR_FTP_UNAVAILABLE, HTTP_SERVICE_UNAVAILABLE, fwd->request);
3515
3516             break;
3517
3518         case SENT_CWD:
3519
3520         case SENT_RETR:
3521             if (ctrl.replycode == 550)
3522                 ftperr = new ErrorState(ERR_FTP_NOT_FOUND, HTTP_NOT_FOUND, fwd->request);
3523
3524             break;
3525
3526         default:
3527             break;
3528         }
3529
3530         break;
3531
3532     case ERR_READ_TIMEOUT:
3533         ftperr = new ErrorState(error, HTTP_GATEWAY_TIMEOUT, fwd->request);
3534         break;
3535
3536     default:
3537         ftperr = new ErrorState(error, HTTP_BAD_GATEWAY, fwd->request);
3538         break;
3539     }
3540
3541     if (ftperr == NULL)
3542         ftperr = new ErrorState(ERR_FTP_FAILURE, HTTP_BAD_GATEWAY, fwd->request);
3543
3544     ftperr->xerrno = xerrno;
3545
3546     ftperr->ftp.server_msg = ctrl.message;
3547     ctrl.message = NULL;
3548
3549     if (old_request)
3550         command = old_request;
3551     else
3552         command = ctrl.last_command;
3553
3554     if (command && strncmp(command, "PASS", 4) == 0)
3555         command = "PASS <yourpassword>";
3556
3557     if (old_reply)
3558         reply = old_reply;
3559     else
3560         reply = ctrl.last_reply;
3561
3562     if (command)
3563         ftperr->ftp.request = xstrdup(command);
3564
3565     if (reply)
3566         ftperr->ftp.reply = xstrdup(reply);
3567
3568     entry->replaceHttpReply( ftperr->BuildHttpReply() );
3569     delete ftperr;
3570 }
3571
3572 /// \ingroup ServerProtocolFTPInternal
3573 static void
3574 ftpSendReply(FtpStateData * ftpState)
3575 {
3576     int code = ftpState->ctrl.replycode;
3577     http_status http_code;
3578     err_type err_code = ERR_NONE;
3579
3580     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
3581
3582     if (cbdataReferenceValid(ftpState))
3583         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
3584
3585     if (code == 226 || code == 250) {
3586         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
3587         http_code = (ftpState->mdtm > 0) ? HTTP_ACCEPTED : HTTP_CREATED;
3588     } else if (code == 227) {
3589         err_code = ERR_FTP_PUT_CREATED;
3590         http_code = HTTP_CREATED;
3591     } else {
3592         err_code = ERR_FTP_PUT_ERROR;
3593         http_code = HTTP_INTERNAL_SERVER_ERROR;
3594     }
3595
3596     ErrorState err(err_code, http_code, ftpState->request);
3597
3598     if (ftpState->old_request)
3599         err.ftp.request = xstrdup(ftpState->old_request);
3600     else
3601         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
3602
3603     if (ftpState->old_reply)
3604         err.ftp.reply = xstrdup(ftpState->old_reply);
3605     else if (ftpState->ctrl.last_reply)
3606         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
3607     else
3608         err.ftp.reply = xstrdup("");
3609
3610     // TODO: interpret as FTP-specific error code
3611     err.detailError(code);
3612
3613     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
3614
3615     ftpSendQuit(ftpState);
3616 }
3617
3618 void
3619 FtpStateData::appendSuccessHeader()
3620 {
3621     const char *mime_type = NULL;
3622     const char *mime_enc = NULL;
3623     String urlpath = request->urlpath;
3624     const char *filename = NULL;
3625     const char *t = NULL;
3626
3627     debugs(9, 3, HERE);
3628
3629     if (flags.http_header_sent)
3630         return;
3631
3632     HttpReply *reply = new HttpReply;
3633
3634     flags.http_header_sent = 1;
3635
3636     assert(entry->isEmpty());
3637
3638     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
3639
3640     entry->buffer();    /* released when done processing current data payload */
3641
3642     filename = (t = urlpath.rpos('/')) ? t + 1 : urlpath.termedBuf();
3643
3644     if (flags.isdir) {
3645         mime_type = "text/html";
3646     } else {
3647         switch (typecode) {
3648
3649         case 'I':
3650             mime_type = "application/octet-stream";
3651             mime_enc = mimeGetContentEncoding(filename);
3652             break;
3653
3654         case 'A':
3655             mime_type = "text/plain";
3656             break;
3657
3658         default:
3659             mime_type = mimeGetContentType(filename);
3660             mime_enc = mimeGetContentEncoding(filename);
3661             break;
3662         }
3663     }
3664
3665     /* set standard stuff */
3666
3667     if (0 == getCurrentOffset()) {
3668         /* Full reply */
3669         reply->setHeaders(HTTP_OK, "Gatewaying", mime_type, theSize, mdtm, -2);
3670     } else if (theSize < getCurrentOffset()) {
3671         /*
3672          * DPW 2007-05-04
3673          * offset should not be larger than theSize.  We should
3674          * not be seeing this condition any more because we'll only
3675          * send REST if we know the theSize and if it is less than theSize.
3676          */
3677         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
3678                " current offset=" << getCurrentOffset() <<
3679                ", but theSize=" << theSize <<
3680                ".  assuming full content response");
3681         reply->setHeaders(HTTP_OK, "Gatewaying", mime_type, theSize, mdtm, -2);
3682     } else {
3683         /* Partial reply */
3684         HttpHdrRangeSpec range_spec;
3685         range_spec.offset = getCurrentOffset();
3686         range_spec.length = theSize - getCurrentOffset();
3687         reply->setHeaders(HTTP_PARTIAL_CONTENT, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
3688         httpHeaderAddContRange(&reply->header, range_spec, theSize);
3689     }
3690
3691     /* additional info */
3692     if (mime_enc)
3693         reply->header.putStr(HDR_CONTENT_ENCODING, mime_enc);
3694
3695     setVirginReply(reply);
3696     adaptOrFinalizeReply();
3697 }
3698
3699 void
3700 FtpStateData::haveParsedReplyHeaders()
3701 {
3702     ServerStateData::haveParsedReplyHeaders();
3703
3704     StoreEntry *e = entry;
3705
3706     e->timestampsSet();
3707
3708     if (flags.authenticated) {
3709         /*
3710          * Authenticated requests can't be cached.
3711          */
3712         e->release();
3713     } else if (EBIT_TEST(e->flags, ENTRY_CACHABLE) && !getCurrentOffset()) {
3714         e->setPublicKey();
3715     } else {
3716         e->release();
3717     }
3718 }
3719
3720 HttpReply *
3721 FtpStateData::ftpAuthRequired(HttpRequest * request, const char *realm)
3722 {
3723     ErrorState err(ERR_CACHE_ACCESS_DENIED, HTTP_UNAUTHORIZED, request);
3724     HttpReply *newrep = err.BuildHttpReply();
3725 #if HAVE_AUTH_MODULE_BASIC
3726     /* add Authenticate header */
3727     newrep->header.putAuth("Basic", realm);
3728 #endif
3729     return newrep;
3730 }
3731
3732 /**
3733  \ingroup ServerProtocolFTPAPI
3734  \todo Should be a URL class API call.
3735  *
3736  *  Construct an URI with leading / in PATH portion for use by CWD command
3737  *  possibly others. FTP encodes absolute paths as beginning with '/'
3738  *  after the initial URI path delimiter, which happens to be / itself.
3739  *  This makes FTP absolute URI appear as:  ftp:host:port//root/path
3740  *  To encompass older software which compacts multiple // to / in transit
3741  *  We use standard URI-encoding on the second / making it
3742  *  ftp:host:port/%2froot/path  AKA 'the FTP %2f hack'.
3743  */
3744 const char *
3745 ftpUrlWith2f(HttpRequest * request)
3746 {
3747     String newbuf = "%2f";
3748
3749     if (request->protocol != AnyP::PROTO_FTP)
3750         return NULL;
3751
3752     if ( request->urlpath[0]=='/' ) {
3753         newbuf.append(request->urlpath);
3754         request->urlpath.absorb(newbuf);
3755         safe_free(request->canonical);
3756     } else if ( !strncmp(request->urlpath.termedBuf(), "%2f", 3) ) {
3757         newbuf.append(request->urlpath.substr(1,request->urlpath.size()));
3758         request->urlpath.absorb(newbuf);
3759         safe_free(request->canonical);
3760     }
3761
3762     return urlCanonical(request);
3763 }
3764
3765 void
3766 FtpStateData::printfReplyBody(const char *fmt, ...)
3767 {
3768     va_list args;
3769     va_start (args, fmt);
3770     static char buf[4096];
3771     buf[0] = '\0';
3772     vsnprintf(buf, 4096, fmt, args);
3773     writeReplyBody(buf, strlen(buf));
3774 }
3775
3776 /**
3777  * Call this when there is data from the origin server
3778  * which should be sent to either StoreEntry, or to ICAP...
3779  */
3780 void
3781 FtpStateData::writeReplyBody(const char *dataToWrite, size_t dataLength)
3782 {
3783     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
3784     addVirginReplyBody(dataToWrite, dataLength);
3785 }
3786
3787 /**
3788  * called after we wrote the last byte of the request body
3789  */
3790 void
3791 FtpStateData::doneSendingRequestBody()
3792 {
3793     ServerStateData::doneSendingRequestBody();
3794     debugs(9,3, HERE);
3795     dataComplete();
3796     /* NP: RFC 959  3.3.  DATA CONNECTION MANAGEMENT
3797      * if transfer type is 'stream' call dataComplete()
3798      * otherwise leave open. (reschedule control channel read?)
3799      */
3800 }
3801
3802 /**
3803  * A hack to ensure we do not double-complete on the forward entry.
3804  *
3805  \todo FtpStateData logic should probably be rewritten to avoid
3806  *      double-completion or FwdState should be rewritten to allow it.
3807  */
3808 void
3809 FtpStateData::completeForwarding()
3810 {
3811     if (fwd == NULL || flags.completed_forwarding) {
3812         debugs(9, 3, HERE << "completeForwarding avoids " <<
3813                "double-complete on FD " << ctrl.conn->fd << ", Data FD " << data.conn->fd <<
3814                ", this " << this << ", fwd " << fwd);
3815         return;
3816     }
3817
3818     flags.completed_forwarding = true;
3819     ServerStateData::completeForwarding();
3820 }
3821
3822 /**
3823  * Close the FTP server connection(s). Used by serverComplete().
3824  */
3825 void
3826 FtpStateData::closeServer()
3827 {
3828     if (Comm::IsConnOpen(ctrl.conn)) {
3829         debugs(9,3, HERE << "closing FTP server FD " << ctrl.conn->fd << ", this " << this);
3830         fwd->unregister(ctrl.conn);
3831         ctrl.close();
3832     }
3833
3834     if (Comm::IsConnOpen(data.conn)) {
3835         debugs(9,3, HERE << "closing FTP data FD " << data.conn->fd << ", this " << this);
3836         data.close();
3837     }
3838
3839     debugs(9,3, HERE << "FTP ctrl and data connections closed. this " << this);
3840 }
3841
3842 /**
3843  * Did we close all FTP server connection(s)?
3844  *
3845  \retval true   Both server control and data channels are closed. And not waiting for a new data connection to open.
3846  \retval false  Either control channel or data is still active.
3847  */
3848 bool
3849 FtpStateData::doneWithServer() const
3850 {
3851     return !Comm::IsConnOpen(ctrl.conn) && !Comm::IsConnOpen(data.conn);
3852 }
3853
3854 /**
3855  * Have we lost the FTP server control channel?
3856  *
3857  \retval true   The server control channel is available.
3858  \retval false  The server control channel is not available.
3859  */
3860 bool
3861 FtpStateData::haveControlChannel(const char *caller_name) const
3862 {
3863     if (doneWithServer())
3864         return false;
3865
3866     /* doneWithServer() only checks BOTH channels are closed. */
3867     if (!Comm::IsConnOpen(ctrl.conn)) {
3868         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
3869         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
3870         return false;
3871     }
3872
3873     return true;
3874 }
3875
3876 /**
3877  * Quickly abort the transaction
3878  *
3879  \todo destruction should be sufficient as the destructor should cleanup,
3880  *      including canceling close handlers
3881  */
3882 void
3883 FtpStateData::abortTransaction(const char *reason)
3884 {
3885     debugs(9, 3, HERE << "aborting transaction for " << reason <<
3886            "; FD " << (ctrl.conn!=NULL?ctrl.conn->fd:-1) << ", Data FD " << (data.conn!=NULL?data.conn->fd:-1) << ", this " << this);
3887     if (Comm::IsConnOpen(ctrl.conn)) {
3888         ctrl.conn->close();
3889         return;
3890     }
3891
3892     fwd->handleUnregisteredServerEnd();
3893     mustStop("FtpStateData::abortTransaction");
3894 }
3895
3896 /// creates a data channel Comm close callback
3897 AsyncCall::Pointer
3898 FtpStateData::dataCloser()
3899 {
3900     typedef CommCbMemFunT<FtpStateData, CommCloseCbParams> Dialer;
3901     return JobCallback(9, 5, Dialer, this, FtpStateData::dataClosed);
3902 }
3903
3904 /// configures the channel with a descriptor and registers a close handler
3905 void
3906 FtpChannel::opened(const Comm::ConnectionPointer &newConn, const AsyncCall::Pointer &aCloser)
3907 {
3908     assert(!Comm::IsConnOpen(conn));
3909     assert(closer == NULL);
3910
3911     assert(Comm::IsConnOpen(newConn));
3912     assert(aCloser != NULL);
3913
3914     conn = newConn;
3915     closer = aCloser;
3916     comm_add_close_handler(conn->fd, closer);
3917 }
3918
3919 /// planned close: removes the close handler and calls comm_close
3920 void
3921 FtpChannel::close()
3922 {
3923     // channels with active listeners will be closed when the listener handler dies.
3924     if (Comm::IsConnOpen(conn)) {
3925         comm_remove_close_handler(conn->fd, closer);
3926         conn->close(); // we do not expect to be called back
3927     }
3928     clear();
3929 }
3930
3931 void
3932 FtpChannel::clear()
3933 {
3934     conn = NULL;
3935     closer = NULL;
3936 }