src/clients/FtpGateway.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "clients/forward.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm.h"
  16 #include "comm/ConnOpener.h"
  17 #include "comm/Read.h"
  18 #include "comm/TcpAcceptor.h"
  19 #include "CommCalls.h"
  20 #include "compat/strtoll.h"
  21 #include "errorpage.h"
  22 #include "fd.h"
  23 #include "fde.h"
  24 #include "FwdState.h"
  25 #include "html_quote.h"
  26 #include "HttpHdrContRange.h"
  27 #include "HttpHeader.h"
  28 #include "HttpHeaderRange.h"
  29 #include "HttpReply.h"
  30 #include "ip/tools.h"
  31 #include "MemBuf.h"
  32 #include "mime.h"
  33 #include "rfc1738.h"
  34 #include "SquidConfig.h"
  35 #include "SquidString.h"
  36 #include "SquidTime.h"
  37 #include "StatCounters.h"
  38 #include "Store.h"
  39 #include "tools.h"
  40 #include "URL.h"
  41 #include "util.h"
  42 #include "wordlist.h"
  43
  44 #if USE_DELAY_POOLS
  45 #include "DelayPools.h"
  46 #include "MemObject.h"
  47 #endif
  48
  49 #include <cerrno>
  50
  51 namespace Ftp
  52 {
  53
  54 struct GatewayFlags {
  55
  56     /* passive mode */
  57     bool pasv_supported;  ///< PASV command is allowed
  58     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
  59     bool pasv_only;
  60     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
  61
  62     /* authentication */
  63     bool authenticated;         ///< authentication success
  64     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
  65     bool tried_auth_nopass;     ///< auth tried username with no password already.
  66
  67     /* other */
  68     bool isdir;
  69     bool skip_whitespace;
  70     bool rest_supported;
  71     bool http_header_sent;
  72     bool tried_nlst;
  73     bool need_base_href;
  74     bool dir_slash;
  75     bool root_dir;
  76     bool no_dotdot;
  77     bool binary;
  78     bool try_slash_hack;
  79     bool put;
  80     bool put_mkdir;
  81     bool listformat_unknown;
  82     bool listing;
  83     bool completed_forwarding;
  84 };
  85
  86 class Gateway;
  87 typedef void (StateMethod)(Ftp::Gateway *);
  88
  89 /// FTP Gateway: An FTP client that takes an HTTP request with an ftp:// URI,
  90 /// converts it into one or more FTP commands, and then
  91 /// converts one or more FTP responses into the final HTTP response.
  92 class Gateway : public Ftp::Client
  93 {
  94     CBDATA_CLASS(Gateway);
  95
  96 public:
  97     Gateway(FwdState *);
  98     virtual ~Gateway();
  99     char user[MAX_URL];
 100     char password[MAX_URL];
 101     int password_url;
 102     char *reply_hdr;
 103     int reply_hdr_state;
 104     String clean_url;
 105     String title_url;
 106     String base_href;
 107     int conn_att;
 108     int login_att;
 109     time_t mdtm;
 110     int64_t theSize;
 111     wordlist *pathcomps;
 112     char *filepath;
 113     char *dirpath;
 114     int64_t restart_offset;
 115     char *proxy_host;
 116     size_t list_width;
 117     String cwd_message;
 118     char *old_filepath;
 119     char typecode;
 120     MemBuf listing;     ///< FTP directory listing in HTML format.
 121
 122     GatewayFlags flags;
 123
 124 public:
 125     // these should all be private
 126     virtual void start();
 127     virtual Http::StatusCode failedHttpStatus(err_type &error);
 128     int restartable();
 129     void appendSuccessHeader();
 130     void hackShortcut(StateMethod *nextState);
 131     void unhack();
 132     void readStor();
 133     void parseListing();
 134     MemBuf *htmlifyListEntry(const char *line);
 135     void completedListing(void);
 136
 137     /// create a data channel acceptor and start listening.
 138     void listenForDataChannel(const Comm::ConnectionPointer &conn);
 139
 140     int checkAuth(const HttpHeader * req_hdr);
 141     void checkUrlpath();
 142     void buildTitleUrl();
 143     void writeReplyBody(const char *, size_t len);
 144     void printfReplyBody(const char *fmt, ...);
 145     virtual void completeForwarding();
 146     void processHeadResponse();
 147     void processReplyBody();
 148     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 149     int64_t getCurrentOffset() const { return currentOffset; }
 150
 151     virtual void dataChannelConnected(const CommConnectCbParams &io);
 152     static PF ftpDataWrite;
 153     virtual void timeout(const CommTimeoutCbParams &io);
 154     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 155
 156     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 157     const char *ftpRealm(void);
 158     void loginFailed(void);
 159
 160     virtual void haveParsedReplyHeaders();
 161
 162     virtual bool haveControlChannel(const char *caller_name) const;
 163
 164 protected:
 165     virtual void handleControlReply();
 166     virtual void dataClosed(const CommCloseCbParams &io);
 167
 168 private:
 169     virtual bool mayReadVirginReplyBody() const;
 170     // BodyConsumer for HTTP: consume request body.
 171     virtual void handleRequestBodyProducerAborted();
 172
 173     void loginParser(const SBuf &login, bool escaped);
 174 };
 175
 176 } // namespace Ftp
 177
 178 typedef Ftp::StateMethod FTPSM; // to avoid lots of non-changes
 179
 180 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Gateway);
 181
 182 typedef struct {
 183     char type;
 184     int64_t size;
 185     char *date;
 186     char *name;
 187     char *showname;
 188     char *link;
 189 } ftpListParts;
 190
 191 #define CTRL_BUFLEN 16*1024
 192 static char cbuf[CTRL_BUFLEN];
 193
 194 /*
 195  * State machine functions
 196  * send == state transition
 197  * read == wait for response, and select next state transition
 198  * other == Transition logic
 199  */
 200 static FTPSM ftpReadWelcome;
 201 static FTPSM ftpSendUser;
 202 static FTPSM ftpReadUser;
 203 static FTPSM ftpSendPass;
 204 static FTPSM ftpReadPass;
 205 static FTPSM ftpSendType;
 206 static FTPSM ftpReadType;
 207 static FTPSM ftpSendMdtm;
 208 static FTPSM ftpReadMdtm;
 209 static FTPSM ftpSendSize;
 210 static FTPSM ftpReadSize;
 211 #if 0
 212 static FTPSM ftpSendEPRT;
 213 #endif
 214 static FTPSM ftpReadEPRT;
 215 static FTPSM ftpSendPORT;
 216 static FTPSM ftpReadPORT;
 217 static FTPSM ftpSendPassive;
 218 static FTPSM ftpReadEPSV;
 219 static FTPSM ftpReadPasv;
 220 static FTPSM ftpTraverseDirectory;
 221 static FTPSM ftpListDir;
 222 static FTPSM ftpGetFile;
 223 static FTPSM ftpSendCwd;
 224 static FTPSM ftpReadCwd;
 225 static FTPSM ftpRestOrList;
 226 static FTPSM ftpSendList;
 227 static FTPSM ftpSendNlst;
 228 static FTPSM ftpReadList;
 229 static FTPSM ftpSendRest;
 230 static FTPSM ftpReadRest;
 231 static FTPSM ftpSendRetr;
 232 static FTPSM ftpReadRetr;
 233 static FTPSM ftpReadTransferDone;
 234 static FTPSM ftpSendStor;
 235 static FTPSM ftpReadStor;
 236 static FTPSM ftpWriteTransferDone;
 237 static FTPSM ftpSendReply;
 238 static FTPSM ftpSendMkdir;
 239 static FTPSM ftpReadMkdir;
 240 static FTPSM ftpFail;
 241 static FTPSM ftpSendQuit;
 242 static FTPSM ftpReadQuit;
 243
 244 /************************************************
 245 ** Debugs Levels used here                     **
 246 *************************************************
 247 0   CRITICAL Events
 248 1   IMPORTANT Events
 249     Protocol and Transmission failures.
 250 2   FTP Protocol Chatter
 251 3   Logic Flows
 252 4   Data Parsing Flows
 253 5   Data Dumps
 254 7   ??
 255 ************************************************/
 256
 257 /************************************************
 258 ** State Machine Description (excluding hacks) **
 259 *************************************************
 260 From            To
 261 ---------------------------------------
 262 Welcome         User
 263 User            Pass
 264 Pass            Type
 265 Type            TraverseDirectory / GetFile
 266 TraverseDirectory   Cwd / GetFile / ListDir
 267 Cwd         TraverseDirectory / Mkdir
 268 GetFile         Mdtm
 269 Mdtm            Size
 270 Size            Epsv
 271 ListDir         Epsv
 272 Epsv            FileOrList
 273 FileOrList      Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 274 Rest            Retr
 275 Retr / Nlst / List  DataRead* (on datachannel)
 276 DataRead*       ReadTransferDone
 277 ReadTransferDone    DataTransferDone
 278 Stor            DataWrite* (on datachannel)
 279 DataWrite*      RequestPutBody** (from client)
 280 RequestPutBody**    DataWrite* / WriteTransferDone
 281 WriteTransferDone   DataTransferDone
 282 DataTransferDone    Quit
 283 Quit            -
 284 ************************************************/
 285
 286 FTPSM *FTP_SM_FUNCS[] = {
 287     ftpReadWelcome,     /* BEGIN */
 288     ftpReadUser,        /* SENT_USER */
 289     ftpReadPass,        /* SENT_PASS */
 290     ftpReadType,        /* SENT_TYPE */
 291     ftpReadMdtm,        /* SENT_MDTM */
 292     ftpReadSize,        /* SENT_SIZE */
 293     ftpReadEPRT,        /* SENT_EPRT */
 294     ftpReadPORT,        /* SENT_PORT */
 295     ftpReadEPSV,        /* SENT_EPSV_ALL */
 296     ftpReadEPSV,        /* SENT_EPSV_1 */
 297     ftpReadEPSV,        /* SENT_EPSV_2 */
 298     ftpReadPasv,        /* SENT_PASV */
 299     ftpReadCwd,     /* SENT_CWD */
 300     ftpReadList,        /* SENT_LIST */
 301     ftpReadList,        /* SENT_NLST */
 302     ftpReadRest,        /* SENT_REST */
 303     ftpReadRetr,        /* SENT_RETR */
 304     ftpReadStor,        /* SENT_STOR */
 305     ftpReadQuit,        /* SENT_QUIT */
 306     ftpReadTransferDone,    /* READING_DATA (RETR,LIST,NLST) */
 307     ftpWriteTransferDone,   /* WRITING_DATA (STOR) */
 308     ftpReadMkdir,       /* SENT_MKDIR */
 309     NULL,           /* SENT_FEAT */
 310     NULL,           /* SENT_PWD */
 311     NULL,           /* SENT_CDUP*/
 312     NULL,           /* SENT_DATA_REQUEST */
 313     NULL            /* SENT_COMMAND */
 314 };
 315
 316 /// handler called by Comm when FTP data channel is closed unexpectedly
 317 void
 318 Ftp::Gateway::dataClosed(const CommCloseCbParams &io)
 319 {
 320     Ftp::Client::dataClosed(io);
 321     failed(ERR_FTP_FAILURE, 0);
 322     /* failed closes ctrl.conn and frees ftpState */
 323
 324     /* NP: failure recovery may be possible when its only a data.conn failure.
 325      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 326      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 327      */
 328 }
 329
 330 Ftp::Gateway::Gateway(FwdState *fwdState):
 331     AsyncJob("FtpStateData"),
 332     Ftp::Client(fwdState),
 333     password_url(0),
 334     reply_hdr(NULL),
 335     reply_hdr_state(0),
 336     conn_att(0),
 337     login_att(0),
 338     mdtm(-1),
 339     theSize(-1),
 340     pathcomps(NULL),
 341     filepath(NULL),
 342     dirpath(NULL),
 343     restart_offset(0),
 344     proxy_host(NULL),
 345     list_width(0),
 346     old_filepath(NULL),
 347     typecode('\0')
 348 {
 349     debugs(9, 3, entry->url());
 350
 351     *user = 0;
 352     *password = 0;
 353     memset(&flags, 0, sizeof(flags));
 354
 355     if (Config.Ftp.passive && !flags.pasv_failed)
 356         flags.pasv_supported = 1;
 357
 358     flags.rest_supported = 1;
 359
 360     if (request->method == Http::METHOD_PUT)
 361         flags.put = 1;
 362
 363     initReadBuf();
 364 }
 365
 366 Ftp::Gateway::~Gateway()
 367 {
 368     debugs(9, 3, entry->url());
 369
 370     if (Comm::IsConnOpen(ctrl.conn)) {
 371         debugs(9, DBG_IMPORTANT, "Internal bug: FTP Gateway left open " <<
 372                "control channel " << ctrl.conn);
 373     }
 374
 375     if (reply_hdr) {
 376         memFree(reply_hdr, MEM_8K_BUF);
 377         reply_hdr = NULL;
 378     }
 379
 380     if (pathcomps)
 381         wordlistDestroy(&pathcomps);
 382
 383     cwd_message.clean();
 384     xfree(old_filepath);
 385     title_url.clean();
 386     base_href.clean();
 387     xfree(filepath);
 388     xfree(dirpath);
 389 }
 390
 391 /**
 392  * Parse a possible login username:password pair.
 393  * Produces filled member variables user, password, password_url if anything found.
 394  *
 395  * \param login    a decoded Basic authentication credential token or URI user-info token
 396  * \param escaped  whether to URL-decode the token after extracting user and password
 397  */
 398 void
 399 Ftp::Gateway::loginParser(const SBuf &login, bool escaped)
 400 {
 401     debugs(9, 4, "login=" << login << ", escaped=" << escaped);
 402     debugs(9, 9, "IN : login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 403
 404     if (login.isEmpty())
 405         return;
 406
 407     const SBuf::size_type colonPos = login.find(':');
 408
 409     /* If there was a username part with at least one character use it.
 410      * Ignore 0-length username portion, retain what we have already.
 411      */
 412     if (colonPos == SBuf::npos || colonPos > 0) {
 413         const SBuf userName = login.substr(0, colonPos);
 414         SBuf::size_type upto = userName.copy(user, sizeof(user)-1);
 415         user[upto]='\0';
 416         debugs(9, 9, "found user=" << userName << ' ' <<
 417                (upto != userName.length() ? ", truncated-to=" : ", length=") << upto <<
 418                ", escaped=" << escaped);
 419         if (escaped)
 420             rfc1738_unescape(user);
 421         debugs(9, 9, "found user=" << user << " (" << strlen(user) << ") unescaped.");
 422     }
 423
 424     /* If there was a password part.
 425      * For 0-length password clobber what we have already, this means explicitly none
 426      */
 427     if (colonPos != SBuf::npos) {
 428         const SBuf pass = login.substr(colonPos+1, SBuf::npos);
 429         SBuf::size_type upto = pass.copy(password, sizeof(password)-1);
 430         password[upto]='\0';
 431         debugs(9, 9, "found password=" << pass << " " <<
 432                (upto != pass.length() ? ", truncated-to=" : ", length=") << upto <<
 433                ", escaped=" << escaped);
 434         if (escaped) {
 435             rfc1738_unescape(password);
 436             password_url = 1;
 437         }
 438         debugs(9, 9, "found password=" << password << " (" << strlen(password) << ") unescaped.");
 439     }
 440
 441     debugs(9, 9, "OUT: login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 442 }
 443
 444 void
 445 Ftp::Gateway::listenForDataChannel(const Comm::ConnectionPointer &conn)
 446 {
 447     if (!Comm::IsConnOpen(ctrl.conn)) {
 448         debugs(9, 5, "The control connection to the remote end is closed");
 449         return;
 450     }
 451
 452     assert(!Comm::IsConnOpen(data.conn));
 453
 454     typedef CommCbMemFunT<Gateway, CommAcceptCbParams> AcceptDialer;
 455     typedef AsyncCallT<AcceptDialer> AcceptCall;
 456     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, Ftp::Gateway::ftpAcceptDataConnection));
 457     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 458     const char *note = entry->url();
 459
 460     /* open the conn if its not already open */
 461     if (!Comm::IsConnOpen(conn)) {
 462         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 463         if (!Comm::IsConnOpen(conn)) {
 464             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 465             return;
 466         }
 467         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 468     }
 469
 470     conn->tos = ctrl.conn->tos;
 471     conn->nfmark = ctrl.conn->nfmark;
 472
 473     assert(Comm::IsConnOpen(conn));
 474     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 475
 476     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 477     data.opened(conn, dataCloser());
 478     switchTimeoutToDataChannel();
 479 }
 480
 481 void
 482 Ftp::Gateway::timeout(const CommTimeoutCbParams &io)
 483 {
 484     if (SENT_PASV == state) {
 485         /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */
 486         flags.pasv_supported = false;
 487         debugs(9, DBG_IMPORTANT, "FTP Gateway timeout in SENT_PASV state");
 488
 489         // cancel the data connection setup.
 490         if (data.opener != NULL) {
 491             data.opener->cancel("timeout");
 492             data.opener = NULL;
 493         }
 494         data.close();
 495     }
 496
 497     Ftp::Client::timeout(io);
 498 }
 499
 500 static const char *Month[] = {
 501     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 502     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 503 };
 504
 505 static int
 506 is_month(const char *buf)
 507 {
 508     int i;
 509
 510     for (i = 0; i < 12; ++i)
 511         if (!strcasecmp(buf, Month[i]))
 512             return 1;
 513
 514     return 0;
 515 }
 516
 517 static void
 518 ftpListPartsFree(ftpListParts ** parts)
 519 {
 520     safe_free((*parts)->date);
 521     safe_free((*parts)->name);
 522     safe_free((*parts)->showname);
 523     safe_free((*parts)->link);
 524     safe_free(*parts);
 525 }
 526
 527 #define MAX_TOKENS 64
 528
 529 static ftpListParts *
 530 ftpListParseParts(const char *buf, struct Ftp::GatewayFlags flags)
 531 {
 532     ftpListParts *p = NULL;
 533     char *t = NULL;
 534     const char *ct = NULL;
 535     char *tokens[MAX_TOKENS];
 536     int i;
 537     int n_tokens;
 538     static char tbuf[128];
 539     char *xbuf = NULL;
 540     static int scan_ftp_initialized = 0;
 541     static regex_t scan_ftp_integer;
 542     static regex_t scan_ftp_time;
 543     static regex_t scan_ftp_dostime;
 544     static regex_t scan_ftp_dosdate;
 545
 546     if (!scan_ftp_initialized) {
 547         scan_ftp_initialized = 1;
 548         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 549         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 550         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 551         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 552     }
 553
 554     if (buf == NULL)
 555         return NULL;
 556
 557     if (*buf == '\0')
 558         return NULL;
 559
 560     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 561
 562     n_tokens = 0;
 563
 564     memset(tokens, 0, sizeof(tokens));
 565
 566     xbuf = xstrdup(buf);
 567
 568     if (flags.tried_nlst) {
 569         /* Machine readable format, one name per line */
 570         p->name = xbuf;
 571         p->type = '\0';
 572         return p;
 573     }
 574
 575     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 576         tokens[n_tokens] = xstrdup(t);
 577         ++n_tokens;
 578     }
 579
 580     xfree(xbuf);
 581
 582     /* locate the Month field */
 583     for (i = 3; i < n_tokens - 2; ++i) {
 584         char *size = tokens[i - 1];
 585         char *month = tokens[i];
 586         char *day = tokens[i + 1];
 587         char *year = tokens[i + 2];
 588
 589         if (!is_month(month))
 590             continue;
 591
 592         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 593             continue;
 594
 595         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 596             continue;
 597
 598         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0) /* Yr | hh:mm */
 599             continue;
 600
 601         snprintf(tbuf, 128, "%s %2s %5s",
 602                  month, day, year);
 603
 604         if (!strstr(buf, tbuf))
 605             snprintf(tbuf, 128, "%s %2s %-5s",
 606                      month, day, year);
 607
 608         char const *copyFrom = NULL;
 609
 610         if ((copyFrom = strstr(buf, tbuf))) {
 611             p->type = *tokens[0];
 612             p->size = strtoll(size, NULL, 10);
 613             p->date = xstrdup(tbuf);
 614
 615             if (flags.skip_whitespace) {
 616                 copyFrom += strlen(tbuf);
 617
 618                 while (strchr(w_space, *copyFrom))
 619                     ++copyFrom;
 620             } else {
 621                 /* XXX assumes a single space between date and filename
 622                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 623                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 624                 copyFrom += strlen(tbuf) + 1;
 625             }
 626
 627             p->name = xstrdup(copyFrom);
 628
 629             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 630                 *t = '\0';
 631                 p->link = xstrdup(t + 4);
 632             }
 633
 634             goto found;
 635         }
 636
 637         break;
 638     }
 639
 640     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 641     if (n_tokens > 3 &&
 642             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 643             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 644         if (!strcasecmp(tokens[2], "<dir>")) {
 645             p->type = 'd';
 646         } else {
 647             p->type = '-';
 648             p->size = strtoll(tokens[2], NULL, 10);
 649         }
 650
 651         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 652         p->date = xstrdup(tbuf);
 653
 654         if (p->type == 'd') {
 655             /* Directory.. name begins with first printable after <dir> */
 656             ct = strstr(buf, tokens[2]);
 657             ct += strlen(tokens[2]);
 658
 659             while (xisspace(*ct))
 660                 ++ct;
 661
 662             if (!*ct)
 663                 ct = NULL;
 664         } else {
 665             /* A file. Name begins after size, with a space in between */
 666             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 667             ct = strstr(buf, tbuf);
 668
 669             if (ct) {
 670                 ct += strlen(tokens[2]) + 2;
 671             }
 672         }
 673
 674         p->name = xstrdup(ct ? ct : tokens[3]);
 675         goto found;
 676     }
 677
 678     /* Try EPLF format; carson@lehman.com */
 679     if (buf[0] == '+') {
 680         ct = buf + 1;
 681         p->type = 0;
 682
 683         while (ct && *ct) {
 684             time_t tm;
 685             int l = strcspn(ct, ",");
 686             char *tmp;
 687
 688             if (l < 1)
 689                 goto blank;
 690
 691             switch (*ct) {
 692
 693             case '\t':
 694                 p->name = xstrndup(ct + 1, l + 1);
 695                 break;
 696
 697             case 's':
 698                 p->size = atoi(ct + 1);
 699                 break;
 700
 701             case 'm':
 702                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 703
 704                 if (tmp != ct + 1)
 705                     break;  /* not a valid integer */
 706
 707                 p->date = xstrdup(ctime(&tm));
 708
 709                 *(strstr(p->date, "\n")) = '\0';
 710
 711                 break;
 712
 713             case '/':
 714                 p->type = 'd';
 715
 716                 break;
 717
 718             case 'r':
 719                 p->type = '-';
 720
 721                 break;
 722
 723             case 'i':
 724                 break;
 725
 726             default:
 727                 break;
 728             }
 729
 730 blank:
 731             ct = strstr(ct, ",");
 732
 733             if (ct) {
 734                 ++ct;
 735             }
 736         }
 737
 738         if (p->type == 0) {
 739             p->type = '-';
 740         }
 741
 742         if (p->name)
 743             goto found;
 744         else
 745             safe_free(p->date);
 746     }
 747
 748 found:
 749
 750     for (i = 0; i < n_tokens; ++i)
 751         xfree(tokens[i]);
 752
 753     if (!p->name)
 754         ftpListPartsFree(&p);   /* cleanup */
 755
 756     return p;
 757 }
 758
 759 MemBuf *
 760 Ftp::Gateway::htmlifyListEntry(const char *line)
 761 {
 762     char icon[2048];
 763     char href[2048 + 40];
 764     char text[ 2048];
 765     char size[ 2048];
 766     char chdir[ 2048 + 40];
 767     char view[ 2048 + 40];
 768     char download[ 2048 + 40];
 769     char link[ 2048 + 40];
 770     MemBuf *html;
 771     char prefix[2048];
 772     ftpListParts *parts;
 773     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 774
 775     debugs(9, 7, HERE << " line ={" << line << "}");
 776
 777     if (strlen(line) > 1024) {
 778         html = new MemBuf();
 779         html->init();
 780         html->appendf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 781         return html;
 782     }
 783
 784     if (flags.dir_slash && dirpath && typecode != 'D')
 785         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 786     else
 787         prefix[0] = '\0';
 788
 789     if ((parts = ftpListParseParts(line, flags)) == NULL) {
 790         const char *p;
 791
 792         html = new MemBuf();
 793         html->init();
 794         html->appendf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
 795
 796         for (p = line; *p && xisspace(*p); ++p);
 797         if (*p && !xisspace(*p))
 798             flags.listformat_unknown = 1;
 799
 800         return html;
 801     }
 802
 803     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
 804         ftpListPartsFree(&parts);
 805         return NULL;
 806     }
 807
 808     parts->size += 1023;
 809     parts->size >>= 10;
 810     parts->showname = xstrdup(parts->name);
 811
 812     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
 813     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
 814
 815     xstrncpy(text, parts->showname, 2048);
 816
 817     switch (parts->type) {
 818
 819     case 'd':
 820         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 821                  mimeGetIconURL("internal-dir"),
 822                  "[DIR]");
 823         strcat(href, "/");  /* margin is allocated above */
 824         break;
 825
 826     case 'l':
 827         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 828                  mimeGetIconURL("internal-link"),
 829                  "[LINK]");
 830         /* sometimes there is an 'l' flag, but no "->" link */
 831
 832         if (parts->link) {
 833             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
 834             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
 835                      *link2 != '/' ? prefix : "", link2,
 836                      html_quote(parts->link));
 837             safe_free(link2);
 838         }
 839
 840         break;
 841
 842     case '\0':
 843         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 844                  mimeGetIconURL(parts->name),
 845                  "[UNKNOWN]");
 846         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
 847                  "alt=\"[DIR]\"></a>",
 848                  rfc1738_escape_part(parts->name),
 849                  mimeGetIconURL("internal-dir"));
 850         break;
 851
 852     case '-':
 853
 854     default:
 855         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 856                  mimeGetIconURL(parts->name),
 857                  "[FILE]");
 858         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
 859         break;
 860     }
 861
 862     if (parts->type != 'd') {
 863         if (mimeGetViewOption(parts->name)) {
 864             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
 865                      "alt=\"[VIEW]\"></a>",
 866                      prefix, href, mimeGetIconURL("internal-view"));
 867         }
 868
 869         if (mimeGetDownloadOption(parts->name)) {
 870             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
 871                      "alt=\"[DOWNLOAD]\"></a>",
 872                      prefix, href, mimeGetIconURL("internal-download"));
 873         }
 874     }
 875
 876     /* construct the table row from parts. */
 877     html = new MemBuf();
 878     html->init();
 879     html->appendf("<tr class=\"entry\">"
 880                   "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
 881                   "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
 882                   "<td class=\"date\">%s</td>"
 883                   "<td class=\"size\">%s</td>"
 884                   "<td class=\"actions\">%s%s%s%s</td>"
 885                   "</tr>\n",
 886                   prefix, href, icon,
 887                   prefix, href, html_quote(text),
 888                   parts->date,
 889                   size,
 890                   chdir, view, download, link);
 891
 892     ftpListPartsFree(&parts);
 893     return html;
 894 }
 895
 896 void
 897 Ftp::Gateway::parseListing()
 898 {
 899     char *buf = data.readBuf->content();
 900     char *sbuf;         /* NULL-terminated copy of termedBuf */
 901     char *end;
 902     char *line;
 903     char *s;
 904     MemBuf *t;
 905     size_t linelen;
 906     size_t usable;
 907     size_t len = data.readBuf->contentSize();
 908
 909     if (!len) {
 910         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
 911         return;
 912     }
 913
 914     /*
 915      * We need a NULL-terminated buffer for scanning, ick
 916      */
 917     sbuf = (char *)xmalloc(len + 1);
 918     xstrncpy(sbuf, buf, len + 1);
 919     end = sbuf + len - 1;
 920
 921     while (*end != '\r' && *end != '\n' && end > sbuf)
 922         --end;
 923
 924     usable = end - sbuf;
 925
 926     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
 927
 928     if (usable == 0) {
 929         if (buf[0] == '\0' && len == 1) {
 930             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
 931             data.readBuf->consume(len);
 932         } else {
 933             debugs(9, 3, HERE << "didn't find end for " << entry->url());
 934             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
 935         }
 936         xfree(sbuf);
 937         return;
 938     }
 939
 940     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
 941
 942     line = (char *)memAllocate(MEM_4K_BUF);
 943     ++end;
 944     s = sbuf;
 945     s += strspn(s, crlf);
 946
 947     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
 948         debugs(9, 7, HERE << "s = {" << s << "}");
 949         linelen = strcspn(s, crlf) + 1;
 950
 951         if (linelen < 2)
 952             break;
 953
 954         if (linelen > 4096)
 955             linelen = 4096;
 956
 957         xstrncpy(line, s, linelen);
 958
 959         debugs(9, 7, HERE << "{" << line << "}");
 960
 961         if (!strncmp(line, "total", 5))
 962             continue;
 963
 964         t = htmlifyListEntry(line);
 965
 966         if ( t != NULL) {
 967             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
 968             listing.append(t->content(), t->contentSize());
 969             delete t;
 970         }
 971     }
 972
 973     debugs(9, 7, HERE << "Done.");
 974     data.readBuf->consume(usable);
 975     memFree(line, MEM_4K_BUF);
 976     xfree(sbuf);
 977 }
 978
 979 void
 980 Ftp::Gateway::processReplyBody()
 981 {
 982     debugs(9, 3, status());
 983
 984     if (request->method == Http::METHOD_HEAD && (flags.isdir || theSize != -1)) {
 985         serverComplete();
 986         return;
 987     }
 988
 989     /* Directory listings are special. They write ther own headers via the error objects */
 990     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
 991         appendSuccessHeader();
 992
 993     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 994         /*
 995          * probably was aborted because content length exceeds one
 996          * of the maximum size limits.
 997          */
 998         abortAll("entry aborted after calling appendSuccessHeader()");
 999         return;
1000     }
1001
1002 #if USE_ADAPTATION
1003
1004     if (adaptationAccessCheckPending) {
1005         debugs(9, 3, "returning from Ftp::Gateway::processReplyBody due to adaptationAccessCheckPending");
1006         return;
1007     }
1008
1009 #endif
1010
1011     if (flags.isdir) {
1012         if (!flags.listing) {
1013             flags.listing = 1;
1014             listing.reset();
1015         }
1016         parseListing();
1017         maybeReadVirginBody();
1018         return;
1019     } else if (const int csize = data.readBuf->contentSize()) {
1020         writeReplyBody(data.readBuf->content(), csize);
1021         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1022         data.readBuf->consume(csize);
1023     }
1024
1025     entry->flush();
1026
1027     maybeReadVirginBody();
1028 }
1029
1030 /**
1031  * Locates the FTP user:password login.
1032  *
1033  * Highest to lowest priority:
1034  *  - Checks URL (ftp://user:pass@domain)
1035  *  - Authorization: Basic header
1036  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1037  *
1038  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1039  *
1040  * TODO: we might be able to do something about locating username from other sources:
1041  *       ie, external ACL user=* tag or ident lookup
1042  *
1043  \retval 1  if we have everything needed to complete this request.
1044  \retval 0  if something is missing.
1045  */
1046 int
1047 Ftp::Gateway::checkAuth(const HttpHeader * req_hdr)
1048 {
1049     /* default username */
1050     xstrncpy(user, "anonymous", MAX_URL);
1051
1052 #if HAVE_AUTH_MODULE_BASIC
1053     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1054     const SBuf auth(req_hdr->getAuth(Http::HdrType::AUTHORIZATION, "Basic"));
1055     if (!auth.isEmpty()) {
1056         flags.authenticated = 1;
1057         loginParser(auth, false);
1058     }
1059     /* we fail with authorization-required error later IFF the FTP server requests it */
1060 #endif
1061
1062     /* Test URL login syntax. Overrides any headers received. */
1063     loginParser(request->url.userInfo(), true);
1064
1065     /* name is missing. thats fatal. */
1066     if (!user[0])
1067         fatal("FTP login parsing destroyed username info");
1068
1069     /* name + password == success */
1070     if (password[0])
1071         return 1;
1072
1073     /* Setup default FTP password settings */
1074     /* this has to be done last so that we can have a no-password case above. */
1075     if (!password[0]) {
1076         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1077             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1078             flags.tried_auth_anonymous=1;
1079             return 1;
1080         } else if (!flags.tried_auth_nopass) {
1081             xstrncpy(password, null_string, MAX_URL);
1082             flags.tried_auth_nopass=1;
1083             return 1;
1084         }
1085     }
1086
1087     return 0;           /* different username */
1088 }
1089
1090 void
1091 Ftp::Gateway::checkUrlpath()
1092 {
1093     static SBuf str_type_eq("type=");
1094     auto t = request->url.path().rfind(';');
1095
1096     if (t != SBuf::npos) {
1097         auto filenameEnd = t-1;
1098         if (request->url.path().substr(++t).cmp(str_type_eq, str_type_eq.length()) == 0) {
1099             t += str_type_eq.length();
1100             typecode = (char)xtoupper(request->url.path()[t]);
1101             request->url.path(request->url.path().substr(0,filenameEnd));
1102         }
1103     }
1104
1105     int l = request->url.path().length();
1106     /* check for null path */
1107
1108     if (!l) {
1109         flags.isdir = 1;
1110         flags.root_dir = 1;
1111         flags.need_base_href = 1;   /* Work around broken browsers */
1112     } else if (!request->url.path().cmp("/%2f/")) {
1113         /* UNIX root directory */
1114         flags.isdir = 1;
1115         flags.root_dir = 1;
1116     } else if ((l >= 1) && (request->url.path()[l-1] == '/')) {
1117         /* Directory URL, ending in / */
1118         flags.isdir = 1;
1119
1120         if (l == 1)
1121             flags.root_dir = 1;
1122     } else {
1123         flags.dir_slash = 1;
1124     }
1125 }
1126
1127 void
1128 Ftp::Gateway::buildTitleUrl()
1129 {
1130     title_url = "ftp://";
1131
1132     if (strcmp(user, "anonymous")) {
1133         title_url.append(user);
1134         title_url.append("@");
1135     }
1136
1137     SBuf authority = request->url.authority(request->url.getScheme() != AnyP::PROTO_FTP);
1138
1139     title_url.append(authority.rawContent(), authority.length());
1140     title_url.append(request->url.path().rawContent(), request->url.path().length());
1141
1142     base_href = "ftp://";
1143
1144     if (strcmp(user, "anonymous") != 0) {
1145         base_href.append(rfc1738_escape_part(user));
1146
1147         if (password_url) {
1148             base_href.append(":");
1149             base_href.append(rfc1738_escape_part(password));
1150         }
1151
1152         base_href.append("@");
1153     }
1154
1155     base_href.append(authority.rawContent(), authority.length());
1156     base_href.append(request->url.path().rawContent(), request->url.path().length());
1157     base_href.append("/");
1158 }
1159
1160 void
1161 Ftp::Gateway::start()
1162 {
1163     if (!checkAuth(&request->header)) {
1164         /* create appropriate reply */
1165         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1166         entry->replaceHttpReply(reply);
1167         serverComplete();
1168         return;
1169     }
1170
1171     checkUrlpath();
1172     buildTitleUrl();
1173     debugs(9, 5, "FD " << (ctrl.conn ? ctrl.conn->fd : -1) << " : host=" << request->url.host() <<
1174            ", path=" << request->url.path() << ", user=" << user << ", passwd=" << password);
1175     state = BEGIN;
1176     Ftp::Client::start();
1177 }
1178
1179 /* ====================================================================== */
1180
1181 void
1182 Ftp::Gateway::handleControlReply()
1183 {
1184     Ftp::Client::handleControlReply();
1185     if (ctrl.message == NULL)
1186         return; // didn't get complete reply yet
1187
1188     /* Copy the message except for the last line to cwd_message to be
1189      * printed in error messages.
1190      */
1191     for (wordlist *w = ctrl.message; w && w->next; w = w->next) {
1192         cwd_message.append('\n');
1193         cwd_message.append(w->key);
1194     }
1195
1196     FTP_SM_FUNCS[state] (this);
1197 }
1198
1199 /* ====================================================================== */
1200
1201 static void
1202 ftpReadWelcome(Ftp::Gateway * ftpState)
1203 {
1204     int code = ftpState->ctrl.replycode;
1205     debugs(9, 3, HERE);
1206
1207     if (ftpState->flags.pasv_only)
1208         ++ ftpState->login_att;
1209
1210     if (code == 220) {
1211         if (ftpState->ctrl.message) {
1212             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1213                 ftpState->flags.skip_whitespace = 1;
1214         }
1215
1216         ftpSendUser(ftpState);
1217     } else if (code == 120) {
1218         if (NULL != ftpState->ctrl.message)
1219             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1220
1221         return;
1222     } else {
1223         ftpFail(ftpState);
1224     }
1225 }
1226
1227 /**
1228  * Translate FTP login failure into HTTP error
1229  * this is an attmpt to get the 407 message to show up outside Squid.
1230  * its NOT a general failure. But a correct FTP response type.
1231  */
1232 void
1233 Ftp::Gateway::loginFailed()
1234 {
1235     ErrorState *err = NULL;
1236
1237     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1238         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1239             // 421/426 - Service Overload - retry permitted.
1240             err = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
1241         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1242             // 43x - Invalid or Credential Error - retry challenge required.
1243             err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1244         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1245             // 53x - Credentials Missing - retry challenge required
1246             if (password_url) // but they were in the URI! major fail.
1247                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
1248             else
1249                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1250         }
1251     }
1252
1253     if (!err) {
1254         ftpFail(this);
1255         return;
1256     }
1257
1258     failed(ERR_NONE, ctrl.replycode, err);
1259     // any other problems are general falures.
1260
1261     HttpReply *newrep = err->BuildHttpReply();
1262     delete err;
1263
1264 #if HAVE_AUTH_MODULE_BASIC
1265     /* add Authenticate header */
1266     newrep->header.putAuth("Basic", ftpRealm());
1267 #endif
1268
1269     // add it to the store entry for response....
1270     entry->replaceHttpReply(newrep);
1271     serverComplete();
1272 }
1273
1274 const char *
1275 Ftp::Gateway::ftpRealm()
1276 {
1277     static char realm[8192];
1278
1279     /* This request is not fully authenticated */
1280     if (!request) {
1281         snprintf(realm, 8192, "FTP %s unknown", user);
1282     } else if (request->url.port() == 21) {
1283         snprintf(realm, 8192, "FTP %s %s", user, request->url.host());
1284     } else {
1285         snprintf(realm, 8192, "FTP %s %s port %d", user, request->url.host(), request->url.port());
1286     }
1287     return realm;
1288 }
1289
1290 static void
1291 ftpSendUser(Ftp::Gateway * ftpState)
1292 {
1293     /* check the server control channel is still available */
1294     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1295         return;
1296
1297     if (ftpState->proxy_host != NULL)
1298         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n", ftpState->user, ftpState->request->url.host());
1299     else
1300         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1301
1302     ftpState->writeCommand(cbuf);
1303
1304     ftpState->state = Ftp::Client::SENT_USER;
1305 }
1306
1307 static void
1308 ftpReadUser(Ftp::Gateway * ftpState)
1309 {
1310     int code = ftpState->ctrl.replycode;
1311     debugs(9, 3, HERE);
1312
1313     if (code == 230) {
1314         ftpReadPass(ftpState);
1315     } else if (code == 331) {
1316         ftpSendPass(ftpState);
1317     } else {
1318         ftpState->loginFailed();
1319     }
1320 }
1321
1322 static void
1323 ftpSendPass(Ftp::Gateway * ftpState)
1324 {
1325     /* check the server control channel is still available */
1326     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
1327         return;
1328
1329     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
1330     ftpState->writeCommand(cbuf);
1331     ftpState->state = Ftp::Client::SENT_PASS;
1332 }
1333
1334 static void
1335 ftpReadPass(Ftp::Gateway * ftpState)
1336 {
1337     int code = ftpState->ctrl.replycode;
1338     debugs(9, 3, HERE << "code=" << code);
1339
1340     if (code == 230) {
1341         ftpSendType(ftpState);
1342     } else {
1343         ftpState->loginFailed();
1344     }
1345 }
1346
1347 static void
1348 ftpSendType(Ftp::Gateway * ftpState)
1349 {
1350     /* check the server control channel is still available */
1351     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
1352         return;
1353
1354     /*
1355      * Ref section 3.2.2 of RFC 1738
1356      */
1357     char mode = ftpState->typecode;
1358
1359     switch (mode) {
1360
1361     case 'D':
1362         mode = 'A';
1363         break;
1364
1365     case 'A':
1366
1367     case 'I':
1368         break;
1369
1370     default:
1371
1372         if (ftpState->flags.isdir) {
1373             mode = 'A';
1374         } else {
1375             auto t = ftpState->request->url.path().rfind('/');
1376             // XXX: performance regression, c_str() may reallocate
1377             SBuf filename = ftpState->request->url.path().substr(t != SBuf::npos ? t + 1 : 0);
1378             mode = mimeGetTransferMode(filename.c_str());
1379         }
1380
1381         break;
1382     }
1383
1384     if (mode == 'I')
1385         ftpState->flags.binary = 1;
1386     else
1387         ftpState->flags.binary = 0;
1388
1389     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
1390
1391     ftpState->writeCommand(cbuf);
1392
1393     ftpState->state = Ftp::Client::SENT_TYPE;
1394 }
1395
1396 static void
1397 ftpReadType(Ftp::Gateway * ftpState)
1398 {
1399     int code = ftpState->ctrl.replycode;
1400     char *path;
1401     char *d, *p;
1402     debugs(9, 3, HERE << "code=" << code);
1403
1404     if (code == 200) {
1405         p = path = SBufToCstring(ftpState->request->url.path());
1406
1407         if (*p == '/')
1408             ++p;
1409
1410         while (*p) {
1411             d = p;
1412             p += strcspn(p, "/");
1413
1414             if (*p) {
1415                 *p = '\0';
1416                 ++p;
1417             }
1418
1419             rfc1738_unescape(d);
1420
1421             if (*d)
1422                 wordlistAdd(&ftpState->pathcomps, d);
1423         }
1424
1425         xfree(path);
1426
1427         if (ftpState->pathcomps)
1428             ftpTraverseDirectory(ftpState);
1429         else
1430             ftpListDir(ftpState);
1431     } else {
1432         ftpFail(ftpState);
1433     }
1434 }
1435
1436 static void
1437 ftpTraverseDirectory(Ftp::Gateway * ftpState)
1438 {
1439     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
1440
1441     safe_free(ftpState->dirpath);
1442     ftpState->dirpath = ftpState->filepath;
1443     ftpState->filepath = NULL;
1444
1445     /* Done? */
1446
1447     if (ftpState->pathcomps == NULL) {
1448         debugs(9, 3, HERE << "the final component was a directory");
1449         ftpListDir(ftpState);
1450         return;
1451     }
1452
1453     /* Go to next path component */
1454     ftpState->filepath = wordlistChopHead(& ftpState->pathcomps);
1455
1456     /* Check if we are to CWD or RETR */
1457     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
1458         ftpSendCwd(ftpState);
1459     } else {
1460         debugs(9, 3, HERE << "final component is probably a file");
1461         ftpGetFile(ftpState);
1462         return;
1463     }
1464 }
1465
1466 static void
1467 ftpSendCwd(Ftp::Gateway * ftpState)
1468 {
1469     char *path = NULL;
1470
1471     /* check the server control channel is still available */
1472     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
1473         return;
1474
1475     debugs(9, 3, HERE);
1476
1477     path = ftpState->filepath;
1478
1479     if (!strcmp(path, "..") || !strcmp(path, "/")) {
1480         ftpState->flags.no_dotdot = 1;
1481     } else {
1482         ftpState->flags.no_dotdot = 0;
1483     }
1484
1485     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
1486
1487     ftpState->writeCommand(cbuf);
1488
1489     ftpState->state = Ftp::Client::SENT_CWD;
1490 }
1491
1492 static void
1493 ftpReadCwd(Ftp::Gateway * ftpState)
1494 {
1495     int code = ftpState->ctrl.replycode;
1496     debugs(9, 3, HERE);
1497
1498     if (code >= 200 && code < 300) {
1499         /* CWD OK */
1500         ftpState->unhack();
1501
1502         /* Reset cwd_message to only include the last message */
1503         ftpState->cwd_message.reset("");
1504         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
1505             ftpState->cwd_message.append(' ');
1506             ftpState->cwd_message.append(w->key);
1507         }
1508         ftpState->ctrl.message = NULL;
1509
1510         /* Continue to traverse the path */
1511         ftpTraverseDirectory(ftpState);
1512     } else {
1513         /* CWD FAILED */
1514
1515         if (!ftpState->flags.put)
1516             ftpFail(ftpState);
1517         else
1518             ftpSendMkdir(ftpState);
1519     }
1520 }
1521
1522 static void
1523 ftpSendMkdir(Ftp::Gateway * ftpState)
1524 {
1525     char *path = NULL;
1526
1527     /* check the server control channel is still available */
1528     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
1529         return;
1530
1531     path = ftpState->filepath;
1532     debugs(9, 3, HERE << "with path=" << path);
1533     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
1534     ftpState->writeCommand(cbuf);
1535     ftpState->state = Ftp::Client::SENT_MKDIR;
1536 }
1537
1538 static void
1539 ftpReadMkdir(Ftp::Gateway * ftpState)
1540 {
1541     char *path = ftpState->filepath;
1542     int code = ftpState->ctrl.replycode;
1543
1544     debugs(9, 3, HERE << "path " << path << ", code " << code);
1545
1546     if (code == 257) {      /* success */
1547         ftpSendCwd(ftpState);
1548     } else if (code == 550) {   /* dir exists */
1549
1550         if (ftpState->flags.put_mkdir) {
1551             ftpState->flags.put_mkdir = 1;
1552             ftpSendCwd(ftpState);
1553         } else
1554             ftpSendReply(ftpState);
1555     } else
1556         ftpSendReply(ftpState);
1557 }
1558
1559 static void
1560 ftpGetFile(Ftp::Gateway * ftpState)
1561 {
1562     assert(*ftpState->filepath != '\0');
1563     ftpState->flags.isdir = 0;
1564     ftpSendMdtm(ftpState);
1565 }
1566
1567 static void
1568 ftpListDir(Ftp::Gateway * ftpState)
1569 {
1570     if (ftpState->flags.dir_slash) {
1571         debugs(9, 3, HERE << "Directory path did not end in /");
1572         ftpState->title_url.append("/");
1573         ftpState->flags.isdir = 1;
1574     }
1575
1576     ftpSendPassive(ftpState);
1577 }
1578
1579 static void
1580 ftpSendMdtm(Ftp::Gateway * ftpState)
1581 {
1582     /* check the server control channel is still available */
1583     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
1584         return;
1585
1586     assert(*ftpState->filepath != '\0');
1587     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
1588     ftpState->writeCommand(cbuf);
1589     ftpState->state = Ftp::Client::SENT_MDTM;
1590 }
1591
1592 static void
1593 ftpReadMdtm(Ftp::Gateway * ftpState)
1594 {
1595     int code = ftpState->ctrl.replycode;
1596     debugs(9, 3, HERE);
1597
1598     if (code == 213) {
1599         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
1600         ftpState->unhack();
1601     } else if (code < 0) {
1602         ftpFail(ftpState);
1603         return;
1604     }
1605
1606     ftpSendSize(ftpState);
1607 }
1608
1609 static void
1610 ftpSendSize(Ftp::Gateway * ftpState)
1611 {
1612     /* check the server control channel is still available */
1613     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
1614         return;
1615
1616     /* Only send SIZE for binary transfers. The returned size
1617      * is useless on ASCII transfers */
1618
1619     if (ftpState->flags.binary) {
1620         assert(ftpState->filepath != NULL);
1621         assert(*ftpState->filepath != '\0');
1622         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
1623         ftpState->writeCommand(cbuf);
1624         ftpState->state = Ftp::Client::SENT_SIZE;
1625     } else
1626         /* Skip to next state no non-binary transfers */
1627         ftpSendPassive(ftpState);
1628 }
1629
1630 static void
1631 ftpReadSize(Ftp::Gateway * ftpState)
1632 {
1633     int code = ftpState->ctrl.replycode;
1634     debugs(9, 3, HERE);
1635
1636     if (code == 213) {
1637         ftpState->unhack();
1638         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
1639
1640         if (ftpState->theSize == 0) {
1641             debugs(9, 2, "SIZE reported " <<
1642                    ftpState->ctrl.last_reply << " on " <<
1643                    ftpState->title_url);
1644             ftpState->theSize = -1;
1645         }
1646     } else if (code < 0) {
1647         ftpFail(ftpState);
1648         return;
1649     }
1650
1651     ftpSendPassive(ftpState);
1652 }
1653
1654 static void
1655 ftpReadEPSV(Ftp::Gateway* ftpState)
1656 {
1657     Ip::Address srvAddr; // unused
1658     if (ftpState->handleEpsvReply(srvAddr)) {
1659         if (ftpState->ctrl.message == NULL)
1660             return; // didn't get complete reply yet
1661
1662         ftpState->connectDataChannel();
1663     }
1664 }
1665
1666 /** Send Passive connection request.
1667  * Default method is to use modern EPSV request.
1668  * The failover mechanism should check for previous state and re-call with alternates on failure.
1669  */
1670 static void
1671 ftpSendPassive(Ftp::Gateway * ftpState)
1672 {
1673     /** Checks the server control channel is still available before running. */
1674     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
1675         return;
1676
1677     debugs(9, 3, HERE);
1678
1679     /** \par
1680       * Checks for 'HEAD' method request and passes off for special handling by Ftp::Gateway::processHeadResponse(). */
1681     if (ftpState->request->method == Http::METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
1682         ftpState->processHeadResponse(); // may call serverComplete
1683         return;
1684     }
1685
1686     if (ftpState->sendPassive()) {
1687         // SENT_EPSV_ALL blocks other non-EPSV connections being attempted
1688         if (ftpState->state == Ftp::Client::SENT_EPSV_ALL)
1689             ftpState->flags.epsv_all_sent = true;
1690     }
1691 }
1692
1693 void
1694 Ftp::Gateway::processHeadResponse()
1695 {
1696     debugs(9, 5, HERE << "handling HEAD response");
1697     ftpSendQuit(this);
1698     appendSuccessHeader();
1699
1700     /*
1701      * On rare occasions I'm seeing the entry get aborted after
1702      * readControlReply() and before here, probably when
1703      * trying to write to the client.
1704      */
1705     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1706         abortAll("entry aborted while processing HEAD");
1707         return;
1708     }
1709
1710 #if USE_ADAPTATION
1711     if (adaptationAccessCheckPending) {
1712         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
1713         return;
1714     }
1715 #endif
1716
1717     // processReplyBody calls serverComplete() since there is no body
1718     processReplyBody();
1719 }
1720
1721 static void
1722 ftpReadPasv(Ftp::Gateway * ftpState)
1723 {
1724     Ip::Address srvAddr; // unused
1725     if (ftpState->handlePasvReply(srvAddr))
1726         ftpState->connectDataChannel();
1727     else {
1728         ftpFail(ftpState);
1729         // Currently disabled, does not work correctly:
1730         // ftpSendEPRT(ftpState);
1731         return;
1732     }
1733 }
1734
1735 void
1736 Ftp::Gateway::dataChannelConnected(const CommConnectCbParams &io)
1737 {
1738     debugs(9, 3, HERE);
1739     data.opener = NULL;
1740
1741     if (io.flag != Comm::OK) {
1742         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
1743
1744         // ABORT on timeouts. server may be waiting on a broken TCP link.
1745         if (io.xerrno == Comm::TIMEOUT)
1746             writeCommand("ABOR\r\n");
1747
1748         // try another connection attempt with some other method
1749         ftpSendPassive(this);
1750         return;
1751     }
1752
1753     data.opened(io.conn, dataCloser());
1754     ftpRestOrList(this);
1755 }
1756
1757 static void
1758 ftpOpenListenSocket(Ftp::Gateway * ftpState, int fallback)
1759 {
1760     /// Close old data channels, if any. We may open a new one below.
1761     if (ftpState->data.conn != NULL) {
1762         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
1763             // NP: in fact it points to the control channel. just clear it.
1764             ftpState->data.clear();
1765         else
1766             ftpState->data.close();
1767     }
1768     safe_free(ftpState->data.host);
1769
1770     if (!Comm::IsConnOpen(ftpState->ctrl.conn)) {
1771         debugs(9, 5, "The control connection to the remote end is closed");
1772         return;
1773     }
1774
1775     /*
1776      * Set up a listen socket on the same local address as the
1777      * control connection.
1778      */
1779     Comm::ConnectionPointer temp = new Comm::Connection;
1780     temp->local = ftpState->ctrl.conn->local;
1781
1782     /*
1783      * REUSEADDR is needed in fallback mode, since the same port is
1784      * used for both control and data.
1785      */
1786     if (fallback) {
1787         int on = 1;
1788         errno = 0;
1789         if (setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR,
1790                        (char *) &on, sizeof(on)) == -1) {
1791             int xerrno = errno;
1792             // SO_REUSEADDR is only an optimization, no need to be verbose about error
1793             debugs(9, 4, "setsockopt failed: " << xstrerr(xerrno));
1794         }
1795         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
1796         temp->flags |= COMM_REUSEADDR;
1797     } else {
1798         /* if not running in fallback mode a new port needs to be retrieved */
1799         temp->local.port(0);
1800     }
1801
1802     ftpState->listenForDataChannel(temp);
1803 }
1804
1805 static void
1806 ftpSendPORT(Ftp::Gateway * ftpState)
1807 {
1808     /* check the server control channel is still available */
1809     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
1810         return;
1811
1812     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1813         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
1814         return;
1815     }
1816
1817     debugs(9, 3, HERE);
1818     ftpState->flags.pasv_supported = 0;
1819     ftpOpenListenSocket(ftpState, 0);
1820
1821     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
1822         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.isIPv4() ) {
1823             /* non-IPv4 CANNOT send PORT command.                       */
1824             /* we got here by attempting and failing an EPRT            */
1825             /* using the same reply code should simulate a PORT failure */
1826             ftpReadPORT(ftpState);
1827             return;
1828         }
1829
1830         /* XXX Need to set error message */
1831         ftpFail(ftpState);
1832         return;
1833     }
1834
1835     // pull out the internal IP address bytes to send in PORT command...
1836     // source them from the listen_conn->local
1837
1838     struct addrinfo *AI = NULL;
1839     ftpState->data.listenConn->local.getAddrInfo(AI, AF_INET);
1840     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
1841     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
1842     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
1843              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
1844              portptr[0], portptr[1]);
1845     ftpState->writeCommand(cbuf);
1846     ftpState->state = Ftp::Client::SENT_PORT;
1847
1848     Ip::Address::FreeAddr(AI);
1849 }
1850
1851 static void
1852 ftpReadPORT(Ftp::Gateway * ftpState)
1853 {
1854     int code = ftpState->ctrl.replycode;
1855     debugs(9, 3, HERE);
1856
1857     if (code != 200) {
1858         /* Fall back on using the same port as the control connection */
1859         debugs(9, 3, "PORT not supported by remote end");
1860         ftpOpenListenSocket(ftpState, 1);
1861     }
1862
1863     ftpRestOrList(ftpState);
1864 }
1865
1866 #if 0
1867 static void
1868 ftpSendEPRT(Ftp::Gateway * ftpState)
1869 {
1870     /* check the server control channel is still available */
1871     if (!ftpState || !ftpState->haveControlChannel("ftpSendEPRT"))
1872         return;
1873
1874     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1875         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
1876         return;
1877     }
1878
1879     if (!Config.Ftp.eprt) {
1880         /* Disabled. Switch immediately to attempting old PORT command. */
1881         debugs(9, 3, "EPRT disabled by local administrator");
1882         ftpSendPORT(ftpState);
1883         return;
1884     }
1885
1886     debugs(9, 3, HERE);
1887     ftpState->flags.pasv_supported = 0;
1888
1889     ftpOpenListenSocket(ftpState, 0);
1890     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
1891     if (!Comm::IsConnOpen(ftpState->data.conn)) {
1892         /* XXX Need to set error message */
1893         ftpFail(ftpState);
1894         return;
1895     }
1896
1897     char buf[MAX_IPSTRLEN];
1898
1899     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
1900     /* Which can be used by EITHER protocol. */
1901     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
1902              ( ftpState->data.listenConn->local.isIPv6() ? 2 : 1 ),
1903              ftpState->data.listenConn->local.toStr(buf,MAX_IPSTRLEN),
1904              ftpState->data.listenConn->local.port() );
1905
1906     ftpState->writeCommand(cbuf);
1907     ftpState->state = Ftp::Client::SENT_EPRT;
1908 }
1909 #endif
1910
1911 static void
1912 ftpReadEPRT(Ftp::Gateway * ftpState)
1913 {
1914     int code = ftpState->ctrl.replycode;
1915     debugs(9, 3, HERE);
1916
1917     if (code != 200) {
1918         /* Failover to attempting old PORT command. */
1919         debugs(9, 3, "EPRT not supported by remote end");
1920         ftpSendPORT(ftpState);
1921         return;
1922     }
1923
1924     ftpRestOrList(ftpState);
1925 }
1926
1927 /** "read" handler to accept FTP data connections.
1928  *
1929  \param io    comm accept(2) callback parameters
1930  */
1931 void
1932 Ftp::Gateway::ftpAcceptDataConnection(const CommAcceptCbParams &io)
1933 {
1934     debugs(9, 3, HERE);
1935
1936     if (!Comm::IsConnOpen(ctrl.conn)) { /*Close handlers will cleanup*/
1937         debugs(9, 5, "The control connection to the remote end is closed");
1938         return;
1939     }
1940
1941     if (io.flag != Comm::OK) {
1942         data.listenConn->close();
1943         data.listenConn = NULL;
1944         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
1945         /** \todo Need to send error message on control channel*/
1946         ftpFail(this);
1947         return;
1948     }
1949
1950     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1951         abortAll("entry aborted when accepting data conn");
1952         data.listenConn->close();
1953         data.listenConn = NULL;
1954         io.conn->close();
1955         return;
1956     }
1957
1958     /* data listening conn is no longer even open. abort. */
1959     if (!Comm::IsConnOpen(data.listenConn)) {
1960         data.listenConn = NULL; // ensure that it's cleared and not just closed.
1961         return;
1962     }
1963
1964     /* data listening conn is no longer even open. abort. */
1965     if (!Comm::IsConnOpen(data.conn)) {
1966         data.clear(); // ensure that it's cleared and not just closed.
1967         return;
1968     }
1969
1970     /** \par
1971      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
1972      * made by the remote client which is connected to the FTP control socket.
1973      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
1974      * This prevents third-party hacks, but also third-party load balancing handshakes.
1975      */
1976     if (Config.Ftp.sanitycheck) {
1977         // accept if either our data or ctrl connection is talking to this remote peer.
1978         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
1979             debugs(9, DBG_IMPORTANT,
1980                    "FTP data connection from unexpected server (" <<
1981                    io.conn->remote << "), expecting " <<
1982                    data.conn->remote << " or " << ctrl.conn->remote);
1983
1984             /* close the bad sources connection down ASAP. */
1985             io.conn->close();
1986
1987             /* drop the bad connection (io) by ignoring the attempt. */
1988             return;
1989         }
1990     }
1991
1992     /** On Comm::OK start using the accepted data socket and discard the temporary listen socket. */
1993     data.close();
1994     data.opened(io.conn, dataCloser());
1995     data.addr(io.conn->remote);
1996
1997     debugs(9, 3, HERE << "Connected data socket on " <<
1998            io.conn << ". FD table says: " <<
1999            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
2000            "data-peer= " << fd_table[data.conn->fd].ipaddr);
2001
2002     assert(haveControlChannel("ftpAcceptDataConnection"));
2003     assert(ctrl.message == NULL);
2004
2005     // Ctrl channel operations will determine what happens to this data connection
2006 }
2007
2008 static void
2009 ftpRestOrList(Ftp::Gateway * ftpState)
2010 {
2011     debugs(9, 3, HERE);
2012
2013     if (ftpState->typecode == 'D') {
2014         ftpState->flags.isdir = 1;
2015
2016         if (ftpState->flags.put) {
2017             ftpSendMkdir(ftpState); /* PUT name;type=d */
2018         } else {
2019             ftpSendNlst(ftpState);  /* GET name;type=d  sec 3.2.2 of RFC 1738 */
2020         }
2021     } else if (ftpState->flags.put) {
2022         ftpSendStor(ftpState);
2023     } else if (ftpState->flags.isdir)
2024         ftpSendList(ftpState);
2025     else if (ftpState->restartable())
2026         ftpSendRest(ftpState);
2027     else
2028         ftpSendRetr(ftpState);
2029 }
2030
2031 static void
2032 ftpSendStor(Ftp::Gateway * ftpState)
2033 {
2034     /* check the server control channel is still available */
2035     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
2036         return;
2037
2038     debugs(9, 3, HERE);
2039
2040     if (ftpState->filepath != NULL) {
2041         /* Plain file upload */
2042         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
2043         ftpState->writeCommand(cbuf);
2044         ftpState->state = Ftp::Client::SENT_STOR;
2045     } else if (ftpState->request->header.getInt64(Http::HdrType::CONTENT_LENGTH) > 0) {
2046         /* File upload without a filename. use STOU to generate one */
2047         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
2048         ftpState->writeCommand(cbuf);
2049         ftpState->state = Ftp::Client::SENT_STOR;
2050     } else {
2051         /* No file to transfer. Only create directories if needed */
2052         ftpSendReply(ftpState);
2053     }
2054 }
2055
2056 /// \deprecated use ftpState->readStor() instead.
2057 static void
2058 ftpReadStor(Ftp::Gateway * ftpState)
2059 {
2060     ftpState->readStor();
2061 }
2062
2063 void Ftp::Gateway::readStor()
2064 {
2065     int code = ctrl.replycode;
2066     debugs(9, 3, HERE);
2067
2068     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
2069         if (!originalRequest()->body_pipe) {
2070             debugs(9, 3, "zero-size STOR?");
2071             state = WRITING_DATA; // make ftpWriteTransferDone() responsible
2072             dataComplete(); // XXX: keep in sync with doneSendingRequestBody()
2073             return;
2074         }
2075
2076         if (!startRequestBodyFlow()) { // register to receive body data
2077             ftpFail(this);
2078             return;
2079         }
2080
2081         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
2082         debugs(9, 3, HERE << "starting data transfer");
2083         switchTimeoutToDataChannel();
2084         sendMoreRequestBody();
2085         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
2086         state = WRITING_DATA;
2087         debugs(9, 3, HERE << "writing data channel");
2088     } else if (code == 150) {
2089         /* When client code is 150 with no data channel, Accept data channel. */
2090         debugs(9, 3, "ftpReadStor: accepting data channel");
2091         listenForDataChannel(data.conn);
2092     } else {
2093         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
2094         ftpFail(this);
2095     }
2096 }
2097
2098 static void
2099 ftpSendRest(Ftp::Gateway * ftpState)
2100 {
2101     /* check the server control channel is still available */
2102     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
2103         return;
2104
2105     debugs(9, 3, HERE);
2106
2107     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
2108     ftpState->writeCommand(cbuf);
2109     ftpState->state = Ftp::Client::SENT_REST;
2110 }
2111
2112 int
2113 Ftp::Gateway::restartable()
2114 {
2115     if (restart_offset > 0)
2116         return 1;
2117
2118     if (!request->range)
2119         return 0;
2120
2121     if (!flags.binary)
2122         return 0;
2123
2124     if (theSize <= 0)
2125         return 0;
2126
2127     int64_t desired_offset = request->range->lowestOffset(theSize);
2128
2129     if (desired_offset <= 0)
2130         return 0;
2131
2132     if (desired_offset >= theSize)
2133         return 0;
2134
2135     restart_offset = desired_offset;
2136     return 1;
2137 }
2138
2139 static void
2140 ftpReadRest(Ftp::Gateway * ftpState)
2141 {
2142     int code = ftpState->ctrl.replycode;
2143     debugs(9, 3, HERE);
2144     assert(ftpState->restart_offset > 0);
2145
2146     if (code == 350) {
2147         ftpState->setCurrentOffset(ftpState->restart_offset);
2148         ftpSendRetr(ftpState);
2149     } else if (code > 0) {
2150         debugs(9, 3, HERE << "REST not supported");
2151         ftpState->flags.rest_supported = 0;
2152         ftpSendRetr(ftpState);
2153     } else {
2154         ftpFail(ftpState);
2155     }
2156 }
2157
2158 static void
2159 ftpSendList(Ftp::Gateway * ftpState)
2160 {
2161     /* check the server control channel is still available */
2162     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
2163         return;
2164
2165     debugs(9, 3, HERE);
2166
2167     if (ftpState->filepath) {
2168         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
2169     } else {
2170         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
2171     }
2172
2173     ftpState->writeCommand(cbuf);
2174     ftpState->state = Ftp::Client::SENT_LIST;
2175 }
2176
2177 static void
2178 ftpSendNlst(Ftp::Gateway * ftpState)
2179 {
2180     /* check the server control channel is still available */
2181     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
2182         return;
2183
2184     debugs(9, 3, HERE);
2185
2186     ftpState->flags.tried_nlst = 1;
2187
2188     if (ftpState->filepath) {
2189         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
2190     } else {
2191         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
2192     }
2193
2194     ftpState->writeCommand(cbuf);
2195     ftpState->state = Ftp::Client::SENT_NLST;
2196 }
2197
2198 static void
2199 ftpReadList(Ftp::Gateway * ftpState)
2200 {
2201     int code = ftpState->ctrl.replycode;
2202     debugs(9, 3, HERE);
2203
2204     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2205         /* Begin data transfer */
2206         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2207         ftpState->switchTimeoutToDataChannel();
2208         ftpState->maybeReadVirginBody();
2209         ftpState->state = Ftp::Client::READING_DATA;
2210         return;
2211     } else if (code == 150) {
2212         /* Accept data channel */
2213         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2214         ftpState->listenForDataChannel(ftpState->data.conn);
2215         return;
2216     } else if (!ftpState->flags.tried_nlst && code > 300) {
2217         ftpSendNlst(ftpState);
2218     } else {
2219         ftpFail(ftpState);
2220         return;
2221     }
2222 }
2223
2224 static void
2225 ftpSendRetr(Ftp::Gateway * ftpState)
2226 {
2227     /* check the server control channel is still available */
2228     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
2229         return;
2230
2231     debugs(9, 3, HERE);
2232
2233     assert(ftpState->filepath != NULL);
2234     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
2235     ftpState->writeCommand(cbuf);
2236     ftpState->state = Ftp::Client::SENT_RETR;
2237 }
2238
2239 static void
2240 ftpReadRetr(Ftp::Gateway * ftpState)
2241 {
2242     int code = ftpState->ctrl.replycode;
2243     debugs(9, 3, HERE);
2244
2245     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2246         /* Begin data transfer */
2247         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2248         ftpState->switchTimeoutToDataChannel();
2249         ftpState->maybeReadVirginBody();
2250         ftpState->state = Ftp::Client::READING_DATA;
2251     } else if (code == 150) {
2252         /* Accept data channel */
2253         ftpState->listenForDataChannel(ftpState->data.conn);
2254     } else if (code >= 300) {
2255         if (!ftpState->flags.try_slash_hack) {
2256             /* Try this as a directory missing trailing slash... */
2257             ftpState->hackShortcut(ftpSendCwd);
2258         } else {
2259             ftpFail(ftpState);
2260         }
2261     } else {
2262         ftpFail(ftpState);
2263     }
2264 }
2265
2266 /**
2267  * Generate the HTTP headers and template fluff around an FTP
2268  * directory listing display.
2269  */
2270 void
2271 Ftp::Gateway::completedListing()
2272 {
2273     assert(entry);
2274     entry->lock("Ftp::Gateway");
2275     ErrorState ferr(ERR_DIR_LISTING, Http::scOkay, request);
2276     ferr.ftp.listing = &listing;
2277     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
2278     ferr.ftp.server_msg = ctrl.message;
2279     ctrl.message = NULL;
2280     entry->replaceHttpReply( ferr.BuildHttpReply() );
2281     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2282     entry->flush();
2283     entry->unlock("Ftp::Gateway");
2284 }
2285
2286 static void
2287 ftpReadTransferDone(Ftp::Gateway * ftpState)
2288 {
2289     int code = ftpState->ctrl.replycode;
2290     debugs(9, 3, HERE);
2291
2292     if (code == 226 || code == 250) {
2293         /* Connection closed; retrieval done. */
2294         if (ftpState->flags.listing) {
2295             ftpState->completedListing();
2296             /* QUIT operation handles sending the reply to client */
2297         }
2298         ftpSendQuit(ftpState);
2299     } else {            /* != 226 */
2300         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
2301         ftpState->failed(ERR_FTP_FAILURE, 0);
2302         /* failed closes ctrl.conn and frees ftpState */
2303         return;
2304     }
2305 }
2306
2307 // premature end of the request body
2308 void
2309 Ftp::Gateway::handleRequestBodyProducerAborted()
2310 {
2311     Client::handleRequestBodyProducerAborted();
2312     debugs(9, 3, HERE << "ftpState=" << this);
2313     failed(ERR_READ_ERROR, 0);
2314 }
2315
2316 static void
2317 ftpWriteTransferDone(Ftp::Gateway * ftpState)
2318 {
2319     int code = ftpState->ctrl.replycode;
2320     debugs(9, 3, HERE);
2321
2322     if (!(code == 226 || code == 250)) {
2323         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
2324         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
2325         return;
2326     }
2327
2328     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
2329     ftpSendReply(ftpState);
2330 }
2331
2332 static void
2333 ftpSendQuit(Ftp::Gateway * ftpState)
2334 {
2335     /* check the server control channel is still available */
2336     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
2337         return;
2338
2339     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
2340     ftpState->writeCommand(cbuf);
2341     ftpState->state = Ftp::Client::SENT_QUIT;
2342 }
2343
2344 /** Completes a client FTP operation with success or other page
2345  *  generated and stored in the entry field by the code issuing QUIT.
2346  */
2347 static void
2348 ftpReadQuit(Ftp::Gateway * ftpState)
2349 {
2350     ftpState->serverComplete();
2351 }
2352
2353 static void
2354 ftpTrySlashHack(Ftp::Gateway * ftpState)
2355 {
2356     char *path;
2357     ftpState->flags.try_slash_hack = 1;
2358     /* Free old paths */
2359
2360     debugs(9, 3, HERE);
2361
2362     if (ftpState->pathcomps)
2363         wordlistDestroy(&ftpState->pathcomps);
2364
2365     safe_free(ftpState->filepath);
2366
2367     /* Build the new path (urlpath begins with /) */
2368     path = SBufToCstring(ftpState->request->url.path());
2369
2370     rfc1738_unescape(path);
2371
2372     ftpState->filepath = path;
2373
2374     /* And off we go */
2375     ftpGetFile(ftpState);
2376 }
2377
2378 /**
2379  * Forget hack status. Next error is shown to the user
2380  */
2381 void
2382 Ftp::Gateway::unhack()
2383 {
2384     debugs(9, 3, HERE);
2385
2386     if (old_request != NULL) {
2387         safe_free(old_request);
2388         safe_free(old_reply);
2389     }
2390 }
2391
2392 void
2393 Ftp::Gateway::hackShortcut(FTPSM * nextState)
2394 {
2395     /* Clear some unwanted state */
2396     setCurrentOffset(0);
2397     restart_offset = 0;
2398     /* Save old error message & some state info */
2399
2400     debugs(9, 3, HERE);
2401
2402     if (old_request == NULL) {
2403         old_request = ctrl.last_command;
2404         ctrl.last_command = NULL;
2405         old_reply = ctrl.last_reply;
2406         ctrl.last_reply = NULL;
2407
2408         if (pathcomps == NULL && filepath != NULL)
2409             old_filepath = xstrdup(filepath);
2410     }
2411
2412     /* Jump to the "hack" state */
2413     nextState(this);
2414 }
2415
2416 static void
2417 ftpFail(Ftp::Gateway *ftpState)
2418 {
2419     const bool slashHack = ftpState->request->url.path().caseCmp("/%2f", 4)==0;
2420     int code = ftpState->ctrl.replycode;
2421     err_type error_code = ERR_NONE;
2422
2423     debugs(9, 6, "state " << ftpState->state <<
2424            " reply code " << code << "flags(" <<
2425            (ftpState->flags.isdir?"IS_DIR,":"") <<
2426            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
2427            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
2428            "slashhack=" << (slashHack? "T":"F"));
2429
2430     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
2431     if (!ftpState->flags.isdir &&   /* Not a directory */
2432             !ftpState->flags.try_slash_hack && !slashHack && /* Not doing slash hack */
2433             ftpState->mdtm <= 0 && ftpState->theSize < 0) { /* Not known as a file */
2434
2435         switch (ftpState->state) {
2436
2437         case Ftp::Client::SENT_CWD:
2438
2439         case Ftp::Client::SENT_RETR:
2440             /* Try the / hack */
2441             ftpState->hackShortcut(ftpTrySlashHack);
2442             return;
2443
2444         default:
2445             break;
2446         }
2447     }
2448
2449     Http::StatusCode sc = ftpState->failedHttpStatus(error_code);
2450     ErrorState *ftperr = new ErrorState(error_code, sc, ftpState->fwd->request);
2451     ftpState->failed(error_code, code, ftperr);
2452     ftperr->detailError(code);
2453     HttpReply *newrep = ftperr->BuildHttpReply();
2454     delete ftperr;
2455
2456     ftpState->entry->replaceHttpReply(newrep);
2457     ftpSendQuit(ftpState);
2458 }
2459
2460 Http::StatusCode
2461 Ftp::Gateway::failedHttpStatus(err_type &error)
2462 {
2463     if (error == ERR_NONE) {
2464         switch (state) {
2465
2466         case SENT_USER:
2467
2468         case SENT_PASS:
2469
2470             if (ctrl.replycode > 500) {
2471                 error = ERR_FTP_FORBIDDEN;
2472                 return password_url ? Http::scForbidden : Http::scUnauthorized;
2473             } else if (ctrl.replycode == 421) {
2474                 error = ERR_FTP_UNAVAILABLE;
2475                 return Http::scServiceUnavailable;
2476             }
2477             break;
2478
2479         case SENT_CWD:
2480
2481         case SENT_RETR:
2482             if (ctrl.replycode == 550) {
2483                 error = ERR_FTP_NOT_FOUND;
2484                 return Http::scNotFound;
2485             }
2486             break;
2487
2488         default:
2489             break;
2490         }
2491     }
2492     return Ftp::Client::failedHttpStatus(error);
2493 }
2494
2495 static void
2496 ftpSendReply(Ftp::Gateway * ftpState)
2497 {
2498     int code = ftpState->ctrl.replycode;
2499     Http::StatusCode http_code;
2500     err_type err_code = ERR_NONE;
2501
2502     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
2503
2504     if (cbdataReferenceValid(ftpState))
2505         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
2506
2507     if (code == 226 || code == 250) {
2508         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
2509         http_code = (ftpState->mdtm > 0) ? Http::scAccepted : Http::scCreated;
2510     } else if (code == 227) {
2511         err_code = ERR_FTP_PUT_CREATED;
2512         http_code = Http::scCreated;
2513     } else {
2514         err_code = ERR_FTP_PUT_ERROR;
2515         http_code = Http::scInternalServerError;
2516     }
2517
2518     ErrorState err(err_code, http_code, ftpState->request);
2519
2520     if (ftpState->old_request)
2521         err.ftp.request = xstrdup(ftpState->old_request);
2522     else
2523         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
2524
2525     if (ftpState->old_reply)
2526         err.ftp.reply = xstrdup(ftpState->old_reply);
2527     else if (ftpState->ctrl.last_reply)
2528         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
2529     else
2530         err.ftp.reply = xstrdup("");
2531
2532     // TODO: interpret as FTP-specific error code
2533     err.detailError(code);
2534
2535     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
2536
2537     ftpSendQuit(ftpState);
2538 }
2539
2540 void
2541 Ftp::Gateway::appendSuccessHeader()
2542 {
2543     debugs(9, 3, HERE);
2544
2545     if (flags.http_header_sent)
2546         return;
2547
2548     HttpReply *reply = new HttpReply;
2549
2550     flags.http_header_sent = 1;
2551
2552     assert(entry->isEmpty());
2553
2554     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2555
2556     entry->buffer();    /* released when done processing current data payload */
2557
2558     SBuf urlPath = request->url.path();
2559     auto t = urlPath.rfind('/');
2560     SBuf filename = urlPath.substr(t != SBuf::npos ? t : 0);
2561
2562     const char *mime_type = NULL;
2563     const char *mime_enc = NULL;
2564
2565     if (flags.isdir) {
2566         mime_type = "text/html";
2567     } else {
2568         switch (typecode) {
2569
2570         case 'I':
2571             mime_type = "application/octet-stream";
2572             // XXX: performance regression, c_str() may reallocate
2573             mime_enc = mimeGetContentEncoding(filename.c_str());
2574             break;
2575
2576         case 'A':
2577             mime_type = "text/plain";
2578             break;
2579
2580         default:
2581             // XXX: performance regression, c_str() may reallocate
2582             mime_type = mimeGetContentType(filename.c_str());
2583             mime_enc = mimeGetContentEncoding(filename.c_str());
2584             break;
2585         }
2586     }
2587
2588     /* set standard stuff */
2589
2590     if (0 == getCurrentOffset()) {
2591         /* Full reply */
2592         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2593     } else if (theSize < getCurrentOffset()) {
2594         /*
2595          * DPW 2007-05-04
2596          * offset should not be larger than theSize.  We should
2597          * not be seeing this condition any more because we'll only
2598          * send REST if we know the theSize and if it is less than theSize.
2599          */
2600         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
2601                " current offset=" << getCurrentOffset() <<
2602                ", but theSize=" << theSize <<
2603                ".  assuming full content response");
2604         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2605     } else {
2606         /* Partial reply */
2607         HttpHdrRangeSpec range_spec;
2608         range_spec.offset = getCurrentOffset();
2609         range_spec.length = theSize - getCurrentOffset();
2610         reply->setHeaders(Http::scPartialContent, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
2611         httpHeaderAddContRange(&reply->header, range_spec, theSize);
2612     }
2613
2614     /* additional info */
2615     if (mime_enc)
2616         reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc);
2617
2618     reply->sources |= Http::Message::srcFtp;
2619     setVirginReply(reply);
2620     adaptOrFinalizeReply();
2621 }
2622
2623 void
2624 Ftp::Gateway::haveParsedReplyHeaders()
2625 {
2626     Client::haveParsedReplyHeaders();
2627
2628     StoreEntry *e = entry;
2629
2630     e->timestampsSet();
2631
2632     if (flags.authenticated) {
2633         /*
2634          * Authenticated requests can't be cached.
2635          */
2636         e->release();
2637     } else if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && !getCurrentOffset()) {
2638         e->setPublicKey();
2639     } else {
2640         e->release();
2641     }
2642 }
2643
2644 HttpReply *
2645 Ftp::Gateway::ftpAuthRequired(HttpRequest * request, const char *realm)
2646 {
2647     ErrorState err(ERR_CACHE_ACCESS_DENIED, Http::scUnauthorized, request);
2648     HttpReply *newrep = err.BuildHttpReply();
2649 #if HAVE_AUTH_MODULE_BASIC
2650     /* add Authenticate header */
2651     newrep->header.putAuth("Basic", realm);
2652 #endif
2653     return newrep;
2654 }
2655
2656 const SBuf &
2657 Ftp::UrlWith2f(HttpRequest * request)
2658 {
2659     SBuf newbuf("%2f");
2660
2661     if (request->url.getScheme() != AnyP::PROTO_FTP) {
2662         static const SBuf nil;
2663         return nil;
2664     }
2665
2666     if (request->url.path()[0] == '/') {
2667         newbuf.append(request->url.path());
2668         request->url.path(newbuf);
2669     } else if (!request->url.path().startsWith(newbuf)) {
2670         newbuf.append(request->url.path().substr(1));
2671         request->url.path(newbuf);
2672     }
2673
2674     return request->effectiveRequestUri();
2675 }
2676
2677 void
2678 Ftp::Gateway::printfReplyBody(const char *fmt, ...)
2679 {
2680     va_list args;
2681     va_start (args, fmt);
2682     static char buf[4096];
2683     buf[0] = '\0';
2684     vsnprintf(buf, 4096, fmt, args);
2685     writeReplyBody(buf, strlen(buf));
2686     va_end(args);
2687 }
2688
2689 /**
2690  * Call this when there is data from the origin server
2691  * which should be sent to either StoreEntry, or to ICAP...
2692  */
2693 void
2694 Ftp::Gateway::writeReplyBody(const char *dataToWrite, size_t dataLength)
2695 {
2696     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
2697     addVirginReplyBody(dataToWrite, dataLength);
2698 }
2699
2700 /**
2701  * A hack to ensure we do not double-complete on the forward entry.
2702  *
2703  \todo Ftp::Gateway logic should probably be rewritten to avoid
2704  *  double-completion or FwdState should be rewritten to allow it.
2705  */
2706 void
2707 Ftp::Gateway::completeForwarding()
2708 {
2709     if (fwd == NULL || flags.completed_forwarding) {
2710         debugs(9, 3, "avoid double-complete on FD " <<
2711                (ctrl.conn ? ctrl.conn->fd : -1) << ", Data FD " << data.conn->fd <<
2712                ", this " << this << ", fwd " << fwd);
2713         return;
2714     }
2715
2716     flags.completed_forwarding = true;
2717     Client::completeForwarding();
2718 }
2719
2720 /**
2721  * Have we lost the FTP server control channel?
2722  *
2723  \retval true   The server control channel is available.
2724  \retval false  The server control channel is not available.
2725  */
2726 bool
2727 Ftp::Gateway::haveControlChannel(const char *caller_name) const
2728 {
2729     if (doneWithServer())
2730         return false;
2731
2732     /* doneWithServer() only checks BOTH channels are closed. */
2733     if (!Comm::IsConnOpen(ctrl.conn)) {
2734         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
2735         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
2736         return false;
2737     }
2738
2739     return true;
2740 }
2741
2742 bool
2743 Ftp::Gateway::mayReadVirginReplyBody() const
2744 {
2745     // TODO: Can we do what Ftp::Relay::mayReadVirginReplyBody() does instead?
2746     return !doneWithServer();
2747 }
2748
2749 AsyncJob::Pointer
2750 Ftp::StartGateway(FwdState *const fwdState)
2751 {
2752     return AsyncJob::Start(new Ftp::Gateway(fwdState));
2753 }
2754