src/clients/FtpGateway.cc

   1 /*
   2  * Copyright (C) 1996-2017 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "clients/forward.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm.h"
  16 #include "comm/ConnOpener.h"
  17 #include "comm/Read.h"
  18 #include "comm/TcpAcceptor.h"
  19 #include "CommCalls.h"
  20 #include "compat/strtoll.h"
  21 #include "errorpage.h"
  22 #include "fd.h"
  23 #include "fde.h"
  24 #include "FwdState.h"
  25 #include "html_quote.h"
  26 #include "HttpHdrContRange.h"
  27 #include "HttpHeader.h"
  28 #include "HttpHeaderRange.h"
  29 #include "HttpReply.h"
  30 #include "HttpRequest.h"
  31 #include "ip/tools.h"
  32 #include "MemBuf.h"
  33 #include "mime.h"
  34 #include "rfc1738.h"
  35 #include "SquidConfig.h"
  36 #include "SquidString.h"
  37 #include "SquidTime.h"
  38 #include "StatCounters.h"
  39 #include "Store.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "util.h"
  43 #include "wordlist.h"
  44
  45 #if USE_DELAY_POOLS
  46 #include "DelayPools.h"
  47 #include "MemObject.h"
  48 #endif
  49
  50 #include <cerrno>
  51
  52 namespace Ftp
  53 {
  54
  55 struct GatewayFlags {
  56
  57     /* passive mode */
  58     bool pasv_supported;  ///< PASV command is allowed
  59     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
  60     bool pasv_only;
  61     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
  62
  63     /* authentication */
  64     bool authenticated;         ///< authentication success
  65     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
  66     bool tried_auth_nopass;     ///< auth tried username with no password already.
  67
  68     /* other */
  69     bool isdir;
  70     bool skip_whitespace;
  71     bool rest_supported;
  72     bool http_header_sent;
  73     bool tried_nlst;
  74     bool need_base_href;
  75     bool dir_slash;
  76     bool root_dir;
  77     bool no_dotdot;
  78     bool binary;
  79     bool try_slash_hack;
  80     bool put;
  81     bool put_mkdir;
  82     bool listformat_unknown;
  83     bool listing;
  84     bool completed_forwarding;
  85 };
  86
  87 class Gateway;
  88 typedef void (StateMethod)(Ftp::Gateway *);
  89
  90 /// FTP Gateway: An FTP client that takes an HTTP request with an ftp:// URI,
  91 /// converts it into one or more FTP commands, and then
  92 /// converts one or more FTP responses into the final HTTP response.
  93 class Gateway : public Ftp::Client
  94 {
  95     CBDATA_CLASS(Gateway);
  96
  97 public:
  98     Gateway(FwdState *);
  99     virtual ~Gateway();
 100     char user[MAX_URL];
 101     char password[MAX_URL];
 102     int password_url;
 103     char *reply_hdr;
 104     int reply_hdr_state;
 105     String clean_url;
 106     String title_url;
 107     String base_href;
 108     int conn_att;
 109     int login_att;
 110     time_t mdtm;
 111     int64_t theSize;
 112     wordlist *pathcomps;
 113     char *filepath;
 114     char *dirpath;
 115     int64_t restart_offset;
 116     char *proxy_host;
 117     size_t list_width;
 118     String cwd_message;
 119     char *old_filepath;
 120     char typecode;
 121     MemBuf listing;     ///< FTP directory listing in HTML format.
 122
 123     GatewayFlags flags;
 124
 125 public:
 126     // these should all be private
 127     virtual void start();
 128     virtual Http::StatusCode failedHttpStatus(err_type &error);
 129     int restartable();
 130     void appendSuccessHeader();
 131     void hackShortcut(StateMethod *nextState);
 132     void unhack();
 133     void readStor();
 134     void parseListing();
 135     MemBuf *htmlifyListEntry(const char *line);
 136     void completedListing(void);
 137
 138     /// create a data channel acceptor and start listening.
 139     void listenForDataChannel(const Comm::ConnectionPointer &conn);
 140
 141     int checkAuth(const HttpHeader * req_hdr);
 142     void checkUrlpath();
 143     void buildTitleUrl();
 144     void writeReplyBody(const char *, size_t len);
 145     void printfReplyBody(const char *fmt, ...);
 146     virtual void completeForwarding();
 147     void processHeadResponse();
 148     void processReplyBody();
 149     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 150     int64_t getCurrentOffset() const { return currentOffset; }
 151
 152     virtual void dataChannelConnected(const CommConnectCbParams &io);
 153     static PF ftpDataWrite;
 154     virtual void timeout(const CommTimeoutCbParams &io);
 155     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 156
 157     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 158     const char *ftpRealm(void);
 159     void loginFailed(void);
 160
 161     virtual void haveParsedReplyHeaders();
 162
 163     virtual bool haveControlChannel(const char *caller_name) const;
 164
 165 protected:
 166     virtual void handleControlReply();
 167     virtual void dataClosed(const CommCloseCbParams &io);
 168
 169 private:
 170     virtual bool mayReadVirginReplyBody() const;
 171     // BodyConsumer for HTTP: consume request body.
 172     virtual void handleRequestBodyProducerAborted();
 173
 174     void loginParser(const SBuf &login, bool escaped);
 175 };
 176
 177 } // namespace Ftp
 178
 179 typedef Ftp::StateMethod FTPSM; // to avoid lots of non-changes
 180
 181 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Gateway);
 182
 183 typedef struct {
 184     char type;
 185     int64_t size;
 186     char *date;
 187     char *name;
 188     char *showname;
 189     char *link;
 190 } ftpListParts;
 191
 192 #define CTRL_BUFLEN 1024
 193 static char cbuf[CTRL_BUFLEN];
 194
 195 /*
 196  * State machine functions
 197  * send == state transition
 198  * read == wait for response, and select next state transition
 199  * other == Transition logic
 200  */
 201 static FTPSM ftpReadWelcome;
 202 static FTPSM ftpSendUser;
 203 static FTPSM ftpReadUser;
 204 static FTPSM ftpSendPass;
 205 static FTPSM ftpReadPass;
 206 static FTPSM ftpSendType;
 207 static FTPSM ftpReadType;
 208 static FTPSM ftpSendMdtm;
 209 static FTPSM ftpReadMdtm;
 210 static FTPSM ftpSendSize;
 211 static FTPSM ftpReadSize;
 212 #if 0
 213 static FTPSM ftpSendEPRT;
 214 #endif
 215 static FTPSM ftpReadEPRT;
 216 static FTPSM ftpSendPORT;
 217 static FTPSM ftpReadPORT;
 218 static FTPSM ftpSendPassive;
 219 static FTPSM ftpReadEPSV;
 220 static FTPSM ftpReadPasv;
 221 static FTPSM ftpTraverseDirectory;
 222 static FTPSM ftpListDir;
 223 static FTPSM ftpGetFile;
 224 static FTPSM ftpSendCwd;
 225 static FTPSM ftpReadCwd;
 226 static FTPSM ftpRestOrList;
 227 static FTPSM ftpSendList;
 228 static FTPSM ftpSendNlst;
 229 static FTPSM ftpReadList;
 230 static FTPSM ftpSendRest;
 231 static FTPSM ftpReadRest;
 232 static FTPSM ftpSendRetr;
 233 static FTPSM ftpReadRetr;
 234 static FTPSM ftpReadTransferDone;
 235 static FTPSM ftpSendStor;
 236 static FTPSM ftpReadStor;
 237 static FTPSM ftpWriteTransferDone;
 238 static FTPSM ftpSendReply;
 239 static FTPSM ftpSendMkdir;
 240 static FTPSM ftpReadMkdir;
 241 static FTPSM ftpFail;
 242 static FTPSM ftpSendQuit;
 243 static FTPSM ftpReadQuit;
 244
 245 /************************************************
 246 ** Debugs Levels used here                     **
 247 *************************************************
 248 0   CRITICAL Events
 249 1   IMPORTANT Events
 250     Protocol and Transmission failures.
 251 2   FTP Protocol Chatter
 252 3   Logic Flows
 253 4   Data Parsing Flows
 254 5   Data Dumps
 255 7   ??
 256 ************************************************/
 257
 258 /************************************************
 259 ** State Machine Description (excluding hacks) **
 260 *************************************************
 261 From            To
 262 ---------------------------------------
 263 Welcome         User
 264 User            Pass
 265 Pass            Type
 266 Type            TraverseDirectory / GetFile
 267 TraverseDirectory   Cwd / GetFile / ListDir
 268 Cwd         TraverseDirectory / Mkdir
 269 GetFile         Mdtm
 270 Mdtm            Size
 271 Size            Epsv
 272 ListDir         Epsv
 273 Epsv            FileOrList
 274 FileOrList      Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 275 Rest            Retr
 276 Retr / Nlst / List  DataRead* (on datachannel)
 277 DataRead*       ReadTransferDone
 278 ReadTransferDone    DataTransferDone
 279 Stor            DataWrite* (on datachannel)
 280 DataWrite*      RequestPutBody** (from client)
 281 RequestPutBody**    DataWrite* / WriteTransferDone
 282 WriteTransferDone   DataTransferDone
 283 DataTransferDone    Quit
 284 Quit            -
 285 ************************************************/
 286
 287 FTPSM *FTP_SM_FUNCS[] = {
 288     ftpReadWelcome,     /* BEGIN */
 289     ftpReadUser,        /* SENT_USER */
 290     ftpReadPass,        /* SENT_PASS */
 291     ftpReadType,        /* SENT_TYPE */
 292     ftpReadMdtm,        /* SENT_MDTM */
 293     ftpReadSize,        /* SENT_SIZE */
 294     ftpReadEPRT,        /* SENT_EPRT */
 295     ftpReadPORT,        /* SENT_PORT */
 296     ftpReadEPSV,        /* SENT_EPSV_ALL */
 297     ftpReadEPSV,        /* SENT_EPSV_1 */
 298     ftpReadEPSV,        /* SENT_EPSV_2 */
 299     ftpReadPasv,        /* SENT_PASV */
 300     ftpReadCwd,     /* SENT_CWD */
 301     ftpReadList,        /* SENT_LIST */
 302     ftpReadList,        /* SENT_NLST */
 303     ftpReadRest,        /* SENT_REST */
 304     ftpReadRetr,        /* SENT_RETR */
 305     ftpReadStor,        /* SENT_STOR */
 306     ftpReadQuit,        /* SENT_QUIT */
 307     ftpReadTransferDone,    /* READING_DATA (RETR,LIST,NLST) */
 308     ftpWriteTransferDone,   /* WRITING_DATA (STOR) */
 309     ftpReadMkdir,       /* SENT_MKDIR */
 310     NULL,           /* SENT_FEAT */
 311     NULL,           /* SENT_PWD */
 312     NULL,           /* SENT_CDUP*/
 313     NULL,           /* SENT_DATA_REQUEST */
 314     NULL            /* SENT_COMMAND */
 315 };
 316
 317 /// handler called by Comm when FTP data channel is closed unexpectedly
 318 void
 319 Ftp::Gateway::dataClosed(const CommCloseCbParams &io)
 320 {
 321     Ftp::Client::dataClosed(io);
 322     failed(ERR_FTP_FAILURE, 0);
 323     /* failed closes ctrl.conn and frees ftpState */
 324
 325     /* NP: failure recovery may be possible when its only a data.conn failure.
 326      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 327      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 328      */
 329 }
 330
 331 Ftp::Gateway::Gateway(FwdState *fwdState):
 332     AsyncJob("FtpStateData"),
 333     Ftp::Client(fwdState),
 334     password_url(0),
 335     reply_hdr(NULL),
 336     reply_hdr_state(0),
 337     conn_att(0),
 338     login_att(0),
 339     mdtm(-1),
 340     theSize(-1),
 341     pathcomps(NULL),
 342     filepath(NULL),
 343     dirpath(NULL),
 344     restart_offset(0),
 345     proxy_host(NULL),
 346     list_width(0),
 347     old_filepath(NULL),
 348     typecode('\0')
 349 {
 350     debugs(9, 3, entry->url());
 351
 352     *user = 0;
 353     *password = 0;
 354     memset(&flags, 0, sizeof(flags));
 355
 356     if (Config.Ftp.passive && !flags.pasv_failed)
 357         flags.pasv_supported = 1;
 358
 359     flags.rest_supported = 1;
 360
 361     if (request->method == Http::METHOD_PUT)
 362         flags.put = 1;
 363
 364     initReadBuf();
 365 }
 366
 367 Ftp::Gateway::~Gateway()
 368 {
 369     debugs(9, 3, entry->url());
 370
 371     if (Comm::IsConnOpen(ctrl.conn)) {
 372         debugs(9, DBG_IMPORTANT, "Internal bug: FTP Gateway left open " <<
 373                "control channel " << ctrl.conn);
 374     }
 375
 376     if (reply_hdr) {
 377         memFree(reply_hdr, MEM_8K_BUF);
 378         reply_hdr = NULL;
 379     }
 380
 381     if (pathcomps)
 382         wordlistDestroy(&pathcomps);
 383
 384     cwd_message.clean();
 385     xfree(old_filepath);
 386     title_url.clean();
 387     base_href.clean();
 388     xfree(filepath);
 389     xfree(dirpath);
 390 }
 391
 392 /**
 393  * Parse a possible login username:password pair.
 394  * Produces filled member variables user, password, password_url if anything found.
 395  *
 396  * \param login    a decoded Basic authentication credential token or URI user-info token
 397  * \param escaped  whether to URL-decode the token after extracting user and password
 398  */
 399 void
 400 Ftp::Gateway::loginParser(const SBuf &login, bool escaped)
 401 {
 402     debugs(9, 4, "login=" << login << ", escaped=" << escaped);
 403     debugs(9, 9, "IN : login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 404
 405     if (login.isEmpty())
 406         return;
 407
 408     const SBuf::size_type colonPos = login.find(':');
 409
 410     /* If there was a username part with at least one character use it.
 411      * Ignore 0-length username portion, retain what we have already.
 412      */
 413     if (colonPos == SBuf::npos || colonPos > 0) {
 414         const SBuf userName = login.substr(0, colonPos);
 415         SBuf::size_type upto = userName.copy(user, sizeof(user)-1);
 416         user[upto]='\0';
 417         debugs(9, 9, "found user=" << userName << ' ' <<
 418                (upto != userName.length() ? ", truncated-to=" : ", length=") << upto <<
 419                ", escaped=" << escaped);
 420         if (escaped)
 421             rfc1738_unescape(user);
 422         debugs(9, 9, "found user=" << user << " (" << strlen(user) << ") unescaped.");
 423     }
 424
 425     /* If there was a password part.
 426      * For 0-length password clobber what we have already, this means explicitly none
 427      */
 428     if (colonPos != SBuf::npos) {
 429         const SBuf pass = login.substr(colonPos+1, SBuf::npos);
 430         SBuf::size_type upto = pass.copy(password, sizeof(password)-1);
 431         password[upto]='\0';
 432         debugs(9, 9, "found password=" << pass << " " <<
 433                (upto != pass.length() ? ", truncated-to=" : ", length=") << upto <<
 434                ", escaped=" << escaped);
 435         if (escaped) {
 436             rfc1738_unescape(password);
 437             password_url = 1;
 438         }
 439         debugs(9, 9, "found password=" << password << " (" << strlen(password) << ") unescaped.");
 440     }
 441
 442     debugs(9, 9, "OUT: login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 443 }
 444
 445 void
 446 Ftp::Gateway::listenForDataChannel(const Comm::ConnectionPointer &conn)
 447 {
 448     if (!Comm::IsConnOpen(ctrl.conn)) {
 449         debugs(9, 5, "The control connection to the remote end is closed");
 450         return;
 451     }
 452
 453     assert(!Comm::IsConnOpen(data.conn));
 454
 455     typedef CommCbMemFunT<Gateway, CommAcceptCbParams> AcceptDialer;
 456     typedef AsyncCallT<AcceptDialer> AcceptCall;
 457     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, Ftp::Gateway::ftpAcceptDataConnection));
 458     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 459     const char *note = entry->url();
 460
 461     /* open the conn if its not already open */
 462     if (!Comm::IsConnOpen(conn)) {
 463         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 464         if (!Comm::IsConnOpen(conn)) {
 465             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 466             return;
 467         }
 468         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 469     }
 470
 471     conn->tos = ctrl.conn->tos;
 472     conn->nfmark = ctrl.conn->nfmark;
 473
 474     assert(Comm::IsConnOpen(conn));
 475     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 476
 477     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 478     data.opened(conn, dataCloser());
 479     switchTimeoutToDataChannel();
 480 }
 481
 482 void
 483 Ftp::Gateway::timeout(const CommTimeoutCbParams &io)
 484 {
 485     if (SENT_PASV == state) {
 486         /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */
 487         flags.pasv_supported = false;
 488         debugs(9, DBG_IMPORTANT, "FTP Gateway timeout in SENT_PASV state");
 489
 490         // cancel the data connection setup.
 491         if (data.opener != NULL) {
 492             data.opener->cancel("timeout");
 493             data.opener = NULL;
 494         }
 495         data.close();
 496     }
 497
 498     Ftp::Client::timeout(io);
 499 }
 500
 501 static const char *Month[] = {
 502     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 503     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 504 };
 505
 506 static int
 507 is_month(const char *buf)
 508 {
 509     int i;
 510
 511     for (i = 0; i < 12; ++i)
 512         if (!strcasecmp(buf, Month[i]))
 513             return 1;
 514
 515     return 0;
 516 }
 517
 518 static void
 519 ftpListPartsFree(ftpListParts ** parts)
 520 {
 521     safe_free((*parts)->date);
 522     safe_free((*parts)->name);
 523     safe_free((*parts)->showname);
 524     safe_free((*parts)->link);
 525     safe_free(*parts);
 526 }
 527
 528 #define MAX_TOKENS 64
 529
 530 static ftpListParts *
 531 ftpListParseParts(const char *buf, struct Ftp::GatewayFlags flags)
 532 {
 533     ftpListParts *p = NULL;
 534     char *t = NULL;
 535     const char *ct = NULL;
 536     char *tokens[MAX_TOKENS];
 537     int i;
 538     int n_tokens;
 539     static char tbuf[128];
 540     char *xbuf = NULL;
 541     static int scan_ftp_initialized = 0;
 542     static regex_t scan_ftp_integer;
 543     static regex_t scan_ftp_time;
 544     static regex_t scan_ftp_dostime;
 545     static regex_t scan_ftp_dosdate;
 546
 547     if (!scan_ftp_initialized) {
 548         scan_ftp_initialized = 1;
 549         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 550         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 551         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 552         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 553     }
 554
 555     if (buf == NULL)
 556         return NULL;
 557
 558     if (*buf == '\0')
 559         return NULL;
 560
 561     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 562
 563     n_tokens = 0;
 564
 565     memset(tokens, 0, sizeof(tokens));
 566
 567     xbuf = xstrdup(buf);
 568
 569     if (flags.tried_nlst) {
 570         /* Machine readable format, one name per line */
 571         p->name = xbuf;
 572         p->type = '\0';
 573         return p;
 574     }
 575
 576     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 577         tokens[n_tokens] = xstrdup(t);
 578         ++n_tokens;
 579     }
 580
 581     xfree(xbuf);
 582
 583     /* locate the Month field */
 584     for (i = 3; i < n_tokens - 2; ++i) {
 585         char *size = tokens[i - 1];
 586         char *month = tokens[i];
 587         char *day = tokens[i + 1];
 588         char *year = tokens[i + 2];
 589
 590         if (!is_month(month))
 591             continue;
 592
 593         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 594             continue;
 595
 596         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 597             continue;
 598
 599         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0) /* Yr | hh:mm */
 600             continue;
 601
 602         snprintf(tbuf, 128, "%s %2s %5s",
 603                  month, day, year);
 604
 605         if (!strstr(buf, tbuf))
 606             snprintf(tbuf, 128, "%s %2s %-5s",
 607                      month, day, year);
 608
 609         char const *copyFrom = NULL;
 610
 611         if ((copyFrom = strstr(buf, tbuf))) {
 612             p->type = *tokens[0];
 613             p->size = strtoll(size, NULL, 10);
 614             p->date = xstrdup(tbuf);
 615
 616             if (flags.skip_whitespace) {
 617                 copyFrom += strlen(tbuf);
 618
 619                 while (strchr(w_space, *copyFrom))
 620                     ++copyFrom;
 621             } else {
 622                 /* XXX assumes a single space between date and filename
 623                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 624                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 625                 copyFrom += strlen(tbuf) + 1;
 626             }
 627
 628             p->name = xstrdup(copyFrom);
 629
 630             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 631                 *t = '\0';
 632                 p->link = xstrdup(t + 4);
 633             }
 634
 635             goto found;
 636         }
 637
 638         break;
 639     }
 640
 641     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 642     if (n_tokens > 3 &&
 643             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 644             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 645         if (!strcasecmp(tokens[2], "<dir>")) {
 646             p->type = 'd';
 647         } else {
 648             p->type = '-';
 649             p->size = strtoll(tokens[2], NULL, 10);
 650         }
 651
 652         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 653         p->date = xstrdup(tbuf);
 654
 655         if (p->type == 'd') {
 656             /* Directory.. name begins with first printable after <dir> */
 657             ct = strstr(buf, tokens[2]);
 658             ct += strlen(tokens[2]);
 659
 660             while (xisspace(*ct))
 661                 ++ct;
 662
 663             if (!*ct)
 664                 ct = NULL;
 665         } else {
 666             /* A file. Name begins after size, with a space in between */
 667             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 668             ct = strstr(buf, tbuf);
 669
 670             if (ct) {
 671                 ct += strlen(tokens[2]) + 2;
 672             }
 673         }
 674
 675         p->name = xstrdup(ct ? ct : tokens[3]);
 676         goto found;
 677     }
 678
 679     /* Try EPLF format; carson@lehman.com */
 680     if (buf[0] == '+') {
 681         ct = buf + 1;
 682         p->type = 0;
 683
 684         while (ct && *ct) {
 685             time_t tm;
 686             int l = strcspn(ct, ",");
 687             char *tmp;
 688
 689             if (l < 1)
 690                 goto blank;
 691
 692             switch (*ct) {
 693
 694             case '\t':
 695                 p->name = xstrndup(ct + 1, l + 1);
 696                 break;
 697
 698             case 's':
 699                 p->size = atoi(ct + 1);
 700                 break;
 701
 702             case 'm':
 703                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 704
 705                 if (tmp != ct + 1)
 706                     break;  /* not a valid integer */
 707
 708                 p->date = xstrdup(ctime(&tm));
 709
 710                 *(strstr(p->date, "\n")) = '\0';
 711
 712                 break;
 713
 714             case '/':
 715                 p->type = 'd';
 716
 717                 break;
 718
 719             case 'r':
 720                 p->type = '-';
 721
 722                 break;
 723
 724             case 'i':
 725                 break;
 726
 727             default:
 728                 break;
 729             }
 730
 731 blank:
 732             ct = strstr(ct, ",");
 733
 734             if (ct) {
 735                 ++ct;
 736             }
 737         }
 738
 739         if (p->type == 0) {
 740             p->type = '-';
 741         }
 742
 743         if (p->name)
 744             goto found;
 745         else
 746             safe_free(p->date);
 747     }
 748
 749 found:
 750
 751     for (i = 0; i < n_tokens; ++i)
 752         xfree(tokens[i]);
 753
 754     if (!p->name)
 755         ftpListPartsFree(&p);   /* cleanup */
 756
 757     return p;
 758 }
 759
 760 MemBuf *
 761 Ftp::Gateway::htmlifyListEntry(const char *line)
 762 {
 763     char icon[2048];
 764     char href[2048 + 40];
 765     char text[ 2048];
 766     char size[ 2048];
 767     char chdir[ 2048 + 40];
 768     char view[ 2048 + 40];
 769     char download[ 2048 + 40];
 770     char link[ 2048 + 40];
 771     MemBuf *html;
 772     char prefix[2048];
 773     ftpListParts *parts;
 774     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 775
 776     debugs(9, 7, HERE << " line ={" << line << "}");
 777
 778     if (strlen(line) > 1024) {
 779         html = new MemBuf();
 780         html->init();
 781         html->appendf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 782         return html;
 783     }
 784
 785     if (flags.dir_slash && dirpath && typecode != 'D')
 786         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 787     else
 788         prefix[0] = '\0';
 789
 790     if ((parts = ftpListParseParts(line, flags)) == NULL) {
 791         const char *p;
 792
 793         html = new MemBuf();
 794         html->init();
 795         html->appendf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
 796
 797         for (p = line; *p && xisspace(*p); ++p);
 798         if (*p && !xisspace(*p))
 799             flags.listformat_unknown = 1;
 800
 801         return html;
 802     }
 803
 804     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
 805         ftpListPartsFree(&parts);
 806         return NULL;
 807     }
 808
 809     parts->size += 1023;
 810     parts->size >>= 10;
 811     parts->showname = xstrdup(parts->name);
 812
 813     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
 814     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
 815
 816     xstrncpy(text, parts->showname, 2048);
 817
 818     switch (parts->type) {
 819
 820     case 'd':
 821         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 822                  mimeGetIconURL("internal-dir"),
 823                  "[DIR]");
 824         strcat(href, "/");  /* margin is allocated above */
 825         break;
 826
 827     case 'l':
 828         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 829                  mimeGetIconURL("internal-link"),
 830                  "[LINK]");
 831         /* sometimes there is an 'l' flag, but no "->" link */
 832
 833         if (parts->link) {
 834             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
 835             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
 836                      *link2 != '/' ? prefix : "", link2,
 837                      html_quote(parts->link));
 838             safe_free(link2);
 839         }
 840
 841         break;
 842
 843     case '\0':
 844         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 845                  mimeGetIconURL(parts->name),
 846                  "[UNKNOWN]");
 847         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
 848                  "alt=\"[DIR]\"></a>",
 849                  rfc1738_escape_part(parts->name),
 850                  mimeGetIconURL("internal-dir"));
 851         break;
 852
 853     case '-':
 854
 855     default:
 856         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 857                  mimeGetIconURL(parts->name),
 858                  "[FILE]");
 859         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
 860         break;
 861     }
 862
 863     if (parts->type != 'd') {
 864         if (mimeGetViewOption(parts->name)) {
 865             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
 866                      "alt=\"[VIEW]\"></a>",
 867                      prefix, href, mimeGetIconURL("internal-view"));
 868         }
 869
 870         if (mimeGetDownloadOption(parts->name)) {
 871             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
 872                      "alt=\"[DOWNLOAD]\"></a>",
 873                      prefix, href, mimeGetIconURL("internal-download"));
 874         }
 875     }
 876
 877     /* construct the table row from parts. */
 878     html = new MemBuf();
 879     html->init();
 880     html->appendf("<tr class=\"entry\">"
 881                   "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
 882                   "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
 883                   "<td class=\"date\">%s</td>"
 884                   "<td class=\"size\">%s</td>"
 885                   "<td class=\"actions\">%s%s%s%s</td>"
 886                   "</tr>\n",
 887                   prefix, href, icon,
 888                   prefix, href, html_quote(text),
 889                   parts->date,
 890                   size,
 891                   chdir, view, download, link);
 892
 893     ftpListPartsFree(&parts);
 894     return html;
 895 }
 896
 897 void
 898 Ftp::Gateway::parseListing()
 899 {
 900     char *buf = data.readBuf->content();
 901     char *sbuf;         /* NULL-terminated copy of termedBuf */
 902     char *end;
 903     char *line;
 904     char *s;
 905     MemBuf *t;
 906     size_t linelen;
 907     size_t usable;
 908     size_t len = data.readBuf->contentSize();
 909
 910     if (!len) {
 911         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
 912         return;
 913     }
 914
 915     /*
 916      * We need a NULL-terminated buffer for scanning, ick
 917      */
 918     sbuf = (char *)xmalloc(len + 1);
 919     xstrncpy(sbuf, buf, len + 1);
 920     end = sbuf + len - 1;
 921
 922     while (*end != '\r' && *end != '\n' && end > sbuf)
 923         --end;
 924
 925     usable = end - sbuf;
 926
 927     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
 928
 929     if (usable == 0) {
 930         if (buf[0] == '\0' && len == 1) {
 931             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
 932             data.readBuf->consume(len);
 933         } else {
 934             debugs(9, 3, HERE << "didn't find end for " << entry->url());
 935             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
 936         }
 937         xfree(sbuf);
 938         return;
 939     }
 940
 941     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
 942
 943     line = (char *)memAllocate(MEM_4K_BUF);
 944     ++end;
 945     s = sbuf;
 946     s += strspn(s, crlf);
 947
 948     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
 949         debugs(9, 7, HERE << "s = {" << s << "}");
 950         linelen = strcspn(s, crlf) + 1;
 951
 952         if (linelen < 2)
 953             break;
 954
 955         if (linelen > 4096)
 956             linelen = 4096;
 957
 958         xstrncpy(line, s, linelen);
 959
 960         debugs(9, 7, HERE << "{" << line << "}");
 961
 962         if (!strncmp(line, "total", 5))
 963             continue;
 964
 965         t = htmlifyListEntry(line);
 966
 967         if ( t != NULL) {
 968             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
 969             listing.append(t->content(), t->contentSize());
 970             delete t;
 971         }
 972     }
 973
 974     debugs(9, 7, HERE << "Done.");
 975     data.readBuf->consume(usable);
 976     memFree(line, MEM_4K_BUF);
 977     xfree(sbuf);
 978 }
 979
 980 void
 981 Ftp::Gateway::processReplyBody()
 982 {
 983     debugs(9, 3, status());
 984
 985     if (request->method == Http::METHOD_HEAD && (flags.isdir || theSize != -1)) {
 986         serverComplete();
 987         return;
 988     }
 989
 990     /* Directory listings are special. They write ther own headers via the error objects */
 991     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
 992         appendSuccessHeader();
 993
 994     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 995         /*
 996          * probably was aborted because content length exceeds one
 997          * of the maximum size limits.
 998          */
 999         abortAll("entry aborted after calling appendSuccessHeader()");
1000         return;
1001     }
1002
1003 #if USE_ADAPTATION
1004
1005     if (adaptationAccessCheckPending) {
1006         debugs(9, 3, "returning from Ftp::Gateway::processReplyBody due to adaptationAccessCheckPending");
1007         return;
1008     }
1009
1010 #endif
1011
1012     if (flags.isdir) {
1013         if (!flags.listing) {
1014             flags.listing = 1;
1015             listing.reset();
1016         }
1017         parseListing();
1018         maybeReadVirginBody();
1019         return;
1020     } else if (const int csize = data.readBuf->contentSize()) {
1021         writeReplyBody(data.readBuf->content(), csize);
1022         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1023         data.readBuf->consume(csize);
1024     }
1025
1026     entry->flush();
1027
1028     maybeReadVirginBody();
1029 }
1030
1031 /**
1032  * Locates the FTP user:password login.
1033  *
1034  * Highest to lowest priority:
1035  *  - Checks URL (ftp://user:pass@domain)
1036  *  - Authorization: Basic header
1037  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1038  *
1039  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1040  *
1041  * TODO: we might be able to do something about locating username from other sources:
1042  *       ie, external ACL user=* tag or ident lookup
1043  *
1044  \retval 1  if we have everything needed to complete this request.
1045  \retval 0  if something is missing.
1046  */
1047 int
1048 Ftp::Gateway::checkAuth(const HttpHeader * req_hdr)
1049 {
1050     /* default username */
1051     xstrncpy(user, "anonymous", MAX_URL);
1052
1053 #if HAVE_AUTH_MODULE_BASIC
1054     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1055     const SBuf auth(req_hdr->getAuth(Http::HdrType::AUTHORIZATION, "Basic"));
1056     if (!auth.isEmpty()) {
1057         flags.authenticated = 1;
1058         loginParser(auth, false);
1059     }
1060     /* we fail with authorization-required error later IFF the FTP server requests it */
1061 #endif
1062
1063     /* Test URL login syntax. Overrides any headers received. */
1064     loginParser(request->url.userInfo(), true);
1065
1066     /* name is missing. thats fatal. */
1067     if (!user[0])
1068         fatal("FTP login parsing destroyed username info");
1069
1070     /* name + password == success */
1071     if (password[0])
1072         return 1;
1073
1074     /* Setup default FTP password settings */
1075     /* this has to be done last so that we can have a no-password case above. */
1076     if (!password[0]) {
1077         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1078             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1079             flags.tried_auth_anonymous=1;
1080             return 1;
1081         } else if (!flags.tried_auth_nopass) {
1082             xstrncpy(password, null_string, MAX_URL);
1083             flags.tried_auth_nopass=1;
1084             return 1;
1085         }
1086     }
1087
1088     return 0;           /* different username */
1089 }
1090
1091 void
1092 Ftp::Gateway::checkUrlpath()
1093 {
1094     static SBuf str_type_eq("type=");
1095     auto t = request->url.path().rfind(';');
1096
1097     if (t != SBuf::npos) {
1098         auto filenameEnd = t-1;
1099         if (request->url.path().substr(++t).cmp(str_type_eq, str_type_eq.length()) == 0) {
1100             t += str_type_eq.length();
1101             typecode = (char)xtoupper(request->url.path()[t]);
1102             request->url.path(request->url.path().substr(0,filenameEnd));
1103         }
1104     }
1105
1106     int l = request->url.path().length();
1107     /* check for null path */
1108
1109     if (!l) {
1110         flags.isdir = 1;
1111         flags.root_dir = 1;
1112         flags.need_base_href = 1;   /* Work around broken browsers */
1113     } else if (!request->url.path().cmp("/%2f/")) {
1114         /* UNIX root directory */
1115         flags.isdir = 1;
1116         flags.root_dir = 1;
1117     } else if ((l >= 1) && (request->url.path()[l-1] == '/')) {
1118         /* Directory URL, ending in / */
1119         flags.isdir = 1;
1120
1121         if (l == 1)
1122             flags.root_dir = 1;
1123     } else {
1124         flags.dir_slash = 1;
1125     }
1126 }
1127
1128 void
1129 Ftp::Gateway::buildTitleUrl()
1130 {
1131     title_url = "ftp://";
1132
1133     if (strcmp(user, "anonymous")) {
1134         title_url.append(user);
1135         title_url.append("@");
1136     }
1137
1138     SBuf authority = request->url.authority(request->url.getScheme() != AnyP::PROTO_FTP);
1139
1140     title_url.append(authority.rawContent(), authority.length());
1141     title_url.append(request->url.path().rawContent(), request->url.path().length());
1142
1143     base_href = "ftp://";
1144
1145     if (strcmp(user, "anonymous") != 0) {
1146         base_href.append(rfc1738_escape_part(user));
1147
1148         if (password_url) {
1149             base_href.append(":");
1150             base_href.append(rfc1738_escape_part(password));
1151         }
1152
1153         base_href.append("@");
1154     }
1155
1156     base_href.append(authority.rawContent(), authority.length());
1157     base_href.append(request->url.path().rawContent(), request->url.path().length());
1158     base_href.append("/");
1159 }
1160
1161 void
1162 Ftp::Gateway::start()
1163 {
1164     if (!checkAuth(&request->header)) {
1165         /* create appropriate reply */
1166         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1167         entry->replaceHttpReply(reply);
1168         serverComplete();
1169         return;
1170     }
1171
1172     checkUrlpath();
1173     buildTitleUrl();
1174     debugs(9, 5, "FD " << (ctrl.conn ? ctrl.conn->fd : -1) << " : host=" << request->url.host() <<
1175            ", path=" << request->url.path() << ", user=" << user << ", passwd=" << password);
1176     state = BEGIN;
1177     Ftp::Client::start();
1178 }
1179
1180 /* ====================================================================== */
1181
1182 void
1183 Ftp::Gateway::handleControlReply()
1184 {
1185     Ftp::Client::handleControlReply();
1186     if (ctrl.message == NULL)
1187         return; // didn't get complete reply yet
1188
1189     /* Copy the message except for the last line to cwd_message to be
1190      * printed in error messages.
1191      */
1192     for (wordlist *w = ctrl.message; w && w->next; w = w->next) {
1193         cwd_message.append('\n');
1194         cwd_message.append(w->key);
1195     }
1196
1197     FTP_SM_FUNCS[state] (this);
1198 }
1199
1200 /* ====================================================================== */
1201
1202 static void
1203 ftpReadWelcome(Ftp::Gateway * ftpState)
1204 {
1205     int code = ftpState->ctrl.replycode;
1206     debugs(9, 3, HERE);
1207
1208     if (ftpState->flags.pasv_only)
1209         ++ ftpState->login_att;
1210
1211     if (code == 220) {
1212         if (ftpState->ctrl.message) {
1213             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1214                 ftpState->flags.skip_whitespace = 1;
1215         }
1216
1217         ftpSendUser(ftpState);
1218     } else if (code == 120) {
1219         if (NULL != ftpState->ctrl.message)
1220             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1221
1222         return;
1223     } else {
1224         ftpFail(ftpState);
1225     }
1226 }
1227
1228 /**
1229  * Translate FTP login failure into HTTP error
1230  * this is an attmpt to get the 407 message to show up outside Squid.
1231  * its NOT a general failure. But a correct FTP response type.
1232  */
1233 void
1234 Ftp::Gateway::loginFailed()
1235 {
1236     ErrorState *err = NULL;
1237
1238     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1239         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1240             // 421/426 - Service Overload - retry permitted.
1241             err = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
1242         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1243             // 43x - Invalid or Credential Error - retry challenge required.
1244             err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1245         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1246             // 53x - Credentials Missing - retry challenge required
1247             if (password_url) // but they were in the URI! major fail.
1248                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
1249             else
1250                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1251         }
1252     }
1253
1254     if (!err) {
1255         ftpFail(this);
1256         return;
1257     }
1258
1259     failed(ERR_NONE, ctrl.replycode, err);
1260     // any other problems are general falures.
1261
1262     HttpReply *newrep = err->BuildHttpReply();
1263     delete err;
1264
1265 #if HAVE_AUTH_MODULE_BASIC
1266     /* add Authenticate header */
1267     newrep->header.putAuth("Basic", ftpRealm());
1268 #endif
1269
1270     // add it to the store entry for response....
1271     entry->replaceHttpReply(newrep);
1272     serverComplete();
1273 }
1274
1275 const char *
1276 Ftp::Gateway::ftpRealm()
1277 {
1278     static char realm[8192];
1279
1280     /* This request is not fully authenticated */
1281     if (!request) {
1282         snprintf(realm, 8192, "FTP %s unknown", user);
1283     } else if (request->url.port() == 21) {
1284         snprintf(realm, 8192, "FTP %s %s", user, request->url.host());
1285     } else {
1286         snprintf(realm, 8192, "FTP %s %s port %d", user, request->url.host(), request->url.port());
1287     }
1288     return realm;
1289 }
1290
1291 static void
1292 ftpSendUser(Ftp::Gateway * ftpState)
1293 {
1294     /* check the server control channel is still available */
1295     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1296         return;
1297
1298     if (ftpState->proxy_host != NULL)
1299         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n", ftpState->user, ftpState->request->url.host());
1300     else
1301         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1302
1303     ftpState->writeCommand(cbuf);
1304
1305     ftpState->state = Ftp::Client::SENT_USER;
1306 }
1307
1308 static void
1309 ftpReadUser(Ftp::Gateway * ftpState)
1310 {
1311     int code = ftpState->ctrl.replycode;
1312     debugs(9, 3, HERE);
1313
1314     if (code == 230) {
1315         ftpReadPass(ftpState);
1316     } else if (code == 331) {
1317         ftpSendPass(ftpState);
1318     } else {
1319         ftpState->loginFailed();
1320     }
1321 }
1322
1323 static void
1324 ftpSendPass(Ftp::Gateway * ftpState)
1325 {
1326     /* check the server control channel is still available */
1327     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
1328         return;
1329
1330     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
1331     ftpState->writeCommand(cbuf);
1332     ftpState->state = Ftp::Client::SENT_PASS;
1333 }
1334
1335 static void
1336 ftpReadPass(Ftp::Gateway * ftpState)
1337 {
1338     int code = ftpState->ctrl.replycode;
1339     debugs(9, 3, HERE << "code=" << code);
1340
1341     if (code == 230) {
1342         ftpSendType(ftpState);
1343     } else {
1344         ftpState->loginFailed();
1345     }
1346 }
1347
1348 static void
1349 ftpSendType(Ftp::Gateway * ftpState)
1350 {
1351     /* check the server control channel is still available */
1352     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
1353         return;
1354
1355     /*
1356      * Ref section 3.2.2 of RFC 1738
1357      */
1358     char mode = ftpState->typecode;
1359
1360     switch (mode) {
1361
1362     case 'D':
1363         mode = 'A';
1364         break;
1365
1366     case 'A':
1367
1368     case 'I':
1369         break;
1370
1371     default:
1372
1373         if (ftpState->flags.isdir) {
1374             mode = 'A';
1375         } else {
1376             auto t = ftpState->request->url.path().rfind('/');
1377             // XXX: performance regression, c_str() may reallocate
1378             SBuf filename = ftpState->request->url.path().substr(t != SBuf::npos ? t + 1 : 0);
1379             mode = mimeGetTransferMode(filename.c_str());
1380         }
1381
1382         break;
1383     }
1384
1385     if (mode == 'I')
1386         ftpState->flags.binary = 1;
1387     else
1388         ftpState->flags.binary = 0;
1389
1390     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
1391
1392     ftpState->writeCommand(cbuf);
1393
1394     ftpState->state = Ftp::Client::SENT_TYPE;
1395 }
1396
1397 static void
1398 ftpReadType(Ftp::Gateway * ftpState)
1399 {
1400     int code = ftpState->ctrl.replycode;
1401     char *path;
1402     char *d, *p;
1403     debugs(9, 3, HERE << "code=" << code);
1404
1405     if (code == 200) {
1406         p = path = SBufToCstring(ftpState->request->url.path());
1407
1408         if (*p == '/')
1409             ++p;
1410
1411         while (*p) {
1412             d = p;
1413             p += strcspn(p, "/");
1414
1415             if (*p) {
1416                 *p = '\0';
1417                 ++p;
1418             }
1419
1420             rfc1738_unescape(d);
1421
1422             if (*d)
1423                 wordlistAdd(&ftpState->pathcomps, d);
1424         }
1425
1426         xfree(path);
1427
1428         if (ftpState->pathcomps)
1429             ftpTraverseDirectory(ftpState);
1430         else
1431             ftpListDir(ftpState);
1432     } else {
1433         ftpFail(ftpState);
1434     }
1435 }
1436
1437 static void
1438 ftpTraverseDirectory(Ftp::Gateway * ftpState)
1439 {
1440     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
1441
1442     safe_free(ftpState->dirpath);
1443     ftpState->dirpath = ftpState->filepath;
1444     ftpState->filepath = NULL;
1445
1446     /* Done? */
1447
1448     if (ftpState->pathcomps == NULL) {
1449         debugs(9, 3, HERE << "the final component was a directory");
1450         ftpListDir(ftpState);
1451         return;
1452     }
1453
1454     /* Go to next path component */
1455     ftpState->filepath = wordlistChopHead(& ftpState->pathcomps);
1456
1457     /* Check if we are to CWD or RETR */
1458     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
1459         ftpSendCwd(ftpState);
1460     } else {
1461         debugs(9, 3, HERE << "final component is probably a file");
1462         ftpGetFile(ftpState);
1463         return;
1464     }
1465 }
1466
1467 static void
1468 ftpSendCwd(Ftp::Gateway * ftpState)
1469 {
1470     char *path = NULL;
1471
1472     /* check the server control channel is still available */
1473     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
1474         return;
1475
1476     debugs(9, 3, HERE);
1477
1478     path = ftpState->filepath;
1479
1480     if (!strcmp(path, "..") || !strcmp(path, "/")) {
1481         ftpState->flags.no_dotdot = 1;
1482     } else {
1483         ftpState->flags.no_dotdot = 0;
1484     }
1485
1486     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
1487
1488     ftpState->writeCommand(cbuf);
1489
1490     ftpState->state = Ftp::Client::SENT_CWD;
1491 }
1492
1493 static void
1494 ftpReadCwd(Ftp::Gateway * ftpState)
1495 {
1496     int code = ftpState->ctrl.replycode;
1497     debugs(9, 3, HERE);
1498
1499     if (code >= 200 && code < 300) {
1500         /* CWD OK */
1501         ftpState->unhack();
1502
1503         /* Reset cwd_message to only include the last message */
1504         ftpState->cwd_message.reset("");
1505         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
1506             ftpState->cwd_message.append(' ');
1507             ftpState->cwd_message.append(w->key);
1508         }
1509         ftpState->ctrl.message = NULL;
1510
1511         /* Continue to traverse the path */
1512         ftpTraverseDirectory(ftpState);
1513     } else {
1514         /* CWD FAILED */
1515
1516         if (!ftpState->flags.put)
1517             ftpFail(ftpState);
1518         else
1519             ftpSendMkdir(ftpState);
1520     }
1521 }
1522
1523 static void
1524 ftpSendMkdir(Ftp::Gateway * ftpState)
1525 {
1526     char *path = NULL;
1527
1528     /* check the server control channel is still available */
1529     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
1530         return;
1531
1532     path = ftpState->filepath;
1533     debugs(9, 3, HERE << "with path=" << path);
1534     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
1535     ftpState->writeCommand(cbuf);
1536     ftpState->state = Ftp::Client::SENT_MKDIR;
1537 }
1538
1539 static void
1540 ftpReadMkdir(Ftp::Gateway * ftpState)
1541 {
1542     char *path = ftpState->filepath;
1543     int code = ftpState->ctrl.replycode;
1544
1545     debugs(9, 3, HERE << "path " << path << ", code " << code);
1546
1547     if (code == 257) {      /* success */
1548         ftpSendCwd(ftpState);
1549     } else if (code == 550) {   /* dir exists */
1550
1551         if (ftpState->flags.put_mkdir) {
1552             ftpState->flags.put_mkdir = 1;
1553             ftpSendCwd(ftpState);
1554         } else
1555             ftpSendReply(ftpState);
1556     } else
1557         ftpSendReply(ftpState);
1558 }
1559
1560 static void
1561 ftpGetFile(Ftp::Gateway * ftpState)
1562 {
1563     assert(*ftpState->filepath != '\0');
1564     ftpState->flags.isdir = 0;
1565     ftpSendMdtm(ftpState);
1566 }
1567
1568 static void
1569 ftpListDir(Ftp::Gateway * ftpState)
1570 {
1571     if (ftpState->flags.dir_slash) {
1572         debugs(9, 3, HERE << "Directory path did not end in /");
1573         ftpState->title_url.append("/");
1574         ftpState->flags.isdir = 1;
1575     }
1576
1577     ftpSendPassive(ftpState);
1578 }
1579
1580 static void
1581 ftpSendMdtm(Ftp::Gateway * ftpState)
1582 {
1583     /* check the server control channel is still available */
1584     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
1585         return;
1586
1587     assert(*ftpState->filepath != '\0');
1588     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
1589     ftpState->writeCommand(cbuf);
1590     ftpState->state = Ftp::Client::SENT_MDTM;
1591 }
1592
1593 static void
1594 ftpReadMdtm(Ftp::Gateway * ftpState)
1595 {
1596     int code = ftpState->ctrl.replycode;
1597     debugs(9, 3, HERE);
1598
1599     if (code == 213) {
1600         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
1601         ftpState->unhack();
1602     } else if (code < 0) {
1603         ftpFail(ftpState);
1604         return;
1605     }
1606
1607     ftpSendSize(ftpState);
1608 }
1609
1610 static void
1611 ftpSendSize(Ftp::Gateway * ftpState)
1612 {
1613     /* check the server control channel is still available */
1614     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
1615         return;
1616
1617     /* Only send SIZE for binary transfers. The returned size
1618      * is useless on ASCII transfers */
1619
1620     if (ftpState->flags.binary) {
1621         assert(ftpState->filepath != NULL);
1622         assert(*ftpState->filepath != '\0');
1623         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
1624         ftpState->writeCommand(cbuf);
1625         ftpState->state = Ftp::Client::SENT_SIZE;
1626     } else
1627         /* Skip to next state no non-binary transfers */
1628         ftpSendPassive(ftpState);
1629 }
1630
1631 static void
1632 ftpReadSize(Ftp::Gateway * ftpState)
1633 {
1634     int code = ftpState->ctrl.replycode;
1635     debugs(9, 3, HERE);
1636
1637     if (code == 213) {
1638         ftpState->unhack();
1639         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
1640
1641         if (ftpState->theSize == 0) {
1642             debugs(9, 2, "SIZE reported " <<
1643                    ftpState->ctrl.last_reply << " on " <<
1644                    ftpState->title_url);
1645             ftpState->theSize = -1;
1646         }
1647     } else if (code < 0) {
1648         ftpFail(ftpState);
1649         return;
1650     }
1651
1652     ftpSendPassive(ftpState);
1653 }
1654
1655 static void
1656 ftpReadEPSV(Ftp::Gateway* ftpState)
1657 {
1658     Ip::Address srvAddr; // unused
1659     if (ftpState->handleEpsvReply(srvAddr)) {
1660         if (ftpState->ctrl.message == NULL)
1661             return; // didn't get complete reply yet
1662
1663         ftpState->connectDataChannel();
1664     }
1665 }
1666
1667 /** Send Passive connection request.
1668  * Default method is to use modern EPSV request.
1669  * The failover mechanism should check for previous state and re-call with alternates on failure.
1670  */
1671 static void
1672 ftpSendPassive(Ftp::Gateway * ftpState)
1673 {
1674     /** Checks the server control channel is still available before running. */
1675     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
1676         return;
1677
1678     debugs(9, 3, HERE);
1679
1680     /** \par
1681       * Checks for 'HEAD' method request and passes off for special handling by Ftp::Gateway::processHeadResponse(). */
1682     if (ftpState->request->method == Http::METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
1683         ftpState->processHeadResponse(); // may call serverComplete
1684         return;
1685     }
1686
1687     if (ftpState->sendPassive()) {
1688         // SENT_EPSV_ALL blocks other non-EPSV connections being attempted
1689         if (ftpState->state == Ftp::Client::SENT_EPSV_ALL)
1690             ftpState->flags.epsv_all_sent = true;
1691     }
1692 }
1693
1694 void
1695 Ftp::Gateway::processHeadResponse()
1696 {
1697     debugs(9, 5, HERE << "handling HEAD response");
1698     ftpSendQuit(this);
1699     appendSuccessHeader();
1700
1701     /*
1702      * On rare occasions I'm seeing the entry get aborted after
1703      * readControlReply() and before here, probably when
1704      * trying to write to the client.
1705      */
1706     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1707         abortAll("entry aborted while processing HEAD");
1708         return;
1709     }
1710
1711 #if USE_ADAPTATION
1712     if (adaptationAccessCheckPending) {
1713         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
1714         return;
1715     }
1716 #endif
1717
1718     // processReplyBody calls serverComplete() since there is no body
1719     processReplyBody();
1720 }
1721
1722 static void
1723 ftpReadPasv(Ftp::Gateway * ftpState)
1724 {
1725     Ip::Address srvAddr; // unused
1726     if (ftpState->handlePasvReply(srvAddr))
1727         ftpState->connectDataChannel();
1728     else {
1729         ftpFail(ftpState);
1730         // Currently disabled, does not work correctly:
1731         // ftpSendEPRT(ftpState);
1732         return;
1733     }
1734 }
1735
1736 void
1737 Ftp::Gateway::dataChannelConnected(const CommConnectCbParams &io)
1738 {
1739     debugs(9, 3, HERE);
1740     data.opener = NULL;
1741
1742     if (io.flag != Comm::OK) {
1743         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
1744
1745         // ABORT on timeouts. server may be waiting on a broken TCP link.
1746         if (io.xerrno == Comm::TIMEOUT)
1747             writeCommand("ABOR");
1748
1749         // try another connection attempt with some other method
1750         ftpSendPassive(this);
1751         return;
1752     }
1753
1754     data.opened(io.conn, dataCloser());
1755     ftpRestOrList(this);
1756 }
1757
1758 static void
1759 ftpOpenListenSocket(Ftp::Gateway * ftpState, int fallback)
1760 {
1761     /// Close old data channels, if any. We may open a new one below.
1762     if (ftpState->data.conn != NULL) {
1763         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
1764             // NP: in fact it points to the control channel. just clear it.
1765             ftpState->data.clear();
1766         else
1767             ftpState->data.close();
1768     }
1769     safe_free(ftpState->data.host);
1770
1771     if (!Comm::IsConnOpen(ftpState->ctrl.conn)) {
1772         debugs(9, 5, "The control connection to the remote end is closed");
1773         return;
1774     }
1775
1776     /*
1777      * Set up a listen socket on the same local address as the
1778      * control connection.
1779      */
1780     Comm::ConnectionPointer temp = new Comm::Connection;
1781     temp->local = ftpState->ctrl.conn->local;
1782
1783     /*
1784      * REUSEADDR is needed in fallback mode, since the same port is
1785      * used for both control and data.
1786      */
1787     if (fallback) {
1788         int on = 1;
1789         errno = 0;
1790         if (setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR,
1791                        (char *) &on, sizeof(on)) == -1) {
1792             int xerrno = errno;
1793             // SO_REUSEADDR is only an optimization, no need to be verbose about error
1794             debugs(9, 4, "setsockopt failed: " << xstrerr(xerrno));
1795         }
1796         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
1797         temp->flags |= COMM_REUSEADDR;
1798     } else {
1799         /* if not running in fallback mode a new port needs to be retrieved */
1800         temp->local.port(0);
1801     }
1802
1803     ftpState->listenForDataChannel(temp);
1804 }
1805
1806 static void
1807 ftpSendPORT(Ftp::Gateway * ftpState)
1808 {
1809     /* check the server control channel is still available */
1810     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
1811         return;
1812
1813     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1814         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
1815         return;
1816     }
1817
1818     debugs(9, 3, HERE);
1819     ftpState->flags.pasv_supported = 0;
1820     ftpOpenListenSocket(ftpState, 0);
1821
1822     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
1823         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.isIPv4() ) {
1824             /* non-IPv4 CANNOT send PORT command.                       */
1825             /* we got here by attempting and failing an EPRT            */
1826             /* using the same reply code should simulate a PORT failure */
1827             ftpReadPORT(ftpState);
1828             return;
1829         }
1830
1831         /* XXX Need to set error message */
1832         ftpFail(ftpState);
1833         return;
1834     }
1835
1836     // pull out the internal IP address bytes to send in PORT command...
1837     // source them from the listen_conn->local
1838
1839     struct addrinfo *AI = NULL;
1840     ftpState->data.listenConn->local.getAddrInfo(AI, AF_INET);
1841     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
1842     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
1843     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
1844              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
1845              portptr[0], portptr[1]);
1846     ftpState->writeCommand(cbuf);
1847     ftpState->state = Ftp::Client::SENT_PORT;
1848
1849     Ip::Address::FreeAddr(AI);
1850 }
1851
1852 static void
1853 ftpReadPORT(Ftp::Gateway * ftpState)
1854 {
1855     int code = ftpState->ctrl.replycode;
1856     debugs(9, 3, HERE);
1857
1858     if (code != 200) {
1859         /* Fall back on using the same port as the control connection */
1860         debugs(9, 3, "PORT not supported by remote end");
1861         ftpOpenListenSocket(ftpState, 1);
1862     }
1863
1864     ftpRestOrList(ftpState);
1865 }
1866
1867 #if 0
1868 static void
1869 ftpSendEPRT(Ftp::Gateway * ftpState)
1870 {
1871     /* check the server control channel is still available */
1872     if (!ftpState || !ftpState->haveControlChannel("ftpSendEPRT"))
1873         return;
1874
1875     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1876         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
1877         return;
1878     }
1879
1880     if (!Config.Ftp.eprt) {
1881         /* Disabled. Switch immediately to attempting old PORT command. */
1882         debugs(9, 3, "EPRT disabled by local administrator");
1883         ftpSendPORT(ftpState);
1884         return;
1885     }
1886
1887     debugs(9, 3, HERE);
1888     ftpState->flags.pasv_supported = 0;
1889
1890     ftpOpenListenSocket(ftpState, 0);
1891     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
1892     if (!Comm::IsConnOpen(ftpState->data.conn)) {
1893         /* XXX Need to set error message */
1894         ftpFail(ftpState);
1895         return;
1896     }
1897
1898     char buf[MAX_IPSTRLEN];
1899
1900     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
1901     /* Which can be used by EITHER protocol. */
1902     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
1903              ( ftpState->data.listenConn->local.isIPv6() ? 2 : 1 ),
1904              ftpState->data.listenConn->local.toStr(buf,MAX_IPSTRLEN),
1905              ftpState->data.listenConn->local.port() );
1906
1907     ftpState->writeCommand(cbuf);
1908     ftpState->state = Ftp::Client::SENT_EPRT;
1909 }
1910 #endif
1911
1912 static void
1913 ftpReadEPRT(Ftp::Gateway * ftpState)
1914 {
1915     int code = ftpState->ctrl.replycode;
1916     debugs(9, 3, HERE);
1917
1918     if (code != 200) {
1919         /* Failover to attempting old PORT command. */
1920         debugs(9, 3, "EPRT not supported by remote end");
1921         ftpSendPORT(ftpState);
1922         return;
1923     }
1924
1925     ftpRestOrList(ftpState);
1926 }
1927
1928 /** "read" handler to accept FTP data connections.
1929  *
1930  \param io    comm accept(2) callback parameters
1931  */
1932 void
1933 Ftp::Gateway::ftpAcceptDataConnection(const CommAcceptCbParams &io)
1934 {
1935     debugs(9, 3, HERE);
1936
1937     if (!Comm::IsConnOpen(ctrl.conn)) { /*Close handlers will cleanup*/
1938         debugs(9, 5, "The control connection to the remote end is closed");
1939         return;
1940     }
1941
1942     if (io.flag != Comm::OK) {
1943         data.listenConn->close();
1944         data.listenConn = NULL;
1945         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
1946         /** \todo Need to send error message on control channel*/
1947         ftpFail(this);
1948         return;
1949     }
1950
1951     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1952         abortAll("entry aborted when accepting data conn");
1953         data.listenConn->close();
1954         data.listenConn = NULL;
1955         io.conn->close();
1956         return;
1957     }
1958
1959     /* data listening conn is no longer even open. abort. */
1960     if (!Comm::IsConnOpen(data.listenConn)) {
1961         data.listenConn = NULL; // ensure that it's cleared and not just closed.
1962         return;
1963     }
1964
1965     /* data listening conn is no longer even open. abort. */
1966     if (!Comm::IsConnOpen(data.conn)) {
1967         data.clear(); // ensure that it's cleared and not just closed.
1968         return;
1969     }
1970
1971     /** \par
1972      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
1973      * made by the remote client which is connected to the FTP control socket.
1974      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
1975      * This prevents third-party hacks, but also third-party load balancing handshakes.
1976      */
1977     if (Config.Ftp.sanitycheck) {
1978         // accept if either our data or ctrl connection is talking to this remote peer.
1979         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
1980             debugs(9, DBG_IMPORTANT,
1981                    "FTP data connection from unexpected server (" <<
1982                    io.conn->remote << "), expecting " <<
1983                    data.conn->remote << " or " << ctrl.conn->remote);
1984
1985             /* close the bad sources connection down ASAP. */
1986             io.conn->close();
1987
1988             /* drop the bad connection (io) by ignoring the attempt. */
1989             return;
1990         }
1991     }
1992
1993     /** On Comm::OK start using the accepted data socket and discard the temporary listen socket. */
1994     data.close();
1995     data.opened(io.conn, dataCloser());
1996     data.addr(io.conn->remote);
1997
1998     debugs(9, 3, HERE << "Connected data socket on " <<
1999            io.conn << ". FD table says: " <<
2000            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
2001            "data-peer= " << fd_table[data.conn->fd].ipaddr);
2002
2003     assert(haveControlChannel("ftpAcceptDataConnection"));
2004     assert(ctrl.message == NULL);
2005
2006     // Ctrl channel operations will determine what happens to this data connection
2007 }
2008
2009 static void
2010 ftpRestOrList(Ftp::Gateway * ftpState)
2011 {
2012     debugs(9, 3, HERE);
2013
2014     if (ftpState->typecode == 'D') {
2015         ftpState->flags.isdir = 1;
2016
2017         if (ftpState->flags.put) {
2018             ftpSendMkdir(ftpState); /* PUT name;type=d */
2019         } else {
2020             ftpSendNlst(ftpState);  /* GET name;type=d  sec 3.2.2 of RFC 1738 */
2021         }
2022     } else if (ftpState->flags.put) {
2023         ftpSendStor(ftpState);
2024     } else if (ftpState->flags.isdir)
2025         ftpSendList(ftpState);
2026     else if (ftpState->restartable())
2027         ftpSendRest(ftpState);
2028     else
2029         ftpSendRetr(ftpState);
2030 }
2031
2032 static void
2033 ftpSendStor(Ftp::Gateway * ftpState)
2034 {
2035     /* check the server control channel is still available */
2036     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
2037         return;
2038
2039     debugs(9, 3, HERE);
2040
2041     if (ftpState->filepath != NULL) {
2042         /* Plain file upload */
2043         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
2044         ftpState->writeCommand(cbuf);
2045         ftpState->state = Ftp::Client::SENT_STOR;
2046     } else if (ftpState->request->header.getInt64(Http::HdrType::CONTENT_LENGTH) > 0) {
2047         /* File upload without a filename. use STOU to generate one */
2048         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
2049         ftpState->writeCommand(cbuf);
2050         ftpState->state = Ftp::Client::SENT_STOR;
2051     } else {
2052         /* No file to transfer. Only create directories if needed */
2053         ftpSendReply(ftpState);
2054     }
2055 }
2056
2057 /// \deprecated use ftpState->readStor() instead.
2058 static void
2059 ftpReadStor(Ftp::Gateway * ftpState)
2060 {
2061     ftpState->readStor();
2062 }
2063
2064 void Ftp::Gateway::readStor()
2065 {
2066     int code = ctrl.replycode;
2067     debugs(9, 3, HERE);
2068
2069     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
2070         if (!originalRequest()->body_pipe) {
2071             debugs(9, 3, "zero-size STOR?");
2072             state = WRITING_DATA; // make ftpWriteTransferDone() responsible
2073             dataComplete(); // XXX: keep in sync with doneSendingRequestBody()
2074             return;
2075         }
2076
2077         if (!startRequestBodyFlow()) { // register to receive body data
2078             ftpFail(this);
2079             return;
2080         }
2081
2082         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
2083         debugs(9, 3, HERE << "starting data transfer");
2084         switchTimeoutToDataChannel();
2085         sendMoreRequestBody();
2086         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
2087         state = WRITING_DATA;
2088         debugs(9, 3, HERE << "writing data channel");
2089     } else if (code == 150) {
2090         /* When client code is 150 with no data channel, Accept data channel. */
2091         debugs(9, 3, "ftpReadStor: accepting data channel");
2092         listenForDataChannel(data.conn);
2093     } else {
2094         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
2095         ftpFail(this);
2096     }
2097 }
2098
2099 static void
2100 ftpSendRest(Ftp::Gateway * ftpState)
2101 {
2102     /* check the server control channel is still available */
2103     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
2104         return;
2105
2106     debugs(9, 3, HERE);
2107
2108     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
2109     ftpState->writeCommand(cbuf);
2110     ftpState->state = Ftp::Client::SENT_REST;
2111 }
2112
2113 int
2114 Ftp::Gateway::restartable()
2115 {
2116     if (restart_offset > 0)
2117         return 1;
2118
2119     if (!request->range)
2120         return 0;
2121
2122     if (!flags.binary)
2123         return 0;
2124
2125     if (theSize <= 0)
2126         return 0;
2127
2128     int64_t desired_offset = request->range->lowestOffset(theSize);
2129
2130     if (desired_offset <= 0)
2131         return 0;
2132
2133     if (desired_offset >= theSize)
2134         return 0;
2135
2136     restart_offset = desired_offset;
2137     return 1;
2138 }
2139
2140 static void
2141 ftpReadRest(Ftp::Gateway * ftpState)
2142 {
2143     int code = ftpState->ctrl.replycode;
2144     debugs(9, 3, HERE);
2145     assert(ftpState->restart_offset > 0);
2146
2147     if (code == 350) {
2148         ftpState->setCurrentOffset(ftpState->restart_offset);
2149         ftpSendRetr(ftpState);
2150     } else if (code > 0) {
2151         debugs(9, 3, HERE << "REST not supported");
2152         ftpState->flags.rest_supported = 0;
2153         ftpSendRetr(ftpState);
2154     } else {
2155         ftpFail(ftpState);
2156     }
2157 }
2158
2159 static void
2160 ftpSendList(Ftp::Gateway * ftpState)
2161 {
2162     /* check the server control channel is still available */
2163     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
2164         return;
2165
2166     debugs(9, 3, HERE);
2167
2168     if (ftpState->filepath) {
2169         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
2170     } else {
2171         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
2172     }
2173
2174     ftpState->writeCommand(cbuf);
2175     ftpState->state = Ftp::Client::SENT_LIST;
2176 }
2177
2178 static void
2179 ftpSendNlst(Ftp::Gateway * ftpState)
2180 {
2181     /* check the server control channel is still available */
2182     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
2183         return;
2184
2185     debugs(9, 3, HERE);
2186
2187     ftpState->flags.tried_nlst = 1;
2188
2189     if (ftpState->filepath) {
2190         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
2191     } else {
2192         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
2193     }
2194
2195     ftpState->writeCommand(cbuf);
2196     ftpState->state = Ftp::Client::SENT_NLST;
2197 }
2198
2199 static void
2200 ftpReadList(Ftp::Gateway * ftpState)
2201 {
2202     int code = ftpState->ctrl.replycode;
2203     debugs(9, 3, HERE);
2204
2205     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2206         /* Begin data transfer */
2207         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2208         ftpState->switchTimeoutToDataChannel();
2209         ftpState->maybeReadVirginBody();
2210         ftpState->state = Ftp::Client::READING_DATA;
2211         return;
2212     } else if (code == 150) {
2213         /* Accept data channel */
2214         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2215         ftpState->listenForDataChannel(ftpState->data.conn);
2216         return;
2217     } else if (!ftpState->flags.tried_nlst && code > 300) {
2218         ftpSendNlst(ftpState);
2219     } else {
2220         ftpFail(ftpState);
2221         return;
2222     }
2223 }
2224
2225 static void
2226 ftpSendRetr(Ftp::Gateway * ftpState)
2227 {
2228     /* check the server control channel is still available */
2229     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
2230         return;
2231
2232     debugs(9, 3, HERE);
2233
2234     assert(ftpState->filepath != NULL);
2235     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
2236     ftpState->writeCommand(cbuf);
2237     ftpState->state = Ftp::Client::SENT_RETR;
2238 }
2239
2240 static void
2241 ftpReadRetr(Ftp::Gateway * ftpState)
2242 {
2243     int code = ftpState->ctrl.replycode;
2244     debugs(9, 3, HERE);
2245
2246     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2247         /* Begin data transfer */
2248         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2249         ftpState->switchTimeoutToDataChannel();
2250         ftpState->maybeReadVirginBody();
2251         ftpState->state = Ftp::Client::READING_DATA;
2252     } else if (code == 150) {
2253         /* Accept data channel */
2254         ftpState->listenForDataChannel(ftpState->data.conn);
2255     } else if (code >= 300) {
2256         if (!ftpState->flags.try_slash_hack) {
2257             /* Try this as a directory missing trailing slash... */
2258             ftpState->hackShortcut(ftpSendCwd);
2259         } else {
2260             ftpFail(ftpState);
2261         }
2262     } else {
2263         ftpFail(ftpState);
2264     }
2265 }
2266
2267 /**
2268  * Generate the HTTP headers and template fluff around an FTP
2269  * directory listing display.
2270  */
2271 void
2272 Ftp::Gateway::completedListing()
2273 {
2274     assert(entry);
2275     entry->lock("Ftp::Gateway");
2276     ErrorState ferr(ERR_DIR_LISTING, Http::scOkay, request);
2277     ferr.ftp.listing = &listing;
2278     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
2279     ferr.ftp.server_msg = ctrl.message;
2280     ctrl.message = NULL;
2281     entry->replaceHttpReply( ferr.BuildHttpReply() );
2282     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2283     entry->flush();
2284     entry->unlock("Ftp::Gateway");
2285 }
2286
2287 static void
2288 ftpReadTransferDone(Ftp::Gateway * ftpState)
2289 {
2290     int code = ftpState->ctrl.replycode;
2291     debugs(9, 3, HERE);
2292
2293     if (code == 226 || code == 250) {
2294         /* Connection closed; retrieval done. */
2295         if (ftpState->flags.listing) {
2296             ftpState->completedListing();
2297             /* QUIT operation handles sending the reply to client */
2298         }
2299         ftpSendQuit(ftpState);
2300     } else {            /* != 226 */
2301         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
2302         ftpState->failed(ERR_FTP_FAILURE, 0);
2303         /* failed closes ctrl.conn and frees ftpState */
2304         return;
2305     }
2306 }
2307
2308 // premature end of the request body
2309 void
2310 Ftp::Gateway::handleRequestBodyProducerAborted()
2311 {
2312     Client::handleRequestBodyProducerAborted();
2313     debugs(9, 3, HERE << "ftpState=" << this);
2314     failed(ERR_READ_ERROR, 0);
2315 }
2316
2317 static void
2318 ftpWriteTransferDone(Ftp::Gateway * ftpState)
2319 {
2320     int code = ftpState->ctrl.replycode;
2321     debugs(9, 3, HERE);
2322
2323     if (!(code == 226 || code == 250)) {
2324         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
2325         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
2326         return;
2327     }
2328
2329     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
2330     ftpSendReply(ftpState);
2331 }
2332
2333 static void
2334 ftpSendQuit(Ftp::Gateway * ftpState)
2335 {
2336     /* check the server control channel is still available */
2337     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
2338         return;
2339
2340     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
2341     ftpState->writeCommand(cbuf);
2342     ftpState->state = Ftp::Client::SENT_QUIT;
2343 }
2344
2345 /** Completes a client FTP operation with success or other page
2346  *  generated and stored in the entry field by the code issuing QUIT.
2347  */
2348 static void
2349 ftpReadQuit(Ftp::Gateway * ftpState)
2350 {
2351     ftpState->serverComplete();
2352 }
2353
2354 static void
2355 ftpTrySlashHack(Ftp::Gateway * ftpState)
2356 {
2357     char *path;
2358     ftpState->flags.try_slash_hack = 1;
2359     /* Free old paths */
2360
2361     debugs(9, 3, HERE);
2362
2363     if (ftpState->pathcomps)
2364         wordlistDestroy(&ftpState->pathcomps);
2365
2366     safe_free(ftpState->filepath);
2367
2368     /* Build the new path (urlpath begins with /) */
2369     path = SBufToCstring(ftpState->request->url.path());
2370
2371     rfc1738_unescape(path);
2372
2373     ftpState->filepath = path;
2374
2375     /* And off we go */
2376     ftpGetFile(ftpState);
2377 }
2378
2379 /**
2380  * Forget hack status. Next error is shown to the user
2381  */
2382 void
2383 Ftp::Gateway::unhack()
2384 {
2385     debugs(9, 3, HERE);
2386
2387     if (old_request != NULL) {
2388         safe_free(old_request);
2389         safe_free(old_reply);
2390     }
2391 }
2392
2393 void
2394 Ftp::Gateway::hackShortcut(FTPSM * nextState)
2395 {
2396     /* Clear some unwanted state */
2397     setCurrentOffset(0);
2398     restart_offset = 0;
2399     /* Save old error message & some state info */
2400
2401     debugs(9, 3, HERE);
2402
2403     if (old_request == NULL) {
2404         old_request = ctrl.last_command;
2405         ctrl.last_command = NULL;
2406         old_reply = ctrl.last_reply;
2407         ctrl.last_reply = NULL;
2408
2409         if (pathcomps == NULL && filepath != NULL)
2410             old_filepath = xstrdup(filepath);
2411     }
2412
2413     /* Jump to the "hack" state */
2414     nextState(this);
2415 }
2416
2417 static void
2418 ftpFail(Ftp::Gateway *ftpState)
2419 {
2420     const bool slashHack = ftpState->request->url.path().caseCmp("/%2f", 4)==0;
2421     int code = ftpState->ctrl.replycode;
2422     err_type error_code = ERR_NONE;
2423
2424     debugs(9, 6, "state " << ftpState->state <<
2425            " reply code " << code << "flags(" <<
2426            (ftpState->flags.isdir?"IS_DIR,":"") <<
2427            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
2428            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
2429            "slashhack=" << (slashHack? "T":"F"));
2430
2431     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
2432     if (!ftpState->flags.isdir &&   /* Not a directory */
2433             !ftpState->flags.try_slash_hack && !slashHack && /* Not doing slash hack */
2434             ftpState->mdtm <= 0 && ftpState->theSize < 0) { /* Not known as a file */
2435
2436         switch (ftpState->state) {
2437
2438         case Ftp::Client::SENT_CWD:
2439
2440         case Ftp::Client::SENT_RETR:
2441             /* Try the / hack */
2442             ftpState->hackShortcut(ftpTrySlashHack);
2443             return;
2444
2445         default:
2446             break;
2447         }
2448     }
2449
2450     Http::StatusCode sc = ftpState->failedHttpStatus(error_code);
2451     ErrorState *ftperr = new ErrorState(error_code, sc, ftpState->fwd->request);
2452     ftpState->failed(error_code, code, ftperr);
2453     ftperr->detailError(code);
2454     HttpReply *newrep = ftperr->BuildHttpReply();
2455     delete ftperr;
2456
2457     ftpState->entry->replaceHttpReply(newrep);
2458     ftpSendQuit(ftpState);
2459 }
2460
2461 Http::StatusCode
2462 Ftp::Gateway::failedHttpStatus(err_type &error)
2463 {
2464     if (error == ERR_NONE) {
2465         switch (state) {
2466
2467         case SENT_USER:
2468
2469         case SENT_PASS:
2470
2471             if (ctrl.replycode > 500) {
2472                 error = ERR_FTP_FORBIDDEN;
2473                 return password_url ? Http::scForbidden : Http::scUnauthorized;
2474             } else if (ctrl.replycode == 421) {
2475                 error = ERR_FTP_UNAVAILABLE;
2476                 return Http::scServiceUnavailable;
2477             }
2478             break;
2479
2480         case SENT_CWD:
2481
2482         case SENT_RETR:
2483             if (ctrl.replycode == 550) {
2484                 error = ERR_FTP_NOT_FOUND;
2485                 return Http::scNotFound;
2486             }
2487             break;
2488
2489         default:
2490             break;
2491         }
2492     }
2493     return Ftp::Client::failedHttpStatus(error);
2494 }
2495
2496 static void
2497 ftpSendReply(Ftp::Gateway * ftpState)
2498 {
2499     int code = ftpState->ctrl.replycode;
2500     Http::StatusCode http_code;
2501     err_type err_code = ERR_NONE;
2502
2503     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
2504
2505     if (cbdataReferenceValid(ftpState))
2506         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
2507
2508     if (code == 226 || code == 250) {
2509         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
2510         http_code = (ftpState->mdtm > 0) ? Http::scAccepted : Http::scCreated;
2511     } else if (code == 227) {
2512         err_code = ERR_FTP_PUT_CREATED;
2513         http_code = Http::scCreated;
2514     } else {
2515         err_code = ERR_FTP_PUT_ERROR;
2516         http_code = Http::scInternalServerError;
2517     }
2518
2519     ErrorState err(err_code, http_code, ftpState->request);
2520
2521     if (ftpState->old_request)
2522         err.ftp.request = xstrdup(ftpState->old_request);
2523     else
2524         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
2525
2526     if (ftpState->old_reply)
2527         err.ftp.reply = xstrdup(ftpState->old_reply);
2528     else if (ftpState->ctrl.last_reply)
2529         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
2530     else
2531         err.ftp.reply = xstrdup("");
2532
2533     // TODO: interpret as FTP-specific error code
2534     err.detailError(code);
2535
2536     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
2537
2538     ftpSendQuit(ftpState);
2539 }
2540
2541 void
2542 Ftp::Gateway::appendSuccessHeader()
2543 {
2544     debugs(9, 3, HERE);
2545
2546     if (flags.http_header_sent)
2547         return;
2548
2549     HttpReply *reply = new HttpReply;
2550
2551     flags.http_header_sent = 1;
2552
2553     assert(entry->isEmpty());
2554
2555     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2556
2557     entry->buffer();    /* released when done processing current data payload */
2558
2559     SBuf urlPath = request->url.path();
2560     auto t = urlPath.rfind('/');
2561     SBuf filename = urlPath.substr(t != SBuf::npos ? t : 0);
2562
2563     const char *mime_type = NULL;
2564     const char *mime_enc = NULL;
2565
2566     if (flags.isdir) {
2567         mime_type = "text/html";
2568     } else {
2569         switch (typecode) {
2570
2571         case 'I':
2572             mime_type = "application/octet-stream";
2573             // XXX: performance regression, c_str() may reallocate
2574             mime_enc = mimeGetContentEncoding(filename.c_str());
2575             break;
2576
2577         case 'A':
2578             mime_type = "text/plain";
2579             break;
2580
2581         default:
2582             // XXX: performance regression, c_str() may reallocate
2583             mime_type = mimeGetContentType(filename.c_str());
2584             mime_enc = mimeGetContentEncoding(filename.c_str());
2585             break;
2586         }
2587     }
2588
2589     /* set standard stuff */
2590
2591     if (0 == getCurrentOffset()) {
2592         /* Full reply */
2593         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2594     } else if (theSize < getCurrentOffset()) {
2595         /*
2596          * DPW 2007-05-04
2597          * offset should not be larger than theSize.  We should
2598          * not be seeing this condition any more because we'll only
2599          * send REST if we know the theSize and if it is less than theSize.
2600          */
2601         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
2602                " current offset=" << getCurrentOffset() <<
2603                ", but theSize=" << theSize <<
2604                ".  assuming full content response");
2605         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2606     } else {
2607         /* Partial reply */
2608         HttpHdrRangeSpec range_spec;
2609         range_spec.offset = getCurrentOffset();
2610         range_spec.length = theSize - getCurrentOffset();
2611         reply->setHeaders(Http::scPartialContent, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
2612         httpHeaderAddContRange(&reply->header, range_spec, theSize);
2613     }
2614
2615     /* additional info */
2616     if (mime_enc)
2617         reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc);
2618
2619     reply->sources |= HttpMsg::srcFtp;
2620     setVirginReply(reply);
2621     adaptOrFinalizeReply();
2622 }
2623
2624 void
2625 Ftp::Gateway::haveParsedReplyHeaders()
2626 {
2627     Client::haveParsedReplyHeaders();
2628
2629     StoreEntry *e = entry;
2630
2631     e->timestampsSet();
2632
2633     if (flags.authenticated) {
2634         /*
2635          * Authenticated requests can't be cached.
2636          */
2637         e->release();
2638     } else if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && !getCurrentOffset()) {
2639         e->setPublicKey();
2640     } else {
2641         e->release();
2642     }
2643 }
2644
2645 HttpReply *
2646 Ftp::Gateway::ftpAuthRequired(HttpRequest * request, const char *realm)
2647 {
2648     ErrorState err(ERR_CACHE_ACCESS_DENIED, Http::scUnauthorized, request);
2649     HttpReply *newrep = err.BuildHttpReply();
2650 #if HAVE_AUTH_MODULE_BASIC
2651     /* add Authenticate header */
2652     newrep->header.putAuth("Basic", realm);
2653 #endif
2654     return newrep;
2655 }
2656
2657 const SBuf &
2658 Ftp::UrlWith2f(HttpRequest * request)
2659 {
2660     SBuf newbuf("%2f");
2661
2662     if (request->url.getScheme() != AnyP::PROTO_FTP) {
2663         static const SBuf nil;
2664         return nil;
2665     }
2666
2667     if (request->url.path()[0] == '/') {
2668         newbuf.append(request->url.path());
2669         request->url.path(newbuf);
2670     } else if (!request->url.path().startsWith(newbuf)) {
2671         newbuf.append(request->url.path().substr(1));
2672         request->url.path(newbuf);
2673     }
2674
2675     return request->effectiveRequestUri();
2676 }
2677
2678 void
2679 Ftp::Gateway::printfReplyBody(const char *fmt, ...)
2680 {
2681     va_list args;
2682     va_start (args, fmt);
2683     static char buf[4096];
2684     buf[0] = '\0';
2685     vsnprintf(buf, 4096, fmt, args);
2686     writeReplyBody(buf, strlen(buf));
2687     va_end(args);
2688 }
2689
2690 /**
2691  * Call this when there is data from the origin server
2692  * which should be sent to either StoreEntry, or to ICAP...
2693  */
2694 void
2695 Ftp::Gateway::writeReplyBody(const char *dataToWrite, size_t dataLength)
2696 {
2697     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
2698     addVirginReplyBody(dataToWrite, dataLength);
2699 }
2700
2701 /**
2702  * A hack to ensure we do not double-complete on the forward entry.
2703  *
2704  \todo Ftp::Gateway logic should probably be rewritten to avoid
2705  *  double-completion or FwdState should be rewritten to allow it.
2706  */
2707 void
2708 Ftp::Gateway::completeForwarding()
2709 {
2710     if (fwd == NULL || flags.completed_forwarding) {
2711         debugs(9, 3, "avoid double-complete on FD " <<
2712                (ctrl.conn ? ctrl.conn->fd : -1) << ", Data FD " << data.conn->fd <<
2713                ", this " << this << ", fwd " << fwd);
2714         return;
2715     }
2716
2717     flags.completed_forwarding = true;
2718     Client::completeForwarding();
2719 }
2720
2721 /**
2722  * Have we lost the FTP server control channel?
2723  *
2724  \retval true   The server control channel is available.
2725  \retval false  The server control channel is not available.
2726  */
2727 bool
2728 Ftp::Gateway::haveControlChannel(const char *caller_name) const
2729 {
2730     if (doneWithServer())
2731         return false;
2732
2733     /* doneWithServer() only checks BOTH channels are closed. */
2734     if (!Comm::IsConnOpen(ctrl.conn)) {
2735         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
2736         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
2737         return false;
2738     }
2739
2740     return true;
2741 }
2742
2743 bool
2744 Ftp::Gateway::mayReadVirginReplyBody() const
2745 {
2746     // TODO: Can we do what Ftp::Relay::mayReadVirginReplyBody() does instead?
2747     return !doneWithServer();
2748 }
2749
2750 AsyncJob::Pointer
2751 Ftp::StartGateway(FwdState *const fwdState)
2752 {
2753     return AsyncJob::Start(new Ftp::Gateway(fwdState));
2754 }
2755