src/clients/FtpGateway.cc

   1 /*
   2  * Copyright (C) 1996-2015 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "clients/forward.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm.h"
  16 #include "comm/ConnOpener.h"
  17 #include "comm/Read.h"
  18 #include "comm/TcpAcceptor.h"
  19 #include "CommCalls.h"
  20 #include "compat/strtoll.h"
  21 #include "errorpage.h"
  22 #include "fd.h"
  23 #include "fde.h"
  24 #include "FwdState.h"
  25 #include "html_quote.h"
  26 #include "HttpHdrContRange.h"
  27 #include "HttpHeader.h"
  28 #include "HttpHeaderRange.h"
  29 #include "HttpReply.h"
  30 #include "HttpRequest.h"
  31 #include "ip/tools.h"
  32 #include "MemBuf.h"
  33 #include "mime.h"
  34 #include "rfc1738.h"
  35 #include "SquidConfig.h"
  36 #include "SquidString.h"
  37 #include "SquidTime.h"
  38 #include "StatCounters.h"
  39 #include "Store.h"
  40 #include "tools.h"
  41 #include "URL.h"
  42 #include "util.h"
  43 #include "wordlist.h"
  44
  45 #if USE_DELAY_POOLS
  46 #include "DelayPools.h"
  47 #include "MemObject.h"
  48 #endif
  49
  50 #include <cerrno>
  51
  52 namespace Ftp
  53 {
  54
  55 struct GatewayFlags {
  56
  57     /* passive mode */
  58     bool pasv_supported;  ///< PASV command is allowed
  59     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
  60     bool pasv_only;
  61     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
  62
  63     /* authentication */
  64     bool authenticated;         ///< authentication success
  65     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
  66     bool tried_auth_nopass;     ///< auth tried username with no password already.
  67
  68     /* other */
  69     bool isdir;
  70     bool skip_whitespace;
  71     bool rest_supported;
  72     bool http_header_sent;
  73     bool tried_nlst;
  74     bool need_base_href;
  75     bool dir_slash;
  76     bool root_dir;
  77     bool no_dotdot;
  78     bool binary;
  79     bool try_slash_hack;
  80     bool put;
  81     bool put_mkdir;
  82     bool listformat_unknown;
  83     bool listing;
  84     bool completed_forwarding;
  85 };
  86
  87 class Gateway;
  88 typedef void (StateMethod)(Ftp::Gateway *);
  89
  90 /// FTP Gateway: An FTP client that takes an HTTP request with an ftp:// URI,
  91 /// converts it into one or more FTP commands, and then
  92 /// converts one or more FTP responses into the final HTTP response.
  93 class Gateway : public Ftp::Client
  94 {
  95     CBDATA_CLASS(Gateway);
  96
  97 public:
  98     Gateway(FwdState *);
  99     virtual ~Gateway();
 100     char user[MAX_URL];
 101     char password[MAX_URL];
 102     int password_url;
 103     char *reply_hdr;
 104     int reply_hdr_state;
 105     String clean_url;
 106     String title_url;
 107     String base_href;
 108     int conn_att;
 109     int login_att;
 110     time_t mdtm;
 111     int64_t theSize;
 112     wordlist *pathcomps;
 113     char *filepath;
 114     char *dirpath;
 115     int64_t restart_offset;
 116     char *proxy_host;
 117     size_t list_width;
 118     String cwd_message;
 119     char *old_filepath;
 120     char typecode;
 121     MemBuf listing;     ///< FTP directory listing in HTML format.
 122
 123     GatewayFlags flags;
 124
 125 public:
 126     // these should all be private
 127     virtual void start();
 128     virtual Http::StatusCode failedHttpStatus(err_type &error);
 129     int restartable();
 130     void appendSuccessHeader();
 131     void hackShortcut(StateMethod *nextState);
 132     void unhack();
 133     void readStor();
 134     void parseListing();
 135     MemBuf *htmlifyListEntry(const char *line);
 136     void completedListing(void);
 137
 138     /// create a data channel acceptor and start listening.
 139     void listenForDataChannel(const Comm::ConnectionPointer &conn);
 140
 141     int checkAuth(const HttpHeader * req_hdr);
 142     void checkUrlpath();
 143     void buildTitleUrl();
 144     void writeReplyBody(const char *, size_t len);
 145     void printfReplyBody(const char *fmt, ...);
 146     virtual void completeForwarding();
 147     void processHeadResponse();
 148     void processReplyBody();
 149     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 150     int64_t getCurrentOffset() const { return currentOffset; }
 151
 152     virtual void dataChannelConnected(const CommConnectCbParams &io);
 153     static PF ftpDataWrite;
 154     virtual void timeout(const CommTimeoutCbParams &io);
 155     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 156
 157     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 158     const char *ftpRealm(void);
 159     void loginFailed(void);
 160
 161     virtual void haveParsedReplyHeaders();
 162
 163     virtual bool haveControlChannel(const char *caller_name) const;
 164
 165 protected:
 166     virtual void handleControlReply();
 167     virtual void dataClosed(const CommCloseCbParams &io);
 168
 169 private:
 170     virtual bool mayReadVirginReplyBody() const;
 171     // BodyConsumer for HTTP: consume request body.
 172     virtual void handleRequestBodyProducerAborted();
 173
 174     void loginParser(const SBuf &login, bool escaped);
 175 };
 176
 177 } // namespace Ftp
 178
 179 typedef Ftp::StateMethod FTPSM; // to avoid lots of non-changes
 180
 181 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Gateway);
 182
 183 typedef struct {
 184     char type;
 185     int64_t size;
 186     char *date;
 187     char *name;
 188     char *showname;
 189     char *link;
 190 } ftpListParts;
 191
 192 #define CTRL_BUFLEN 1024
 193 static char cbuf[CTRL_BUFLEN];
 194
 195 /*
 196  * State machine functions
 197  * send == state transition
 198  * read == wait for response, and select next state transition
 199  * other == Transition logic
 200  */
 201 static FTPSM ftpReadWelcome;
 202 static FTPSM ftpSendUser;
 203 static FTPSM ftpReadUser;
 204 static FTPSM ftpSendPass;
 205 static FTPSM ftpReadPass;
 206 static FTPSM ftpSendType;
 207 static FTPSM ftpReadType;
 208 static FTPSM ftpSendMdtm;
 209 static FTPSM ftpReadMdtm;
 210 static FTPSM ftpSendSize;
 211 static FTPSM ftpReadSize;
 212 static FTPSM ftpSendEPRT;
 213 static FTPSM ftpReadEPRT;
 214 static FTPSM ftpSendPORT;
 215 static FTPSM ftpReadPORT;
 216 static FTPSM ftpSendPassive;
 217 static FTPSM ftpReadEPSV;
 218 static FTPSM ftpReadPasv;
 219 static FTPSM ftpTraverseDirectory;
 220 static FTPSM ftpListDir;
 221 static FTPSM ftpGetFile;
 222 static FTPSM ftpSendCwd;
 223 static FTPSM ftpReadCwd;
 224 static FTPSM ftpRestOrList;
 225 static FTPSM ftpSendList;
 226 static FTPSM ftpSendNlst;
 227 static FTPSM ftpReadList;
 228 static FTPSM ftpSendRest;
 229 static FTPSM ftpReadRest;
 230 static FTPSM ftpSendRetr;
 231 static FTPSM ftpReadRetr;
 232 static FTPSM ftpReadTransferDone;
 233 static FTPSM ftpSendStor;
 234 static FTPSM ftpReadStor;
 235 static FTPSM ftpWriteTransferDone;
 236 static FTPSM ftpSendReply;
 237 static FTPSM ftpSendMkdir;
 238 static FTPSM ftpReadMkdir;
 239 static FTPSM ftpFail;
 240 static FTPSM ftpSendQuit;
 241 static FTPSM ftpReadQuit;
 242
 243 /************************************************
 244 ** Debugs Levels used here                     **
 245 *************************************************
 246 0   CRITICAL Events
 247 1   IMPORTANT Events
 248     Protocol and Transmission failures.
 249 2   FTP Protocol Chatter
 250 3   Logic Flows
 251 4   Data Parsing Flows
 252 5   Data Dumps
 253 7   ??
 254 ************************************************/
 255
 256 /************************************************
 257 ** State Machine Description (excluding hacks) **
 258 *************************************************
 259 From            To
 260 ---------------------------------------
 261 Welcome         User
 262 User            Pass
 263 Pass            Type
 264 Type            TraverseDirectory / GetFile
 265 TraverseDirectory   Cwd / GetFile / ListDir
 266 Cwd         TraverseDirectory / Mkdir
 267 GetFile         Mdtm
 268 Mdtm            Size
 269 Size            Epsv
 270 ListDir         Epsv
 271 Epsv            FileOrList
 272 FileOrList      Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 273 Rest            Retr
 274 Retr / Nlst / List  DataRead* (on datachannel)
 275 DataRead*       ReadTransferDone
 276 ReadTransferDone    DataTransferDone
 277 Stor            DataWrite* (on datachannel)
 278 DataWrite*      RequestPutBody** (from client)
 279 RequestPutBody**    DataWrite* / WriteTransferDone
 280 WriteTransferDone   DataTransferDone
 281 DataTransferDone    Quit
 282 Quit            -
 283 ************************************************/
 284
 285 FTPSM *FTP_SM_FUNCS[] = {
 286     ftpReadWelcome,     /* BEGIN */
 287     ftpReadUser,        /* SENT_USER */
 288     ftpReadPass,        /* SENT_PASS */
 289     ftpReadType,        /* SENT_TYPE */
 290     ftpReadMdtm,        /* SENT_MDTM */
 291     ftpReadSize,        /* SENT_SIZE */
 292     ftpReadEPRT,        /* SENT_EPRT */
 293     ftpReadPORT,        /* SENT_PORT */
 294     ftpReadEPSV,        /* SENT_EPSV_ALL */
 295     ftpReadEPSV,        /* SENT_EPSV_1 */
 296     ftpReadEPSV,        /* SENT_EPSV_2 */
 297     ftpReadPasv,        /* SENT_PASV */
 298     ftpReadCwd,     /* SENT_CWD */
 299     ftpReadList,        /* SENT_LIST */
 300     ftpReadList,        /* SENT_NLST */
 301     ftpReadRest,        /* SENT_REST */
 302     ftpReadRetr,        /* SENT_RETR */
 303     ftpReadStor,        /* SENT_STOR */
 304     ftpReadQuit,        /* SENT_QUIT */
 305     ftpReadTransferDone,    /* READING_DATA (RETR,LIST,NLST) */
 306     ftpWriteTransferDone,   /* WRITING_DATA (STOR) */
 307     ftpReadMkdir,       /* SENT_MKDIR */
 308     NULL,           /* SENT_FEAT */
 309     NULL,           /* SENT_PWD */
 310     NULL,           /* SENT_CDUP*/
 311     NULL,           /* SENT_DATA_REQUEST */
 312     NULL            /* SENT_COMMAND */
 313 };
 314
 315 /// handler called by Comm when FTP data channel is closed unexpectedly
 316 void
 317 Ftp::Gateway::dataClosed(const CommCloseCbParams &io)
 318 {
 319     Ftp::Client::dataClosed(io);
 320     failed(ERR_FTP_FAILURE, 0);
 321     /* failed closes ctrl.conn and frees ftpState */
 322
 323     /* NP: failure recovery may be possible when its only a data.conn failure.
 324      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 325      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 326      */
 327 }
 328
 329 Ftp::Gateway::Gateway(FwdState *fwdState):
 330     AsyncJob("FtpStateData"),
 331     Ftp::Client(fwdState),
 332     password_url(0),
 333     reply_hdr(NULL),
 334     reply_hdr_state(0),
 335     conn_att(0),
 336     login_att(0),
 337     mdtm(-1),
 338     theSize(-1),
 339     pathcomps(NULL),
 340     filepath(NULL),
 341     dirpath(NULL),
 342     restart_offset(0),
 343     proxy_host(NULL),
 344     list_width(0),
 345     old_filepath(NULL),
 346     typecode('\0')
 347 {
 348     debugs(9, 3, entry->url());
 349
 350     *user = 0;
 351     *password = 0;
 352     memset(&flags, 0, sizeof(flags));
 353
 354     if (Config.Ftp.passive && !flags.pasv_failed)
 355         flags.pasv_supported = 1;
 356
 357     flags.rest_supported = 1;
 358
 359     if (request->method == Http::METHOD_PUT)
 360         flags.put = 1;
 361
 362     initReadBuf();
 363 }
 364
 365 Ftp::Gateway::~Gateway()
 366 {
 367     debugs(9, 3, entry->url());
 368
 369     if (Comm::IsConnOpen(ctrl.conn)) {
 370         debugs(9, DBG_IMPORTANT, "Internal bug: FTP Gateway left open " <<
 371                "control channel " << ctrl.conn);
 372     }
 373
 374     if (reply_hdr) {
 375         memFree(reply_hdr, MEM_8K_BUF);
 376         reply_hdr = NULL;
 377     }
 378
 379     if (pathcomps)
 380         wordlistDestroy(&pathcomps);
 381
 382     cwd_message.clean();
 383     xfree(old_filepath);
 384     title_url.clean();
 385     base_href.clean();
 386     xfree(filepath);
 387     xfree(dirpath);
 388 }
 389
 390 /**
 391  * Parse a possible login username:password pair.
 392  * Produces filled member variables user, password, password_url if anything found.
 393  *
 394  * \param login    a decoded Basic authentication credential token or URI user-info token
 395  * \param escaped  whether to URL-decode the token after extracting user and password
 396  */
 397 void
 398 Ftp::Gateway::loginParser(const SBuf &login, bool escaped)
 399 {
 400     debugs(9, 4, "login=" << login << ", escaped=" << escaped);
 401     debugs(9, 9, "IN : login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 402
 403     if (login.isEmpty())
 404         return;
 405
 406     const SBuf::size_type colonPos = login.find(':');
 407
 408     /* If there was a username part with at least one character use it.
 409      * Ignore 0-length username portion, retain what we have already.
 410      */
 411     if (colonPos == SBuf::npos || colonPos > 0) {
 412         const SBuf userName = login.substr(0, colonPos);
 413         SBuf::size_type upto = userName.copy(user, sizeof(user)-1);
 414         user[upto]='\0';
 415         debugs(9, 9, "found user=" << userName << ' ' <<
 416                (upto != userName.length() ? ", truncated-to=" : ", length=") << upto <<
 417                ", escaped=" << escaped);
 418         if (escaped)
 419             rfc1738_unescape(user);
 420         debugs(9, 9, "found user=" << user << " (" << strlen(user) << ") unescaped.");
 421     }
 422
 423     /* If there was a password part.
 424      * For 0-length password clobber what we have already, this means explicitly none
 425      */
 426     if (colonPos != SBuf::npos) {
 427         const SBuf pass = login.substr(colonPos+1, SBuf::npos);
 428         SBuf::size_type upto = pass.copy(password, sizeof(password)-1);
 429         password[upto]='\0';
 430         debugs(9, 9, "found password=" << pass << " " <<
 431                (upto != pass.length() ? ", truncated-to=" : ", length=") << upto <<
 432                ", escaped=" << escaped);
 433         if (escaped) {
 434             rfc1738_unescape(password);
 435             password_url = 1;
 436         }
 437         debugs(9, 9, "found password=" << password << " (" << strlen(password) << ") unescaped.");
 438     }
 439
 440     debugs(9, 9, "OUT: login=" << login << ", escaped=" << escaped << ", user=" << user << ", password=" << password);
 441 }
 442
 443 void
 444 Ftp::Gateway::listenForDataChannel(const Comm::ConnectionPointer &conn)
 445 {
 446     assert(!Comm::IsConnOpen(data.conn));
 447
 448     typedef CommCbMemFunT<Gateway, CommAcceptCbParams> AcceptDialer;
 449     typedef AsyncCallT<AcceptDialer> AcceptCall;
 450     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, Ftp::Gateway::ftpAcceptDataConnection));
 451     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 452     const char *note = entry->url();
 453
 454     /* open the conn if its not already open */
 455     if (!Comm::IsConnOpen(conn)) {
 456         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 457         if (!Comm::IsConnOpen(conn)) {
 458             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 459             return;
 460         }
 461         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 462     }
 463
 464     conn->tos = ctrl.conn->tos;
 465     conn->nfmark = ctrl.conn->nfmark;
 466
 467     assert(Comm::IsConnOpen(conn));
 468     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 469
 470     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 471     data.opened(conn, dataCloser());
 472     switchTimeoutToDataChannel();
 473 }
 474
 475 void
 476 Ftp::Gateway::timeout(const CommTimeoutCbParams &io)
 477 {
 478     if (SENT_PASV == state) {
 479         /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */
 480         flags.pasv_supported = false;
 481         debugs(9, DBG_IMPORTANT, "FTP Gateway timeout in SENT_PASV state");
 482
 483         // cancel the data connection setup.
 484         if (data.opener != NULL) {
 485             data.opener->cancel("timeout");
 486             data.opener = NULL;
 487         }
 488         data.close();
 489     }
 490
 491     Ftp::Client::timeout(io);
 492 }
 493
 494 static const char *Month[] = {
 495     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 496     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 497 };
 498
 499 static int
 500 is_month(const char *buf)
 501 {
 502     int i;
 503
 504     for (i = 0; i < 12; ++i)
 505         if (!strcasecmp(buf, Month[i]))
 506             return 1;
 507
 508     return 0;
 509 }
 510
 511 static void
 512 ftpListPartsFree(ftpListParts ** parts)
 513 {
 514     safe_free((*parts)->date);
 515     safe_free((*parts)->name);
 516     safe_free((*parts)->showname);
 517     safe_free((*parts)->link);
 518     safe_free(*parts);
 519 }
 520
 521 #define MAX_TOKENS 64
 522
 523 static ftpListParts *
 524 ftpListParseParts(const char *buf, struct Ftp::GatewayFlags flags)
 525 {
 526     ftpListParts *p = NULL;
 527     char *t = NULL;
 528     const char *ct = NULL;
 529     char *tokens[MAX_TOKENS];
 530     int i;
 531     int n_tokens;
 532     static char tbuf[128];
 533     char *xbuf = NULL;
 534     static int scan_ftp_initialized = 0;
 535     static regex_t scan_ftp_integer;
 536     static regex_t scan_ftp_time;
 537     static regex_t scan_ftp_dostime;
 538     static regex_t scan_ftp_dosdate;
 539
 540     if (!scan_ftp_initialized) {
 541         scan_ftp_initialized = 1;
 542         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 543         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 544         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 545         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 546     }
 547
 548     if (buf == NULL)
 549         return NULL;
 550
 551     if (*buf == '\0')
 552         return NULL;
 553
 554     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 555
 556     n_tokens = 0;
 557
 558     memset(tokens, 0, sizeof(tokens));
 559
 560     xbuf = xstrdup(buf);
 561
 562     if (flags.tried_nlst) {
 563         /* Machine readable format, one name per line */
 564         p->name = xbuf;
 565         p->type = '\0';
 566         return p;
 567     }
 568
 569     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 570         tokens[n_tokens] = xstrdup(t);
 571         ++n_tokens;
 572     }
 573
 574     xfree(xbuf);
 575
 576     /* locate the Month field */
 577     for (i = 3; i < n_tokens - 2; ++i) {
 578         char *size = tokens[i - 1];
 579         char *month = tokens[i];
 580         char *day = tokens[i + 1];
 581         char *year = tokens[i + 2];
 582
 583         if (!is_month(month))
 584             continue;
 585
 586         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 587             continue;
 588
 589         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 590             continue;
 591
 592         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0) /* Yr | hh:mm */
 593             continue;
 594
 595         snprintf(tbuf, 128, "%s %2s %5s",
 596                  month, day, year);
 597
 598         if (!strstr(buf, tbuf))
 599             snprintf(tbuf, 128, "%s %2s %-5s",
 600                      month, day, year);
 601
 602         char const *copyFrom = NULL;
 603
 604         if ((copyFrom = strstr(buf, tbuf))) {
 605             p->type = *tokens[0];
 606             p->size = strtoll(size, NULL, 10);
 607             p->date = xstrdup(tbuf);
 608
 609             if (flags.skip_whitespace) {
 610                 copyFrom += strlen(tbuf);
 611
 612                 while (strchr(w_space, *copyFrom))
 613                     ++copyFrom;
 614             } else {
 615                 /* XXX assumes a single space between date and filename
 616                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 617                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 618                 copyFrom += strlen(tbuf) + 1;
 619             }
 620
 621             p->name = xstrdup(copyFrom);
 622
 623             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 624                 *t = '\0';
 625                 p->link = xstrdup(t + 4);
 626             }
 627
 628             goto found;
 629         }
 630
 631         break;
 632     }
 633
 634     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 635     if (n_tokens > 3 &&
 636             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 637             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 638         if (!strcasecmp(tokens[2], "<dir>")) {
 639             p->type = 'd';
 640         } else {
 641             p->type = '-';
 642             p->size = strtoll(tokens[2], NULL, 10);
 643         }
 644
 645         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 646         p->date = xstrdup(tbuf);
 647
 648         if (p->type == 'd') {
 649             /* Directory.. name begins with first printable after <dir> */
 650             ct = strstr(buf, tokens[2]);
 651             ct += strlen(tokens[2]);
 652
 653             while (xisspace(*ct))
 654                 ++ct;
 655
 656             if (!*ct)
 657                 ct = NULL;
 658         } else {
 659             /* A file. Name begins after size, with a space in between */
 660             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 661             ct = strstr(buf, tbuf);
 662
 663             if (ct) {
 664                 ct += strlen(tokens[2]) + 2;
 665             }
 666         }
 667
 668         p->name = xstrdup(ct ? ct : tokens[3]);
 669         goto found;
 670     }
 671
 672     /* Try EPLF format; carson@lehman.com */
 673     if (buf[0] == '+') {
 674         ct = buf + 1;
 675         p->type = 0;
 676
 677         while (ct && *ct) {
 678             time_t tm;
 679             int l = strcspn(ct, ",");
 680             char *tmp;
 681
 682             if (l < 1)
 683                 goto blank;
 684
 685             switch (*ct) {
 686
 687             case '\t':
 688                 p->name = xstrndup(ct + 1, l + 1);
 689                 break;
 690
 691             case 's':
 692                 p->size = atoi(ct + 1);
 693                 break;
 694
 695             case 'm':
 696                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 697
 698                 if (tmp != ct + 1)
 699                     break;  /* not a valid integer */
 700
 701                 p->date = xstrdup(ctime(&tm));
 702
 703                 *(strstr(p->date, "\n")) = '\0';
 704
 705                 break;
 706
 707             case '/':
 708                 p->type = 'd';
 709
 710                 break;
 711
 712             case 'r':
 713                 p->type = '-';
 714
 715                 break;
 716
 717             case 'i':
 718                 break;
 719
 720             default:
 721                 break;
 722             }
 723
 724 blank:
 725             ct = strstr(ct, ",");
 726
 727             if (ct) {
 728                 ++ct;
 729             }
 730         }
 731
 732         if (p->type == 0) {
 733             p->type = '-';
 734         }
 735
 736         if (p->name)
 737             goto found;
 738         else
 739             safe_free(p->date);
 740     }
 741
 742 found:
 743
 744     for (i = 0; i < n_tokens; ++i)
 745         xfree(tokens[i]);
 746
 747     if (!p->name)
 748         ftpListPartsFree(&p);   /* cleanup */
 749
 750     return p;
 751 }
 752
 753 MemBuf *
 754 Ftp::Gateway::htmlifyListEntry(const char *line)
 755 {
 756     char icon[2048];
 757     char href[2048 + 40];
 758     char text[ 2048];
 759     char size[ 2048];
 760     char chdir[ 2048 + 40];
 761     char view[ 2048 + 40];
 762     char download[ 2048 + 40];
 763     char link[ 2048 + 40];
 764     MemBuf *html;
 765     char prefix[2048];
 766     ftpListParts *parts;
 767     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 768
 769     debugs(9, 7, HERE << " line ={" << line << "}");
 770
 771     if (strlen(line) > 1024) {
 772         html = new MemBuf();
 773         html->init();
 774         html->appendf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 775         return html;
 776     }
 777
 778     if (flags.dir_slash && dirpath && typecode != 'D')
 779         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 780     else
 781         prefix[0] = '\0';
 782
 783     if ((parts = ftpListParseParts(line, flags)) == NULL) {
 784         const char *p;
 785
 786         html = new MemBuf();
 787         html->init();
 788         html->appendf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
 789
 790         for (p = line; *p && xisspace(*p); ++p);
 791         if (*p && !xisspace(*p))
 792             flags.listformat_unknown = 1;
 793
 794         return html;
 795     }
 796
 797     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
 798         ftpListPartsFree(&parts);
 799         return NULL;
 800     }
 801
 802     parts->size += 1023;
 803     parts->size >>= 10;
 804     parts->showname = xstrdup(parts->name);
 805
 806     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
 807     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
 808
 809     xstrncpy(text, parts->showname, 2048);
 810
 811     switch (parts->type) {
 812
 813     case 'd':
 814         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 815                  mimeGetIconURL("internal-dir"),
 816                  "[DIR]");
 817         strcat(href, "/");  /* margin is allocated above */
 818         break;
 819
 820     case 'l':
 821         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 822                  mimeGetIconURL("internal-link"),
 823                  "[LINK]");
 824         /* sometimes there is an 'l' flag, but no "->" link */
 825
 826         if (parts->link) {
 827             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
 828             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
 829                      *link2 != '/' ? prefix : "", link2,
 830                      html_quote(parts->link));
 831             safe_free(link2);
 832         }
 833
 834         break;
 835
 836     case '\0':
 837         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 838                  mimeGetIconURL(parts->name),
 839                  "[UNKNOWN]");
 840         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
 841                  "alt=\"[DIR]\"></a>",
 842                  rfc1738_escape_part(parts->name),
 843                  mimeGetIconURL("internal-dir"));
 844         break;
 845
 846     case '-':
 847
 848     default:
 849         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 850                  mimeGetIconURL(parts->name),
 851                  "[FILE]");
 852         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
 853         break;
 854     }
 855
 856     if (parts->type != 'd') {
 857         if (mimeGetViewOption(parts->name)) {
 858             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
 859                      "alt=\"[VIEW]\"></a>",
 860                      prefix, href, mimeGetIconURL("internal-view"));
 861         }
 862
 863         if (mimeGetDownloadOption(parts->name)) {
 864             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
 865                      "alt=\"[DOWNLOAD]\"></a>",
 866                      prefix, href, mimeGetIconURL("internal-download"));
 867         }
 868     }
 869
 870     /* construct the table row from parts. */
 871     html = new MemBuf();
 872     html->init();
 873     html->appendf("<tr class=\"entry\">"
 874                   "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
 875                   "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
 876                   "<td class=\"date\">%s</td>"
 877                   "<td class=\"size\">%s</td>"
 878                   "<td class=\"actions\">%s%s%s%s</td>"
 879                   "</tr>\n",
 880                   prefix, href, icon,
 881                   prefix, href, html_quote(text),
 882                   parts->date,
 883                   size,
 884                   chdir, view, download, link);
 885
 886     ftpListPartsFree(&parts);
 887     return html;
 888 }
 889
 890 void
 891 Ftp::Gateway::parseListing()
 892 {
 893     char *buf = data.readBuf->content();
 894     char *sbuf;         /* NULL-terminated copy of termedBuf */
 895     char *end;
 896     char *line;
 897     char *s;
 898     MemBuf *t;
 899     size_t linelen;
 900     size_t usable;
 901     size_t len = data.readBuf->contentSize();
 902
 903     if (!len) {
 904         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
 905         return;
 906     }
 907
 908     /*
 909      * We need a NULL-terminated buffer for scanning, ick
 910      */
 911     sbuf = (char *)xmalloc(len + 1);
 912     xstrncpy(sbuf, buf, len + 1);
 913     end = sbuf + len - 1;
 914
 915     while (*end != '\r' && *end != '\n' && end > sbuf)
 916         --end;
 917
 918     usable = end - sbuf;
 919
 920     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
 921
 922     if (usable == 0) {
 923         if (buf[0] == '\0' && len == 1) {
 924             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
 925             data.readBuf->consume(len);
 926         } else {
 927             debugs(9, 3, HERE << "didn't find end for " << entry->url());
 928             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
 929         }
 930         xfree(sbuf);
 931         return;
 932     }
 933
 934     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
 935
 936     line = (char *)memAllocate(MEM_4K_BUF);
 937     ++end;
 938     s = sbuf;
 939     s += strspn(s, crlf);
 940
 941     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
 942         debugs(9, 7, HERE << "s = {" << s << "}");
 943         linelen = strcspn(s, crlf) + 1;
 944
 945         if (linelen < 2)
 946             break;
 947
 948         if (linelen > 4096)
 949             linelen = 4096;
 950
 951         xstrncpy(line, s, linelen);
 952
 953         debugs(9, 7, HERE << "{" << line << "}");
 954
 955         if (!strncmp(line, "total", 5))
 956             continue;
 957
 958         t = htmlifyListEntry(line);
 959
 960         if ( t != NULL) {
 961             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
 962             listing.append(t->content(), t->contentSize());
 963             delete t;
 964         }
 965     }
 966
 967     debugs(9, 7, HERE << "Done.");
 968     data.readBuf->consume(usable);
 969     memFree(line, MEM_4K_BUF);
 970     xfree(sbuf);
 971 }
 972
 973 void
 974 Ftp::Gateway::processReplyBody()
 975 {
 976     debugs(9, 3, status());
 977
 978     if (request->method == Http::METHOD_HEAD && (flags.isdir || theSize != -1)) {
 979         serverComplete();
 980         return;
 981     }
 982
 983     /* Directory listings are special. They write ther own headers via the error objects */
 984     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
 985         appendSuccessHeader();
 986
 987     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 988         /*
 989          * probably was aborted because content length exceeds one
 990          * of the maximum size limits.
 991          */
 992         abortTransaction("entry aborted after calling appendSuccessHeader()");
 993         return;
 994     }
 995
 996 #if USE_ADAPTATION
 997
 998     if (adaptationAccessCheckPending) {
 999         debugs(9, 3, "returning from Ftp::Gateway::processReplyBody due to adaptationAccessCheckPending");
1000         return;
1001     }
1002
1003 #endif
1004
1005     if (flags.isdir) {
1006         if (!flags.listing) {
1007             flags.listing = 1;
1008             listing.reset();
1009         }
1010         parseListing();
1011         maybeReadVirginBody();
1012         return;
1013     } else if (const int csize = data.readBuf->contentSize()) {
1014         writeReplyBody(data.readBuf->content(), csize);
1015         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1016         data.readBuf->consume(csize);
1017     }
1018
1019     entry->flush();
1020
1021     maybeReadVirginBody();
1022 }
1023
1024 /**
1025  * Locates the FTP user:password login.
1026  *
1027  * Highest to lowest priority:
1028  *  - Checks URL (ftp://user:pass@domain)
1029  *  - Authorization: Basic header
1030  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1031  *
1032  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1033  *
1034  * TODO: we might be able to do something about locating username from other sources:
1035  *       ie, external ACL user=* tag or ident lookup
1036  *
1037  \retval 1  if we have everything needed to complete this request.
1038  \retval 0  if something is missing.
1039  */
1040 int
1041 Ftp::Gateway::checkAuth(const HttpHeader * req_hdr)
1042 {
1043     /* default username */
1044     xstrncpy(user, "anonymous", MAX_URL);
1045
1046 #if HAVE_AUTH_MODULE_BASIC
1047     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1048     const SBuf auth(req_hdr->getAuth(Http::HdrType::AUTHORIZATION, "Basic"));
1049     if (!auth.isEmpty()) {
1050         flags.authenticated = 1;
1051         loginParser(auth, false);
1052     }
1053     /* we fail with authorization-required error later IFF the FTP server requests it */
1054 #endif
1055
1056     /* Test URL login syntax. Overrides any headers received. */
1057     loginParser(request->url.userInfo(), true);
1058
1059     /* name is missing. thats fatal. */
1060     if (!user[0])
1061         fatal("FTP login parsing destroyed username info");
1062
1063     /* name + password == success */
1064     if (password[0])
1065         return 1;
1066
1067     /* Setup default FTP password settings */
1068     /* this has to be done last so that we can have a no-password case above. */
1069     if (!password[0]) {
1070         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1071             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1072             flags.tried_auth_anonymous=1;
1073             return 1;
1074         } else if (!flags.tried_auth_nopass) {
1075             xstrncpy(password, null_string, MAX_URL);
1076             flags.tried_auth_nopass=1;
1077             return 1;
1078         }
1079     }
1080
1081     return 0;           /* different username */
1082 }
1083
1084 void
1085 Ftp::Gateway::checkUrlpath()
1086 {
1087     static SBuf str_type_eq("type=");
1088     auto t = request->url.path().rfind(';');
1089
1090     if (t != SBuf::npos) {
1091         auto filenameEnd = t-1;
1092         if (request->url.path().substr(++t).cmp(str_type_eq, str_type_eq.length()) == 0) {
1093             t += str_type_eq.length();
1094             typecode = (char)xtoupper(request->url.path()[t]);
1095             request->url.path(request->url.path().substr(0,filenameEnd));
1096         }
1097     }
1098
1099     int l = request->url.path().length();
1100     /* check for null path */
1101
1102     if (!l) {
1103         flags.isdir = 1;
1104         flags.root_dir = 1;
1105         flags.need_base_href = 1;   /* Work around broken browsers */
1106     } else if (!request->url.path().cmp("/%2f/")) {
1107         /* UNIX root directory */
1108         flags.isdir = 1;
1109         flags.root_dir = 1;
1110     } else if ((l >= 1) && (request->url.path()[l-1] == '/')) {
1111         /* Directory URL, ending in / */
1112         flags.isdir = 1;
1113
1114         if (l == 1)
1115             flags.root_dir = 1;
1116     } else {
1117         flags.dir_slash = 1;
1118     }
1119 }
1120
1121 void
1122 Ftp::Gateway::buildTitleUrl()
1123 {
1124     title_url = "ftp://";
1125
1126     if (strcmp(user, "anonymous")) {
1127         title_url.append(user);
1128         title_url.append("@");
1129     }
1130
1131     SBuf authority = request->url.authority(request->url.getScheme() != AnyP::PROTO_FTP);
1132
1133     title_url.append(authority.rawContent(), authority.length());
1134     title_url.append(request->url.path().rawContent(), request->url.path().length());
1135
1136     base_href = "ftp://";
1137
1138     if (strcmp(user, "anonymous") != 0) {
1139         base_href.append(rfc1738_escape_part(user));
1140
1141         if (password_url) {
1142             base_href.append(":");
1143             base_href.append(rfc1738_escape_part(password));
1144         }
1145
1146         base_href.append("@");
1147     }
1148
1149     base_href.append(authority.rawContent(), authority.length());
1150     base_href.append(request->url.path().rawContent(), request->url.path().length());
1151     base_href.append("/");
1152 }
1153
1154 void
1155 Ftp::Gateway::start()
1156 {
1157     if (!checkAuth(&request->header)) {
1158         /* create appropriate reply */
1159         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1160         entry->replaceHttpReply(reply);
1161         serverComplete();
1162         return;
1163     }
1164
1165     checkUrlpath();
1166     buildTitleUrl();
1167     debugs(9, 5, "FD " << ctrl.conn->fd << " : host=" << request->url.host() <<
1168            ", path=" << request->url.path() << ", user=" << user << ", passwd=" << password);
1169     state = BEGIN;
1170     Ftp::Client::start();
1171 }
1172
1173 /* ====================================================================== */
1174
1175 void
1176 Ftp::Gateway::handleControlReply()
1177 {
1178     Ftp::Client::handleControlReply();
1179     if (ctrl.message == NULL)
1180         return; // didn't get complete reply yet
1181
1182     /* Copy the message except for the last line to cwd_message to be
1183      * printed in error messages.
1184      */
1185     for (wordlist *w = ctrl.message; w && w->next; w = w->next) {
1186         cwd_message.append('\n');
1187         cwd_message.append(w->key);
1188     }
1189
1190     FTP_SM_FUNCS[state] (this);
1191 }
1192
1193 /* ====================================================================== */
1194
1195 static void
1196 ftpReadWelcome(Ftp::Gateway * ftpState)
1197 {
1198     int code = ftpState->ctrl.replycode;
1199     debugs(9, 3, HERE);
1200
1201     if (ftpState->flags.pasv_only)
1202         ++ ftpState->login_att;
1203
1204     if (code == 220) {
1205         if (ftpState->ctrl.message) {
1206             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1207                 ftpState->flags.skip_whitespace = 1;
1208         }
1209
1210         ftpSendUser(ftpState);
1211     } else if (code == 120) {
1212         if (NULL != ftpState->ctrl.message)
1213             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1214
1215         return;
1216     } else {
1217         ftpFail(ftpState);
1218     }
1219 }
1220
1221 /**
1222  * Translate FTP login failure into HTTP error
1223  * this is an attmpt to get the 407 message to show up outside Squid.
1224  * its NOT a general failure. But a correct FTP response type.
1225  */
1226 void
1227 Ftp::Gateway::loginFailed()
1228 {
1229     ErrorState *err = NULL;
1230     const char *command, *reply;
1231
1232     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1233         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1234             // 421/426 - Service Overload - retry permitted.
1235             err = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
1236         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1237             // 43x - Invalid or Credential Error - retry challenge required.
1238             err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1239         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1240             // 53x - Credentials Missing - retry challenge required
1241             if (password_url) // but they were in the URI! major fail.
1242                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
1243             else
1244                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1245         }
1246     }
1247
1248     // any other problems are general falures.
1249     if (!err) {
1250         ftpFail(this);
1251         return;
1252     }
1253
1254     err->ftp.server_msg = ctrl.message;
1255
1256     ctrl.message = NULL;
1257
1258     if (old_request)
1259         command = old_request;
1260     else
1261         command = ctrl.last_command;
1262
1263     if (command && strncmp(command, "PASS", 4) == 0)
1264         command = "PASS <yourpassword>";
1265
1266     if (old_reply)
1267         reply = old_reply;
1268     else
1269         reply = ctrl.last_reply;
1270
1271     if (command)
1272         err->ftp.request = xstrdup(command);
1273
1274     if (reply)
1275         err->ftp.reply = xstrdup(reply);
1276
1277     HttpReply *newrep = err->BuildHttpReply();
1278     delete err;
1279
1280 #if HAVE_AUTH_MODULE_BASIC
1281     /* add Authenticate header */
1282     newrep->header.putAuth("Basic", ftpRealm());
1283 #endif
1284
1285     // add it to the store entry for response....
1286     entry->replaceHttpReply(newrep);
1287     serverComplete();
1288 }
1289
1290 const char *
1291 Ftp::Gateway::ftpRealm()
1292 {
1293     static char realm[8192];
1294
1295     /* This request is not fully authenticated */
1296     if (!request) {
1297         snprintf(realm, 8192, "FTP %s unknown", user);
1298     } else if (request->url.port() == 21) {
1299         snprintf(realm, 8192, "FTP %s %s", user, request->url.host());
1300     } else {
1301         snprintf(realm, 8192, "FTP %s %s port %d", user, request->url.host(), request->url.port());
1302     }
1303     return realm;
1304 }
1305
1306 static void
1307 ftpSendUser(Ftp::Gateway * ftpState)
1308 {
1309     /* check the server control channel is still available */
1310     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1311         return;
1312
1313     if (ftpState->proxy_host != NULL)
1314         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n", ftpState->user, ftpState->request->url.host());
1315     else
1316         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1317
1318     ftpState->writeCommand(cbuf);
1319
1320     ftpState->state = Ftp::Client::SENT_USER;
1321 }
1322
1323 static void
1324 ftpReadUser(Ftp::Gateway * ftpState)
1325 {
1326     int code = ftpState->ctrl.replycode;
1327     debugs(9, 3, HERE);
1328
1329     if (code == 230) {
1330         ftpReadPass(ftpState);
1331     } else if (code == 331) {
1332         ftpSendPass(ftpState);
1333     } else {
1334         ftpState->loginFailed();
1335     }
1336 }
1337
1338 static void
1339 ftpSendPass(Ftp::Gateway * ftpState)
1340 {
1341     /* check the server control channel is still available */
1342     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
1343         return;
1344
1345     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
1346     ftpState->writeCommand(cbuf);
1347     ftpState->state = Ftp::Client::SENT_PASS;
1348 }
1349
1350 static void
1351 ftpReadPass(Ftp::Gateway * ftpState)
1352 {
1353     int code = ftpState->ctrl.replycode;
1354     debugs(9, 3, HERE << "code=" << code);
1355
1356     if (code == 230) {
1357         ftpSendType(ftpState);
1358     } else {
1359         ftpState->loginFailed();
1360     }
1361 }
1362
1363 static void
1364 ftpSendType(Ftp::Gateway * ftpState)
1365 {
1366     /* check the server control channel is still available */
1367     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
1368         return;
1369
1370     /*
1371      * Ref section 3.2.2 of RFC 1738
1372      */
1373     char mode = ftpState->typecode;
1374
1375     switch (mode) {
1376
1377     case 'D':
1378         mode = 'A';
1379         break;
1380
1381     case 'A':
1382
1383     case 'I':
1384         break;
1385
1386     default:
1387
1388         if (ftpState->flags.isdir) {
1389             mode = 'A';
1390         } else {
1391             auto t = ftpState->request->url.path().rfind('/');
1392             // XXX: performance regression, c_str() may reallocate
1393             SBuf filename = ftpState->request->url.path().substr(t != SBuf::npos ? t + 1 : 0);
1394             mode = mimeGetTransferMode(filename.c_str());
1395         }
1396
1397         break;
1398     }
1399
1400     if (mode == 'I')
1401         ftpState->flags.binary = 1;
1402     else
1403         ftpState->flags.binary = 0;
1404
1405     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
1406
1407     ftpState->writeCommand(cbuf);
1408
1409     ftpState->state = Ftp::Client::SENT_TYPE;
1410 }
1411
1412 static void
1413 ftpReadType(Ftp::Gateway * ftpState)
1414 {
1415     int code = ftpState->ctrl.replycode;
1416     char *path;
1417     char *d, *p;
1418     debugs(9, 3, HERE << "code=" << code);
1419
1420     if (code == 200) {
1421         p = path = SBufToCstring(ftpState->request->url.path());
1422
1423         if (*p == '/')
1424             ++p;
1425
1426         while (*p) {
1427             d = p;
1428             p += strcspn(p, "/");
1429
1430             if (*p) {
1431                 *p = '\0';
1432                 ++p;
1433             }
1434
1435             rfc1738_unescape(d);
1436
1437             if (*d)
1438                 wordlistAdd(&ftpState->pathcomps, d);
1439         }
1440
1441         xfree(path);
1442
1443         if (ftpState->pathcomps)
1444             ftpTraverseDirectory(ftpState);
1445         else
1446             ftpListDir(ftpState);
1447     } else {
1448         ftpFail(ftpState);
1449     }
1450 }
1451
1452 static void
1453 ftpTraverseDirectory(Ftp::Gateway * ftpState)
1454 {
1455     wordlist *w;
1456     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
1457
1458     safe_free(ftpState->dirpath);
1459     ftpState->dirpath = ftpState->filepath;
1460     ftpState->filepath = NULL;
1461
1462     /* Done? */
1463
1464     if (ftpState->pathcomps == NULL) {
1465         debugs(9, 3, HERE << "the final component was a directory");
1466         ftpListDir(ftpState);
1467         return;
1468     }
1469
1470     /* Go to next path component */
1471     w = ftpState->pathcomps;
1472
1473     ftpState->filepath = w->key;
1474
1475     ftpState->pathcomps = w->next;
1476
1477     delete w;
1478
1479     /* Check if we are to CWD or RETR */
1480     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
1481         ftpSendCwd(ftpState);
1482     } else {
1483         debugs(9, 3, HERE << "final component is probably a file");
1484         ftpGetFile(ftpState);
1485         return;
1486     }
1487 }
1488
1489 static void
1490 ftpSendCwd(Ftp::Gateway * ftpState)
1491 {
1492     char *path = NULL;
1493
1494     /* check the server control channel is still available */
1495     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
1496         return;
1497
1498     debugs(9, 3, HERE);
1499
1500     path = ftpState->filepath;
1501
1502     if (!strcmp(path, "..") || !strcmp(path, "/")) {
1503         ftpState->flags.no_dotdot = 1;
1504     } else {
1505         ftpState->flags.no_dotdot = 0;
1506     }
1507
1508     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
1509
1510     ftpState->writeCommand(cbuf);
1511
1512     ftpState->state = Ftp::Client::SENT_CWD;
1513 }
1514
1515 static void
1516 ftpReadCwd(Ftp::Gateway * ftpState)
1517 {
1518     int code = ftpState->ctrl.replycode;
1519     debugs(9, 3, HERE);
1520
1521     if (code >= 200 && code < 300) {
1522         /* CWD OK */
1523         ftpState->unhack();
1524
1525         /* Reset cwd_message to only include the last message */
1526         ftpState->cwd_message.reset("");
1527         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
1528             ftpState->cwd_message.append(' ');
1529             ftpState->cwd_message.append(w->key);
1530         }
1531         ftpState->ctrl.message = NULL;
1532
1533         /* Continue to traverse the path */
1534         ftpTraverseDirectory(ftpState);
1535     } else {
1536         /* CWD FAILED */
1537
1538         if (!ftpState->flags.put)
1539             ftpFail(ftpState);
1540         else
1541             ftpSendMkdir(ftpState);
1542     }
1543 }
1544
1545 static void
1546 ftpSendMkdir(Ftp::Gateway * ftpState)
1547 {
1548     char *path = NULL;
1549
1550     /* check the server control channel is still available */
1551     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
1552         return;
1553
1554     path = ftpState->filepath;
1555     debugs(9, 3, HERE << "with path=" << path);
1556     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
1557     ftpState->writeCommand(cbuf);
1558     ftpState->state = Ftp::Client::SENT_MKDIR;
1559 }
1560
1561 static void
1562 ftpReadMkdir(Ftp::Gateway * ftpState)
1563 {
1564     char *path = ftpState->filepath;
1565     int code = ftpState->ctrl.replycode;
1566
1567     debugs(9, 3, HERE << "path " << path << ", code " << code);
1568
1569     if (code == 257) {      /* success */
1570         ftpSendCwd(ftpState);
1571     } else if (code == 550) {   /* dir exists */
1572
1573         if (ftpState->flags.put_mkdir) {
1574             ftpState->flags.put_mkdir = 1;
1575             ftpSendCwd(ftpState);
1576         } else
1577             ftpSendReply(ftpState);
1578     } else
1579         ftpSendReply(ftpState);
1580 }
1581
1582 static void
1583 ftpGetFile(Ftp::Gateway * ftpState)
1584 {
1585     assert(*ftpState->filepath != '\0');
1586     ftpState->flags.isdir = 0;
1587     ftpSendMdtm(ftpState);
1588 }
1589
1590 static void
1591 ftpListDir(Ftp::Gateway * ftpState)
1592 {
1593     if (ftpState->flags.dir_slash) {
1594         debugs(9, 3, HERE << "Directory path did not end in /");
1595         ftpState->title_url.append("/");
1596         ftpState->flags.isdir = 1;
1597     }
1598
1599     ftpSendPassive(ftpState);
1600 }
1601
1602 static void
1603 ftpSendMdtm(Ftp::Gateway * ftpState)
1604 {
1605     /* check the server control channel is still available */
1606     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
1607         return;
1608
1609     assert(*ftpState->filepath != '\0');
1610     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
1611     ftpState->writeCommand(cbuf);
1612     ftpState->state = Ftp::Client::SENT_MDTM;
1613 }
1614
1615 static void
1616 ftpReadMdtm(Ftp::Gateway * ftpState)
1617 {
1618     int code = ftpState->ctrl.replycode;
1619     debugs(9, 3, HERE);
1620
1621     if (code == 213) {
1622         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
1623         ftpState->unhack();
1624     } else if (code < 0) {
1625         ftpFail(ftpState);
1626         return;
1627     }
1628
1629     ftpSendSize(ftpState);
1630 }
1631
1632 static void
1633 ftpSendSize(Ftp::Gateway * ftpState)
1634 {
1635     /* check the server control channel is still available */
1636     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
1637         return;
1638
1639     /* Only send SIZE for binary transfers. The returned size
1640      * is useless on ASCII transfers */
1641
1642     if (ftpState->flags.binary) {
1643         assert(ftpState->filepath != NULL);
1644         assert(*ftpState->filepath != '\0');
1645         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
1646         ftpState->writeCommand(cbuf);
1647         ftpState->state = Ftp::Client::SENT_SIZE;
1648     } else
1649         /* Skip to next state no non-binary transfers */
1650         ftpSendPassive(ftpState);
1651 }
1652
1653 static void
1654 ftpReadSize(Ftp::Gateway * ftpState)
1655 {
1656     int code = ftpState->ctrl.replycode;
1657     debugs(9, 3, HERE);
1658
1659     if (code == 213) {
1660         ftpState->unhack();
1661         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
1662
1663         if (ftpState->theSize == 0) {
1664             debugs(9, 2, "SIZE reported " <<
1665                    ftpState->ctrl.last_reply << " on " <<
1666                    ftpState->title_url);
1667             ftpState->theSize = -1;
1668         }
1669     } else if (code < 0) {
1670         ftpFail(ftpState);
1671         return;
1672     }
1673
1674     ftpSendPassive(ftpState);
1675 }
1676
1677 static void
1678 ftpReadEPSV(Ftp::Gateway* ftpState)
1679 {
1680     Ip::Address srvAddr; // unused
1681     if (ftpState->handleEpsvReply(srvAddr)) {
1682         if (ftpState->ctrl.message == NULL)
1683             return; // didn't get complete reply yet
1684
1685         ftpState->connectDataChannel();
1686     }
1687 }
1688
1689 /** Send Passive connection request.
1690  * Default method is to use modern EPSV request.
1691  * The failover mechanism should check for previous state and re-call with alternates on failure.
1692  */
1693 static void
1694 ftpSendPassive(Ftp::Gateway * ftpState)
1695 {
1696     /** Checks the server control channel is still available before running. */
1697     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
1698         return;
1699
1700     debugs(9, 3, HERE);
1701
1702     /** \par
1703       * Checks for 'HEAD' method request and passes off for special handling by Ftp::Gateway::processHeadResponse(). */
1704     if (ftpState->request->method == Http::METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
1705         ftpState->processHeadResponse(); // may call serverComplete
1706         return;
1707     }
1708
1709     if (ftpState->sendPassive()) {
1710         // SENT_EPSV_ALL blocks other non-EPSV connections being attempted
1711         if (ftpState->state == Ftp::Client::SENT_EPSV_ALL)
1712             ftpState->flags.epsv_all_sent = true;
1713     }
1714 }
1715
1716 void
1717 Ftp::Gateway::processHeadResponse()
1718 {
1719     debugs(9, 5, HERE << "handling HEAD response");
1720     ftpSendQuit(this);
1721     appendSuccessHeader();
1722
1723     /*
1724      * On rare occasions I'm seeing the entry get aborted after
1725      * readControlReply() and before here, probably when
1726      * trying to write to the client.
1727      */
1728     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1729         abortTransaction("entry aborted while processing HEAD");
1730         return;
1731     }
1732
1733 #if USE_ADAPTATION
1734     if (adaptationAccessCheckPending) {
1735         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
1736         return;
1737     }
1738 #endif
1739
1740     // processReplyBody calls serverComplete() since there is no body
1741     processReplyBody();
1742 }
1743
1744 static void
1745 ftpReadPasv(Ftp::Gateway * ftpState)
1746 {
1747     Ip::Address srvAddr; // unused
1748     if (ftpState->handlePasvReply(srvAddr))
1749         ftpState->connectDataChannel();
1750     else {
1751         ftpSendEPRT(ftpState);
1752         return;
1753     }
1754 }
1755
1756 void
1757 Ftp::Gateway::dataChannelConnected(const CommConnectCbParams &io)
1758 {
1759     debugs(9, 3, HERE);
1760     data.opener = NULL;
1761
1762     if (io.flag != Comm::OK) {
1763         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
1764
1765         // ABORT on timeouts. server may be waiting on a broken TCP link.
1766         if (io.xerrno == Comm::TIMEOUT)
1767             writeCommand("ABOR");
1768
1769         // try another connection attempt with some other method
1770         ftpSendPassive(this);
1771         return;
1772     }
1773
1774     data.opened(io.conn, dataCloser());
1775     ftpRestOrList(this);
1776 }
1777
1778 static void
1779 ftpOpenListenSocket(Ftp::Gateway * ftpState, int fallback)
1780 {
1781     /// Close old data channels, if any. We may open a new one below.
1782     if (ftpState->data.conn != NULL) {
1783         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
1784             // NP: in fact it points to the control channel. just clear it.
1785             ftpState->data.clear();
1786         else
1787             ftpState->data.close();
1788     }
1789     safe_free(ftpState->data.host);
1790
1791     /*
1792      * Set up a listen socket on the same local address as the
1793      * control connection.
1794      */
1795     Comm::ConnectionPointer temp = new Comm::Connection;
1796     temp->local = ftpState->ctrl.conn->local;
1797
1798     /*
1799      * REUSEADDR is needed in fallback mode, since the same port is
1800      * used for both control and data.
1801      */
1802     if (fallback) {
1803         int on = 1;
1804         errno = 0;
1805         if (setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR,
1806                        (char *) &on, sizeof(on)) == -1) {
1807             // SO_REUSEADDR is only an optimization, no need to be verbose about error
1808             debugs(9, 4, "setsockopt failed: " << xstrerror());
1809         }
1810         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
1811         temp->flags |= COMM_REUSEADDR;
1812     } else {
1813         /* if not running in fallback mode a new port needs to be retrieved */
1814         temp->local.port(0);
1815     }
1816
1817     ftpState->listenForDataChannel(temp);
1818 }
1819
1820 static void
1821 ftpSendPORT(Ftp::Gateway * ftpState)
1822 {
1823     /* check the server control channel is still available */
1824     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
1825         return;
1826
1827     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1828         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
1829         return;
1830     }
1831
1832     debugs(9, 3, HERE);
1833     ftpState->flags.pasv_supported = 0;
1834     ftpOpenListenSocket(ftpState, 0);
1835
1836     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
1837         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.isIPv4() ) {
1838             /* non-IPv4 CANNOT send PORT command.                       */
1839             /* we got here by attempting and failing an EPRT            */
1840             /* using the same reply code should simulate a PORT failure */
1841             ftpReadPORT(ftpState);
1842             return;
1843         }
1844
1845         /* XXX Need to set error message */
1846         ftpFail(ftpState);
1847         return;
1848     }
1849
1850     // pull out the internal IP address bytes to send in PORT command...
1851     // source them from the listen_conn->local
1852
1853     struct addrinfo *AI = NULL;
1854     ftpState->data.listenConn->local.getAddrInfo(AI, AF_INET);
1855     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
1856     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
1857     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
1858              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
1859              portptr[0], portptr[1]);
1860     ftpState->writeCommand(cbuf);
1861     ftpState->state = Ftp::Client::SENT_PORT;
1862
1863     Ip::Address::FreeAddr(AI);
1864 }
1865
1866 static void
1867 ftpReadPORT(Ftp::Gateway * ftpState)
1868 {
1869     int code = ftpState->ctrl.replycode;
1870     debugs(9, 3, HERE);
1871
1872     if (code != 200) {
1873         /* Fall back on using the same port as the control connection */
1874         debugs(9, 3, "PORT not supported by remote end");
1875         ftpOpenListenSocket(ftpState, 1);
1876     }
1877
1878     ftpRestOrList(ftpState);
1879 }
1880
1881 static void
1882 ftpSendEPRT(Ftp::Gateway * ftpState)
1883 {
1884     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1885         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
1886         return;
1887     }
1888
1889     if (!Config.Ftp.eprt) {
1890         /* Disabled. Switch immediately to attempting old PORT command. */
1891         debugs(9, 3, "EPRT disabled by local administrator");
1892         ftpSendPORT(ftpState);
1893         return;
1894     }
1895
1896     debugs(9, 3, HERE);
1897     ftpState->flags.pasv_supported = 0;
1898
1899     ftpOpenListenSocket(ftpState, 0);
1900     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
1901     if (!Comm::IsConnOpen(ftpState->data.conn)) {
1902         /* XXX Need to set error message */
1903         ftpFail(ftpState);
1904         return;
1905     }
1906
1907     char buf[MAX_IPSTRLEN];
1908
1909     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
1910     /* Which can be used by EITHER protocol. */
1911     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
1912              ( ftpState->data.listenConn->local.isIPv6() ? 2 : 1 ),
1913              ftpState->data.listenConn->local.toStr(buf,MAX_IPSTRLEN),
1914              ftpState->data.listenConn->local.port() );
1915
1916     ftpState->writeCommand(cbuf);
1917     ftpState->state = Ftp::Client::SENT_EPRT;
1918 }
1919
1920 static void
1921 ftpReadEPRT(Ftp::Gateway * ftpState)
1922 {
1923     int code = ftpState->ctrl.replycode;
1924     debugs(9, 3, HERE);
1925
1926     if (code != 200) {
1927         /* Failover to attempting old PORT command. */
1928         debugs(9, 3, "EPRT not supported by remote end");
1929         ftpSendPORT(ftpState);
1930         return;
1931     }
1932
1933     ftpRestOrList(ftpState);
1934 }
1935
1936 /** "read" handler to accept FTP data connections.
1937  *
1938  \param io    comm accept(2) callback parameters
1939  */
1940 void
1941 Ftp::Gateway::ftpAcceptDataConnection(const CommAcceptCbParams &io)
1942 {
1943     debugs(9, 3, HERE);
1944
1945     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1946         abortTransaction("entry aborted when accepting data conn");
1947         data.listenConn->close();
1948         data.listenConn = NULL;
1949         return;
1950     }
1951
1952     if (io.flag != Comm::OK) {
1953         data.listenConn->close();
1954         data.listenConn = NULL;
1955         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
1956         /** \todo Need to send error message on control channel*/
1957         ftpFail(this);
1958         return;
1959     }
1960
1961     /* data listening conn is no longer even open. abort. */
1962     if (!Comm::IsConnOpen(data.listenConn)) {
1963         data.listenConn = NULL; // ensure that it's cleared and not just closed.
1964         return;
1965     }
1966
1967     /* data listening conn is no longer even open. abort. */
1968     if (!Comm::IsConnOpen(data.conn)) {
1969         data.clear(); // ensure that it's cleared and not just closed.
1970         return;
1971     }
1972
1973     /** \par
1974      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
1975      * made by the remote client which is connected to the FTP control socket.
1976      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
1977      * This prevents third-party hacks, but also third-party load balancing handshakes.
1978      */
1979     if (Config.Ftp.sanitycheck) {
1980         // accept if either our data or ctrl connection is talking to this remote peer.
1981         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
1982             debugs(9, DBG_IMPORTANT,
1983                    "FTP data connection from unexpected server (" <<
1984                    io.conn->remote << "), expecting " <<
1985                    data.conn->remote << " or " << ctrl.conn->remote);
1986
1987             /* close the bad sources connection down ASAP. */
1988             io.conn->close();
1989
1990             /* drop the bad connection (io) by ignoring the attempt. */
1991             return;
1992         }
1993     }
1994
1995     /** On Comm::OK start using the accepted data socket and discard the temporary listen socket. */
1996     data.close();
1997     data.opened(io.conn, dataCloser());
1998     data.addr(io.conn->remote);
1999
2000     debugs(9, 3, HERE << "Connected data socket on " <<
2001            io.conn << ". FD table says: " <<
2002            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
2003            "data-peer= " << fd_table[data.conn->fd].ipaddr);
2004
2005     assert(haveControlChannel("ftpAcceptDataConnection"));
2006     assert(ctrl.message == NULL);
2007
2008     // Ctrl channel operations will determine what happens to this data connection
2009 }
2010
2011 static void
2012 ftpRestOrList(Ftp::Gateway * ftpState)
2013 {
2014     debugs(9, 3, HERE);
2015
2016     if (ftpState->typecode == 'D') {
2017         ftpState->flags.isdir = 1;
2018
2019         if (ftpState->flags.put) {
2020             ftpSendMkdir(ftpState); /* PUT name;type=d */
2021         } else {
2022             ftpSendNlst(ftpState);  /* GET name;type=d  sec 3.2.2 of RFC 1738 */
2023         }
2024     } else if (ftpState->flags.put) {
2025         ftpSendStor(ftpState);
2026     } else if (ftpState->flags.isdir)
2027         ftpSendList(ftpState);
2028     else if (ftpState->restartable())
2029         ftpSendRest(ftpState);
2030     else
2031         ftpSendRetr(ftpState);
2032 }
2033
2034 static void
2035 ftpSendStor(Ftp::Gateway * ftpState)
2036 {
2037     /* check the server control channel is still available */
2038     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
2039         return;
2040
2041     debugs(9, 3, HERE);
2042
2043     if (ftpState->filepath != NULL) {
2044         /* Plain file upload */
2045         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
2046         ftpState->writeCommand(cbuf);
2047         ftpState->state = Ftp::Client::SENT_STOR;
2048     } else if (ftpState->request->header.getInt64(Http::HdrType::CONTENT_LENGTH) > 0) {
2049         /* File upload without a filename. use STOU to generate one */
2050         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
2051         ftpState->writeCommand(cbuf);
2052         ftpState->state = Ftp::Client::SENT_STOR;
2053     } else {
2054         /* No file to transfer. Only create directories if needed */
2055         ftpSendReply(ftpState);
2056     }
2057 }
2058
2059 /// \deprecated use ftpState->readStor() instead.
2060 static void
2061 ftpReadStor(Ftp::Gateway * ftpState)
2062 {
2063     ftpState->readStor();
2064 }
2065
2066 void Ftp::Gateway::readStor()
2067 {
2068     int code = ctrl.replycode;
2069     debugs(9, 3, HERE);
2070
2071     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
2072         if (!originalRequest()->body_pipe) {
2073             debugs(9, 3, "zero-size STOR?");
2074             state = WRITING_DATA; // make ftpWriteTransferDone() responsible
2075             dataComplete(); // XXX: keep in sync with doneSendingRequestBody()
2076             return;
2077         }
2078
2079         if (!startRequestBodyFlow()) { // register to receive body data
2080             ftpFail(this);
2081             return;
2082         }
2083
2084         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
2085         debugs(9, 3, HERE << "starting data transfer");
2086         switchTimeoutToDataChannel();
2087         sendMoreRequestBody();
2088         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
2089         state = WRITING_DATA;
2090         debugs(9, 3, HERE << "writing data channel");
2091     } else if (code == 150) {
2092         /* When client code is 150 with no data channel, Accept data channel. */
2093         debugs(9, 3, "ftpReadStor: accepting data channel");
2094         listenForDataChannel(data.conn);
2095     } else {
2096         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
2097         ftpFail(this);
2098     }
2099 }
2100
2101 static void
2102 ftpSendRest(Ftp::Gateway * ftpState)
2103 {
2104     /* check the server control channel is still available */
2105     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
2106         return;
2107
2108     debugs(9, 3, HERE);
2109
2110     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
2111     ftpState->writeCommand(cbuf);
2112     ftpState->state = Ftp::Client::SENT_REST;
2113 }
2114
2115 int
2116 Ftp::Gateway::restartable()
2117 {
2118     if (restart_offset > 0)
2119         return 1;
2120
2121     if (!request->range)
2122         return 0;
2123
2124     if (!flags.binary)
2125         return 0;
2126
2127     if (theSize <= 0)
2128         return 0;
2129
2130     int64_t desired_offset = request->range->lowestOffset(theSize);
2131
2132     if (desired_offset <= 0)
2133         return 0;
2134
2135     if (desired_offset >= theSize)
2136         return 0;
2137
2138     restart_offset = desired_offset;
2139     return 1;
2140 }
2141
2142 static void
2143 ftpReadRest(Ftp::Gateway * ftpState)
2144 {
2145     int code = ftpState->ctrl.replycode;
2146     debugs(9, 3, HERE);
2147     assert(ftpState->restart_offset > 0);
2148
2149     if (code == 350) {
2150         ftpState->setCurrentOffset(ftpState->restart_offset);
2151         ftpSendRetr(ftpState);
2152     } else if (code > 0) {
2153         debugs(9, 3, HERE << "REST not supported");
2154         ftpState->flags.rest_supported = 0;
2155         ftpSendRetr(ftpState);
2156     } else {
2157         ftpFail(ftpState);
2158     }
2159 }
2160
2161 static void
2162 ftpSendList(Ftp::Gateway * ftpState)
2163 {
2164     /* check the server control channel is still available */
2165     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
2166         return;
2167
2168     debugs(9, 3, HERE);
2169
2170     if (ftpState->filepath) {
2171         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
2172     } else {
2173         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
2174     }
2175
2176     ftpState->writeCommand(cbuf);
2177     ftpState->state = Ftp::Client::SENT_LIST;
2178 }
2179
2180 static void
2181 ftpSendNlst(Ftp::Gateway * ftpState)
2182 {
2183     /* check the server control channel is still available */
2184     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
2185         return;
2186
2187     debugs(9, 3, HERE);
2188
2189     ftpState->flags.tried_nlst = 1;
2190
2191     if (ftpState->filepath) {
2192         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
2193     } else {
2194         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
2195     }
2196
2197     ftpState->writeCommand(cbuf);
2198     ftpState->state = Ftp::Client::SENT_NLST;
2199 }
2200
2201 static void
2202 ftpReadList(Ftp::Gateway * ftpState)
2203 {
2204     int code = ftpState->ctrl.replycode;
2205     debugs(9, 3, HERE);
2206
2207     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2208         /* Begin data transfer */
2209         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2210         ftpState->switchTimeoutToDataChannel();
2211         ftpState->maybeReadVirginBody();
2212         ftpState->state = Ftp::Client::READING_DATA;
2213         return;
2214     } else if (code == 150) {
2215         /* Accept data channel */
2216         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2217         ftpState->listenForDataChannel(ftpState->data.conn);
2218         return;
2219     } else if (!ftpState->flags.tried_nlst && code > 300) {
2220         ftpSendNlst(ftpState);
2221     } else {
2222         ftpFail(ftpState);
2223         return;
2224     }
2225 }
2226
2227 static void
2228 ftpSendRetr(Ftp::Gateway * ftpState)
2229 {
2230     /* check the server control channel is still available */
2231     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
2232         return;
2233
2234     debugs(9, 3, HERE);
2235
2236     assert(ftpState->filepath != NULL);
2237     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
2238     ftpState->writeCommand(cbuf);
2239     ftpState->state = Ftp::Client::SENT_RETR;
2240 }
2241
2242 static void
2243 ftpReadRetr(Ftp::Gateway * ftpState)
2244 {
2245     int code = ftpState->ctrl.replycode;
2246     debugs(9, 3, HERE);
2247
2248     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2249         /* Begin data transfer */
2250         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2251         ftpState->switchTimeoutToDataChannel();
2252         ftpState->maybeReadVirginBody();
2253         ftpState->state = Ftp::Client::READING_DATA;
2254     } else if (code == 150) {
2255         /* Accept data channel */
2256         ftpState->listenForDataChannel(ftpState->data.conn);
2257     } else if (code >= 300) {
2258         if (!ftpState->flags.try_slash_hack) {
2259             /* Try this as a directory missing trailing slash... */
2260             ftpState->hackShortcut(ftpSendCwd);
2261         } else {
2262             ftpFail(ftpState);
2263         }
2264     } else {
2265         ftpFail(ftpState);
2266     }
2267 }
2268
2269 /**
2270  * Generate the HTTP headers and template fluff around an FTP
2271  * directory listing display.
2272  */
2273 void
2274 Ftp::Gateway::completedListing()
2275 {
2276     assert(entry);
2277     entry->lock("Ftp::Gateway");
2278     ErrorState ferr(ERR_DIR_LISTING, Http::scOkay, request);
2279     ferr.ftp.listing = &listing;
2280     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
2281     ferr.ftp.server_msg = ctrl.message;
2282     ctrl.message = NULL;
2283     entry->replaceHttpReply( ferr.BuildHttpReply() );
2284     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2285     entry->flush();
2286     entry->unlock("Ftp::Gateway");
2287 }
2288
2289 static void
2290 ftpReadTransferDone(Ftp::Gateway * ftpState)
2291 {
2292     int code = ftpState->ctrl.replycode;
2293     debugs(9, 3, HERE);
2294
2295     if (code == 226 || code == 250) {
2296         /* Connection closed; retrieval done. */
2297         if (ftpState->flags.listing) {
2298             ftpState->completedListing();
2299             /* QUIT operation handles sending the reply to client */
2300         }
2301         ftpSendQuit(ftpState);
2302     } else {            /* != 226 */
2303         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
2304         ftpState->failed(ERR_FTP_FAILURE, 0);
2305         /* failed closes ctrl.conn and frees ftpState */
2306         return;
2307     }
2308 }
2309
2310 // premature end of the request body
2311 void
2312 Ftp::Gateway::handleRequestBodyProducerAborted()
2313 {
2314     Client::handleRequestBodyProducerAborted();
2315     debugs(9, 3, HERE << "ftpState=" << this);
2316     failed(ERR_READ_ERROR, 0);
2317 }
2318
2319 static void
2320 ftpWriteTransferDone(Ftp::Gateway * ftpState)
2321 {
2322     int code = ftpState->ctrl.replycode;
2323     debugs(9, 3, HERE);
2324
2325     if (!(code == 226 || code == 250)) {
2326         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
2327         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
2328         return;
2329     }
2330
2331     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
2332     ftpSendReply(ftpState);
2333 }
2334
2335 static void
2336 ftpSendQuit(Ftp::Gateway * ftpState)
2337 {
2338     /* check the server control channel is still available */
2339     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
2340         return;
2341
2342     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
2343     ftpState->writeCommand(cbuf);
2344     ftpState->state = Ftp::Client::SENT_QUIT;
2345 }
2346
2347 /** Completes a client FTP operation with success or other page
2348  *  generated and stored in the entry field by the code issuing QUIT.
2349  */
2350 static void
2351 ftpReadQuit(Ftp::Gateway * ftpState)
2352 {
2353     ftpState->serverComplete();
2354 }
2355
2356 static void
2357 ftpTrySlashHack(Ftp::Gateway * ftpState)
2358 {
2359     char *path;
2360     ftpState->flags.try_slash_hack = 1;
2361     /* Free old paths */
2362
2363     debugs(9, 3, HERE);
2364
2365     if (ftpState->pathcomps)
2366         wordlistDestroy(&ftpState->pathcomps);
2367
2368     safe_free(ftpState->filepath);
2369
2370     /* Build the new path (urlpath begins with /) */
2371     path = SBufToCstring(ftpState->request->url.path());
2372
2373     rfc1738_unescape(path);
2374
2375     ftpState->filepath = path;
2376
2377     /* And off we go */
2378     ftpGetFile(ftpState);
2379 }
2380
2381 /**
2382  * Forget hack status. Next error is shown to the user
2383  */
2384 void
2385 Ftp::Gateway::unhack()
2386 {
2387     debugs(9, 3, HERE);
2388
2389     if (old_request != NULL) {
2390         safe_free(old_request);
2391         safe_free(old_reply);
2392     }
2393 }
2394
2395 void
2396 Ftp::Gateway::hackShortcut(FTPSM * nextState)
2397 {
2398     /* Clear some unwanted state */
2399     setCurrentOffset(0);
2400     restart_offset = 0;
2401     /* Save old error message & some state info */
2402
2403     debugs(9, 3, HERE);
2404
2405     if (old_request == NULL) {
2406         old_request = ctrl.last_command;
2407         ctrl.last_command = NULL;
2408         old_reply = ctrl.last_reply;
2409         ctrl.last_reply = NULL;
2410
2411         if (pathcomps == NULL && filepath != NULL)
2412             old_filepath = xstrdup(filepath);
2413     }
2414
2415     /* Jump to the "hack" state */
2416     nextState(this);
2417 }
2418
2419 static void
2420 ftpFail(Ftp::Gateway *ftpState)
2421 {
2422     const bool slashHack = ftpState->request->url.path().caseCmp("/%2f", 4)==0;
2423     debugs(9, 6, "flags(" <<
2424            (ftpState->flags.isdir?"IS_DIR,":"") <<
2425            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
2426            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
2427            "slashhack=" << (slashHack? "T":"F"));
2428
2429     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
2430     if (!ftpState->flags.isdir &&   /* Not a directory */
2431             !ftpState->flags.try_slash_hack && !slashHack && /* Not doing slash hack */
2432             ftpState->mdtm <= 0 && ftpState->theSize < 0) { /* Not known as a file */
2433
2434         switch (ftpState->state) {
2435
2436         case Ftp::Client::SENT_CWD:
2437
2438         case Ftp::Client::SENT_RETR:
2439             /* Try the / hack */
2440             ftpState->hackShortcut(ftpTrySlashHack);
2441             return;
2442
2443         default:
2444             break;
2445         }
2446     }
2447
2448     ftpState->failed(ERR_NONE, 0);
2449     /* failed() closes ctrl.conn and frees this */
2450 }
2451
2452 Http::StatusCode
2453 Ftp::Gateway::failedHttpStatus(err_type &error)
2454 {
2455     if (error == ERR_NONE) {
2456         switch (state) {
2457
2458         case SENT_USER:
2459
2460         case SENT_PASS:
2461
2462             if (ctrl.replycode > 500) {
2463                 error = ERR_FTP_FORBIDDEN;
2464                 return password_url ? Http::scForbidden : Http::scUnauthorized;
2465             } else if (ctrl.replycode == 421) {
2466                 error = ERR_FTP_UNAVAILABLE;
2467                 return Http::scServiceUnavailable;
2468             }
2469             break;
2470
2471         case SENT_CWD:
2472
2473         case SENT_RETR:
2474             if (ctrl.replycode == 550) {
2475                 error = ERR_FTP_NOT_FOUND;
2476                 return Http::scNotFound;
2477             }
2478             break;
2479
2480         default:
2481             break;
2482         }
2483     }
2484     return Ftp::Client::failedHttpStatus(error);
2485 }
2486
2487 static void
2488 ftpSendReply(Ftp::Gateway * ftpState)
2489 {
2490     int code = ftpState->ctrl.replycode;
2491     Http::StatusCode http_code;
2492     err_type err_code = ERR_NONE;
2493
2494     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
2495
2496     if (cbdataReferenceValid(ftpState))
2497         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
2498
2499     if (code == 226 || code == 250) {
2500         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
2501         http_code = (ftpState->mdtm > 0) ? Http::scAccepted : Http::scCreated;
2502     } else if (code == 227) {
2503         err_code = ERR_FTP_PUT_CREATED;
2504         http_code = Http::scCreated;
2505     } else {
2506         err_code = ERR_FTP_PUT_ERROR;
2507         http_code = Http::scInternalServerError;
2508     }
2509
2510     ErrorState err(err_code, http_code, ftpState->request);
2511
2512     if (ftpState->old_request)
2513         err.ftp.request = xstrdup(ftpState->old_request);
2514     else
2515         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
2516
2517     if (ftpState->old_reply)
2518         err.ftp.reply = xstrdup(ftpState->old_reply);
2519     else if (ftpState->ctrl.last_reply)
2520         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
2521     else
2522         err.ftp.reply = xstrdup("");
2523
2524     // TODO: interpret as FTP-specific error code
2525     err.detailError(code);
2526
2527     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
2528
2529     ftpSendQuit(ftpState);
2530 }
2531
2532 void
2533 Ftp::Gateway::appendSuccessHeader()
2534 {
2535     debugs(9, 3, HERE);
2536
2537     if (flags.http_header_sent)
2538         return;
2539
2540     HttpReply *reply = new HttpReply;
2541
2542     flags.http_header_sent = 1;
2543
2544     assert(entry->isEmpty());
2545
2546     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2547
2548     entry->buffer();    /* released when done processing current data payload */
2549
2550     SBuf urlPath = request->url.path();
2551     auto t = urlPath.rfind('/');
2552     SBuf filename = urlPath.substr(t != SBuf::npos ? t : 0);
2553
2554     const char *mime_type = NULL;
2555     const char *mime_enc = NULL;
2556
2557     if (flags.isdir) {
2558         mime_type = "text/html";
2559     } else {
2560         switch (typecode) {
2561
2562         case 'I':
2563             mime_type = "application/octet-stream";
2564             // XXX: performance regression, c_str() may reallocate
2565             mime_enc = mimeGetContentEncoding(filename.c_str());
2566             break;
2567
2568         case 'A':
2569             mime_type = "text/plain";
2570             break;
2571
2572         default:
2573             // XXX: performance regression, c_str() may reallocate
2574             mime_type = mimeGetContentType(filename.c_str());
2575             mime_enc = mimeGetContentEncoding(filename.c_str());
2576             break;
2577         }
2578     }
2579
2580     /* set standard stuff */
2581
2582     if (0 == getCurrentOffset()) {
2583         /* Full reply */
2584         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2585     } else if (theSize < getCurrentOffset()) {
2586         /*
2587          * DPW 2007-05-04
2588          * offset should not be larger than theSize.  We should
2589          * not be seeing this condition any more because we'll only
2590          * send REST if we know the theSize and if it is less than theSize.
2591          */
2592         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
2593                " current offset=" << getCurrentOffset() <<
2594                ", but theSize=" << theSize <<
2595                ".  assuming full content response");
2596         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2597     } else {
2598         /* Partial reply */
2599         HttpHdrRangeSpec range_spec;
2600         range_spec.offset = getCurrentOffset();
2601         range_spec.length = theSize - getCurrentOffset();
2602         reply->setHeaders(Http::scPartialContent, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
2603         httpHeaderAddContRange(&reply->header, range_spec, theSize);
2604     }
2605
2606     /* additional info */
2607     if (mime_enc)
2608         reply->header.putStr(Http::HdrType::CONTENT_ENCODING, mime_enc);
2609
2610     setVirginReply(reply);
2611     adaptOrFinalizeReply();
2612 }
2613
2614 void
2615 Ftp::Gateway::haveParsedReplyHeaders()
2616 {
2617     Client::haveParsedReplyHeaders();
2618
2619     StoreEntry *e = entry;
2620
2621     e->timestampsSet();
2622
2623     if (flags.authenticated) {
2624         /*
2625          * Authenticated requests can't be cached.
2626          */
2627         e->release();
2628     } else if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && !getCurrentOffset()) {
2629         e->setPublicKey();
2630     } else {
2631         e->release();
2632     }
2633 }
2634
2635 HttpReply *
2636 Ftp::Gateway::ftpAuthRequired(HttpRequest * request, const char *realm)
2637 {
2638     ErrorState err(ERR_CACHE_ACCESS_DENIED, Http::scUnauthorized, request);
2639     HttpReply *newrep = err.BuildHttpReply();
2640 #if HAVE_AUTH_MODULE_BASIC
2641     /* add Authenticate header */
2642     newrep->header.putAuth("Basic", realm);
2643 #endif
2644     return newrep;
2645 }
2646
2647 const SBuf &
2648 Ftp::UrlWith2f(HttpRequest * request)
2649 {
2650     SBuf newbuf("%2f");
2651
2652     if (request->url.getScheme() != AnyP::PROTO_FTP) {
2653         static const SBuf nil;
2654         return nil;
2655     }
2656
2657     if (request->url.path()[0] == '/') {
2658         newbuf.append(request->url.path());
2659         request->url.path(newbuf);
2660     } else if (!request->url.path().startsWith(newbuf)) {
2661         newbuf.append(request->url.path().substr(1));
2662         request->url.path(newbuf);
2663     }
2664
2665     return request->effectiveRequestUri();
2666 }
2667
2668 void
2669 Ftp::Gateway::printfReplyBody(const char *fmt, ...)
2670 {
2671     va_list args;
2672     va_start (args, fmt);
2673     static char buf[4096];
2674     buf[0] = '\0';
2675     vsnprintf(buf, 4096, fmt, args);
2676     writeReplyBody(buf, strlen(buf));
2677     va_end(args);
2678 }
2679
2680 /**
2681  * Call this when there is data from the origin server
2682  * which should be sent to either StoreEntry, or to ICAP...
2683  */
2684 void
2685 Ftp::Gateway::writeReplyBody(const char *dataToWrite, size_t dataLength)
2686 {
2687     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
2688     addVirginReplyBody(dataToWrite, dataLength);
2689 }
2690
2691 /**
2692  * A hack to ensure we do not double-complete on the forward entry.
2693  *
2694  \todo Ftp::Gateway logic should probably be rewritten to avoid
2695  *  double-completion or FwdState should be rewritten to allow it.
2696  */
2697 void
2698 Ftp::Gateway::completeForwarding()
2699 {
2700     if (fwd == NULL || flags.completed_forwarding) {
2701         debugs(9, 3, HERE << "completeForwarding avoids " <<
2702                "double-complete on FD " << ctrl.conn->fd << ", Data FD " << data.conn->fd <<
2703                ", this " << this << ", fwd " << fwd);
2704         return;
2705     }
2706
2707     flags.completed_forwarding = true;
2708     Client::completeForwarding();
2709 }
2710
2711 /**
2712  * Have we lost the FTP server control channel?
2713  *
2714  \retval true   The server control channel is available.
2715  \retval false  The server control channel is not available.
2716  */
2717 bool
2718 Ftp::Gateway::haveControlChannel(const char *caller_name) const
2719 {
2720     if (doneWithServer())
2721         return false;
2722
2723     /* doneWithServer() only checks BOTH channels are closed. */
2724     if (!Comm::IsConnOpen(ctrl.conn)) {
2725         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
2726         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
2727         return false;
2728     }
2729
2730     return true;
2731 }
2732
2733 bool
2734 Ftp::Gateway::mayReadVirginReplyBody() const
2735 {
2736     // TODO: Can we do what Ftp::Relay::mayReadVirginReplyBody() does instead?
2737     return !doneWithServer();
2738 }
2739
2740 AsyncJob::Pointer
2741 Ftp::StartGateway(FwdState *const fwdState)
2742 {
2743     return AsyncJob::Start(new Ftp::Gateway(fwdState));
2744 }
2745