src/clients/FtpGateway.cc

   1 /*
   2  * Copyright (C) 1996-2014 The Squid Software Foundation and contributors
   3  *
   4  * Squid software is distributed under GPLv2+ license and includes
   5  * contributions from numerous individuals and organizations.
   6  * Please see the COPYING and CONTRIBUTORS files for details.
   7  */
   8
   9 /* DEBUG: section 09    File Transfer Protocol (FTP) */
  10
  11 #include "squid.h"
  12 #include "acl/FilledChecklist.h"
  13 #include "clients/forward.h"
  14 #include "clients/FtpClient.h"
  15 #include "comm.h"
  16 #include "comm/ConnOpener.h"
  17 #include "comm/Read.h"
  18 #include "comm/TcpAcceptor.h"
  19 #include "CommCalls.h"
  20 #include "compat/strtoll.h"
  21 #include "errorpage.h"
  22 #include "fd.h"
  23 #include "fde.h"
  24 #include "FwdState.h"
  25 #include "html_quote.h"
  26 #include "HttpHdrContRange.h"
  27 #include "HttpHeader.h"
  28 #include "HttpHeaderRange.h"
  29 #include "HttpReply.h"
  30 #include "HttpRequest.h"
  31 #include "ip/tools.h"
  32 #include "Mem.h"
  33 #include "MemBuf.h"
  34 #include "mime.h"
  35 #include "rfc1738.h"
  36 #include "SquidConfig.h"
  37 #include "SquidString.h"
  38 #include "SquidTime.h"
  39 #include "StatCounters.h"
  40 #include "Store.h"
  41 #include "tools.h"
  42 #include "URL.h"
  43 #include "wordlist.h"
  44
  45 #if USE_DELAY_POOLS
  46 #include "DelayPools.h"
  47 #include "MemObject.h"
  48 #endif
  49
  50 #include <cerrno>
  51
  52 namespace Ftp
  53 {
  54
  55 struct GatewayFlags {
  56
  57     /* passive mode */
  58     bool pasv_supported;  ///< PASV command is allowed
  59     bool epsv_all_sent;   ///< EPSV ALL has been used. Must abort on failures.
  60     bool pasv_only;
  61     bool pasv_failed;  // was FwdState::flags.ftp_pasv_failed
  62
  63     /* authentication */
  64     bool authenticated;         ///< authentication success
  65     bool tried_auth_anonymous;  ///< auth has tried to use anonymous credentials already.
  66     bool tried_auth_nopass;     ///< auth tried username with no password already.
  67
  68     /* other */
  69     bool isdir;
  70     bool skip_whitespace;
  71     bool rest_supported;
  72     bool http_header_sent;
  73     bool tried_nlst;
  74     bool need_base_href;
  75     bool dir_slash;
  76     bool root_dir;
  77     bool no_dotdot;
  78     bool binary;
  79     bool try_slash_hack;
  80     bool put;
  81     bool put_mkdir;
  82     bool listformat_unknown;
  83     bool listing;
  84     bool completed_forwarding;
  85 };
  86
  87 class Gateway;
  88 typedef void (StateMethod)(Ftp::Gateway *);
  89
  90 /// FTP Gateway: An FTP client that takes an HTTP request with an ftp:// URI,
  91 /// converts it into one or more FTP commands, and then
  92 /// converts one or more FTP responses into the final HTTP response.
  93 class Gateway : public Ftp::Client
  94 {
  95 public:
  96     Gateway(FwdState *);
  97     virtual ~Gateway();
  98     char user[MAX_URL];
  99     char password[MAX_URL];
 100     int password_url;
 101     char *reply_hdr;
 102     int reply_hdr_state;
 103     String clean_url;
 104     String title_url;
 105     String base_href;
 106     int conn_att;
 107     int login_att;
 108     time_t mdtm;
 109     int64_t theSize;
 110     wordlist *pathcomps;
 111     char *filepath;
 112     char *dirpath;
 113     int64_t restart_offset;
 114     char *proxy_host;
 115     size_t list_width;
 116     String cwd_message;
 117     char *old_filepath;
 118     char typecode;
 119     MemBuf listing;             ///< FTP directory listing in HTML format.
 120
 121     GatewayFlags flags;
 122
 123 public:
 124     // these should all be private
 125     virtual void start();
 126     virtual Http::StatusCode failedHttpStatus(err_type &error);
 127     void loginParser(const char *, int escaped);
 128     int restartable();
 129     void appendSuccessHeader();
 130     void hackShortcut(StateMethod *nextState);
 131     void unhack();
 132     void readStor();
 133     void parseListing();
 134     MemBuf *htmlifyListEntry(const char *line);
 135     void completedListing(void);
 136
 137     /// create a data channel acceptor and start listening.
 138     void listenForDataChannel(const Comm::ConnectionPointer &conn);
 139
 140     int checkAuth(const HttpHeader * req_hdr);
 141     void checkUrlpath();
 142     void buildTitleUrl();
 143     void writeReplyBody(const char *, size_t len);
 144     void printfReplyBody(const char *fmt, ...);
 145     virtual void completeForwarding();
 146     void processHeadResponse();
 147     void processReplyBody();
 148     void setCurrentOffset(int64_t offset) { currentOffset = offset; }
 149     int64_t getCurrentOffset() const { return currentOffset; }
 150
 151     virtual void dataChannelConnected(const CommConnectCbParams &io);
 152     static PF ftpDataWrite;
 153     virtual void timeout(const CommTimeoutCbParams &io);
 154     void ftpAcceptDataConnection(const CommAcceptCbParams &io);
 155
 156     static HttpReply *ftpAuthRequired(HttpRequest * request, const char *realm);
 157     const char *ftpRealm(void);
 158     void loginFailed(void);
 159
 160     virtual void haveParsedReplyHeaders();
 161
 162     virtual bool haveControlChannel(const char *caller_name) const;
 163
 164 protected:
 165     virtual void handleControlReply();
 166     virtual void dataClosed(const CommCloseCbParams &io);
 167
 168 private:
 169     virtual bool mayReadVirginReplyBody() const;
 170     // BodyConsumer for HTTP: consume request body.
 171     virtual void handleRequestBodyProducerAborted();
 172
 173     CBDATA_CLASS2(Gateway);
 174 };
 175
 176 } // namespace Ftp
 177
 178 typedef Ftp::StateMethod FTPSM; // to avoid lots of non-changes
 179
 180 CBDATA_NAMESPACED_CLASS_INIT(Ftp, Gateway);
 181
 182 typedef struct {
 183     char type;
 184     int64_t size;
 185     char *date;
 186     char *name;
 187     char *showname;
 188     char *link;
 189 } ftpListParts;
 190
 191 #define FTP_LOGIN_ESCAPED       1
 192
 193 #define FTP_LOGIN_NOT_ESCAPED   0
 194
 195 #define CTRL_BUFLEN 1024
 196 static char cbuf[CTRL_BUFLEN];
 197
 198 /*
 199  * State machine functions
 200  * send == state transition
 201  * read == wait for response, and select next state transition
 202  * other == Transition logic
 203  */
 204 static FTPSM ftpReadWelcome;
 205 static FTPSM ftpSendUser;
 206 static FTPSM ftpReadUser;
 207 static FTPSM ftpSendPass;
 208 static FTPSM ftpReadPass;
 209 static FTPSM ftpSendType;
 210 static FTPSM ftpReadType;
 211 static FTPSM ftpSendMdtm;
 212 static FTPSM ftpReadMdtm;
 213 static FTPSM ftpSendSize;
 214 static FTPSM ftpReadSize;
 215 static FTPSM ftpSendEPRT;
 216 static FTPSM ftpReadEPRT;
 217 static FTPSM ftpSendPORT;
 218 static FTPSM ftpReadPORT;
 219 static FTPSM ftpSendPassive;
 220 static FTPSM ftpReadEPSV;
 221 static FTPSM ftpReadPasv;
 222 static FTPSM ftpTraverseDirectory;
 223 static FTPSM ftpListDir;
 224 static FTPSM ftpGetFile;
 225 static FTPSM ftpSendCwd;
 226 static FTPSM ftpReadCwd;
 227 static FTPSM ftpRestOrList;
 228 static FTPSM ftpSendList;
 229 static FTPSM ftpSendNlst;
 230 static FTPSM ftpReadList;
 231 static FTPSM ftpSendRest;
 232 static FTPSM ftpReadRest;
 233 static FTPSM ftpSendRetr;
 234 static FTPSM ftpReadRetr;
 235 static FTPSM ftpReadTransferDone;
 236 static FTPSM ftpSendStor;
 237 static FTPSM ftpReadStor;
 238 static FTPSM ftpWriteTransferDone;
 239 static FTPSM ftpSendReply;
 240 static FTPSM ftpSendMkdir;
 241 static FTPSM ftpReadMkdir;
 242 static FTPSM ftpFail;
 243 static FTPSM ftpSendQuit;
 244 static FTPSM ftpReadQuit;
 245
 246 /************************************************
 247 ** Debugs Levels used here                     **
 248 *************************************************
 249 0       CRITICAL Events
 250 1       IMPORTANT Events
 251         Protocol and Transmission failures.
 252 2       FTP Protocol Chatter
 253 3       Logic Flows
 254 4       Data Parsing Flows
 255 5       Data Dumps
 256 7       ??
 257 ************************************************/
 258
 259 /************************************************
 260 ** State Machine Description (excluding hacks) **
 261 *************************************************
 262 From                    To
 263 ---------------------------------------
 264 Welcome                 User
 265 User                    Pass
 266 Pass                    Type
 267 Type                    TraverseDirectory / GetFile
 268 TraverseDirectory       Cwd / GetFile / ListDir
 269 Cwd                     TraverseDirectory / Mkdir
 270 GetFile                 Mdtm
 271 Mdtm                    Size
 272 Size                    Epsv
 273 ListDir                 Epsv
 274 Epsv                    FileOrList
 275 FileOrList              Rest / Retr / Nlst / List / Mkdir (PUT /xxx;type=d)
 276 Rest                    Retr
 277 Retr / Nlst / List      DataRead* (on datachannel)
 278 DataRead*               ReadTransferDone
 279 ReadTransferDone        DataTransferDone
 280 Stor                    DataWrite* (on datachannel)
 281 DataWrite*              RequestPutBody** (from client)
 282 RequestPutBody**        DataWrite* / WriteTransferDone
 283 WriteTransferDone       DataTransferDone
 284 DataTransferDone        Quit
 285 Quit                    -
 286 ************************************************/
 287
 288 FTPSM *FTP_SM_FUNCS[] = {
 289     ftpReadWelcome,             /* BEGIN */
 290     ftpReadUser,                /* SENT_USER */
 291     ftpReadPass,                /* SENT_PASS */
 292     ftpReadType,                /* SENT_TYPE */
 293     ftpReadMdtm,                /* SENT_MDTM */
 294     ftpReadSize,                /* SENT_SIZE */
 295     ftpReadEPRT,                /* SENT_EPRT */
 296     ftpReadPORT,                /* SENT_PORT */
 297     ftpReadEPSV,                /* SENT_EPSV_ALL */
 298     ftpReadEPSV,                /* SENT_EPSV_1 */
 299     ftpReadEPSV,                /* SENT_EPSV_2 */
 300     ftpReadPasv,                /* SENT_PASV */
 301     ftpReadCwd,         /* SENT_CWD */
 302     ftpReadList,                /* SENT_LIST */
 303     ftpReadList,                /* SENT_NLST */
 304     ftpReadRest,                /* SENT_REST */
 305     ftpReadRetr,                /* SENT_RETR */
 306     ftpReadStor,                /* SENT_STOR */
 307     ftpReadQuit,                /* SENT_QUIT */
 308     ftpReadTransferDone,        /* READING_DATA (RETR,LIST,NLST) */
 309     ftpWriteTransferDone,       /* WRITING_DATA (STOR) */
 310     ftpReadMkdir,               /* SENT_MKDIR */
 311     NULL,                       /* SENT_FEAT */
 312     NULL,                       /* SENT_PWD */
 313     NULL,                       /* SENT_CDUP*/
 314     NULL,                       /* SENT_DATA_REQUEST */
 315     NULL                        /* SENT_COMMAND */
 316 };
 317
 318 /// handler called by Comm when FTP data channel is closed unexpectedly
 319 void
 320 Ftp::Gateway::dataClosed(const CommCloseCbParams &io)
 321 {
 322     Ftp::Client::dataClosed(io);
 323     failed(ERR_FTP_FAILURE, 0);
 324     /* failed closes ctrl.conn and frees ftpState */
 325
 326     /* NP: failure recovery may be possible when its only a data.conn failure.
 327      *     if the ctrl.conn is still fine, we can send ABOR down it and retry.
 328      *     Just need to watch out for wider Squid states like shutting down or reconfigure.
 329      */
 330 }
 331
 332 Ftp::Gateway::Gateway(FwdState *fwdState):
 333         AsyncJob("FtpStateData"),
 334         Ftp::Client(fwdState)
 335 {
 336     const char *url = entry->url();
 337     debugs(9, 3, HERE << "'" << url << "'" );
 338     theSize = -1;
 339     mdtm = -1;
 340
 341     if (Config.Ftp.passive && !flags.pasv_failed)
 342         flags.pasv_supported = 1;
 343
 344     flags.rest_supported = 1;
 345
 346     if (request->method == Http::METHOD_PUT)
 347         flags.put = 1;
 348
 349     initReadBuf();
 350 }
 351
 352 Ftp::Gateway::~Gateway()
 353 {
 354     debugs(9, 3, HERE << entry->url()  );
 355
 356     if (Comm::IsConnOpen(ctrl.conn)) {
 357         debugs(9, DBG_IMPORTANT, "Internal bug: FTP Gateway left open " <<
 358                "control channel " << ctrl.conn);
 359     }
 360
 361     if (reply_hdr) {
 362         memFree(reply_hdr, MEM_8K_BUF);
 363         reply_hdr = NULL;
 364     }
 365
 366     if (pathcomps)
 367         wordlistDestroy(&pathcomps);
 368
 369     cwd_message.clean();
 370
 371     safe_free(old_filepath);
 372
 373     title_url.clean();
 374
 375     base_href.clean();
 376
 377     safe_free(filepath);
 378
 379     safe_free(dirpath);
 380 }
 381
 382 /**
 383  * Parse a possible login username:password pair.
 384  * Produces filled member variables user, password, password_url if anything found.
 385  */
 386 void
 387 Ftp::Gateway::loginParser(const char *login, int escaped)
 388 {
 389     const char *u = NULL; // end of the username sub-string
 390     int len;              // length of the current sub-string to handle.
 391
 392     int total_len = strlen(login);
 393
 394     debugs(9, 4, HERE << ": login='" << login << "', escaped=" << escaped);
 395     debugs(9, 9, HERE << ": IN : login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 396
 397     if ((u = strchr(login, ':'))) {
 398
 399         /* if there was a username part */
 400         if (u > login) {
 401             len = u - login;
 402             ++u; // jump off the delimiter.
 403             if (len > MAX_URL)
 404                 len = MAX_URL-1;
 405             xstrncpy(user, login, len +1);
 406             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<"), escaped=" << escaped);
 407             if (escaped)
 408                 rfc1738_unescape(user);
 409             debugs(9, 9, HERE << ": found user='" << user << "'(" << len <<") unescaped.");
 410         }
 411
 412         /* if there was a password part */
 413         len = login + total_len - u;
 414         if ( len > 0) {
 415             if (len > MAX_URL)
 416                 len = MAX_URL -1;
 417             xstrncpy(password, u, len +1);
 418             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<"), escaped=" << escaped);
 419             if (escaped) {
 420                 rfc1738_unescape(password);
 421                 password_url = 1;
 422             }
 423             debugs(9, 9, HERE << ": found password='" << password << "'(" << len <<") unescaped.");
 424         }
 425     } else if (login[0]) {
 426         /* no password, just username */
 427         if (total_len > MAX_URL)
 428             total_len = MAX_URL -1;
 429         xstrncpy(user, login, total_len +1);
 430         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<"), escaped=" << escaped);
 431         if (escaped)
 432             rfc1738_unescape(user);
 433         debugs(9, 9, HERE << ": found user='" << user << "'(" << total_len <<") unescaped.");
 434     }
 435
 436     debugs(9, 9, HERE << ": OUT: login='" << login << "', escaped=" << escaped << ", user=" << user << ", password=" << password);
 437 }
 438
 439 void
 440 Ftp::Gateway::listenForDataChannel(const Comm::ConnectionPointer &conn)
 441 {
 442     assert(!Comm::IsConnOpen(data.conn));
 443
 444     typedef CommCbMemFunT<Gateway, CommAcceptCbParams> AcceptDialer;
 445     typedef AsyncCallT<AcceptDialer> AcceptCall;
 446     RefCount<AcceptCall> call = static_cast<AcceptCall*>(JobCallback(11, 5, AcceptDialer, this, Ftp::Gateway::ftpAcceptDataConnection));
 447     Subscription::Pointer sub = new CallSubscription<AcceptCall>(call);
 448     const char *note = entry->url();
 449
 450     /* open the conn if its not already open */
 451     if (!Comm::IsConnOpen(conn)) {
 452         conn->fd = comm_open_listener(SOCK_STREAM, IPPROTO_TCP, conn->local, conn->flags, note);
 453         if (!Comm::IsConnOpen(conn)) {
 454             debugs(5, DBG_CRITICAL, HERE << "comm_open_listener failed:" << conn->local << " error: " << errno);
 455             return;
 456         }
 457         debugs(9, 3, HERE << "Unconnected data socket created on " << conn);
 458     }
 459
 460     conn->tos = ctrl.conn->tos;
 461     conn->nfmark = ctrl.conn->nfmark;
 462
 463     assert(Comm::IsConnOpen(conn));
 464     AsyncJob::Start(new Comm::TcpAcceptor(conn, note, sub));
 465
 466     // Ensure we have a copy of the FD opened for listening and a close handler on it.
 467     data.opened(conn, dataCloser());
 468     switchTimeoutToDataChannel();
 469 }
 470
 471 void
 472 Ftp::Gateway::timeout(const CommTimeoutCbParams &io)
 473 {
 474     if (SENT_PASV == state) {
 475         /* stupid ftp.netscape.com, of FTP server behind stupid firewall rules */
 476         flags.pasv_supported = false;
 477         debugs(9, DBG_IMPORTANT, "FTP Gateway timeout in SENT_PASV state");
 478
 479         // cancel the data connection setup.
 480         if (data.opener != NULL) {
 481             data.opener->cancel("timeout");
 482             data.opener = NULL;
 483         }
 484         data.close();
 485     }
 486
 487     Ftp::Client::timeout(io);
 488 }
 489
 490 static const char *Month[] = {
 491     "Jan", "Feb", "Mar", "Apr", "May", "Jun",
 492     "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"
 493 };
 494
 495 static int
 496 is_month(const char *buf)
 497 {
 498     int i;
 499
 500     for (i = 0; i < 12; ++i)
 501         if (!strcasecmp(buf, Month[i]))
 502             return 1;
 503
 504     return 0;
 505 }
 506
 507 static void
 508 ftpListPartsFree(ftpListParts ** parts)
 509 {
 510     safe_free((*parts)->date);
 511     safe_free((*parts)->name);
 512     safe_free((*parts)->showname);
 513     safe_free((*parts)->link);
 514     safe_free(*parts);
 515 }
 516
 517 #define MAX_TOKENS 64
 518
 519 static ftpListParts *
 520 ftpListParseParts(const char *buf, struct Ftp::GatewayFlags flags)
 521 {
 522     ftpListParts *p = NULL;
 523     char *t = NULL;
 524     const char *ct = NULL;
 525     char *tokens[MAX_TOKENS];
 526     int i;
 527     int n_tokens;
 528     static char tbuf[128];
 529     char *xbuf = NULL;
 530     static int scan_ftp_initialized = 0;
 531     static regex_t scan_ftp_integer;
 532     static regex_t scan_ftp_time;
 533     static regex_t scan_ftp_dostime;
 534     static regex_t scan_ftp_dosdate;
 535
 536     if (!scan_ftp_initialized) {
 537         scan_ftp_initialized = 1;
 538         regcomp(&scan_ftp_integer, "^[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 539         regcomp(&scan_ftp_time, "^[0123456789:]+$", REG_EXTENDED | REG_NOSUB);
 540         regcomp(&scan_ftp_dosdate, "^[0123456789]+-[0123456789]+-[0123456789]+$", REG_EXTENDED | REG_NOSUB);
 541         regcomp(&scan_ftp_dostime, "^[0123456789]+:[0123456789]+[AP]M$", REG_EXTENDED | REG_NOSUB | REG_ICASE);
 542     }
 543
 544     if (buf == NULL)
 545         return NULL;
 546
 547     if (*buf == '\0')
 548         return NULL;
 549
 550     p = (ftpListParts *)xcalloc(1, sizeof(ftpListParts));
 551
 552     n_tokens = 0;
 553
 554     memset(tokens, 0, sizeof(tokens));
 555
 556     xbuf = xstrdup(buf);
 557
 558     if (flags.tried_nlst) {
 559         /* Machine readable format, one name per line */
 560         p->name = xbuf;
 561         p->type = '\0';
 562         return p;
 563     }
 564
 565     for (t = strtok(xbuf, w_space); t && n_tokens < MAX_TOKENS; t = strtok(NULL, w_space)) {
 566         tokens[n_tokens] = xstrdup(t);
 567         ++n_tokens;
 568     }
 569
 570     xfree(xbuf);
 571
 572     /* locate the Month field */
 573     for (i = 3; i < n_tokens - 2; ++i) {
 574         char *size = tokens[i - 1];
 575         char *month = tokens[i];
 576         char *day = tokens[i + 1];
 577         char *year = tokens[i + 2];
 578
 579         if (!is_month(month))
 580             continue;
 581
 582         if (regexec(&scan_ftp_integer, size, 0, NULL, 0) != 0)
 583             continue;
 584
 585         if (regexec(&scan_ftp_integer, day, 0, NULL, 0) != 0)
 586             continue;
 587
 588         if (regexec(&scan_ftp_time, year, 0, NULL, 0) != 0)     /* Yr | hh:mm */
 589             continue;
 590
 591         snprintf(tbuf, 128, "%s %2s %5s",
 592                  month, day, year);
 593
 594         if (!strstr(buf, tbuf))
 595             snprintf(tbuf, 128, "%s %2s %-5s",
 596                      month, day, year);
 597
 598         char const *copyFrom = NULL;
 599
 600         if ((copyFrom = strstr(buf, tbuf))) {
 601             p->type = *tokens[0];
 602             p->size = strtoll(size, NULL, 10);
 603             p->date = xstrdup(tbuf);
 604
 605             if (flags.skip_whitespace) {
 606                 copyFrom += strlen(tbuf);
 607
 608                 while (strchr(w_space, *copyFrom))
 609                     ++copyFrom;
 610             } else {
 611                 /* XXX assumes a single space between date and filename
 612                  * suggested by:  Nathan.Bailey@cc.monash.edu.au and
 613                  * Mike Battersby <mike@starbug.bofh.asn.au> */
 614                 copyFrom += strlen(tbuf) + 1;
 615             }
 616
 617             p->name = xstrdup(copyFrom);
 618
 619             if (p->type == 'l' && (t = strstr(p->name, " -> "))) {
 620                 *t = '\0';
 621                 p->link = xstrdup(t + 4);
 622             }
 623
 624             goto found;
 625         }
 626
 627         break;
 628     }
 629
 630     /* try it as a DOS listing, 04-05-70 09:33PM ... */
 631     if (n_tokens > 3 &&
 632             regexec(&scan_ftp_dosdate, tokens[0], 0, NULL, 0) == 0 &&
 633             regexec(&scan_ftp_dostime, tokens[1], 0, NULL, 0) == 0) {
 634         if (!strcasecmp(tokens[2], "<dir>")) {
 635             p->type = 'd';
 636         } else {
 637             p->type = '-';
 638             p->size = strtoll(tokens[2], NULL, 10);
 639         }
 640
 641         snprintf(tbuf, 128, "%s %s", tokens[0], tokens[1]);
 642         p->date = xstrdup(tbuf);
 643
 644         if (p->type == 'd') {
 645             /* Directory.. name begins with first printable after <dir> */
 646             ct = strstr(buf, tokens[2]);
 647             ct += strlen(tokens[2]);
 648
 649             while (xisspace(*ct))
 650                 ++ct;
 651
 652             if (!*ct)
 653                 ct = NULL;
 654         } else {
 655             /* A file. Name begins after size, with a space in between */
 656             snprintf(tbuf, 128, " %s %s", tokens[2], tokens[3]);
 657             ct = strstr(buf, tbuf);
 658
 659             if (ct) {
 660                 ct += strlen(tokens[2]) + 2;
 661             }
 662         }
 663
 664         p->name = xstrdup(ct ? ct : tokens[3]);
 665         goto found;
 666     }
 667
 668     /* Try EPLF format; carson@lehman.com */
 669     if (buf[0] == '+') {
 670         ct = buf + 1;
 671         p->type = 0;
 672
 673         while (ct && *ct) {
 674             time_t tm;
 675             int l = strcspn(ct, ",");
 676             char *tmp;
 677
 678             if (l < 1)
 679                 goto blank;
 680
 681             switch (*ct) {
 682
 683             case '\t':
 684                 p->name = xstrndup(ct + 1, l + 1);
 685                 break;
 686
 687             case 's':
 688                 p->size = atoi(ct + 1);
 689                 break;
 690
 691             case 'm':
 692                 tm = (time_t) strtol(ct + 1, &tmp, 0);
 693
 694                 if (tmp != ct + 1)
 695                     break;      /* not a valid integer */
 696
 697                 p->date = xstrdup(ctime(&tm));
 698
 699                 *(strstr(p->date, "\n")) = '\0';
 700
 701                 break;
 702
 703             case '/':
 704                 p->type = 'd';
 705
 706                 break;
 707
 708             case 'r':
 709                 p->type = '-';
 710
 711                 break;
 712
 713             case 'i':
 714                 break;
 715
 716             default:
 717                 break;
 718             }
 719
 720 blank:
 721             ct = strstr(ct, ",");
 722
 723             if (ct) {
 724                 ++ct;
 725             }
 726         }
 727
 728         if (p->type == 0) {
 729             p->type = '-';
 730         }
 731
 732         if (p->name)
 733             goto found;
 734         else
 735             safe_free(p->date);
 736     }
 737
 738 found:
 739
 740     for (i = 0; i < n_tokens; ++i)
 741         xfree(tokens[i]);
 742
 743     if (!p->name)
 744         ftpListPartsFree(&p);   /* cleanup */
 745
 746     return p;
 747 }
 748
 749 MemBuf *
 750 Ftp::Gateway::htmlifyListEntry(const char *line)
 751 {
 752     char icon[2048];
 753     char href[2048 + 40];
 754     char text[ 2048];
 755     char size[ 2048];
 756     char chdir[ 2048 + 40];
 757     char view[ 2048 + 40];
 758     char download[ 2048 + 40];
 759     char link[ 2048 + 40];
 760     MemBuf *html;
 761     char prefix[2048];
 762     ftpListParts *parts;
 763     *icon = *href = *text = *size = *chdir = *view = *download = *link = '\0';
 764
 765     debugs(9, 7, HERE << " line ={" << line << "}");
 766
 767     if (strlen(line) > 1024) {
 768         html = new MemBuf();
 769         html->init();
 770         html->Printf("<tr><td colspan=\"5\">%s</td></tr>\n", line);
 771         return html;
 772     }
 773
 774     if (flags.dir_slash && dirpath && typecode != 'D')
 775         snprintf(prefix, 2048, "%s/", rfc1738_escape_part(dirpath));
 776     else
 777         prefix[0] = '\0';
 778
 779     if ((parts = ftpListParseParts(line, flags)) == NULL) {
 780         const char *p;
 781
 782         html = new MemBuf();
 783         html->init();
 784         html->Printf("<tr class=\"entry\"><td colspan=\"5\">%s</td></tr>\n", line);
 785
 786         for (p = line; *p && xisspace(*p); ++p);
 787         if (*p && !xisspace(*p))
 788             flags.listformat_unknown = 1;
 789
 790         return html;
 791     }
 792
 793     if (!strcmp(parts->name, ".") || !strcmp(parts->name, "..")) {
 794         ftpListPartsFree(&parts);
 795         return NULL;
 796     }
 797
 798     parts->size += 1023;
 799     parts->size >>= 10;
 800     parts->showname = xstrdup(parts->name);
 801
 802     /* {icon} {text} . . . {date}{size}{chdir}{view}{download}{link}\n  */
 803     xstrncpy(href, rfc1738_escape_part(parts->name), 2048);
 804
 805     xstrncpy(text, parts->showname, 2048);
 806
 807     switch (parts->type) {
 808
 809     case 'd':
 810         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 811                  mimeGetIconURL("internal-dir"),
 812                  "[DIR]");
 813         strcat(href, "/");      /* margin is allocated above */
 814         break;
 815
 816     case 'l':
 817         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 818                  mimeGetIconURL("internal-link"),
 819                  "[LINK]");
 820         /* sometimes there is an 'l' flag, but no "->" link */
 821
 822         if (parts->link) {
 823             char *link2 = xstrdup(html_quote(rfc1738_escape(parts->link)));
 824             snprintf(link, 2048, " -&gt; <a href=\"%s%s\">%s</a>",
 825                      *link2 != '/' ? prefix : "", link2,
 826                      html_quote(parts->link));
 827             safe_free(link2);
 828         }
 829
 830         break;
 831
 832     case '\0':
 833         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 834                  mimeGetIconURL(parts->name),
 835                  "[UNKNOWN]");
 836         snprintf(chdir, 2048, "<a href=\"%s/;type=d\"><img border=\"0\" src=\"%s\" "
 837                  "alt=\"[DIR]\"></a>",
 838                  rfc1738_escape_part(parts->name),
 839                  mimeGetIconURL("internal-dir"));
 840         break;
 841
 842     case '-':
 843
 844     default:
 845         snprintf(icon, 2048, "<img border=\"0\" src=\"%s\" alt=\"%-6s\">",
 846                  mimeGetIconURL(parts->name),
 847                  "[FILE]");
 848         snprintf(size, 2048, " %6" PRId64 "k", parts->size);
 849         break;
 850     }
 851
 852     if (parts->type != 'd') {
 853         if (mimeGetViewOption(parts->name)) {
 854             snprintf(view, 2048, "<a href=\"%s%s;type=a\"><img border=\"0\" src=\"%s\" "
 855                      "alt=\"[VIEW]\"></a>",
 856                      prefix, href, mimeGetIconURL("internal-view"));
 857         }
 858
 859         if (mimeGetDownloadOption(parts->name)) {
 860             snprintf(download, 2048, "<a href=\"%s%s;type=i\"><img border=\"0\" src=\"%s\" "
 861                      "alt=\"[DOWNLOAD]\"></a>",
 862                      prefix, href, mimeGetIconURL("internal-download"));
 863         }
 864     }
 865
 866     /* construct the table row from parts. */
 867     html = new MemBuf();
 868     html->init();
 869     html->Printf("<tr class=\"entry\">"
 870                  "<td class=\"icon\"><a href=\"%s%s\">%s</a></td>"
 871                  "<td class=\"filename\"><a href=\"%s%s\">%s</a></td>"
 872                  "<td class=\"date\">%s</td>"
 873                  "<td class=\"size\">%s</td>"
 874                  "<td class=\"actions\">%s%s%s%s</td>"
 875                  "</tr>\n",
 876                  prefix, href, icon,
 877                  prefix, href, html_quote(text),
 878                  parts->date,
 879                  size,
 880                  chdir, view, download, link);
 881
 882     ftpListPartsFree(&parts);
 883     return html;
 884 }
 885
 886 void
 887 Ftp::Gateway::parseListing()
 888 {
 889     char *buf = data.readBuf->content();
 890     char *sbuf;                 /* NULL-terminated copy of termedBuf */
 891     char *end;
 892     char *line;
 893     char *s;
 894     MemBuf *t;
 895     size_t linelen;
 896     size_t usable;
 897     size_t len = data.readBuf->contentSize();
 898
 899     if (!len) {
 900         debugs(9, 3, HERE << "no content to parse for " << entry->url()  );
 901         return;
 902     }
 903
 904     /*
 905      * We need a NULL-terminated buffer for scanning, ick
 906      */
 907     sbuf = (char *)xmalloc(len + 1);
 908     xstrncpy(sbuf, buf, len + 1);
 909     end = sbuf + len - 1;
 910
 911     while (*end != '\r' && *end != '\n' && end > sbuf)
 912         --end;
 913
 914     usable = end - sbuf;
 915
 916     debugs(9, 3, HERE << "usable = " << usable << " of " << len << " bytes.");
 917
 918     if (usable == 0) {
 919         if (buf[0] == '\0' && len == 1) {
 920             debugs(9, 3, HERE << "NIL ends data from " << entry->url() << " transfer problem?");
 921             data.readBuf->consume(len);
 922         } else {
 923             debugs(9, 3, HERE << "didn't find end for " << entry->url());
 924             debugs(9, 3, HERE << "buffer remains (" << len << " bytes) '" << rfc1738_do_escape(buf,0) << "'");
 925         }
 926         xfree(sbuf);
 927         return;
 928     }
 929
 930     debugs(9, 3, HERE << (unsigned long int)len << " bytes to play with");
 931
 932     line = (char *)memAllocate(MEM_4K_BUF);
 933     ++end;
 934     s = sbuf;
 935     s += strspn(s, crlf);
 936
 937     for (; s < end; s += strcspn(s, crlf), s += strspn(s, crlf)) {
 938         debugs(9, 7, HERE << "s = {" << s << "}");
 939         linelen = strcspn(s, crlf) + 1;
 940
 941         if (linelen < 2)
 942             break;
 943
 944         if (linelen > 4096)
 945             linelen = 4096;
 946
 947         xstrncpy(line, s, linelen);
 948
 949         debugs(9, 7, HERE << "{" << line << "}");
 950
 951         if (!strncmp(line, "total", 5))
 952             continue;
 953
 954         t = htmlifyListEntry(line);
 955
 956         if ( t != NULL) {
 957             debugs(9, 7, HERE << "listing append: t = {" << t->contentSize() << ", '" << t->content() << "'}");
 958             listing.append(t->content(), t->contentSize());
 959 //leak?            delete t;
 960         }
 961     }
 962
 963     debugs(9, 7, HERE << "Done.");
 964     data.readBuf->consume(usable);
 965     memFree(line, MEM_4K_BUF);
 966     xfree(sbuf);
 967 }
 968
 969 void
 970 Ftp::Gateway::processReplyBody()
 971 {
 972     debugs(9, 3, status());
 973
 974     if (request->method == Http::METHOD_HEAD && (flags.isdir || theSize != -1)) {
 975         serverComplete();
 976         return;
 977     }
 978
 979     /* Directory listings are special. They write ther own headers via the error objects */
 980     if (!flags.http_header_sent && data.readBuf->contentSize() >= 0 && !flags.isdir)
 981         appendSuccessHeader();
 982
 983     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
 984         /*
 985          * probably was aborted because content length exceeds one
 986          * of the maximum size limits.
 987          */
 988         abortTransaction("entry aborted after calling appendSuccessHeader()");
 989         return;
 990     }
 991
 992 #if USE_ADAPTATION
 993
 994     if (adaptationAccessCheckPending) {
 995         debugs(9, 3, "returning from Ftp::Gateway::processReplyBody due to adaptationAccessCheckPending");
 996         return;
 997     }
 998
 999 #endif
1000
1001     if (flags.isdir) {
1002         if (!flags.listing) {
1003             flags.listing = 1;
1004             listing.reset();
1005         }
1006         parseListing();
1007         maybeReadVirginBody();
1008         return;
1009     } else if (const int csize = data.readBuf->contentSize()) {
1010         writeReplyBody(data.readBuf->content(), csize);
1011         debugs(9, 5, HERE << "consuming " << csize << " bytes of readBuf");
1012         data.readBuf->consume(csize);
1013     }
1014
1015     entry->flush();
1016
1017     maybeReadVirginBody();
1018 }
1019
1020 /**
1021  * Locates the FTP user:password login.
1022  *
1023  * Highest to lowest priority:
1024  *  - Checks URL (ftp://user:pass@domain)
1025  *  - Authorization: Basic header
1026  *  - squid.conf anonymous-FTP settings (default: anonymous:Squid@).
1027  *
1028  * Special Case: A username-only may be provided in the URL and password in the HTTP headers.
1029  *
1030  * TODO: we might be able to do something about locating username from other sources:
1031  *       ie, external ACL user=* tag or ident lookup
1032  *
1033  \retval 1      if we have everything needed to complete this request.
1034  \retval 0      if something is missing.
1035  */
1036 int
1037 Ftp::Gateway::checkAuth(const HttpHeader * req_hdr)
1038 {
1039     /* default username */
1040     xstrncpy(user, "anonymous", MAX_URL);
1041
1042 #if HAVE_AUTH_MODULE_BASIC
1043     /* Check HTTP Authorization: headers (better than defaults, but less than URL) */
1044     const char *auth;
1045     if ( (auth = req_hdr->getAuth(HDR_AUTHORIZATION, "Basic")) ) {
1046         flags.authenticated = 1;
1047         loginParser(auth, FTP_LOGIN_NOT_ESCAPED);
1048     }
1049     /* we fail with authorization-required error later IFF the FTP server requests it */
1050 #endif
1051
1052     /* Test URL login syntax. Overrides any headers received. */
1053     loginParser(request->login, FTP_LOGIN_ESCAPED);
1054
1055     /* name is missing. thats fatal. */
1056     if (!user[0])
1057         fatal("FTP login parsing destroyed username info");
1058
1059     /* name + password == success */
1060     if (password[0])
1061         return 1;
1062
1063     /* Setup default FTP password settings */
1064     /* this has to be done last so that we can have a no-password case above. */
1065     if (!password[0]) {
1066         if (strcmp(user, "anonymous") == 0 && !flags.tried_auth_anonymous) {
1067             xstrncpy(password, Config.Ftp.anon_user, MAX_URL);
1068             flags.tried_auth_anonymous=1;
1069             return 1;
1070         } else if (!flags.tried_auth_nopass) {
1071             xstrncpy(password, null_string, MAX_URL);
1072             flags.tried_auth_nopass=1;
1073             return 1;
1074         }
1075     }
1076
1077     return 0;                   /* different username */
1078 }
1079
1080 static String str_type_eq;
1081 void
1082 Ftp::Gateway::checkUrlpath()
1083 {
1084     int l;
1085     size_t t;
1086
1087     if (str_type_eq.size()==0) //hack. String doesn't support global-static
1088         str_type_eq="type=";
1089
1090     if ((t = request->urlpath.rfind(';')) != String::npos) {
1091         if (request->urlpath.substr(t+1,t+1+str_type_eq.size())==str_type_eq) {
1092             typecode = (char)xtoupper(request->urlpath[t+str_type_eq.size()+1]);
1093             request->urlpath.cut(t);
1094         }
1095     }
1096
1097     l = request->urlpath.size();
1098     /* check for null path */
1099
1100     if (!l) {
1101         flags.isdir = 1;
1102         flags.root_dir = 1;
1103         flags.need_base_href = 1;       /* Work around broken browsers */
1104     } else if (!request->urlpath.cmp("/%2f/")) {
1105         /* UNIX root directory */
1106         flags.isdir = 1;
1107         flags.root_dir = 1;
1108     } else if ((l >= 1) && (request->urlpath[l - 1] == '/')) {
1109         /* Directory URL, ending in / */
1110         flags.isdir = 1;
1111
1112         if (l == 1)
1113             flags.root_dir = 1;
1114     } else {
1115         flags.dir_slash = 1;
1116     }
1117 }
1118
1119 void
1120 Ftp::Gateway::buildTitleUrl()
1121 {
1122     title_url = "ftp://";
1123
1124     if (strcmp(user, "anonymous")) {
1125         title_url.append(user);
1126         title_url.append("@");
1127     }
1128
1129     title_url.append(request->GetHost());
1130
1131     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1132         title_url.append(":");
1133         title_url.append(xitoa(request->port));
1134     }
1135
1136     title_url.append (request->urlpath);
1137
1138     base_href = "ftp://";
1139
1140     if (strcmp(user, "anonymous") != 0) {
1141         base_href.append(rfc1738_escape_part(user));
1142
1143         if (password_url) {
1144             base_href.append (":");
1145             base_href.append(rfc1738_escape_part(password));
1146         }
1147
1148         base_href.append("@");
1149     }
1150
1151     base_href.append(request->GetHost());
1152
1153     if (request->port != urlDefaultPort(AnyP::PROTO_FTP)) {
1154         base_href.append(":");
1155         base_href.append(xitoa(request->port));
1156     }
1157
1158     base_href.append(request->urlpath);
1159     base_href.append("/");
1160 }
1161
1162 void
1163 Ftp::Gateway::start()
1164 {
1165     if (!checkAuth(&request->header)) {
1166         /* create appropriate reply */
1167         HttpReply *reply = ftpAuthRequired(request, ftpRealm());
1168         entry->replaceHttpReply(reply);
1169         serverComplete();
1170         return;
1171     }
1172
1173     checkUrlpath();
1174     buildTitleUrl();
1175     debugs(9, 5, HERE << "FD " << ctrl.conn->fd << " : host=" << request->GetHost() <<
1176            ", path=" << request->urlpath << ", user=" << user << ", passwd=" << password);
1177     state = BEGIN;
1178     Ftp::Client::start();
1179 }
1180
1181 /* ====================================================================== */
1182
1183 void
1184 Ftp::Gateway::handleControlReply()
1185 {
1186     Ftp::Client::handleControlReply();
1187     if (ctrl.message == NULL)
1188         return; // didn't get complete reply yet
1189
1190     /* Copy the message except for the last line to cwd_message to be
1191      * printed in error messages.
1192      */
1193     for (wordlist *w = ctrl.message; w && w->next; w = w->next) {
1194         cwd_message.append('\n');
1195         cwd_message.append(w->key);
1196     }
1197
1198     FTP_SM_FUNCS[state] (this);
1199 }
1200
1201 /* ====================================================================== */
1202
1203 static void
1204 ftpReadWelcome(Ftp::Gateway * ftpState)
1205 {
1206     int code = ftpState->ctrl.replycode;
1207     debugs(9, 3, HERE);
1208
1209     if (ftpState->flags.pasv_only)
1210         ++ ftpState->login_att;
1211
1212     if (code == 220) {
1213         if (ftpState->ctrl.message) {
1214             if (strstr(ftpState->ctrl.message->key, "NetWare"))
1215                 ftpState->flags.skip_whitespace = 1;
1216         }
1217
1218         ftpSendUser(ftpState);
1219     } else if (code == 120) {
1220         if (NULL != ftpState->ctrl.message)
1221             debugs(9, DBG_IMPORTANT, "FTP server is busy: " << ftpState->ctrl.message->key);
1222
1223         return;
1224     } else {
1225         ftpFail(ftpState);
1226     }
1227 }
1228
1229 /**
1230  * Translate FTP login failure into HTTP error
1231  * this is an attmpt to get the 407 message to show up outside Squid.
1232  * its NOT a general failure. But a correct FTP response type.
1233  */
1234 void
1235 Ftp::Gateway::loginFailed()
1236 {
1237     ErrorState *err = NULL;
1238     const char *command, *reply;
1239
1240     if ((state == SENT_USER || state == SENT_PASS) && ctrl.replycode >= 400) {
1241         if (ctrl.replycode == 421 || ctrl.replycode == 426) {
1242             // 421/426 - Service Overload - retry permitted.
1243             err = new ErrorState(ERR_FTP_UNAVAILABLE, Http::scServiceUnavailable, fwd->request);
1244         } else if (ctrl.replycode >= 430 && ctrl.replycode <= 439) {
1245             // 43x - Invalid or Credential Error - retry challenge required.
1246             err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1247         } else if (ctrl.replycode >= 530 && ctrl.replycode <= 539) {
1248             // 53x - Credentials Missing - retry challenge required
1249             if (password_url) // but they were in the URI! major fail.
1250                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scForbidden, fwd->request);
1251             else
1252                 err = new ErrorState(ERR_FTP_FORBIDDEN, Http::scUnauthorized, fwd->request);
1253         }
1254     }
1255
1256     // any other problems are general falures.
1257     if (!err) {
1258         ftpFail(this);
1259         return;
1260     }
1261
1262     err->ftp.server_msg = ctrl.message;
1263
1264     ctrl.message = NULL;
1265
1266     if (old_request)
1267         command = old_request;
1268     else
1269         command = ctrl.last_command;
1270
1271     if (command && strncmp(command, "PASS", 4) == 0)
1272         command = "PASS <yourpassword>";
1273
1274     if (old_reply)
1275         reply = old_reply;
1276     else
1277         reply = ctrl.last_reply;
1278
1279     if (command)
1280         err->ftp.request = xstrdup(command);
1281
1282     if (reply)
1283         err->ftp.reply = xstrdup(reply);
1284
1285     HttpReply *newrep = err->BuildHttpReply();
1286     delete err;
1287
1288 #if HAVE_AUTH_MODULE_BASIC
1289     /* add Authenticate header */
1290     newrep->header.putAuth("Basic", ftpRealm());
1291 #endif
1292
1293     // add it to the store entry for response....
1294     entry->replaceHttpReply(newrep);
1295     serverComplete();
1296 }
1297
1298 const char *
1299 Ftp::Gateway::ftpRealm()
1300 {
1301     static char realm[8192];
1302
1303     /* This request is not fully authenticated */
1304     if (!request) {
1305         snprintf(realm, 8192, "FTP %s unknown", user);
1306     } else if (request->port == 21) {
1307         snprintf(realm, 8192, "FTP %s %s", user, request->GetHost());
1308     } else {
1309         snprintf(realm, 8192, "FTP %s %s port %d", user, request->GetHost(), request->port);
1310     }
1311     return realm;
1312 }
1313
1314 static void
1315 ftpSendUser(Ftp::Gateway * ftpState)
1316 {
1317     /* check the server control channel is still available */
1318     if (!ftpState || !ftpState->haveControlChannel("ftpSendUser"))
1319         return;
1320
1321     if (ftpState->proxy_host != NULL)
1322         snprintf(cbuf, CTRL_BUFLEN, "USER %s@%s\r\n",
1323                  ftpState->user,
1324                  ftpState->request->GetHost());
1325     else
1326         snprintf(cbuf, CTRL_BUFLEN, "USER %s\r\n", ftpState->user);
1327
1328     ftpState->writeCommand(cbuf);
1329
1330     ftpState->state = Ftp::Client::SENT_USER;
1331 }
1332
1333 static void
1334 ftpReadUser(Ftp::Gateway * ftpState)
1335 {
1336     int code = ftpState->ctrl.replycode;
1337     debugs(9, 3, HERE);
1338
1339     if (code == 230) {
1340         ftpReadPass(ftpState);
1341     } else if (code == 331) {
1342         ftpSendPass(ftpState);
1343     } else {
1344         ftpState->loginFailed();
1345     }
1346 }
1347
1348 static void
1349 ftpSendPass(Ftp::Gateway * ftpState)
1350 {
1351     /* check the server control channel is still available */
1352     if (!ftpState || !ftpState->haveControlChannel("ftpSendPass"))
1353         return;
1354
1355     snprintf(cbuf, CTRL_BUFLEN, "PASS %s\r\n", ftpState->password);
1356     ftpState->writeCommand(cbuf);
1357     ftpState->state = Ftp::Client::SENT_PASS;
1358 }
1359
1360 static void
1361 ftpReadPass(Ftp::Gateway * ftpState)
1362 {
1363     int code = ftpState->ctrl.replycode;
1364     debugs(9, 3, HERE << "code=" << code);
1365
1366     if (code == 230) {
1367         ftpSendType(ftpState);
1368     } else {
1369         ftpState->loginFailed();
1370     }
1371 }
1372
1373 static void
1374 ftpSendType(Ftp::Gateway * ftpState)
1375 {
1376     const char *t;
1377     const char *filename;
1378     char mode;
1379
1380     /* check the server control channel is still available */
1381     if (!ftpState || !ftpState->haveControlChannel("ftpSendType"))
1382         return;
1383
1384     /*
1385      * Ref section 3.2.2 of RFC 1738
1386      */
1387     mode = ftpState->typecode;
1388
1389     switch (mode) {
1390
1391     case 'D':
1392         mode = 'A';
1393         break;
1394
1395     case 'A':
1396
1397     case 'I':
1398         break;
1399
1400     default:
1401
1402         if (ftpState->flags.isdir) {
1403             mode = 'A';
1404         } else {
1405             t = ftpState->request->urlpath.rpos('/');
1406             filename = t ? t + 1 : ftpState->request->urlpath.termedBuf();
1407             mode = mimeGetTransferMode(filename);
1408         }
1409
1410         break;
1411     }
1412
1413     if (mode == 'I')
1414         ftpState->flags.binary = 1;
1415     else
1416         ftpState->flags.binary = 0;
1417
1418     snprintf(cbuf, CTRL_BUFLEN, "TYPE %c\r\n", mode);
1419
1420     ftpState->writeCommand(cbuf);
1421
1422     ftpState->state = Ftp::Client::SENT_TYPE;
1423 }
1424
1425 static void
1426 ftpReadType(Ftp::Gateway * ftpState)
1427 {
1428     int code = ftpState->ctrl.replycode;
1429     char *path;
1430     char *d, *p;
1431     debugs(9, 3, HERE << "code=" << code);
1432
1433     if (code == 200) {
1434         p = path = xstrdup(ftpState->request->urlpath.termedBuf());
1435
1436         if (*p == '/')
1437             ++p;
1438
1439         while (*p) {
1440             d = p;
1441             p += strcspn(p, "/");
1442
1443             if (*p) {
1444                 *p = '\0';
1445                 ++p;
1446             }
1447
1448             rfc1738_unescape(d);
1449
1450             if (*d)
1451                 wordlistAdd(&ftpState->pathcomps, d);
1452         }
1453
1454         xfree(path);
1455
1456         if (ftpState->pathcomps)
1457             ftpTraverseDirectory(ftpState);
1458         else
1459             ftpListDir(ftpState);
1460     } else {
1461         ftpFail(ftpState);
1462     }
1463 }
1464
1465 static void
1466 ftpTraverseDirectory(Ftp::Gateway * ftpState)
1467 {
1468     wordlist *w;
1469     debugs(9, 4, HERE << (ftpState->filepath ? ftpState->filepath : "<NULL>"));
1470
1471     safe_free(ftpState->dirpath);
1472     ftpState->dirpath = ftpState->filepath;
1473     ftpState->filepath = NULL;
1474
1475     /* Done? */
1476
1477     if (ftpState->pathcomps == NULL) {
1478         debugs(9, 3, HERE << "the final component was a directory");
1479         ftpListDir(ftpState);
1480         return;
1481     }
1482
1483     /* Go to next path component */
1484     w = ftpState->pathcomps;
1485
1486     ftpState->filepath = w->key;
1487
1488     ftpState->pathcomps = w->next;
1489
1490     delete w;
1491
1492     /* Check if we are to CWD or RETR */
1493     if (ftpState->pathcomps != NULL || ftpState->flags.isdir) {
1494         ftpSendCwd(ftpState);
1495     } else {
1496         debugs(9, 3, HERE << "final component is probably a file");
1497         ftpGetFile(ftpState);
1498         return;
1499     }
1500 }
1501
1502 static void
1503 ftpSendCwd(Ftp::Gateway * ftpState)
1504 {
1505     char *path = NULL;
1506
1507     /* check the server control channel is still available */
1508     if (!ftpState || !ftpState->haveControlChannel("ftpSendCwd"))
1509         return;
1510
1511     debugs(9, 3, HERE);
1512
1513     path = ftpState->filepath;
1514
1515     if (!strcmp(path, "..") || !strcmp(path, "/")) {
1516         ftpState->flags.no_dotdot = 1;
1517     } else {
1518         ftpState->flags.no_dotdot = 0;
1519     }
1520
1521     snprintf(cbuf, CTRL_BUFLEN, "CWD %s\r\n", path);
1522
1523     ftpState->writeCommand(cbuf);
1524
1525     ftpState->state = Ftp::Client::SENT_CWD;
1526 }
1527
1528 static void
1529 ftpReadCwd(Ftp::Gateway * ftpState)
1530 {
1531     int code = ftpState->ctrl.replycode;
1532     debugs(9, 3, HERE);
1533
1534     if (code >= 200 && code < 300) {
1535         /* CWD OK */
1536         ftpState->unhack();
1537
1538         /* Reset cwd_message to only include the last message */
1539         ftpState->cwd_message.reset("");
1540         for (wordlist *w = ftpState->ctrl.message; w; w = w->next) {
1541             ftpState->cwd_message.append(' ');
1542             ftpState->cwd_message.append(w->key);
1543         }
1544         ftpState->ctrl.message = NULL;
1545
1546         /* Continue to traverse the path */
1547         ftpTraverseDirectory(ftpState);
1548     } else {
1549         /* CWD FAILED */
1550
1551         if (!ftpState->flags.put)
1552             ftpFail(ftpState);
1553         else
1554             ftpSendMkdir(ftpState);
1555     }
1556 }
1557
1558 static void
1559 ftpSendMkdir(Ftp::Gateway * ftpState)
1560 {
1561     char *path = NULL;
1562
1563     /* check the server control channel is still available */
1564     if (!ftpState || !ftpState->haveControlChannel("ftpSendMkdir"))
1565         return;
1566
1567     path = ftpState->filepath;
1568     debugs(9, 3, HERE << "with path=" << path);
1569     snprintf(cbuf, CTRL_BUFLEN, "MKD %s\r\n", path);
1570     ftpState->writeCommand(cbuf);
1571     ftpState->state = Ftp::Client::SENT_MKDIR;
1572 }
1573
1574 static void
1575 ftpReadMkdir(Ftp::Gateway * ftpState)
1576 {
1577     char *path = ftpState->filepath;
1578     int code = ftpState->ctrl.replycode;
1579
1580     debugs(9, 3, HERE << "path " << path << ", code " << code);
1581
1582     if (code == 257) {          /* success */
1583         ftpSendCwd(ftpState);
1584     } else if (code == 550) {   /* dir exists */
1585
1586         if (ftpState->flags.put_mkdir) {
1587             ftpState->flags.put_mkdir = 1;
1588             ftpSendCwd(ftpState);
1589         } else
1590             ftpSendReply(ftpState);
1591     } else
1592         ftpSendReply(ftpState);
1593 }
1594
1595 static void
1596 ftpGetFile(Ftp::Gateway * ftpState)
1597 {
1598     assert(*ftpState->filepath != '\0');
1599     ftpState->flags.isdir = 0;
1600     ftpSendMdtm(ftpState);
1601 }
1602
1603 static void
1604 ftpListDir(Ftp::Gateway * ftpState)
1605 {
1606     if (ftpState->flags.dir_slash) {
1607         debugs(9, 3, HERE << "Directory path did not end in /");
1608         ftpState->title_url.append("/");
1609         ftpState->flags.isdir = 1;
1610     }
1611
1612     ftpSendPassive(ftpState);
1613 }
1614
1615 static void
1616 ftpSendMdtm(Ftp::Gateway * ftpState)
1617 {
1618     /* check the server control channel is still available */
1619     if (!ftpState || !ftpState->haveControlChannel("ftpSendMdtm"))
1620         return;
1621
1622     assert(*ftpState->filepath != '\0');
1623     snprintf(cbuf, CTRL_BUFLEN, "MDTM %s\r\n", ftpState->filepath);
1624     ftpState->writeCommand(cbuf);
1625     ftpState->state = Ftp::Client::SENT_MDTM;
1626 }
1627
1628 static void
1629 ftpReadMdtm(Ftp::Gateway * ftpState)
1630 {
1631     int code = ftpState->ctrl.replycode;
1632     debugs(9, 3, HERE);
1633
1634     if (code == 213) {
1635         ftpState->mdtm = parse_iso3307_time(ftpState->ctrl.last_reply);
1636         ftpState->unhack();
1637     } else if (code < 0) {
1638         ftpFail(ftpState);
1639         return;
1640     }
1641
1642     ftpSendSize(ftpState);
1643 }
1644
1645 static void
1646 ftpSendSize(Ftp::Gateway * ftpState)
1647 {
1648     /* check the server control channel is still available */
1649     if (!ftpState || !ftpState->haveControlChannel("ftpSendSize"))
1650         return;
1651
1652     /* Only send SIZE for binary transfers. The returned size
1653      * is useless on ASCII transfers */
1654
1655     if (ftpState->flags.binary) {
1656         assert(ftpState->filepath != NULL);
1657         assert(*ftpState->filepath != '\0');
1658         snprintf(cbuf, CTRL_BUFLEN, "SIZE %s\r\n", ftpState->filepath);
1659         ftpState->writeCommand(cbuf);
1660         ftpState->state = Ftp::Client::SENT_SIZE;
1661     } else
1662         /* Skip to next state no non-binary transfers */
1663         ftpSendPassive(ftpState);
1664 }
1665
1666 static void
1667 ftpReadSize(Ftp::Gateway * ftpState)
1668 {
1669     int code = ftpState->ctrl.replycode;
1670     debugs(9, 3, HERE);
1671
1672     if (code == 213) {
1673         ftpState->unhack();
1674         ftpState->theSize = strtoll(ftpState->ctrl.last_reply, NULL, 10);
1675
1676         if (ftpState->theSize == 0) {
1677             debugs(9, 2, "SIZE reported " <<
1678                    ftpState->ctrl.last_reply << " on " <<
1679                    ftpState->title_url);
1680             ftpState->theSize = -1;
1681         }
1682     } else if (code < 0) {
1683         ftpFail(ftpState);
1684         return;
1685     }
1686
1687     ftpSendPassive(ftpState);
1688 }
1689
1690 static void
1691 ftpReadEPSV(Ftp::Gateway* ftpState)
1692 {
1693     Ip::Address srvAddr; // unused
1694     if (ftpState->handleEpsvReply(srvAddr)) {
1695         if (ftpState->ctrl.message == NULL)
1696             return; // didn't get complete reply yet
1697
1698         ftpState->connectDataChannel();
1699     }
1700 }
1701
1702 /** Send Passive connection request.
1703  * Default method is to use modern EPSV request.
1704  * The failover mechanism should check for previous state and re-call with alternates on failure.
1705  */
1706 static void
1707 ftpSendPassive(Ftp::Gateway * ftpState)
1708 {
1709     /** Checks the server control channel is still available before running. */
1710     if (!ftpState || !ftpState->haveControlChannel("ftpSendPassive"))
1711         return;
1712
1713     debugs(9, 3, HERE);
1714
1715     /** \par
1716       * Checks for 'HEAD' method request and passes off for special handling by Ftp::Gateway::processHeadResponse(). */
1717     if (ftpState->request->method == Http::METHOD_HEAD && (ftpState->flags.isdir || ftpState->theSize != -1)) {
1718         ftpState->processHeadResponse(); // may call serverComplete
1719         return;
1720     }
1721
1722     if (ftpState->sendPassive()) {
1723         // SENT_EPSV_ALL blocks other non-EPSV connections being attempted
1724         if (ftpState->state == Ftp::Client::SENT_EPSV_ALL)
1725             ftpState->flags.epsv_all_sent = true;
1726     }
1727 }
1728
1729 void
1730 Ftp::Gateway::processHeadResponse()
1731 {
1732     debugs(9, 5, HERE << "handling HEAD response");
1733     ftpSendQuit(this);
1734     appendSuccessHeader();
1735
1736     /*
1737      * On rare occasions I'm seeing the entry get aborted after
1738      * readControlReply() and before here, probably when
1739      * trying to write to the client.
1740      */
1741     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1742         abortTransaction("entry aborted while processing HEAD");
1743         return;
1744     }
1745
1746 #if USE_ADAPTATION
1747     if (adaptationAccessCheckPending) {
1748         debugs(9,3, HERE << "returning due to adaptationAccessCheckPending");
1749         return;
1750     }
1751 #endif
1752
1753     // processReplyBody calls serverComplete() since there is no body
1754     processReplyBody();
1755 }
1756
1757 static void
1758 ftpReadPasv(Ftp::Gateway * ftpState)
1759 {
1760     Ip::Address srvAddr; // unused
1761     if (ftpState->handlePasvReply(srvAddr))
1762         ftpState->connectDataChannel();
1763     else {
1764         ftpSendEPRT(ftpState);
1765         return;
1766     }
1767 }
1768
1769 void
1770 Ftp::Gateway::dataChannelConnected(const CommConnectCbParams &io)
1771 {
1772     debugs(9, 3, HERE);
1773     data.opener = NULL;
1774
1775     if (io.flag != Comm::OK) {
1776         debugs(9, 2, HERE << "Failed to connect. Retrying via another method.");
1777
1778         // ABORT on timeouts. server may be waiting on a broken TCP link.
1779         if (io.xerrno == Comm::TIMEOUT)
1780             writeCommand("ABOR");
1781
1782         // try another connection attempt with some other method
1783         ftpSendPassive(this);
1784         return;
1785     }
1786
1787     data.opened(io.conn, dataCloser());
1788     ftpRestOrList(this);
1789 }
1790
1791 static void
1792 ftpOpenListenSocket(Ftp::Gateway * ftpState, int fallback)
1793 {
1794     /// Close old data channels, if any. We may open a new one below.
1795     if (ftpState->data.conn != NULL) {
1796         if ((ftpState->data.conn->flags & COMM_REUSEADDR))
1797             // NP: in fact it points to the control channel. just clear it.
1798             ftpState->data.clear();
1799         else
1800             ftpState->data.close();
1801     }
1802     safe_free(ftpState->data.host);
1803
1804     /*
1805      * Set up a listen socket on the same local address as the
1806      * control connection.
1807      */
1808     Comm::ConnectionPointer temp = new Comm::Connection;
1809     temp->local = ftpState->ctrl.conn->local;
1810
1811     /*
1812      * REUSEADDR is needed in fallback mode, since the same port is
1813      * used for both control and data.
1814      */
1815     if (fallback) {
1816         int on = 1;
1817         setsockopt(ftpState->ctrl.conn->fd, SOL_SOCKET, SO_REUSEADDR, (char *) &on, sizeof(on));
1818         ftpState->ctrl.conn->flags |= COMM_REUSEADDR;
1819         temp->flags |= COMM_REUSEADDR;
1820     } else {
1821         /* if not running in fallback mode a new port needs to be retrieved */
1822         temp->local.port(0);
1823     }
1824
1825     ftpState->listenForDataChannel(temp);
1826 }
1827
1828 static void
1829 ftpSendPORT(Ftp::Gateway * ftpState)
1830 {
1831     /* check the server control channel is still available */
1832     if (!ftpState || !ftpState->haveControlChannel("ftpSendPort"))
1833         return;
1834
1835     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1836         debugs(9, DBG_IMPORTANT, "FTP does not allow PORT method after 'EPSV ALL' has been sent.");
1837         return;
1838     }
1839
1840     debugs(9, 3, HERE);
1841     ftpState->flags.pasv_supported = 0;
1842     ftpOpenListenSocket(ftpState, 0);
1843
1844     if (!Comm::IsConnOpen(ftpState->data.listenConn)) {
1845         if ( ftpState->data.listenConn != NULL && !ftpState->data.listenConn->local.isIPv4() ) {
1846             /* non-IPv4 CANNOT send PORT command.                       */
1847             /* we got here by attempting and failing an EPRT            */
1848             /* using the same reply code should simulate a PORT failure */
1849             ftpReadPORT(ftpState);
1850             return;
1851         }
1852
1853         /* XXX Need to set error message */
1854         ftpFail(ftpState);
1855         return;
1856     }
1857
1858     // pull out the internal IP address bytes to send in PORT command...
1859     // source them from the listen_conn->local
1860
1861     struct addrinfo *AI = NULL;
1862     ftpState->data.listenConn->local.getAddrInfo(AI, AF_INET);
1863     unsigned char *addrptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_addr;
1864     unsigned char *portptr = (unsigned char *) &((struct sockaddr_in*)AI->ai_addr)->sin_port;
1865     snprintf(cbuf, CTRL_BUFLEN, "PORT %d,%d,%d,%d,%d,%d\r\n",
1866              addrptr[0], addrptr[1], addrptr[2], addrptr[3],
1867              portptr[0], portptr[1]);
1868     ftpState->writeCommand(cbuf);
1869     ftpState->state = Ftp::Client::SENT_PORT;
1870
1871     Ip::Address::FreeAddrInfo(AI);
1872 }
1873
1874 static void
1875 ftpReadPORT(Ftp::Gateway * ftpState)
1876 {
1877     int code = ftpState->ctrl.replycode;
1878     debugs(9, 3, HERE);
1879
1880     if (code != 200) {
1881         /* Fall back on using the same port as the control connection */
1882         debugs(9, 3, "PORT not supported by remote end");
1883         ftpOpenListenSocket(ftpState, 1);
1884     }
1885
1886     ftpRestOrList(ftpState);
1887 }
1888
1889 static void
1890 ftpSendEPRT(Ftp::Gateway * ftpState)
1891 {
1892     if (Config.Ftp.epsv_all && ftpState->flags.epsv_all_sent) {
1893         debugs(9, DBG_IMPORTANT, "FTP does not allow EPRT method after 'EPSV ALL' has been sent.");
1894         return;
1895     }
1896
1897     if (!Config.Ftp.eprt) {
1898         /* Disabled. Switch immediately to attempting old PORT command. */
1899         debugs(9, 3, "EPRT disabled by local administrator");
1900         ftpSendPORT(ftpState);
1901         return;
1902     }
1903
1904     debugs(9, 3, HERE);
1905     ftpState->flags.pasv_supported = 0;
1906
1907     ftpOpenListenSocket(ftpState, 0);
1908     debugs(9, 3, "Listening for FTP data connection with FD " << ftpState->data.conn);
1909     if (!Comm::IsConnOpen(ftpState->data.conn)) {
1910         /* XXX Need to set error message */
1911         ftpFail(ftpState);
1912         return;
1913     }
1914
1915     char buf[MAX_IPSTRLEN];
1916
1917     /* RFC 2428 defines EPRT as IPv6 equivalent to IPv4 PORT command. */
1918     /* Which can be used by EITHER protocol. */
1919     snprintf(cbuf, CTRL_BUFLEN, "EPRT |%d|%s|%d|\r\n",
1920              ( ftpState->data.listenConn->local.isIPv6() ? 2 : 1 ),
1921              ftpState->data.listenConn->local.toStr(buf,MAX_IPSTRLEN),
1922              ftpState->data.listenConn->local.port() );
1923
1924     ftpState->writeCommand(cbuf);
1925     ftpState->state = Ftp::Client::SENT_EPRT;
1926 }
1927
1928 static void
1929 ftpReadEPRT(Ftp::Gateway * ftpState)
1930 {
1931     int code = ftpState->ctrl.replycode;
1932     debugs(9, 3, HERE);
1933
1934     if (code != 200) {
1935         /* Failover to attempting old PORT command. */
1936         debugs(9, 3, "EPRT not supported by remote end");
1937         ftpSendPORT(ftpState);
1938         return;
1939     }
1940
1941     ftpRestOrList(ftpState);
1942 }
1943
1944 /** "read" handler to accept FTP data connections.
1945  *
1946  \param io    comm accept(2) callback parameters
1947  */
1948 void
1949 Ftp::Gateway::ftpAcceptDataConnection(const CommAcceptCbParams &io)
1950 {
1951     debugs(9, 3, HERE);
1952
1953     if (EBIT_TEST(entry->flags, ENTRY_ABORTED)) {
1954         abortTransaction("entry aborted when accepting data conn");
1955         data.listenConn->close();
1956         data.listenConn = NULL;
1957         return;
1958     }
1959
1960     if (io.flag != Comm::OK) {
1961         data.listenConn->close();
1962         data.listenConn = NULL;
1963         debugs(9, DBG_IMPORTANT, "FTP AcceptDataConnection: " << io.conn << ": " << xstrerr(io.xerrno));
1964         /** \todo Need to send error message on control channel*/
1965         ftpFail(this);
1966         return;
1967     }
1968
1969     /* data listening conn is no longer even open. abort. */
1970     if (!Comm::IsConnOpen(data.listenConn)) {
1971         data.listenConn = NULL; // ensure that it's cleared and not just closed.
1972         return;
1973     }
1974
1975     /* data listening conn is no longer even open. abort. */
1976     if (!Comm::IsConnOpen(data.conn)) {
1977         data.clear(); // ensure that it's cleared and not just closed.
1978         return;
1979     }
1980
1981     /** \par
1982      * When squid.conf ftp_sanitycheck is enabled, check the new connection is actually being
1983      * made by the remote client which is connected to the FTP control socket.
1984      * Or the one which we were told to listen for by control channel messages (may differ under NAT).
1985      * This prevents third-party hacks, but also third-party load balancing handshakes.
1986      */
1987     if (Config.Ftp.sanitycheck) {
1988         // accept if either our data or ctrl connection is talking to this remote peer.
1989         if (data.conn->remote != io.conn->remote && ctrl.conn->remote != io.conn->remote) {
1990             debugs(9, DBG_IMPORTANT,
1991                    "FTP data connection from unexpected server (" <<
1992                    io.conn->remote << "), expecting " <<
1993                    data.conn->remote << " or " << ctrl.conn->remote);
1994
1995             /* close the bad sources connection down ASAP. */
1996             io.conn->close();
1997
1998             /* drop the bad connection (io) by ignoring the attempt. */
1999             return;
2000         }
2001     }
2002
2003     /** On Comm::OK start using the accepted data socket and discard the temporary listen socket. */
2004     data.close();
2005     data.opened(io.conn, dataCloser());
2006     data.addr(io.conn->remote);
2007
2008     debugs(9, 3, HERE << "Connected data socket on " <<
2009            io.conn << ". FD table says: " <<
2010            "ctrl-peer= " << fd_table[ctrl.conn->fd].ipaddr << ", " <<
2011            "data-peer= " << fd_table[data.conn->fd].ipaddr);
2012
2013     assert(haveControlChannel("ftpAcceptDataConnection"));
2014     assert(ctrl.message == NULL);
2015
2016     // Ctrl channel operations will determine what happens to this data connection
2017 }
2018
2019 static void
2020 ftpRestOrList(Ftp::Gateway * ftpState)
2021 {
2022     debugs(9, 3, HERE);
2023
2024     if (ftpState->typecode == 'D') {
2025         ftpState->flags.isdir = 1;
2026
2027         if (ftpState->flags.put) {
2028             ftpSendMkdir(ftpState);     /* PUT name;type=d */
2029         } else {
2030             ftpSendNlst(ftpState);      /* GET name;type=d  sec 3.2.2 of RFC 1738 */
2031         }
2032     } else if (ftpState->flags.put) {
2033         ftpSendStor(ftpState);
2034     } else if (ftpState->flags.isdir)
2035         ftpSendList(ftpState);
2036     else if (ftpState->restartable())
2037         ftpSendRest(ftpState);
2038     else
2039         ftpSendRetr(ftpState);
2040 }
2041
2042 static void
2043 ftpSendStor(Ftp::Gateway * ftpState)
2044 {
2045     /* check the server control channel is still available */
2046     if (!ftpState || !ftpState->haveControlChannel("ftpSendStor"))
2047         return;
2048
2049     debugs(9, 3, HERE);
2050
2051     if (ftpState->filepath != NULL) {
2052         /* Plain file upload */
2053         snprintf(cbuf, CTRL_BUFLEN, "STOR %s\r\n", ftpState->filepath);
2054         ftpState->writeCommand(cbuf);
2055         ftpState->state = Ftp::Client::SENT_STOR;
2056     } else if (ftpState->request->header.getInt64(HDR_CONTENT_LENGTH) > 0) {
2057         /* File upload without a filename. use STOU to generate one */
2058         snprintf(cbuf, CTRL_BUFLEN, "STOU\r\n");
2059         ftpState->writeCommand(cbuf);
2060         ftpState->state = Ftp::Client::SENT_STOR;
2061     } else {
2062         /* No file to transfer. Only create directories if needed */
2063         ftpSendReply(ftpState);
2064     }
2065 }
2066
2067 /// \deprecated use ftpState->readStor() instead.
2068 static void
2069 ftpReadStor(Ftp::Gateway * ftpState)
2070 {
2071     ftpState->readStor();
2072 }
2073
2074 void Ftp::Gateway::readStor()
2075 {
2076     int code = ctrl.replycode;
2077     debugs(9, 3, HERE);
2078
2079     if (code == 125 || (code == 150 && Comm::IsConnOpen(data.conn))) {
2080         if (!originalRequest()->body_pipe) {
2081             debugs(9, 3, "zero-size STOR?");
2082             state = WRITING_DATA; // make ftpWriteTransferDone() responsible
2083             dataComplete(); // XXX: keep in sync with doneSendingRequestBody()
2084             return;
2085         }
2086
2087         if (!startRequestBodyFlow()) { // register to receive body data
2088             ftpFail(this);
2089             return;
2090         }
2091
2092         /* When client status is 125, or 150 and the data connection is open, Begin data transfer. */
2093         debugs(9, 3, HERE << "starting data transfer");
2094         switchTimeoutToDataChannel();
2095         sendMoreRequestBody();
2096         fwd->dontRetry(true); // dont permit re-trying if the body was sent.
2097         state = WRITING_DATA;
2098         debugs(9, 3, HERE << "writing data channel");
2099     } else if (code == 150) {
2100         /* When client code is 150 with no data channel, Accept data channel. */
2101         debugs(9, 3, "ftpReadStor: accepting data channel");
2102         listenForDataChannel(data.conn);
2103     } else {
2104         debugs(9, DBG_IMPORTANT, HERE << "Unexpected reply code "<< std::setfill('0') << std::setw(3) << code);
2105         ftpFail(this);
2106     }
2107 }
2108
2109 static void
2110 ftpSendRest(Ftp::Gateway * ftpState)
2111 {
2112     /* check the server control channel is still available */
2113     if (!ftpState || !ftpState->haveControlChannel("ftpSendRest"))
2114         return;
2115
2116     debugs(9, 3, HERE);
2117
2118     snprintf(cbuf, CTRL_BUFLEN, "REST %" PRId64 "\r\n", ftpState->restart_offset);
2119     ftpState->writeCommand(cbuf);
2120     ftpState->state = Ftp::Client::SENT_REST;
2121 }
2122
2123 int
2124 Ftp::Gateway::restartable()
2125 {
2126     if (restart_offset > 0)
2127         return 1;
2128
2129     if (!request->range)
2130         return 0;
2131
2132     if (!flags.binary)
2133         return 0;
2134
2135     if (theSize <= 0)
2136         return 0;
2137
2138     int64_t desired_offset = request->range->lowestOffset(theSize);
2139
2140     if (desired_offset <= 0)
2141         return 0;
2142
2143     if (desired_offset >= theSize)
2144         return 0;
2145
2146     restart_offset = desired_offset;
2147     return 1;
2148 }
2149
2150 static void
2151 ftpReadRest(Ftp::Gateway * ftpState)
2152 {
2153     int code = ftpState->ctrl.replycode;
2154     debugs(9, 3, HERE);
2155     assert(ftpState->restart_offset > 0);
2156
2157     if (code == 350) {
2158         ftpState->setCurrentOffset(ftpState->restart_offset);
2159         ftpSendRetr(ftpState);
2160     } else if (code > 0) {
2161         debugs(9, 3, HERE << "REST not supported");
2162         ftpState->flags.rest_supported = 0;
2163         ftpSendRetr(ftpState);
2164     } else {
2165         ftpFail(ftpState);
2166     }
2167 }
2168
2169 static void
2170 ftpSendList(Ftp::Gateway * ftpState)
2171 {
2172     /* check the server control channel is still available */
2173     if (!ftpState || !ftpState->haveControlChannel("ftpSendList"))
2174         return;
2175
2176     debugs(9, 3, HERE);
2177
2178     if (ftpState->filepath) {
2179         snprintf(cbuf, CTRL_BUFLEN, "LIST %s\r\n", ftpState->filepath);
2180     } else {
2181         snprintf(cbuf, CTRL_BUFLEN, "LIST\r\n");
2182     }
2183
2184     ftpState->writeCommand(cbuf);
2185     ftpState->state = Ftp::Client::SENT_LIST;
2186 }
2187
2188 static void
2189 ftpSendNlst(Ftp::Gateway * ftpState)
2190 {
2191     /* check the server control channel is still available */
2192     if (!ftpState || !ftpState->haveControlChannel("ftpSendNlst"))
2193         return;
2194
2195     debugs(9, 3, HERE);
2196
2197     ftpState->flags.tried_nlst = 1;
2198
2199     if (ftpState->filepath) {
2200         snprintf(cbuf, CTRL_BUFLEN, "NLST %s\r\n", ftpState->filepath);
2201     } else {
2202         snprintf(cbuf, CTRL_BUFLEN, "NLST\r\n");
2203     }
2204
2205     ftpState->writeCommand(cbuf);
2206     ftpState->state = Ftp::Client::SENT_NLST;
2207 }
2208
2209 static void
2210 ftpReadList(Ftp::Gateway * ftpState)
2211 {
2212     int code = ftpState->ctrl.replycode;
2213     debugs(9, 3, HERE);
2214
2215     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2216         /* Begin data transfer */
2217         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2218         ftpState->switchTimeoutToDataChannel();
2219         ftpState->maybeReadVirginBody();
2220         ftpState->state = Ftp::Client::READING_DATA;
2221         return;
2222     } else if (code == 150) {
2223         /* Accept data channel */
2224         debugs(9, 3, HERE << "accept data channel from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2225         ftpState->listenForDataChannel(ftpState->data.conn);
2226         return;
2227     } else if (!ftpState->flags.tried_nlst && code > 300) {
2228         ftpSendNlst(ftpState);
2229     } else {
2230         ftpFail(ftpState);
2231         return;
2232     }
2233 }
2234
2235 static void
2236 ftpSendRetr(Ftp::Gateway * ftpState)
2237 {
2238     /* check the server control channel is still available */
2239     if (!ftpState || !ftpState->haveControlChannel("ftpSendRetr"))
2240         return;
2241
2242     debugs(9, 3, HERE);
2243
2244     assert(ftpState->filepath != NULL);
2245     snprintf(cbuf, CTRL_BUFLEN, "RETR %s\r\n", ftpState->filepath);
2246     ftpState->writeCommand(cbuf);
2247     ftpState->state = Ftp::Client::SENT_RETR;
2248 }
2249
2250 static void
2251 ftpReadRetr(Ftp::Gateway * ftpState)
2252 {
2253     int code = ftpState->ctrl.replycode;
2254     debugs(9, 3, HERE);
2255
2256     if (code == 125 || (code == 150 && Comm::IsConnOpen(ftpState->data.conn))) {
2257         /* Begin data transfer */
2258         debugs(9, 3, HERE << "begin data transfer from " << ftpState->data.conn->remote << " (" << ftpState->data.conn->local << ")");
2259         ftpState->switchTimeoutToDataChannel();
2260         ftpState->maybeReadVirginBody();
2261         ftpState->state = Ftp::Client::READING_DATA;
2262     } else if (code == 150) {
2263         /* Accept data channel */
2264         ftpState->listenForDataChannel(ftpState->data.conn);
2265     } else if (code >= 300) {
2266         if (!ftpState->flags.try_slash_hack) {
2267             /* Try this as a directory missing trailing slash... */
2268             ftpState->hackShortcut(ftpSendCwd);
2269         } else {
2270             ftpFail(ftpState);
2271         }
2272     } else {
2273         ftpFail(ftpState);
2274     }
2275 }
2276
2277 /**
2278  * Generate the HTTP headers and template fluff around an FTP
2279  * directory listing display.
2280  */
2281 void
2282 Ftp::Gateway::completedListing()
2283 {
2284     assert(entry);
2285     entry->lock("Ftp::Gateway");
2286     ErrorState ferr(ERR_DIR_LISTING, Http::scOkay, request);
2287     ferr.ftp.listing = &listing;
2288     ferr.ftp.cwd_msg = xstrdup(cwd_message.size()? cwd_message.termedBuf() : "");
2289     ferr.ftp.server_msg = ctrl.message;
2290     ctrl.message = NULL;
2291     entry->replaceHttpReply( ferr.BuildHttpReply() );
2292     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2293     entry->flush();
2294     entry->unlock("Ftp::Gateway");
2295 }
2296
2297 static void
2298 ftpReadTransferDone(Ftp::Gateway * ftpState)
2299 {
2300     int code = ftpState->ctrl.replycode;
2301     debugs(9, 3, HERE);
2302
2303     if (code == 226 || code == 250) {
2304         /* Connection closed; retrieval done. */
2305         if (ftpState->flags.listing) {
2306             ftpState->completedListing();
2307             /* QUIT operation handles sending the reply to client */
2308         }
2309         ftpSendQuit(ftpState);
2310     } else {                    /* != 226 */
2311         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after reading data");
2312         ftpState->failed(ERR_FTP_FAILURE, 0);
2313         /* failed closes ctrl.conn and frees ftpState */
2314         return;
2315     }
2316 }
2317
2318 // premature end of the request body
2319 void
2320 Ftp::Gateway::handleRequestBodyProducerAborted()
2321 {
2322     Client::handleRequestBodyProducerAborted();
2323     debugs(9, 3, HERE << "ftpState=" << this);
2324     failed(ERR_READ_ERROR, 0);
2325 }
2326
2327 static void
2328 ftpWriteTransferDone(Ftp::Gateway * ftpState)
2329 {
2330     int code = ftpState->ctrl.replycode;
2331     debugs(9, 3, HERE);
2332
2333     if (!(code == 226 || code == 250)) {
2334         debugs(9, DBG_IMPORTANT, HERE << "Got code " << code << " after sending data");
2335         ftpState->failed(ERR_FTP_PUT_ERROR, 0);
2336         return;
2337     }
2338
2339     ftpState->entry->timestampsSet();   /* XXX Is this needed? */
2340     ftpSendReply(ftpState);
2341 }
2342
2343 static void
2344 ftpSendQuit(Ftp::Gateway * ftpState)
2345 {
2346     /* check the server control channel is still available */
2347     if (!ftpState || !ftpState->haveControlChannel("ftpSendQuit"))
2348         return;
2349
2350     snprintf(cbuf, CTRL_BUFLEN, "QUIT\r\n");
2351     ftpState->writeCommand(cbuf);
2352     ftpState->state = Ftp::Client::SENT_QUIT;
2353 }
2354
2355 /** Completes a client FTP operation with success or other page
2356  *  generated and stored in the entry field by the code issuing QUIT.
2357  */
2358 static void
2359 ftpReadQuit(Ftp::Gateway * ftpState)
2360 {
2361     ftpState->serverComplete();
2362 }
2363
2364 static void
2365 ftpTrySlashHack(Ftp::Gateway * ftpState)
2366 {
2367     char *path;
2368     ftpState->flags.try_slash_hack = 1;
2369     /* Free old paths */
2370
2371     debugs(9, 3, HERE);
2372
2373     if (ftpState->pathcomps)
2374         wordlistDestroy(&ftpState->pathcomps);
2375
2376     safe_free(ftpState->filepath);
2377
2378     /* Build the new path (urlpath begins with /) */
2379     path = xstrdup(ftpState->request->urlpath.termedBuf());
2380
2381     rfc1738_unescape(path);
2382
2383     ftpState->filepath = path;
2384
2385     /* And off we go */
2386     ftpGetFile(ftpState);
2387 }
2388
2389 /**
2390  * Forget hack status. Next error is shown to the user
2391  */
2392 void
2393 Ftp::Gateway::unhack()
2394 {
2395     debugs(9, 3, HERE);
2396
2397     if (old_request != NULL) {
2398         safe_free(old_request);
2399         safe_free(old_reply);
2400     }
2401 }
2402
2403 void
2404 Ftp::Gateway::hackShortcut(FTPSM * nextState)
2405 {
2406     /* Clear some unwanted state */
2407     setCurrentOffset(0);
2408     restart_offset = 0;
2409     /* Save old error message & some state info */
2410
2411     debugs(9, 3, HERE);
2412
2413     if (old_request == NULL) {
2414         old_request = ctrl.last_command;
2415         ctrl.last_command = NULL;
2416         old_reply = ctrl.last_reply;
2417         ctrl.last_reply = NULL;
2418
2419         if (pathcomps == NULL && filepath != NULL)
2420             old_filepath = xstrdup(filepath);
2421     }
2422
2423     /* Jump to the "hack" state */
2424     nextState(this);
2425 }
2426
2427 static void
2428 ftpFail(Ftp::Gateway *ftpState)
2429 {
2430     debugs(9, 6, HERE << "flags(" <<
2431            (ftpState->flags.isdir?"IS_DIR,":"") <<
2432            (ftpState->flags.try_slash_hack?"TRY_SLASH_HACK":"") << "), " <<
2433            "mdtm=" << ftpState->mdtm << ", size=" << ftpState->theSize <<
2434            "slashhack=" << (ftpState->request->urlpath.caseCmp("/%2f", 4)==0? "T":"F") );
2435
2436     /* Try the / hack to support "Netscape" FTP URL's for retreiving files */
2437     if (!ftpState->flags.isdir &&       /* Not a directory */
2438             !ftpState->flags.try_slash_hack &&  /* Not in slash hack */
2439             ftpState->mdtm <= 0 && ftpState->theSize < 0 &&     /* Not known as a file */
2440             ftpState->request->urlpath.caseCmp("/%2f", 4) != 0) {       /* No slash encoded */
2441
2442         switch (ftpState->state) {
2443
2444         case Ftp::Client::SENT_CWD:
2445
2446         case Ftp::Client::SENT_RETR:
2447             /* Try the / hack */
2448             ftpState->hackShortcut(ftpTrySlashHack);
2449             return;
2450
2451         default:
2452             break;
2453         }
2454     }
2455
2456     ftpState->failed(ERR_NONE, 0);
2457     /* failed() closes ctrl.conn and frees this */
2458 }
2459
2460 Http::StatusCode
2461 Ftp::Gateway::failedHttpStatus(err_type &error)
2462 {
2463     if (error == ERR_NONE) {
2464         switch (state) {
2465
2466         case SENT_USER:
2467
2468         case SENT_PASS:
2469
2470             if (ctrl.replycode > 500) {
2471                 error = ERR_FTP_FORBIDDEN;
2472                 return password_url ? Http::scForbidden : Http::scUnauthorized;
2473             } else if (ctrl.replycode == 421) {
2474                 error = ERR_FTP_UNAVAILABLE;
2475                 return Http::scServiceUnavailable;
2476             }
2477             break;
2478
2479         case SENT_CWD:
2480
2481         case SENT_RETR:
2482             if (ctrl.replycode == 550) {
2483                 error = ERR_FTP_NOT_FOUND;
2484                 return Http::scNotFound;
2485             }
2486             break;
2487
2488         default:
2489             break;
2490         }
2491     }
2492     return Ftp::Client::failedHttpStatus(error);
2493 }
2494
2495 static void
2496 ftpSendReply(Ftp::Gateway * ftpState)
2497 {
2498     int code = ftpState->ctrl.replycode;
2499     Http::StatusCode http_code;
2500     err_type err_code = ERR_NONE;
2501
2502     debugs(9, 3, HERE << ftpState->entry->url() << ", code " << code);
2503
2504     if (cbdataReferenceValid(ftpState))
2505         debugs(9, 5, HERE << "ftpState (" << ftpState << ") is valid!");
2506
2507     if (code == 226 || code == 250) {
2508         err_code = (ftpState->mdtm > 0) ? ERR_FTP_PUT_MODIFIED : ERR_FTP_PUT_CREATED;
2509         http_code = (ftpState->mdtm > 0) ? Http::scAccepted : Http::scCreated;
2510     } else if (code == 227) {
2511         err_code = ERR_FTP_PUT_CREATED;
2512         http_code = Http::scCreated;
2513     } else {
2514         err_code = ERR_FTP_PUT_ERROR;
2515         http_code = Http::scInternalServerError;
2516     }
2517
2518     ErrorState err(err_code, http_code, ftpState->request);
2519
2520     if (ftpState->old_request)
2521         err.ftp.request = xstrdup(ftpState->old_request);
2522     else
2523         err.ftp.request = xstrdup(ftpState->ctrl.last_command);
2524
2525     if (ftpState->old_reply)
2526         err.ftp.reply = xstrdup(ftpState->old_reply);
2527     else if (ftpState->ctrl.last_reply)
2528         err.ftp.reply = xstrdup(ftpState->ctrl.last_reply);
2529     else
2530         err.ftp.reply = xstrdup("");
2531
2532     // TODO: interpret as FTP-specific error code
2533     err.detailError(code);
2534
2535     ftpState->entry->replaceHttpReply( err.BuildHttpReply() );
2536
2537     ftpSendQuit(ftpState);
2538 }
2539
2540 void
2541 Ftp::Gateway::appendSuccessHeader()
2542 {
2543     const char *mime_type = NULL;
2544     const char *mime_enc = NULL;
2545     String urlpath = request->urlpath;
2546     const char *filename = NULL;
2547     const char *t = NULL;
2548
2549     debugs(9, 3, HERE);
2550
2551     if (flags.http_header_sent)
2552         return;
2553
2554     HttpReply *reply = new HttpReply;
2555
2556     flags.http_header_sent = 1;
2557
2558     assert(entry->isEmpty());
2559
2560     EBIT_CLR(entry->flags, ENTRY_FWD_HDR_WAIT);
2561
2562     entry->buffer();    /* released when done processing current data payload */
2563
2564     filename = (t = urlpath.rpos('/')) ? t + 1 : urlpath.termedBuf();
2565
2566     if (flags.isdir) {
2567         mime_type = "text/html";
2568     } else {
2569         switch (typecode) {
2570
2571         case 'I':
2572             mime_type = "application/octet-stream";
2573             mime_enc = mimeGetContentEncoding(filename);
2574             break;
2575
2576         case 'A':
2577             mime_type = "text/plain";
2578             break;
2579
2580         default:
2581             mime_type = mimeGetContentType(filename);
2582             mime_enc = mimeGetContentEncoding(filename);
2583             break;
2584         }
2585     }
2586
2587     /* set standard stuff */
2588
2589     if (0 == getCurrentOffset()) {
2590         /* Full reply */
2591         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2592     } else if (theSize < getCurrentOffset()) {
2593         /*
2594          * DPW 2007-05-04
2595          * offset should not be larger than theSize.  We should
2596          * not be seeing this condition any more because we'll only
2597          * send REST if we know the theSize and if it is less than theSize.
2598          */
2599         debugs(0,DBG_CRITICAL,HERE << "Whoops! " <<
2600                " current offset=" << getCurrentOffset() <<
2601                ", but theSize=" << theSize <<
2602                ".  assuming full content response");
2603         reply->setHeaders(Http::scOkay, "Gatewaying", mime_type, theSize, mdtm, -2);
2604     } else {
2605         /* Partial reply */
2606         HttpHdrRangeSpec range_spec;
2607         range_spec.offset = getCurrentOffset();
2608         range_spec.length = theSize - getCurrentOffset();
2609         reply->setHeaders(Http::scPartialContent, "Gatewaying", mime_type, theSize - getCurrentOffset(), mdtm, -2);
2610         httpHeaderAddContRange(&reply->header, range_spec, theSize);
2611     }
2612
2613     /* additional info */
2614     if (mime_enc)
2615         reply->header.putStr(HDR_CONTENT_ENCODING, mime_enc);
2616
2617     setVirginReply(reply);
2618     adaptOrFinalizeReply();
2619 }
2620
2621 void
2622 Ftp::Gateway::haveParsedReplyHeaders()
2623 {
2624     Client::haveParsedReplyHeaders();
2625
2626     StoreEntry *e = entry;
2627
2628     e->timestampsSet();
2629
2630     if (flags.authenticated) {
2631         /*
2632          * Authenticated requests can't be cached.
2633          */
2634         e->release();
2635     } else if (!EBIT_TEST(e->flags, RELEASE_REQUEST) && !getCurrentOffset()) {
2636         e->setPublicKey();
2637     } else {
2638         e->release();
2639     }
2640 }
2641
2642 HttpReply *
2643 Ftp::Gateway::ftpAuthRequired(HttpRequest * request, const char *realm)
2644 {
2645     ErrorState err(ERR_CACHE_ACCESS_DENIED, Http::scUnauthorized, request);
2646     HttpReply *newrep = err.BuildHttpReply();
2647 #if HAVE_AUTH_MODULE_BASIC
2648     /* add Authenticate header */
2649     newrep->header.putAuth("Basic", realm);
2650 #endif
2651     return newrep;
2652 }
2653
2654 const char *
2655 Ftp::UrlWith2f(HttpRequest * request)
2656 {
2657     String newbuf = "%2f";
2658
2659     if (request->url.getScheme() != AnyP::PROTO_FTP)
2660         return NULL;
2661
2662     if ( request->urlpath[0]=='/' ) {
2663         newbuf.append(request->urlpath);
2664         request->urlpath.absorb(newbuf);
2665         safe_free(request->canonical);
2666     } else if ( !strncmp(request->urlpath.termedBuf(), "%2f", 3) ) {
2667         newbuf.append(request->urlpath.substr(1,request->urlpath.size()));
2668         request->urlpath.absorb(newbuf);
2669         safe_free(request->canonical);
2670     }
2671
2672     return urlCanonical(request);
2673 }
2674
2675 void
2676 Ftp::Gateway::printfReplyBody(const char *fmt, ...)
2677 {
2678     va_list args;
2679     va_start (args, fmt);
2680     static char buf[4096];
2681     buf[0] = '\0';
2682     vsnprintf(buf, 4096, fmt, args);
2683     writeReplyBody(buf, strlen(buf));
2684     va_end(args);
2685 }
2686
2687 /**
2688  * Call this when there is data from the origin server
2689  * which should be sent to either StoreEntry, or to ICAP...
2690  */
2691 void
2692 Ftp::Gateway::writeReplyBody(const char *dataToWrite, size_t dataLength)
2693 {
2694     debugs(9, 5, HERE << "writing " << dataLength << " bytes to the reply");
2695     addVirginReplyBody(dataToWrite, dataLength);
2696 }
2697
2698 /**
2699  * A hack to ensure we do not double-complete on the forward entry.
2700  *
2701  \todo Ftp::Gateway logic should probably be rewritten to avoid
2702  *      double-completion or FwdState should be rewritten to allow it.
2703  */
2704 void
2705 Ftp::Gateway::completeForwarding()
2706 {
2707     if (fwd == NULL || flags.completed_forwarding) {
2708         debugs(9, 3, HERE << "completeForwarding avoids " <<
2709                "double-complete on FD " << ctrl.conn->fd << ", Data FD " << data.conn->fd <<
2710                ", this " << this << ", fwd " << fwd);
2711         return;
2712     }
2713
2714     flags.completed_forwarding = true;
2715     Client::completeForwarding();
2716 }
2717
2718 /**
2719  * Have we lost the FTP server control channel?
2720  *
2721  \retval true   The server control channel is available.
2722  \retval false  The server control channel is not available.
2723  */
2724 bool
2725 Ftp::Gateway::haveControlChannel(const char *caller_name) const
2726 {
2727     if (doneWithServer())
2728         return false;
2729
2730     /* doneWithServer() only checks BOTH channels are closed. */
2731     if (!Comm::IsConnOpen(ctrl.conn)) {
2732         debugs(9, DBG_IMPORTANT, "WARNING! FTP Server Control channel is closed, but Data channel still active.");
2733         debugs(9, 2, caller_name << ": attempted on a closed FTP channel.");
2734         return false;
2735     }
2736
2737     return true;
2738 }
2739
2740 bool
2741 Ftp::Gateway::mayReadVirginReplyBody() const
2742 {
2743     // TODO: Can we do what Ftp::Relay::mayReadVirginReplyBody() does instead?
2744     return !doneWithServer();
2745 }
2746
2747 AsyncJob::Pointer
2748 Ftp::StartGateway(FwdState *const fwdState)
2749 {
2750     return AsyncJob::Start(new Ftp::Gateway(fwdState));
2751 }