src/ftp.cc

   1 /* $Id: ftp.cc,v 1.18 1996/04/04 01:30:43 wessels Exp $ */
   2
   3 /*
   4  * DEBUG: Section 9           ftp: FTP
   5  */
   6
   7 #include "squid.h"
   8
   9 #define FTP_DELETE_GAP  (64*1024)
  10 #define READBUFSIZ      4096
  11 #define MAGIC_MARKER    "\004\004\004"  /* No doubt this should be more configurable */
  12 #define MAGIC_MARKER_SZ 3
  13
  14 static char ftpASCII[] = "A";
  15 static char ftpBinary[] = "I";
  16
  17 typedef struct _Ftpdata {
  18     StoreEntry *entry;
  19     char type_id;
  20     char host[SQUIDHOSTNAMELEN + 1];
  21     char request[MAX_URL];
  22     char user[MAX_URL];
  23     char password[MAX_URL];
  24     char *type;
  25     char *mime_hdr;
  26     int ftp_fd;
  27     char *icp_page_ptr;         /* Used to send proxy-http request:
  28                                  * put_free_8k_page(me) if the lifetime
  29                                  * expires */
  30     char *icp_rwd_ptr;          /* When a lifetime expires during the
  31                                  * middle of an icpwrite, don't lose the
  32                                  * icpReadWriteData */
  33     int got_marker;             /* denotes end of successful request */
  34 } FtpData;
  35
  36 static void ftpCloseAndFree(fd, data)
  37      int fd;
  38      FtpData *data;
  39 {
  40     if (fd > -1)
  41         comm_close(fd);
  42     xfree(data);
  43 }
  44
  45 /* XXX: this does not support FTP on a different port! */
  46 int ftp_url_parser(url, data)
  47      char *url;
  48      FtpData *data;
  49 {
  50     static char atypebuf[MAX_URL];
  51     static char hostbuf[MAX_URL];
  52     char *tmp = NULL;
  53     int t;
  54     char *host = data->host;
  55     char *request = data->request;
  56     char *user = data->user;
  57     char *password = data->password;
  58
  59     /* initialize everything */
  60     atypebuf[0] = hostbuf[0] = '\0';
  61     request[0] = host[0] = user[0] = password[0] = '\0';
  62
  63     t = sscanf(url, "%[a-zA-Z]://%[^/]%s", atypebuf, hostbuf, request);
  64     if ((t < 2) ||
  65         !(!strcasecmp(atypebuf, "ftp") || !strcasecmp(atypebuf, "file"))) {
  66         return -1;
  67     } else if (t == 2) {        /* no request */
  68         strcpy(request, "/");
  69     } else {
  70         tmp = url_convert_hex(request);         /* convert %xx to char */
  71         strncpy(request, tmp, MAX_URL);
  72         safe_free(tmp);
  73     }
  74
  75     /* url address format is something like this:
  76      * [ userid [ : password ] @ ] host
  77      * or possibly even
  78      * [ [ userid ] [ : [ password ] ] @ ] host
  79      *
  80      * So we must try to make sense of it.  */
  81
  82     /* XXX: this only support [user:passwd@]host */
  83     t = sscanf(hostbuf, "%[^:]:%[^@]@%s", user, password, host);
  84     if (t < 3) {
  85         strcpy(host, user);     /* no login/passwd information */
  86         strcpy(user, "anonymous");
  87         strcpy(password, "harvest@");
  88     }
  89     /* we need to convert user and password for URL encodings */
  90     tmp = url_convert_hex(user);
  91     strcpy(user, tmp);
  92     safe_free(tmp);
  93
  94     tmp = url_convert_hex(password);
  95     strcpy(password, tmp);
  96     safe_free(tmp);
  97
  98     return 0;
  99 }
 100
 101 int ftpCachable(url)
 102      char *url;
 103 {
 104     stoplist *p = NULL;
 105
 106     /* scan stop list */
 107     p = ftp_stoplist;
 108     while (p) {
 109         if (strstr(url, p->key))
 110             return 0;
 111         p = p->next;
 112     }
 113
 114     /* else cachable */
 115     return 1;
 116 }
 117
 118 /* This will be called when socket lifetime is expired. */
 119 void ftpLifetimeExpire(fd, data)
 120      int fd;
 121      FtpData *data;
 122 {
 123     StoreEntry *entry = NULL;
 124     entry = data->entry;
 125     debug(9, 4, "ftpLifeTimeExpire: FD %d: <URL:%s>\n", fd, entry->url);
 126     if (data->icp_page_ptr) {
 127         put_free_8k_page(data->icp_page_ptr);
 128         data->icp_page_ptr = NULL;
 129     }
 130     safe_free(data->icp_rwd_ptr);
 131     cached_error_entry(entry, ERR_LIFETIME_EXP, NULL);
 132     ftpCloseAndFree(fd, data);
 133 }
 134
 135
 136
 137 /* This will be called when data is ready to be read from fd.  Read until
 138  * error or connection closed. */
 139 int ftpReadReply(fd, data)
 140      int fd;
 141      FtpData *data;
 142 {
 143     static char buf[READBUFSIZ];
 144     int len;
 145     int clen;
 146     int off;
 147     StoreEntry *entry = NULL;
 148
 149     entry = data->entry;
 150     if (entry->flag & DELETE_BEHIND) {
 151         if (storeClientWaiting(entry)) {
 152             /* check if we want to defer reading */
 153             clen = entry->mem_obj->e_current_len;
 154             off = entry->mem_obj->e_lowest_offset;
 155             if ((clen - off) > FTP_DELETE_GAP) {
 156                 debug(9, 3, "ftpReadReply: Read deferred for Object: %s\n",
 157                     entry->url);
 158                 debug(9, 3, "--> Current Gap: %d bytes\n", clen - off);
 159                 /* reschedule, so it will automatically be reactivated when
 160                  * Gap is big enough. */
 161                 comm_set_select_handler(fd,
 162                     COMM_SELECT_READ,
 163                     (PF) ftpReadReply,
 164                     (caddr_t) data);
 165                 comm_set_stall(fd, getStallDelay());    /* dont try reading again for a while */
 166                 return 0;
 167             }
 168         } else {
 169             /* we can terminate connection right now */
 170             cached_error_entry(entry, ERR_NO_CLIENTS_BIG_OBJ, NULL);
 171             ftpCloseAndFree(fd, data);
 172             return 0;
 173         }
 174     }
 175     errno = 0;
 176     len = read(fd, buf, READBUFSIZ);
 177     debug(9, 5, "ftpReadReply: FD %d, Read %d bytes\n", fd, len);
 178
 179     if (len < 0) {
 180         debug(9, 1, "ftpReadReply: read error: %s\n", xstrerror());
 181         if (errno == EAGAIN || errno == EWOULDBLOCK) {
 182             /* reinstall handlers */
 183             /* XXX This may loop forever */
 184             comm_set_select_handler(fd, COMM_SELECT_READ,
 185                 (PF) ftpReadReply, (caddr_t) data);
 186             /* note there is no ftpReadReplyTimeout.  Timeouts are handled
 187              * by `ftpget'. */
 188         } else {
 189             BIT_RESET(entry->flag, CACHABLE);
 190             BIT_SET(entry->flag, RELEASE_REQUEST);
 191             cached_error_entry(entry, ERR_READ_ERROR, xstrerror());
 192             ftpCloseAndFree(fd, data);
 193         }
 194     } else if (len == 0 && entry->mem_obj->e_current_len == 0) {
 195         cached_error_entry(entry,
 196             ERR_ZERO_SIZE_OBJECT,
 197             errno ? xstrerror() : NULL);
 198         ftpCloseAndFree(fd, data);
 199     } else if (len == 0) {
 200         /* Connection closed; retrieval done. */
 201         if (!data->got_marker) {
 202             /* If we didn't see the magic marker, assume the transfer
 203              * failed and arrange so the object gets ejected and
 204              * never gets to disk. */
 205             debug(9, 1, "ftpReadReply: Didn't see magic marker, purging <URL:%s>.\n", entry->url);
 206             entry->expires = cached_curtime + getNegativeTTL();
 207             BIT_RESET(entry->flag, CACHABLE);
 208             BIT_SET(entry->flag, RELEASE_REQUEST);
 209         } else if (!(entry->flag & DELETE_BEHIND)) {
 210             entry->expires = cached_curtime + ttlSet(entry);
 211         }
 212         /* update fdstat and fdtable */
 213         storeComplete(entry);
 214         ftpCloseAndFree(fd, data);
 215     } else if (((entry->mem_obj->e_current_len + len) > getFtpMax()) &&
 216         !(entry->flag & DELETE_BEHIND)) {
 217         /*  accept data, but start to delete behind it */
 218         storeStartDeleteBehind(entry);
 219         storeAppend(entry, buf, len);
 220         comm_set_select_handler(fd,
 221             COMM_SELECT_READ,
 222             (PF) ftpReadReply,
 223             (caddr_t) data);
 224     } else if (entry->flag & CLIENT_ABORT_REQUEST) {
 225         /* append the last bit of info we get */
 226         storeAppend(entry, buf, len);
 227         cached_error_entry(entry, ERR_CLIENT_ABORT, NULL);
 228         ftpCloseAndFree(fd, data);
 229     } else {
 230         /* check for a magic marker at the end of the read */
 231         data->got_marker = 0;
 232         if (len >= MAGIC_MARKER_SZ) {
 233             if (!memcmp(MAGIC_MARKER, buf + len - MAGIC_MARKER_SZ, MAGIC_MARKER_SZ)) {
 234                 data->got_marker = 1;
 235                 len -= MAGIC_MARKER_SZ;
 236             }
 237         }
 238         storeAppend(entry, buf, len);
 239         comm_set_select_handler(fd,
 240             COMM_SELECT_READ,
 241             (PF) ftpReadReply,
 242             (caddr_t) data);
 243         comm_set_select_handler_plus_timeout(fd,
 244             COMM_SELECT_TIMEOUT,
 245             (PF) ftpLifetimeExpire,
 246             (caddr_t) data,
 247             getReadTimeout());
 248     }
 249     return 0;
 250 }
 251
 252 void ftpSendComplete(fd, buf, size, errflag, data)
 253      int fd;
 254      char *buf;
 255      int size;
 256      int errflag;
 257      FtpData *data;
 258 {
 259     StoreEntry *entry = NULL;
 260
 261     entry = data->entry;
 262     debug(9, 5, "ftpSendComplete: FD %d: size %d: errflag %d.\n",
 263         fd, size, errflag);
 264
 265     if (buf) {
 266         put_free_8k_page(buf);  /* Allocated by ftpSendRequest. */
 267         buf = NULL;
 268     }
 269     data->icp_page_ptr = NULL;  /* So lifetime expire doesn't re-free */
 270     data->icp_rwd_ptr = NULL;   /* Don't double free in lifetimeexpire */
 271
 272     if (errflag) {
 273         cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
 274         ftpCloseAndFree(fd, data);
 275         return;
 276     } else {
 277         comm_set_select_handler(data->ftp_fd,
 278             COMM_SELECT_READ,
 279             (PF) ftpReadReply,
 280             (caddr_t) data);
 281         comm_set_select_handler_plus_timeout(data->ftp_fd,
 282             COMM_SELECT_TIMEOUT,
 283             (PF) ftpLifetimeExpire,
 284             (caddr_t) data, getReadTimeout());
 285     }
 286 }
 287
 288 void ftpSendRequest(fd, data)
 289      int fd;
 290      FtpData *data;
 291 {
 292     char *ext = NULL;
 293     ext_table_entry *e = NULL;
 294     int l;
 295     char *path = NULL;
 296     char *mode = NULL;
 297     char *buf = NULL;
 298     static char tbuf[BUFSIZ];
 299     static char opts[BUFSIZ];
 300     static char *space = " ";
 301     char *s = NULL;
 302     int got_timeout = 0;
 303     int got_negttl = 0;
 304     int buflen;
 305
 306     debug(9, 5, "ftpSendRequest: FD %d\n", fd);
 307
 308     buflen = strlen(data->request) + 256;
 309     buf = (char *) get_free_8k_page();
 310     data->icp_page_ptr = buf;
 311     memset(buf, '\0', buflen);
 312
 313     path = data->request;
 314     l = strlen(path);
 315     if (path[l - 1] == '/')
 316         mode = ftpASCII;
 317     else {
 318         if ((ext = strrchr(path, '.')) != NULL) {
 319             ext++;
 320             mode = ((e = mime_ext_to_type(ext)) &&
 321                 strncmp(e->mime_type, "text", 4) == 0) ? ftpASCII :
 322                 ftpBinary;
 323         } else
 324             mode = ftpBinary;
 325     }
 326
 327     /* Remove leading slash from FTP url-path so that we can
 328      *  handle ftp://user:pw@host/path objects where path and /path
 329      *  are quite different.         -DW */
 330     if (!strcmp(path, "/"))
 331         *path = '.';
 332     if (*path == '/')
 333         path++;
 334
 335     /* Start building the buffer ... */
 336
 337     strcat(buf, getFtpProgram());
 338     strcat(buf, space);
 339
 340     strncpy(opts, getFtpOptions(), BUFSIZ);
 341     for (s = strtok(opts, w_space); s; s = strtok(NULL, w_space)) {
 342         strcat(buf, s);
 343         strcat(buf, space);
 344         if (!strncmp(s, "-t", 2))
 345             got_timeout = 1;
 346         if (!strncmp(s, "-n", 2))
 347             got_negttl = 1;
 348     }
 349     if (!got_timeout) {
 350         sprintf(tbuf, "-t %d ", getReadTimeout());
 351         strcat(buf, tbuf);
 352     }
 353     if (!got_negttl) {
 354         sprintf(tbuf, "-n %d ", getNegativeTTL());
 355         strcat(buf, tbuf);
 356     }
 357     strcat(buf, "-h ");         /* httpify */
 358     strcat(buf, "- ");          /* stdout */
 359     strcat(buf, data->host);
 360     strcat(buf, space);
 361     strcat(buf, path);
 362     strcat(buf, space);
 363     strcat(buf, mode);          /* A or I */
 364     strcat(buf, space);
 365     strcat(buf, data->user);
 366     strcat(buf, space);
 367     strcat(buf, data->password);
 368     strcat(buf, space);
 369     debug(9, 5, "ftpSendRequest: FD %d: buf '%s'\n", fd, buf);
 370     data->icp_rwd_ptr = icpWrite(fd, buf, strlen(buf), 30, ftpSendComplete, (caddr_t) data);
 371 }
 372
 373 void ftpConnInProgress(fd, data)
 374      int fd;
 375      FtpData *data;
 376 {
 377     StoreEntry *entry = data->entry;
 378
 379     debug(9, 5, "ftpConnInProgress: FD %d\n", fd);
 380
 381     if (comm_connect(fd, "localhost", 3131) != COMM_OK)
 382         switch (errno) {
 383         case EINPROGRESS:
 384         case EALREADY:
 385             /* schedule this handler again */
 386             comm_set_select_handler(fd,
 387                 COMM_SELECT_WRITE,
 388                 (PF) ftpConnInProgress,
 389                 (caddr_t) data);
 390             return;
 391         case EISCONN:
 392             debug(9, 5, "ftpConnInProgress: FD %d is now connected.", fd);
 393             break;              /* cool, we're connected */
 394         default:
 395             cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
 396             ftpCloseAndFree(fd, data);
 397             return;
 398         }
 399     /* Call the real write handler, now that we're fully connected */
 400     comm_set_select_handler(fd,
 401         COMM_SELECT_WRITE,
 402         (PF) ftpSendRequest,
 403         (caddr_t) data);
 404 }
 405
 406
 407 int ftpStart(unusedfd, url, entry)
 408      int unusedfd;
 409      char *url;
 410      StoreEntry *entry;
 411 {
 412     FtpData *data = NULL;
 413     int status;
 414
 415     debug(9, 3, "FtpStart: FD %d <URL:%s>\n", unusedfd, url);
 416
 417     data = (FtpData *) xcalloc(1, sizeof(FtpData));
 418     data->entry = entry;
 419
 420     /* Parse url. */
 421     if (ftp_url_parser(url, data)) {
 422         cached_error_entry(entry, ERR_INVALID_URL, NULL);
 423         safe_free(data);
 424         return COMM_ERROR;
 425     }
 426     debug(9, 5, "FtpStart: FD %d, host=%s, request=%s, user=%s, passwd=%s\n",
 427         unusedfd, data->host, data->request, data->user, data->password);
 428
 429     data->ftp_fd = comm_open(COMM_NONBLOCKING, 0, 0, url);
 430     if (data->ftp_fd == COMM_ERROR) {
 431         cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
 432         safe_free(data);
 433         return COMM_ERROR;
 434     }
 435     /* Pipe/socket created ok */
 436
 437     /* Now connect ... */
 438     if ((status = comm_connect(data->ftp_fd, "localhost", 3131))) {
 439         if (status != EINPROGRESS) {
 440             cached_error_entry(entry, ERR_CONNECT_FAIL, xstrerror());
 441             ftpCloseAndFree(data->ftp_fd, data);
 442             return COMM_ERROR;
 443         } else {
 444             debug(9, 5, "ftpStart: FD %d: EINPROGRESS.\n", data->ftp_fd);
 445             comm_set_select_handler(data->ftp_fd, COMM_SELECT_LIFETIME,
 446                 (PF) ftpLifetimeExpire, (caddr_t) data);
 447             comm_set_select_handler(data->ftp_fd, COMM_SELECT_WRITE,
 448                 (PF) ftpConnInProgress, (caddr_t) data);
 449             return COMM_OK;
 450         }
 451     }
 452     fdstat_open(data->ftp_fd, Socket);
 453     commSetNonBlocking(data->ftp_fd);
 454     (void) fd_note(data->ftp_fd, entry->url);
 455
 456     /* Install connection complete handler. */
 457     fd_note(data->ftp_fd, entry->url);
 458     comm_set_select_handler(data->ftp_fd,
 459         COMM_SELECT_WRITE,
 460         (PF) ftpSendRequest,
 461         (caddr_t) data);
 462     comm_set_fd_lifetime(data->ftp_fd,
 463         getClientLifetime());
 464     comm_set_select_handler(data->ftp_fd,
 465         COMM_SELECT_LIFETIME,
 466         (PF) ftpLifetimeExpire,
 467         (caddr_t) data);
 468     if (!BIT_TEST(entry->flag, ENTRY_PRIVATE))
 469         storeSetPublicKey(entry);       /* Make it public */
 470
 471     return COMM_OK;
 472 }
 473
 474 int ftpInitialize()
 475 {
 476     int pid;
 477     int fd;
 478     int p[2];
 479     static char pbuf[128];
 480     char *ftpget = getFtpProgram();
 481
 482     if (pipe(p) < 0) {
 483         debug(9, 0, "ftpInitialize: pipe: %s\n", xstrerror());
 484         return -1;
 485     }
 486     if ((pid = fork()) < 0) {
 487         debug(9, 0, "ftpInitialize: fork: %s\n", xstrerror());
 488         return -1;
 489     }
 490     if (pid != 0) {             /* parent */
 491         close(p[0]);
 492         fdstat_open(p[1], Pipe);
 493         fd_note(p[1], "ftpget -S");
 494         fcntl(p[1], F_SETFD, 1);        /* set close-on-exec */
 495         return 0;
 496     }
 497     /* child */
 498     close(0);
 499     dup(p[0]);
 500     close(p[0]);
 501     close(p[1]);
 502     /* inherit stdin,stdout,stderr */
 503     for (fd = 3; fd < fdstat_biggest_fd(); fd++)
 504         (void) close(fd);
 505     sprintf(pbuf, "%d", 3131);
 506     execlp(ftpget, ftpget, "-D26,1", "-S", pbuf, NULL);
 507     debug(9, 0, "ftpInitialize: %s: %s\n", ftpget, xstrerror());
 508     _exit(1);
 509     return (1);                 /* eliminate compiler warning */
 510 }