A generic client-to-server "pump" was added to handle HTTP PUT as

well as POST methods on the client-cache side. Based on "pump"
PUT requests can be made to either HTTP or FTP url's. Code is still
alpha and interoperability with browsers is not guaranteed.
Timeout/close handling should be improved sometime.

diff --git a/ChangeLog b/ChangeLog
index 3ddd968fdea9cb737970a7b8a8fe34cb1c111eca..28afebec5d745f3c804ff62d10322747b0b6e71d 100644 (file)
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,7 @@
+       - A generic client-to-server "pump" was added to handle HTTP PUT as 
+         well as POST methods on the client-cache side. Based on "pump" 
+         PUT requests can be made to either HTTP or FTP url's. Code is still
+         beta and interoperability with browsers etc has not been tested.
        - Fixed ICP bug when we send queries, but expect zero
          replies.
        - Fixed alignment/casting bugs for ICP messages.

diff --git a/doc/debug-sections.txt b/doc/debug-sections.txt
index 1e2b83b0ea173b4bed53edcfe617efa1d5f9cb60..1cd1976a9eba758e3ba520b38ec9b5b8d6c47399 100644 (file)
--- a/doc/debug-sections.txt
+++ b/doc/debug-sections.txt
@@ -64,6 +64,7 @@ section 57    HTTP Status-line
 section 58    HTTP Reply (Response)
 section 59    auto-growing Memory Buffer with printf
 section 60    Packer: A uniform interface to store-like modules
+section 61    Pump module for PUT/POST
 section 62    Generic Histogram
 section 63    Low Level Memory Pool Management
 section 64    HTTP Content-Range Header

diff --git a/src/Makefile.in b/src/Makefile.in
index b89919fe5502efce912a600d24501953fd50efca..66b811f1e9161a1117e7570e933d873289d9463d 100644 (file)
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -1,7 +1,7 @@
 #
 #  Makefile for the Squid Object Cache server
 #
-#  $Id: Makefile.in,v 1.128 1998/03/05 20:55:57 rousskov Exp $
+#  $Id: Makefile.in,v 1.129 1998/03/06 05:43:33 kostas Exp $
 #
 #  Uncomment and customize the following to suit your needs:
 #
@@ -116,6 +116,7 @@ OBJS                = \
                pconn.o \
                peer_select.o \
                proto.o \
+               pump.o \
                redirect.o \
                refresh.o \
                send-announce.o \

diff --git a/src/client.cc b/src/client.cc
index 87fdec90943215bbe91226afd66dc4e050902782..27bd547f0393e27ea76b78d3a55638d8042e85c4 100644 (file)
--- a/src/client.cc
+++ b/src/client.cc
@@ -3,7 +3,7 @@
 
 
 /*
- * $Id: client.cc,v 1.58 1998/03/05 00:42:48 wessels Exp $
+ * $Id: client.cc,v 1.59 1998/03/06 05:43:33 kostas Exp $
  *
  * DEBUG: section 0     WWW Client
  * AUTHOR: Harvest Derived
@@ -118,9 +118,12 @@ static int client_comm_connect(int, char *, u_short, struct timeval *);
 static void usage(const char *progname);
 static int Now(struct timeval *);
 static SIGHDLR catch;
+static SIGHDLR pipe_handler;
+static void set_our_signal();
 static int put_fd;
 static char *put_file = NULL;
 static struct stat p;
+int total_bytes=0;
 
 static void
 usage(const char *progname)
@@ -253,6 +256,7 @@ main(int argc, char *argv[])
        opt_put = 1;
        method = xstrdup("PUT");
        put_fd = open(put_file, O_RDONLY);
+       set_our_signal();
        if (put_fd < 0) {
            fprintf(stderr, "%s: can't open file (%s)\n", argv[0],
                xstrerror());
@@ -341,16 +345,17 @@ main(int argc, char *argv[])
            int x;
            while ((x = read(put_fd, msg, BUFSIZ)) > 0) {
                x = write(conn, msg, x);
+               total_bytes+=x;
                if (x <= 0)
                    break;
            }
-           if (x != 0) {
+           if (x != 0) 
                fprintf(stderr, "client: ERROR: Cannot send file.\n");
-               exit(1);
-           }
+           fprintf(stderr, "TOTAL SENT: %d\n",total_bytes);
            close(put_fd);
        }
        /* Read the data */
+
        while ((len = read(conn, buf, sizeof(buf))) > 0) {
            if (to_stdout)
                fwrite(buf, len, 1, stdout);
@@ -439,3 +444,26 @@ catch(int sig)
     interrupted = 1;
     fprintf(stderr, "Interrupted.\n");
 }
+void
+pipe_handler(int sig)
+{
+       fprintf(stderr,"SIGPIPE received.\n");
+}
+
+static void
+set_our_signal()
+{
+#if HAVE_SIGACTION
+    struct sigaction sa;
+    sa.sa_handler = pipe_handler;
+    sa.sa_flags = SA_RESTART;
+    sigemptyset(&sa.sa_mask);
+    if (sigaction(SIGPIPE, &sa, NULL) < 0) {
+       fprintf(stderr,"Cannot set PIPE signal.\n");
+       exit(-1);
+    }
+#else
+    signal(SIGPIPE, pipe_handler);
+#endif
+
+}

diff --git a/src/client_side.cc b/src/client_side.cc
index 47bd82845f5aaf0a07731f95fea2f47c43a6fbfb..2c3cdb5606ac5500c4309c07c57a0e993e37afcb 100644 (file)
--- a/src/client_side.cc
+++ b/src/client_side.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: client_side.cc,v 1.220 1998/03/05 00:16:24 wessels Exp $
+ * $Id: client_side.cc,v 1.221 1998/03/06 05:43:34 kostas Exp $
  *
  * DEBUG: section 33    Client-side Routines
  * AUTHOR: Duane Wessels
@@ -1261,14 +1261,17 @@ clientProcessRequest(clientHttpRequest * http)
        /* yes, continue */
     } else if (r->protocol != PROTO_HTTP) {
        (void) 0;               /* fallthrough */
+#if OLD_POST_CODE
     } else if (r->method == METHOD_POST) {
        http->log_type = LOG_TCP_MISS;
        passStart(fd, url, r, &http->out.size);
        return;
-    } else if (r->method == METHOD_PUT) {
+#else
+    }
+    if ( r->method == METHOD_PUT || r->method == METHOD_POST )  {
        http->log_type = LOG_TCP_MISS;
-       passStart(fd, url, r, &http->out.size);
-       return;
+       pumpInit(fd, r, http->uri);
+#endif
     }
     http->log_type = clientProcessRequest2(http);
     debug(33, 4) ("clientProcessRequest: %s for '%s'\n",

diff --git a/src/enums.h b/src/enums.h
index 8d41de166959067dce28a0c26450c383221cbd0f..d481cef60a45ae2b64f31762ad3b8257b9ee3af7 100644 (file)
--- a/src/enums.h
+++ b/src/enums.h
@@ -46,6 +46,9 @@ typedef enum {
     ERR_CACHE_ACCESS_DENIED,
     ERR_CACHE_MGR_ACCESS_DENIED,
     ERR_SQUID_SIGNATURE,       /* not really an error */
+    ERR_FTP_PUT_CREATED,       /* !error,a note that the file was created */
+    ERR_FTP_PUT_MODIFIED,      /* modified, !created*/
+    ERR_FTP_PUT_ERROR, 
     ERR_MAX
 } err_type;
 

diff --git a/src/errorpage.cc b/src/errorpage.cc
index d578f5433705253c4e4fdded7325bcc54f6122f6..6d15004904adca42bcaac9c290ab3383262caed4 100644 (file)
--- a/src/errorpage.cc
+++ b/src/errorpage.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: errorpage.cc,v 1.119 1998/02/26 18:00:42 wessels Exp $
+ * $Id: errorpage.cc,v 1.120 1998/03/06 05:43:35 kostas Exp $
  *
  * DEBUG: section 4     Error Generation
  * AUTHOR: Duane Wessels
@@ -190,7 +190,9 @@ errorAppendEntry(StoreEntry * entry, ErrorState * err)
     HttpReply *rep;
 #endif
     MemObject *mem = entry->mem_obj;
+#if 0  /* we might have an ok store for put etc */
     assert(entry->store_status == STORE_PENDING);
+#endif
     assert(mem != NULL);
     assert(mem->inmem_hi == 0);
 #if 0

diff --git a/src/ftp.cc b/src/ftp.cc
index 5327d1bbd25723261e1b3fa557fd83ba0feda72d..dcf66a95fb7feb135fecf7a02a4e7732851dcca4 100644 (file)
--- a/src/ftp.cc
+++ b/src/ftp.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: ftp.cc,v 1.201 1998/03/05 00:42:52 wessels Exp $
+ * $Id: ftp.cc,v 1.202 1998/03/06 05:43:36 kostas Exp $
  *
  * DEBUG: section 9     File Transfer Protocol (FTP)
  * AUTHOR: Harvest Derived
@@ -46,6 +46,8 @@ enum {
     FTP_HTML_HEADER_SENT,
     FTP_BINARY,
     FTP_TRY_SLASH_HACK,
+    FTP_PUT,
+    FTP_PUT_MKDIR,
     FTP_LISTFORMAT_UNKNOWN
 };
 
@@ -66,8 +68,11 @@ typedef enum {
     SENT_NLST,
     SENT_REST,
     SENT_RETR,
+    SENT_STOR,
     SENT_QUIT,
-    READING_DATA
+    READING_DATA,
+    WRITING_DATA,
+    SENT_MKDIR
 } ftp_state_t;
 
 typedef struct _Ftpdata {
@@ -141,6 +146,8 @@ static void ftpAppendSuccessHeader(FtpStateData * ftpState);
 static void ftpAuthRequired(HttpReply * reply, request_t * request, const char *realm);
 static STABH ftpAbort;
 static void ftpHackShortcut(FtpStateData * ftpState, FTPSM * nextState);
+static void ftpPutStart(FtpStateData *);
+static CWCB ftpPutTransferDone;
 
 /* State machine functions
  * send == state transition
@@ -180,7 +187,11 @@ static FTPSM ftpReadQuit;
 static FTPSM ftpFail;
 static FTPSM ftpDataTransferDone;
 static FTPSM ftpRestOrList;
-
+static FTPSM ftpSendStor;
+static FTPSM ftpReadStor;
+static FTPSM ftpSendReply;
+static FTPSM ftpTryMkdir;
+static FTPSM ftpReadMkdir;
 /************************************************
 ** State Machine Description (excluding hacks) **
 *************************************************
@@ -221,8 +232,11 @@ FTPSM *FTP_SM_FUNCS[] =
     ftpReadList,               /* SENT_NLST */
     ftpReadRest,
     ftpReadRetr,
+    ftpReadStor,
     ftpReadQuit,
-    ftpReadTransferDone
+    ftpReadTransferDone,
+    ftpSendReply,
+    ftpReadMkdir
 };
 
 static void
@@ -231,7 +245,7 @@ ftpStateFree(int fdnotused, void *data)
     FtpStateData *ftpState = data;
     if (ftpState == NULL)
        return;
-    debug(9, 3) ("ftpStateFree: %s\n", storeUrl(ftpState->entry));
+    debug(9, 3) ("ftpStateFree: %d\n", storeUrl(ftpState->entry));
     storeUnregisterAbort(ftpState->entry);
     storeUnlockObject(ftpState->entry);
     if (ftpState->reply_hdr) {
@@ -735,8 +749,10 @@ ftpReadComplete(FtpStateData * ftpState)
     /* Connection closed; retrieval done. */
     if (EBIT_TEST(ftpState->flags, FTP_HTML_HEADER_SENT))
        ftpListingFinish(ftpState);
-    storeTimestampsSet(ftpState->entry);
-    storeComplete(ftpState->entry);
+    if (!EBIT_TEST(ftpState->flags, FTP_PUT)) {
+       storeTimestampsSet(ftpState->entry);
+       storeComplete(ftpState->entry);
+    }
     /* expect the "transfer complete" message on the control socket */
     commSetSelect(ftpState->ctrl.fd,
        COMM_SELECT_READ,
@@ -918,6 +934,8 @@ ftpStart(request_t * request, StoreEntry * entry)
     ftpState->data.fd = -1;
     EBIT_SET(ftpState->flags, FTP_PASV_SUPPORTED);
     EBIT_SET(ftpState->flags, FTP_REST_SUPPORTED);
+    if (ftpState->request->method == METHOD_PUT)
+       EBIT_SET(ftpState->flags, FTP_PUT);
     if (!ftpCheckAuth(ftpState, request->headers)) {
        /* This request is not fully authenticated */
        if (request->port == 21) {
@@ -1166,6 +1184,7 @@ ftpReadControlReply(int fd, void *data)
     ftpState->ctrl.last_reply = (*W)->key;
     safe_free(*W);
     ftpState->ctrl.offset = 0;
+    debug(9,8)("ftpReadControlReply: state=%d\n",ftpState->state);
     FTP_SM_FUNCS[ftpState->state] (ftpState);
 }
 
@@ -1304,7 +1323,7 @@ static void
 ftpTraverseDirectory(FtpStateData * ftpState)
 {
     wordlist *w;
-    debug(9, 4) ("ftpTraverseDirectory\n");
+    debug(9, 4) ("ftpTraverseDirectory %s\n", ftpState->filepath);
 
     safe_free(ftpState->filepath);
     /* Done? */
@@ -1358,10 +1377,41 @@ ftpReadCwd(FtpStateData * ftpState)
        ftpTraverseDirectory(ftpState);
     } else {
        /* CWD FAILED */
-       ftpFail(ftpState);
+       if (!EBIT_TEST(ftpState->flags, FTP_PUT)) 
+               ftpFail(ftpState);
+       else
+               ftpTryMkdir(ftpState);
     }
 }
 
+static void
+ftpTryMkdir(FtpStateData *ftpState)
+{
+       char *path=ftpState->filepath;
+       debug(9,3)("ftpTryMkdir: with path=%s\n",path);
+       snprintf(cbuf, 1024, "MKD %s\r\n", path);
+       ftpWriteCommand(cbuf, ftpState);
+       ftpState->state = SENT_MKDIR;
+}
+
+static void
+ftpReadMkdir(FtpStateData *ftpState)
+{
+       char *path=ftpState->filepath;
+       int code = ftpState->ctrl.replycode;
+       
+       debug(9,3)("Here, with code %d\n",path,code);
+       if (code==257) { /* success */
+               ftpSendCwd(ftpState);   
+       } else if (code==550) { /* dir exists */
+               if (EBIT_TEST(ftpState->flags, FTP_PUT_MKDIR)) { 
+                       EBIT_SET(ftpState->flags, FTP_PUT_MKDIR);
+                       ftpSendCwd(ftpState);
+               } else 
+                       ftpSendReply(ftpState);
+       } else ftpSendReply(ftpState);
+}
+
 static void
 ftpGetFile(FtpStateData * ftpState)
 {
@@ -1659,8 +1709,12 @@ ftpAcceptDataConnection(int fd, void *data)
 static void
 ftpRestOrList(FtpStateData * ftpState)
 {
+
     debug(9, 3) ("This is ftpRestOrList\n");
-    if (ftpState->typecode == 'D') {
+    if (EBIT_TEST(ftpState->flags, FTP_PUT)) {
+        debug(9,3)("ftpRestOrList: Sending STOR request...\n");
+       ftpSendStor(ftpState);
+    } else if (ftpState->typecode == 'D') {
        /* XXX This should NOT be here */
        ftpSendNlst(ftpState);  /* sec 3.2.2 of RFC 1738 */
        EBIT_SET(ftpState->flags, FTP_ISDIR);
@@ -1673,6 +1727,36 @@ ftpRestOrList(FtpStateData * ftpState)
        ftpSendRetr(ftpState);
 }
 
+static void
+ftpSendStor(FtpStateData * ftpState)
+{
+    assert(ftpState->filepath != NULL);
+    snprintf(cbuf, 1024, "STOR %s\r\n", ftpState->filepath);
+    ftpWriteCommand(cbuf, ftpState);
+    ftpState->state = SENT_STOR;
+}
+
+static void
+ftpReadStor(FtpStateData * ftpState)
+{
+    int code = ftpState->ctrl.replycode;
+    debug(9, 3) ("This is ftpReadStor\n");
+    if (code >= 100 && code < 200) {
+      ftpPutStart(ftpState);
+      debug(9, 3) ("ftpReadStor: writing data channel\n");
+      ftpState->state = WRITING_DATA;
+    }
+    else if (code==553) { /* directory does not exist, have to create, sigh */
+#if 0
+       ftpTraverseDirectory(ftpState);
+#endif
+       ftpSendReply(ftpState);
+    } else {
+      debug(9, 3) ("ftpReadStor: that's all folks\n");
+       ftpSendReply(ftpState);
+    }
+}
+
 static void
 ftpSendRest(FtpStateData * ftpState)
 {
@@ -1919,6 +2003,62 @@ ftpFail(FtpStateData * ftpState)
     comm_close(ftpState->ctrl.fd);
 }
 
+static void
+ftpPutStart(FtpStateData * ftpState)
+{
+    debug(9, 3) ("ftpPutStart\n");
+    pumpStart(ftpState->data.fd, ftpState->entry , 
+               ftpState->request, ftpPutTransferDone, ftpState);
+}
+
+static void 
+ftpPutTransferDone(int fd, char *bufnotused, size_t size, int errflag, void *data)
+{
+    FtpStateData * ftpState=(FtpStateData *)data;
+    if (ftpState->data.fd >= 0) {
+        comm_close(ftpState->data.fd);
+        ftpState->data.fd = -1;
+    }
+    ftpReadComplete(ftpState);
+}
+
+static void
+ftpSendReply(FtpStateData * ftpState)
+{
+    ErrorState *err;
+    int code=ftpState->ctrl.replycode;
+    int http_code;
+    int err_code=ERR_NONE;
+    debug(9,5)("ftpSendReply for %x (%d)\n", ftpState,code);
+    if (cbdataValid(ftpState))
+    debug(9,5)("ftpSendReply: ftpState (%p) is valid!\n", ftpState);
+
+
+    if (code==226)  {
+        err_code= (ftpState->mdtm>0)?ERR_FTP_PUT_MODIFIED:ERR_FTP_PUT_CREATED;
+       http_code= (ftpState->mdtm>0)?HTTP_ACCEPTED:HTTP_CREATED;
+    } else {
+       err_code = ERR_FTP_PUT_ERROR;
+       http_code=HTTP_INTERNAL_SERVER_ERROR;
+    }
+
+    err = errorCon(err_code, http_code);
+    err->request = requestLink(ftpState->request);
+    if (ftpState->old_request)
+        err->ftp.request = ftpState->old_request;
+    else
+        err->ftp.request = ftpState->ctrl.last_command;
+    if (ftpState->old_reply)
+        err->ftp.reply = ftpState->old_reply;
+    else
+        err->ftp.reply = ftpState->ctrl.last_reply;
+
+    errorAppendEntry(ftpState->entry, err);
+
+    storeBufferFlush(ftpState->entry);
+    comm_close(ftpState->ctrl.fd);
+}
+
 static void
 ftpAppendSuccessHeader(FtpStateData * ftpState)
 {

diff --git a/src/http.cc b/src/http.cc
index b0b444e63e03c89cb833f152ffbd51d6203b3d40..bfdad1baa7f3adf445ee28d587e7b6e3c2d852d6 100644 (file)
--- a/src/http.cc
+++ b/src/http.cc
@@ -1,6 +1,6 @@
 
 /*
- * $Id: http.cc,v 1.246 1998/03/05 00:42:55 wessels Exp $
+ * $Id: http.cc,v 1.247 1998/03/06 05:43:37 kostas Exp $
  *
  * DEBUG: section 11    Hypertext Transfer Protocol (HTTP)
  * AUTHOR: Harvest Derived
@@ -206,6 +206,9 @@ static struct {
 
 static CNCB httpConnectDone;
 static CWCB httpSendComplete;
+static void *sendHeaderDone;
+static CWCB httpSendRequestEntry;
+
 static PF httpReadReply;
 static PF httpSendRequest;
 static PF httpStateFree;
@@ -943,10 +946,11 @@ httpSendRequest(int fd, void *data)
        buflen += req->headers_sz + 1;
     buflen += 512;             /* lots of extra */
 
-    if ((req->method == METHOD_POST || req->method == METHOD_PUT)) {
-       debug_trap("httpSendRequest: should not be handling POST/PUT request");
-       return;
-    }
+    if ((req->method == METHOD_POST || req->method == METHOD_PUT))
+           sendHeaderDone= httpSendRequestEntry;
+       else
+           sendHeaderDone= httpSendComplete;
+
     if (buflen < DISK_PAGE_SIZE) {
        buf = memAllocate(MEM_8K_BUF);
        buftype = BUF_TYPE_8K;
@@ -983,7 +987,7 @@ httpSendRequest(int fd, void *data)
     comm_write(fd,
        buf,
        len,
-       httpSendComplete,
+       sendHeaderDone,
        httpState,
        buftype == BUF_TYPE_8K ? memFree8K : xfree);
 #ifdef BREAKS_PCONN_RESTART
@@ -1334,3 +1338,30 @@ httpReplyHeader(double ver,
     return buf;
 }
 #endif
+
+static void
+httpSendRequestEntry(int fd, char *bufnotused, size_t size, int errflag, void  *data)
+{
+    HttpStateData *httpState = data;
+    StoreEntry *entry = httpState->entry;
+    ErrorState *err;
+    debug(11, 5) ("httpSendRequestEntry: FD %d: size %d: errflag %d.\n",
+        fd, size, errflag);
+    if (size > 0) {
+        fd_bytes(fd, size, FD_WRITE);
+       kb_incr(&Counter.server.all.kbytes_out, size);
+       kb_incr(&Counter.server.http.kbytes_out, size);
+    }
+    if (errflag == COMM_ERR_CLOSING)
+        return;
+    if (errflag) {
+        err = errorCon(ERR_WRITE_ERROR, HTTP_INTERNAL_SERVER_ERROR);
+        err->xerrno = errno;
+        err->request = requestLink(httpState->orig_request);
+        errorAppendEntry(entry, err);
+        comm_close(fd);
+        return;
+    } else {
+       pumpStart(fd, entry, httpState->orig_request , httpSendComplete, httpState);
+    }
+}

diff --git a/src/protos.h b/src/protos.h
index 4f27cf9bd15c7fabf067824891de4efb3b946953..ad7d2cf87be7aa5ab5da211d2da15513a50b426b 100644 (file)
--- a/src/protos.h
+++ b/src/protos.h
@@ -771,6 +771,8 @@ extern void releaseServerSockets(void);
 extern void PrintRusage(void);
 extern void dumpMallocStats(void);
 
+extern void pumpInit(int fd, request_t *r, char *uri);
+extern void pumpStart(int fd, StoreEntry *reply_entry, request_t *r, void *callback, void *cbdata);
 
 extern void unlinkdInit(void);
 extern void unlinkdClose(void);