CURLOPT_UPLOAD_BUFFERSIZE: set upload buffer size

This is step 3 of #2888.

Fixes #2888
Closes #2896

diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 32313499c31549fe5752f31633663132f4d1298c..3685db5841f687518df86cd3e438b7103dfe4361 100644 (file)
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -421,6 +421,8 @@ Size of file to send. \fICURLOPT_INFILESIZE(3)\fP
 Size of file to send. \fICURLOPT_INFILESIZE_LARGE(3)\fP
 .IP CURLOPT_UPLOAD
 Upload data. See \fICURLOPT_UPLOAD(3)\fP
+.IP CURLOPT_UPLOAD_BUFFERSIZE
+Set upload buffer size. See \fICURLOPT_UPLOAD_BUFFERSIZE(3)\fP
 .IP CURLOPT_MIMEPOST
 Post/send MIME data. See \fICURLOPT_MIMEPOST(3)\fP
 .IP CURLOPT_MAXFILESIZE

diff --git a/docs/libcurl/opts/CURLOPT_UPLOAD_BUFFERSIZE.3 b/docs/libcurl/opts/CURLOPT_UPLOAD_BUFFERSIZE.3
new file mode 100644 (file)
index 0000000..c35888c
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_UPLOAD_BUFFERSIZE.3
@@ -0,0 +1,69 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_UPLOAD_BUFFERSIZE 3 "18 Aug 2018" "libcurl 7.62.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_UPLOAD_BUFFERSIZE \- set preferred upload buffer size
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_UPLOAD_BUFFERSIZE, long size);
+.SH DESCRIPTION
+Pass a long specifying your preferred \fIsize\fP (in bytes) for the upload
+buffer in libcurl. It makes libcurl uses a larger buffer that gets passed to
+the next layer in the stack to get sent off. In some setups and for some
+protocols, there's a huge performance benefit of having a larger upload
+buffer.
+
+This is just treated as a request, not an order. You cannot be guaranteed to
+actually get the given size.
+
+The upload buffer size is by default 64 kilobytes. The maximum buffer size
+allowed to be set is 2 megabytes. The minimum buffer size allowed to be set is
+16 kilobytes.
+
+Since curl 7.61.1 the upload buffer is allocated on-demand - so if the handle
+isn't used for upload, this buffer will not be allocated at all.
+.SH DEFAULT
+64 kB
+.SH PROTOCOLS
+All
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  curl_easy_setopt(curl, CURLOPT_URL, "sftp://example.com/foo.bin");
+
+  /* ask libcurl to allocate a larger upload buffer */
+  curl_easy_setopt(curl, CURLOPT_UPLOAD_BUFFERSIZE, 120000L);
+
+  ret = curl_easy_perform(curl);
+
+  curl_easy_cleanup(curl);
+}
+.fi
+.SH AVAILABILITY
+Added in 7.62.0.
+.SH RETURN VALUE
+Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not.
+.SH "SEE ALSO"
+.BR CURLOPT_BUFFERSIZE "(3), " CURLOPT_READFUNCTION "(3), "

diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc
index d65a82073c051615f2a83e95b0c9ceb9034caf75..b7f664a380669e60570a8a02a444df7a4ea4fb27 100644 (file)
--- a/docs/libcurl/opts/Makefile.inc
+++ b/docs/libcurl/opts/Makefile.inc
@@ -327,6 +327,7 @@ man_MANS =                                      \
   CURLOPT_UNIX_SOCKET_PATH.3                    \
   CURLOPT_UNRESTRICTED_AUTH.3                   \
   CURLOPT_UPLOAD.3                              \
+  CURLOPT_UPLOAD_BUFFERSIZE.3                   \
   CURLOPT_URL.3                                 \
   CURLOPT_USERAGENT.3                           \
   CURLOPT_USERNAME.3                            \

diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index 108c3be32609bca0665a8877c632b4764645f6cf..1515e0616590e92cb6002270bef0a635c368cac4 100644 (file)
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -616,6 +616,7 @@ CURLOPT_TRANSFER_ENCODING       7.21.6
 CURLOPT_UNIX_SOCKET_PATH        7.40.0
 CURLOPT_UNRESTRICTED_AUTH       7.10.4
 CURLOPT_UPLOAD                  7.1
+CURLOPT_UPLOAD_BUFFERSIZE       7.62.0
 CURLOPT_URL                     7.1
 CURLOPT_USERAGENT               7.1
 CURLOPT_USERNAME                7.19.1

diff --git a/include/curl/curl.h b/include/curl/curl.h
index 627cf5446db1ed056f465ccdf7ddcc84a0271753..c19613072f20f5c1baa02cbe091ee5681e46eaff 100644 (file)
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -1862,6 +1862,9 @@ typedef enum {
   /* DNS-over-HTTPS URL */
   CINIT(DOH_URL, STRINGPOINT, 279),
 
+  /* Preferred buffer size to use for uploads */
+  CINIT(UPLOAD_BUFFERSIZE, LONG, 280),
+
   CURLOPT_LASTENTRY /* the last unused */
 } CURLoption;
 

diff --git a/lib/file.c b/lib/file.c
index e50e9887693fd49bd5bb1277f20f86b6a39c0b6d..542f34a4523230ad32c575d1eb74a1350068b3a3 100644 (file)
--- a/lib/file.c
+++ b/lib/file.c
@@ -307,7 +307,7 @@ static CURLcode file_upload(struct connectdata *conn)
     size_t nread;
     size_t nwrite;
     size_t readcount;
-    result = Curl_fillreadbuffer(conn, (int)data->set.buffer_size, &readcount);
+    result = Curl_fillreadbuffer(conn, data->set.buffer_size, &readcount);
     if(result)
       break;
 

diff --git a/lib/setopt.c b/lib/setopt.c
index 475a3379808a4044e736503503069ed38294f60f..4a71f9ae4bce7ad8b235624e19898967ec4aebd4 100644 (file)
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -1940,6 +1940,22 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option,
 
     break;
 
+  case CURLOPT_UPLOAD_BUFFERSIZE:
+    /*
+     * The application kindly asks for a differently sized upload buffer.
+     * Cap it to sensible.
+     */
+    arg = va_arg(param, long);
+
+    if(arg > UPLOADBUFFER_MAX)
+      arg = UPLOADBUFFER_MAX;
+    else if(arg < UPLOADBUFFER_MIN)
+      arg = UPLOADBUFFER_MIN;
+
+    data->set.upload_buffer_size = arg;
+    Curl_safefree(data->state.ulbuf); /* force a realloc next opportunity */
+    break;
+
   case CURLOPT_NOSIGNAL:
     /*
      * The application asks not to set any signal() or alarm() handlers,

diff --git a/lib/smb.c b/lib/smb.c
index e4b18fcf5fca9b9323374e13cac88cfc42c776f7..e1209e0995e1618ca3fd448d92586bf9dde39983 100644 (file)
--- a/lib/smb.c
+++ b/lib/smb.c
@@ -610,7 +610,8 @@ static CURLcode smb_send_and_recv(struct connectdata *conn, void **msg)
 
   /* Check if there is data in the transfer buffer */
   if(!smbc->send_size && smbc->upload_size) {
-    size_t nread = smbc->upload_size > UPLOAD_BUFSIZE ? UPLOAD_BUFSIZE :
+    size_t nread = smbc->upload_size > conn->data->set.upload_buffer_size ?
+      conn->data->set.upload_buffer_size :
       smbc->upload_size;
     conn->data->req.upload_fromhere = conn->data->state.ulbuf;
     result = Curl_fillreadbuffer(conn, nread, &nread);

diff --git a/lib/smtp.c b/lib/smtp.c
index ecf10a41ac3746087a201e3a2cadeb4dd6dfacef..50c0b347771a5caf1d7f92e86d63724727fad25e 100644 (file)
--- a/lib/smtp.c
+++ b/lib/smtp.c
@@ -1563,14 +1563,14 @@ CURLcode Curl_smtp_escape_eob(struct connectdata *conn, const ssize_t nread)
   if(!scratch || data->set.crlf) {
     oldscratch = scratch;
 
-    scratch = newscratch = malloc(2 * UPLOAD_BUFSIZE);
+    scratch = newscratch = malloc(2 * data->set.upload_buffer_size);
     if(!newscratch) {
       failf(data, "Failed to alloc scratch buffer!");
 
       return CURLE_OUT_OF_MEMORY;
     }
   }
-  DEBUGASSERT(UPLOAD_BUFSIZE >= nread);
+  DEBUGASSERT(data->set.upload_buffer_size >= (size_t)nread);
 
   /* Have we already sent part of the EOB? */
   eob_sent = smtp->eob;

diff --git a/lib/transfer.c b/lib/transfer.c
index 7159d5c823eb86eda192ad119c5a921d94bfaf4c..63d01efd9377cd8207e35d8f00abe33a30e1dfd7 100644 (file)
--- a/lib/transfer.c
+++ b/lib/transfer.c
@@ -959,7 +959,8 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
             sending_http_headers = FALSE;
         }
 
-        result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount);
+        result = Curl_fillreadbuffer(conn, data->set.upload_buffer_size,
+                                     &fillcount);
         if(result)
           return result;
 
@@ -991,7 +992,7 @@ static CURLcode readwrite_upload(struct Curl_easy *data,
          (data->set.crlf))) {
         /* Do we need to allocate a scratch buffer? */
         if(!data->state.scratch) {
-          data->state.scratch = malloc(2 * UPLOAD_BUFSIZE);
+          data->state.scratch = malloc(2 * data->set.upload_buffer_size);
           if(!data->state.scratch) {
             failf(data, "Failed to alloc scratch buffer!");
 

diff --git a/lib/url.c b/lib/url.c
index 7ffad19b7a3b27ff9e24c2b30da1f17ed9fa0134..a88ca495eb940ffb83abae5777a9882446100121 100644 (file)
--- a/lib/url.c
+++ b/lib/url.c
@@ -526,7 +526,7 @@ CURLcode Curl_init_userdefined(struct Curl_easy *data)
   set->expect_100_timeout = 1000L; /* Wait for a second by default. */
   set->sep_headers = TRUE; /* separated header lists by default */
   set->buffer_size = READBUFFER_SIZE;
-  set->upload_buffer_size = UPLOAD_BUFSIZE;
+  set->upload_buffer_size = UPLOADBUFFER_DEFAULT;
   set->happy_eyeballs_timeout = CURL_HET_DEFAULT;
   set->fnmatch = ZERO_NULL;
   set->maxconnects = DEFAULT_CONNCACHE_SIZE; /* for easy handles */

diff --git a/lib/url.h b/lib/url.h
index a914f56e6c65eb709783ed234c6a52135cb20515..618309d38b2d66c7e11d75bc503f98072047a78c 100644 (file)
--- a/lib/url.h
+++ b/lib/url.h
@@ -27,6 +27,18 @@
 #define READBUFFER_MAX  CURL_MAX_READ_SIZE
 #define READBUFFER_MIN  1024
 
+/* The default upload buffer size, should not be smaller than
+   CURL_MAX_WRITE_SIZE, as it needs to hold a full buffer as could be sent in
+   a write callback.
+
+   The size was 16KB for many years but was bumped to 64KB because it makes
+   libcurl able to do significantly faster uploads in some circumstances. Even
+   larger buffers can help further, but this is deemed a fair memory/speed
+   compromise. */
+#define UPLOADBUFFER_DEFAULT 65536
+#define UPLOADBUFFER_MAX (2*1024*1024)
+#define UPLOADBUFFER_MIN CURL_MAX_WRITE_SIZE
+
 /*
  * Prototypes for library-wide functions provided by url.c
  */

diff --git a/lib/urldata.h b/lib/urldata.h
index 769ab4fa4fd660f5459b92522593d3bb7ce3e6fc..d3a174d24608ed708226ca09c99c264d0edca9d3 100644 (file)
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -142,14 +142,6 @@ typedef ssize_t (Curl_recv)(struct connectdata *conn, /* connection data */
 #include <libssh2_sftp.h>
 #endif /* HAVE_LIBSSH2_H */
 
-/* The upload buffer size, should not be smaller than CURL_MAX_WRITE_SIZE, as
-   it needs to hold a full buffer as could be sent in a write callback.
-
-   The size was 16KB for many years but was bumped to 64KB because it makes
-   libcurl able to do significantly faster uploads in some circumstances. Even
-   larger buffers can help further, but this is deemed a fair memory/speed
-   compromise. */
-#define UPLOAD_BUFSIZE 65536
 
 /* The "master buffer" is for HTTP pipelining */
 #define MASTERBUF_SIZE 16384
@@ -1585,8 +1577,8 @@ struct UserDefined {
   curl_proxytype proxytype; /* what kind of proxy that is in use */
   long dns_cache_timeout; /* DNS cache timeout */
   long buffer_size;      /* size of receive buffer to use */
-  long upload_buffer_size; /* size of upload buffer to use,
-                              keep it >= CURL_MAX_WRITE_SIZE */
+  size_t upload_buffer_size; /* size of upload buffer to use,
+                                keep it >= CURL_MAX_WRITE_SIZE */
   void *private_data; /* application-private data */
 
   struct curl_slist *http200aliases; /* linked list of aliases for http200 */