HTTP: Add --http.max-file-size

Prevents repositories from stagnating Fort with absurdly large files.

Thanks to Koen van Hove for reporting this.

diff --git a/src/config.c b/src/config.c
index e01e387689bd0487f6d578e8b9077f2b707f0af7..9143c015665b4fa0a477aa208eeab378136a9250 100644 (file)
--- a/src/config.c
+++ b/src/config.c
@@ -153,6 +153,11 @@ struct rpki_config {
                unsigned int low_speed_limit;
                /* CURLOPT_LOW_SPEED_TIME for our HTTP transfers. */
                unsigned int low_speed_time;
+               /*
+                * CURLOPT_MAXFILESIZE, except it also works for unknown size
+                * files. (Though this is reactive, not preventive.)
+                */
+               unsigned int max_file_size;
                /* Directory where CA certs to verify peers are found */
                char *ca_path;
        } http;
@@ -635,6 +640,15 @@ static const struct option_field options[] = {
                .min = 0,
                .max = UINT_MAX,
        },
+       {
+               .id = 9011,
+               .name = "http.max-file-size",
+               .type = &gt_uint,
+               .offset = offsetof(struct rpki_config, http.max_file_size),
+               .doc = "Fort will refuse to download files larger than this number of bytes.",
+               .min = 0,
+               .max = 2000000000,
+       },
        {
                .id = 9008,
                .name = "http.ca-path",
@@ -1055,6 +1069,7 @@ set_default_values(void)
        rpki_config.http.transfer_timeout = 0;
        rpki_config.http.low_speed_limit = 30;
        rpki_config.http.low_speed_time = 10;
+       rpki_config.http.max_file_size = 10000000;
        rpki_config.http.ca_path = NULL; /* Use system default */
 
        /*
@@ -1585,6 +1600,12 @@ config_get_http_low_speed_time(void)
        return rpki_config.http.low_speed_time;
 }
 
+long
+config_get_http_max_file_size(void)
+{
+       return rpki_config.http.max_file_size;
+}
+
 char const *
 config_get_http_ca_path(void)
 {

diff --git a/src/config.h b/src/config.h
index 1d7cb8c31afc7ceaa455003df8194d619b303631..bbc0be76480b79d495c30239ee1922de7e38d58c 100644 (file)
--- a/src/config.h
+++ b/src/config.h
@@ -37,6 +37,7 @@ long config_get_http_connect_timeout(void);
 long config_get_http_transfer_timeout(void);
 long config_get_http_low_speed_limit(void);
 long config_get_http_low_speed_time(void);
+long config_get_http_max_file_size(void);
 char const *config_get_http_ca_path(void);
 bool config_get_rsync_enabled(void);
 unsigned int config_get_rsync_priority(void);

diff --git a/src/file.c b/src/file.c
index 9f7debc67815e8225e28dce66bfeb74ba5b91f25..89ae166fc7614c54672813450f4483248805af00 100644 (file)
--- a/src/file.c
+++ b/src/file.c
@@ -39,9 +39,10 @@ file_open(char const *file_name, FILE **result, struct stat *stat)
 }
 
 int
-file_write(char const *file_name, FILE **result, struct stat *stat)
+file_write(char const *file_name, FILE **result)
 {
-       return file_get(file_name, result, stat, "wb");
+       struct stat stat;
+       return file_get(file_name, result, &stat, "wb");
 }
 
 void
@@ -119,13 +120,12 @@ bool
 file_valid(char const *file_name)
 {
        FILE *tmp;
-       struct stat stat;
        int error;
 
        if (file_name == NULL)
                return false;
 
-       error = file_write(file_name, &tmp, &stat);
+       error = file_write(file_name, &tmp);
        if (error)
                return false;
 

diff --git a/src/file.h b/src/file.h
index 1699587c9c23a8a9481324bfe980491961858ca9..0ce44ba5d9cb46ca784f2e6bd570b05ca283f0ad 100644 (file)
--- a/src/file.h
+++ b/src/file.h
@@ -17,7 +17,7 @@ struct file_contents {
 };
 
 int file_open(char const *, FILE **, struct stat *);
-int file_write(char const *, FILE **, struct stat *);
+int file_write(char const *, FILE **);
 void file_close(FILE *);
 
 int file_load(char const *, struct file_contents *);

diff --git a/src/http/http.c b/src/http/http.c
index 9a2f39157e2fc4ff58ec5ba9fdb76576bf9724a5..5ad47efe4972c8cf15e0bbf3be457f672153fe3e 100644 (file)
--- a/src/http/http.c
+++ b/src/http/http.c
@@ -4,7 +4,6 @@
 #include <stdio.h>
 #include <unistd.h>
 #include <curl/curl.h>
-#include <sys/stat.h>
 #include "common.h"
 #include "config.h"
 #include "file.h"
@@ -67,12 +66,52 @@ setopt_long(CURL *curl, CURLoption opt, long value)
        }
 }
 
+struct write_callback_arg {
+       size_t total_bytes;
+       int error;
+       FILE *dst;
+};
+
+static size_t
+write_callback(void *data, size_t size, size_t nmemb, void *userp)
+{
+       struct write_callback_arg *arg = userp;
+
+       arg->total_bytes += size * nmemb;
+       if (arg->total_bytes > config_get_http_max_file_size()) {
+               /*
+                * If the server doesn't provide the file size beforehand,
+                * CURLOPT_MAXFILESIZE doesn't prevent large file downloads.
+                *
+                * Therefore, we cover our asses by way of this reactive
+                * approach. We already reached the size limit, but we're going
+                * to reject the file anyway.
+                */
+               arg->error = -EFBIG;
+               return 0; /* Ugh. See fread(3) */
+       }
+
+       return fwrite(data, size, nmemb, userp);
+}
+
 static void
-setopt_writedata(CURL *curl, FILE *file)
+setopt_writefunction(CURL *curl)
 {
        CURLcode result;
 
-       result = curl_easy_setopt(curl, CURLOPT_WRITEDATA, file);
+       result = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback);
+       if (result != CURLE_OK) {
+               fprintf(stderr, "curl_easy_setopt(%d) returned %d: %s\n",
+                   CURLOPT_WRITEFUNCTION, result, curl_easy_strerror(result));
+       }
+}
+
+static void
+setopt_writedata(CURL *curl, struct write_callback_arg *arg)
+{
+       CURLcode result;
+
+       result = curl_easy_setopt(curl, CURLOPT_WRITEDATA, arg);
        if (result != CURLE_OK) {
                fprintf(stderr, "curl_easy_setopt(%d) returned %d: %s\n",
                    CURLOPT_WRITEDATA, result, curl_easy_strerror(result));
@@ -94,11 +133,13 @@ http_easy_init(struct http_handler *handler)
            config_get_http_connect_timeout());
        setopt_long(result, CURLOPT_TIMEOUT,
            config_get_http_transfer_timeout());
-
        setopt_long(result, CURLOPT_LOW_SPEED_LIMIT,
            config_get_http_low_speed_limit());
        setopt_long(result, CURLOPT_LOW_SPEED_TIME,
            config_get_http_low_speed_time());
+       setopt_long(result, CURLOPT_MAXFILESIZE,
+           config_get_http_max_file_size());
+       setopt_writefunction(result);
 
        /* Always expect HTTPS usage */
        setopt_long(result, CURLOPT_SSL_VERIFYHOST, 2L);
@@ -143,16 +184,27 @@ static int
 http_fetch(struct http_handler *handler, char const *uri, long *response_code,
     long *cond_met, bool log_operation, FILE *file)
 {
+       struct write_callback_arg args;
        CURLcode res, res2;
        long unmet = 0;
 
        handler->errbuf[0] = 0;
        setopt_str(handler->curl, CURLOPT_URL, uri);
-       setopt_writedata(handler->curl, file);
+
+       args.total_bytes = 0;
+       args.error = 0;
+       args.dst = file;
+       setopt_writedata(handler->curl, &args);
 
        pr_val_debug("HTTP GET: %s", uri);
        res = curl_easy_perform(handler->curl);
 
+       if (args.error == -EFBIG) {
+               pr_val_err("The file '%s' is too big (read: %zu bytes). Rejecting.",
+                   uri, args.total_bytes);
+               return -EFBIG;
+       }
+
        res2 = curl_easy_getinfo(handler->curl, CURLINFO_RESPONSE_CODE,
            response_code);
        if (res2 != CURLE_OK) {
@@ -231,11 +283,10 @@ __http_download_file(struct rpki_uri *uri, long *response_code, long ims_value,
 {
        char const *tmp_suffix = "_tmp";
        struct http_handler handler;
-       struct stat stat;
        FILE *out;
        unsigned int retries;
        char const *original_file;
-       char *tmp_file, *tmp;
+       char *tmp_file;
        int error;
 
        retries = 0;
@@ -246,24 +297,18 @@ __http_download_file(struct rpki_uri *uri, long *response_code, long ims_value,
        }
 
        original_file = uri_get_local(uri);
-       tmp_file = strdup(original_file);
+
+       tmp_file = malloc(strlen(original_file) + strlen(tmp_suffix) + 1);
        if (tmp_file == NULL)
                return pr_enomem();
-
-       tmp = realloc(tmp_file, strlen(tmp_file) + strlen(tmp_suffix) + 1);
-       if (tmp == NULL) {
-               error = pr_enomem();
-               goto release_tmp;
-       }
-
-       tmp_file = tmp;
+       strcpy(tmp_file, original_file);
        strcat(tmp_file, tmp_suffix);
 
        error = create_dir_recursive(tmp_file);
        if (error)
                goto release_tmp;
 
-       error = file_write(tmp_file, &out, &stat);
+       error = file_write(tmp_file, &out);
        if (error)
                goto delete_dir;
 
@@ -405,7 +450,6 @@ http_direct_download(char const *remote, char const *dest)
 {
        char const *tmp_suffix = "_tmp";
        struct http_handler handler;
-       struct stat stat;
        FILE *out;
        long response_code;
        long cond_met;
@@ -425,7 +469,7 @@ http_direct_download(char const *remote, char const *dest)
        tmp_file = tmp;
        strcat(tmp_file, tmp_suffix);
 
-       error = file_write(tmp_file, &out, &stat);
+       error = file_write(tmp_file, &out);
        if (error)
                goto release_tmp;
 

diff --git a/src/output_printer.c b/src/output_printer.c
index fcce985723bfc2d0ec54951b6e6baad8b1dfdeba..dcaffc44b9b135d081fbc0e85a8ea95a73fe9d8d 100644 (file)
--- a/src/output_printer.c
+++ b/src/output_printer.c
@@ -19,7 +19,6 @@ static int
 load_output_file(char const *output, FILE **result, bool *fopen)
 {
        FILE *tmp;
-       struct stat stat;
        int error;
 
        if (output == NULL) {
@@ -33,7 +32,7 @@ load_output_file(char const *output, FILE **result, bool *fopen)
                return 0;
        }
 
-       error = file_write(output, &tmp, &stat);
+       error = file_write(output, &tmp);
        if (error) {
                *result = NULL;
                return error;

diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c
index bc67c0e24d54756b2eb700a25b2105cc3baec625..4a16287abae3035ebe8857392a8a4006f26486f1 100644 (file)
--- a/src/rrdp/rrdp_parser.c
+++ b/src/rrdp/rrdp_parser.c
@@ -565,7 +565,6 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len,
     struct visited_uris *visited_uris)
 {
        struct rpki_uri *uri;
-       struct stat stat;
        FILE *out;
        size_t written;
        int error;
@@ -581,7 +580,7 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len,
                return error;
        }
 
-       error = file_write(uri_get_local(uri), &out, &stat);
+       error = file_write(uri_get_local(uri), &out);
        if (error) {
                uri_refput(uri);
                return error;

diff --git a/src/uri.c b/src/uri.c
index d640aa49bedd9a14d92aabc85ab710b38dcaa2b0..93a5db0107080e2bae37ae96ae34bd82fed68648 100644 (file)
--- a/src/uri.c
+++ b/src/uri.c
@@ -568,7 +568,8 @@ uri_get_printable(struct rpki_uri *uri, enum filename_format format)
 }
 
 char const *
-uri_val_get_printable(struct rpki_uri *uri) {
+uri_val_get_printable(struct rpki_uri *uri)
+{
        enum filename_format format;
 
        format = config_get_val_log_filename_format();
@@ -576,7 +577,8 @@ uri_val_get_printable(struct rpki_uri *uri) {
 }
 
 char const *
-uri_op_get_printable(struct rpki_uri *uri) {
+uri_op_get_printable(struct rpki_uri *uri)
+{
        enum filename_format format;
 
        format = config_get_op_log_filename_format();