From: Alberto Leiva Popper Date: Mon, 20 Sep 2021 17:43:26 +0000 (-0500) Subject: HTTP: Add --http.max-file-size X-Git-Tag: 1.5.3~17 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=90fc12820f5000fc6e4fa1370f71753a1c6bb628;p=thirdparty%2FFORT-validator.git HTTP: Add --http.max-file-size Prevents repositories from stagnating Fort with absurdly large files. Thanks to Koen van Hove for reporting this. --- diff --git a/src/config.c b/src/config.c index e01e3876..9143c015 100644 --- a/src/config.c +++ b/src/config.c @@ -153,6 +153,11 @@ struct rpki_config { unsigned int low_speed_limit; /* CURLOPT_LOW_SPEED_TIME for our HTTP transfers. */ unsigned int low_speed_time; + /* + * CURLOPT_MAXFILESIZE, except it also works for unknown size + * files. (Though this is reactive, not preventive.) + */ + unsigned int max_file_size; /* Directory where CA certs to verify peers are found */ char *ca_path; } http; @@ -635,6 +640,15 @@ static const struct option_field options[] = { .min = 0, .max = UINT_MAX, }, + { + .id = 9011, + .name = "http.max-file-size", + .type = >_uint, + .offset = offsetof(struct rpki_config, http.max_file_size), + .doc = "Fort will refuse to download files larger than this number of bytes.", + .min = 0, + .max = 2000000000, + }, { .id = 9008, .name = "http.ca-path", @@ -1055,6 +1069,7 @@ set_default_values(void) rpki_config.http.transfer_timeout = 0; rpki_config.http.low_speed_limit = 30; rpki_config.http.low_speed_time = 10; + rpki_config.http.max_file_size = 10000000; rpki_config.http.ca_path = NULL; /* Use system default */ /* @@ -1585,6 +1600,12 @@ config_get_http_low_speed_time(void) return rpki_config.http.low_speed_time; } +long +config_get_http_max_file_size(void) +{ + return rpki_config.http.max_file_size; +} + char const * config_get_http_ca_path(void) { diff --git a/src/config.h b/src/config.h index 1d7cb8c3..bbc0be76 100644 --- a/src/config.h +++ b/src/config.h @@ -37,6 +37,7 @@ long config_get_http_connect_timeout(void); long config_get_http_transfer_timeout(void); long config_get_http_low_speed_limit(void); long config_get_http_low_speed_time(void); +long config_get_http_max_file_size(void); char const *config_get_http_ca_path(void); bool config_get_rsync_enabled(void); unsigned int config_get_rsync_priority(void); diff --git a/src/file.c b/src/file.c index 9f7debc6..89ae166f 100644 --- a/src/file.c +++ b/src/file.c @@ -39,9 +39,10 @@ file_open(char const *file_name, FILE **result, struct stat *stat) } int -file_write(char const *file_name, FILE **result, struct stat *stat) +file_write(char const *file_name, FILE **result) { - return file_get(file_name, result, stat, "wb"); + struct stat stat; + return file_get(file_name, result, &stat, "wb"); } void @@ -119,13 +120,12 @@ bool file_valid(char const *file_name) { FILE *tmp; - struct stat stat; int error; if (file_name == NULL) return false; - error = file_write(file_name, &tmp, &stat); + error = file_write(file_name, &tmp); if (error) return false; diff --git a/src/file.h b/src/file.h index 1699587c..0ce44ba5 100644 --- a/src/file.h +++ b/src/file.h @@ -17,7 +17,7 @@ struct file_contents { }; int file_open(char const *, FILE **, struct stat *); -int file_write(char const *, FILE **, struct stat *); +int file_write(char const *, FILE **); void file_close(FILE *); int file_load(char const *, struct file_contents *); diff --git a/src/http/http.c b/src/http/http.c index 9a2f3915..5ad47efe 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -4,7 +4,6 @@ #include #include #include -#include #include "common.h" #include "config.h" #include "file.h" @@ -67,12 +66,52 @@ setopt_long(CURL *curl, CURLoption opt, long value) } } +struct write_callback_arg { + size_t total_bytes; + int error; + FILE *dst; +}; + +static size_t +write_callback(void *data, size_t size, size_t nmemb, void *userp) +{ + struct write_callback_arg *arg = userp; + + arg->total_bytes += size * nmemb; + if (arg->total_bytes > config_get_http_max_file_size()) { + /* + * If the server doesn't provide the file size beforehand, + * CURLOPT_MAXFILESIZE doesn't prevent large file downloads. + * + * Therefore, we cover our asses by way of this reactive + * approach. We already reached the size limit, but we're going + * to reject the file anyway. + */ + arg->error = -EFBIG; + return 0; /* Ugh. See fread(3) */ + } + + return fwrite(data, size, nmemb, userp); +} + static void -setopt_writedata(CURL *curl, FILE *file) +setopt_writefunction(CURL *curl) { CURLcode result; - result = curl_easy_setopt(curl, CURLOPT_WRITEDATA, file); + result = curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, write_callback); + if (result != CURLE_OK) { + fprintf(stderr, "curl_easy_setopt(%d) returned %d: %s\n", + CURLOPT_WRITEFUNCTION, result, curl_easy_strerror(result)); + } +} + +static void +setopt_writedata(CURL *curl, struct write_callback_arg *arg) +{ + CURLcode result; + + result = curl_easy_setopt(curl, CURLOPT_WRITEDATA, arg); if (result != CURLE_OK) { fprintf(stderr, "curl_easy_setopt(%d) returned %d: %s\n", CURLOPT_WRITEDATA, result, curl_easy_strerror(result)); @@ -94,11 +133,13 @@ http_easy_init(struct http_handler *handler) config_get_http_connect_timeout()); setopt_long(result, CURLOPT_TIMEOUT, config_get_http_transfer_timeout()); - setopt_long(result, CURLOPT_LOW_SPEED_LIMIT, config_get_http_low_speed_limit()); setopt_long(result, CURLOPT_LOW_SPEED_TIME, config_get_http_low_speed_time()); + setopt_long(result, CURLOPT_MAXFILESIZE, + config_get_http_max_file_size()); + setopt_writefunction(result); /* Always expect HTTPS usage */ setopt_long(result, CURLOPT_SSL_VERIFYHOST, 2L); @@ -143,16 +184,27 @@ static int http_fetch(struct http_handler *handler, char const *uri, long *response_code, long *cond_met, bool log_operation, FILE *file) { + struct write_callback_arg args; CURLcode res, res2; long unmet = 0; handler->errbuf[0] = 0; setopt_str(handler->curl, CURLOPT_URL, uri); - setopt_writedata(handler->curl, file); + + args.total_bytes = 0; + args.error = 0; + args.dst = file; + setopt_writedata(handler->curl, &args); pr_val_debug("HTTP GET: %s", uri); res = curl_easy_perform(handler->curl); + if (args.error == -EFBIG) { + pr_val_err("The file '%s' is too big (read: %zu bytes). Rejecting.", + uri, args.total_bytes); + return -EFBIG; + } + res2 = curl_easy_getinfo(handler->curl, CURLINFO_RESPONSE_CODE, response_code); if (res2 != CURLE_OK) { @@ -231,11 +283,10 @@ __http_download_file(struct rpki_uri *uri, long *response_code, long ims_value, { char const *tmp_suffix = "_tmp"; struct http_handler handler; - struct stat stat; FILE *out; unsigned int retries; char const *original_file; - char *tmp_file, *tmp; + char *tmp_file; int error; retries = 0; @@ -246,24 +297,18 @@ __http_download_file(struct rpki_uri *uri, long *response_code, long ims_value, } original_file = uri_get_local(uri); - tmp_file = strdup(original_file); + + tmp_file = malloc(strlen(original_file) + strlen(tmp_suffix) + 1); if (tmp_file == NULL) return pr_enomem(); - - tmp = realloc(tmp_file, strlen(tmp_file) + strlen(tmp_suffix) + 1); - if (tmp == NULL) { - error = pr_enomem(); - goto release_tmp; - } - - tmp_file = tmp; + strcpy(tmp_file, original_file); strcat(tmp_file, tmp_suffix); error = create_dir_recursive(tmp_file); if (error) goto release_tmp; - error = file_write(tmp_file, &out, &stat); + error = file_write(tmp_file, &out); if (error) goto delete_dir; @@ -405,7 +450,6 @@ http_direct_download(char const *remote, char const *dest) { char const *tmp_suffix = "_tmp"; struct http_handler handler; - struct stat stat; FILE *out; long response_code; long cond_met; @@ -425,7 +469,7 @@ http_direct_download(char const *remote, char const *dest) tmp_file = tmp; strcat(tmp_file, tmp_suffix); - error = file_write(tmp_file, &out, &stat); + error = file_write(tmp_file, &out); if (error) goto release_tmp; diff --git a/src/output_printer.c b/src/output_printer.c index fcce9857..dcaffc44 100644 --- a/src/output_printer.c +++ b/src/output_printer.c @@ -19,7 +19,6 @@ static int load_output_file(char const *output, FILE **result, bool *fopen) { FILE *tmp; - struct stat stat; int error; if (output == NULL) { @@ -33,7 +32,7 @@ load_output_file(char const *output, FILE **result, bool *fopen) return 0; } - error = file_write(output, &tmp, &stat); + error = file_write(output, &tmp); if (error) { *result = NULL; return error; diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c index bc67c0e2..4a16287a 100644 --- a/src/rrdp/rrdp_parser.c +++ b/src/rrdp/rrdp_parser.c @@ -565,7 +565,6 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len, struct visited_uris *visited_uris) { struct rpki_uri *uri; - struct stat stat; FILE *out; size_t written; int error; @@ -581,7 +580,7 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len, return error; } - error = file_write(uri_get_local(uri), &out, &stat); + error = file_write(uri_get_local(uri), &out); if (error) { uri_refput(uri); return error; diff --git a/src/uri.c b/src/uri.c index d640aa49..93a5db01 100644 --- a/src/uri.c +++ b/src/uri.c @@ -568,7 +568,8 @@ uri_get_printable(struct rpki_uri *uri, enum filename_format format) } char const * -uri_val_get_printable(struct rpki_uri *uri) { +uri_val_get_printable(struct rpki_uri *uri) +{ enum filename_format format; format = config_get_val_log_filename_format(); @@ -576,7 +577,8 @@ uri_val_get_printable(struct rpki_uri *uri) { } char const * -uri_op_get_printable(struct rpki_uri *uri) { +uri_op_get_printable(struct rpki_uri *uri) +{ enum filename_format format; format = config_get_op_log_filename_format();