From 2cfd993ada3e9071b57626a9f1084eebcc28472c Mon Sep 17 00:00:00 2001 From: pcarana Date: Tue, 14 Jan 2020 17:42:31 -0600 Subject: [PATCH] Add 'rsync.retry.*' and 'rrdp.retry.*' conf args. +The new arguments are 'rsync.retry.count', 'rsync.retry.interval', 'rrdp.retry.count', and 'rrdp.retry.interval'. Utilized whenever there's an rsync or rrdp sync error, the validator will retry at most '*.retry.count' times, waiting '*.retry.interval' between each retry. +Ensure that HTTP files download returns a negative error in case of error. +Wrap files download function at rrdp_parser. --- src/config.c | 80 +++++++++++++++++++++++++++++++++++++-- src/config.h | 4 ++ src/http/http.c | 8 ++-- src/rrdp/rrdp_parser.c | 37 ++++++++++++++++-- src/rsync/rsync.c | 85 +++++++++++++++++++++++++----------------- 5 files changed, 170 insertions(+), 44 deletions(-) diff --git a/src/config.c b/src/config.c index c7859001..787099ff 100644 --- a/src/config.c +++ b/src/config.c @@ -86,6 +86,13 @@ struct rpki_config { unsigned int priority; /* Synchronization download strategy. */ enum rsync_strategy strategy; + /* Retry conf, utilized on errors */ + struct { + /* Maximum number of retries on error */ + unsigned int count; + /* Interval (in seconds) between each retry */ + unsigned int interval; + } retry; char *program; struct { struct string_array flat; @@ -101,6 +108,13 @@ struct rpki_config { * their accessMethod extension. */ unsigned int priority; + /* Retry conf, utilized on errors */ + struct { + /* Maximum number of retries on error */ + unsigned int count; + /* Interval (in seconds) between each retry */ + unsigned int interval; + } retry; } rrdp; struct { @@ -365,8 +379,24 @@ static const struct option_field options[] = { .type = >_rsync_strategy, .offset = offsetof(struct rpki_config, rsync.strategy), .doc = "RSYNC download strategy", - },{ + }, { .id = 3003, + .name = "rsync.retry.count", + .type = >_uint, + .offset = offsetof(struct rpki_config, rsync.retry.count), + .doc = "Maximum amount of retries whenever there's an RSYNC error", + .min = 0, + .max = UINT_MAX, + }, { + .id = 3004, + .name = "rsync.retry.interval", + .type = >_uint, + .offset = offsetof(struct rpki_config, rsync.retry.interval), + .doc = "Period (in seconds) to wait between retries after an RSYNC error ocurred", + .min = 0, + .max = UINT_MAX, + },{ + .id = 3005, .name = "rsync.program", .type = >_string, .offset = offsetof(struct rpki_config, rsync.program), @@ -374,14 +404,14 @@ static const struct option_field options[] = { .arg_doc = "", .availability = AVAILABILITY_JSON, }, { - .id = 3004, + .id = 3006, .name = "rsync.arguments-recursive", .type = >_string_array, .offset = offsetof(struct rpki_config, rsync.args.recursive), .doc = "RSYNC program arguments that will trigger a recursive RSYNC", .availability = AVAILABILITY_JSON, }, { - .id = 3005, + .id = 3007, .name = "rsync.arguments-flat", .type = >_string_array, .offset = offsetof(struct rpki_config, rsync.args.flat), @@ -404,6 +434,22 @@ static const struct option_field options[] = { .doc = "Priority of execution to fetch repositories files, a higher value means higher priority", .min = 0, .max = 100, + }, { + .id = 10002, + .name = "rrdp.retry.count", + .type = >_uint, + .offset = offsetof(struct rpki_config, rrdp.retry.count), + .doc = "Maximum amount of retries whenever there's an error fetching RRDP files", + .min = 0, + .max = UINT_MAX, + }, { + .id = 10003, + .name = "rrdp.retry.interval", + .type = >_uint, + .offset = offsetof(struct rpki_config, rrdp.retry.interval), + .doc = "Period (in seconds) to wait between retries after an error ocurred fetching RRDP files", + .min = 0, + .max = UINT_MAX, }, /* HTTP requests parameters */ @@ -687,6 +733,8 @@ set_default_values(void) rpki_config.rsync.enabled = true; rpki_config.rsync.priority = 50; rpki_config.rsync.strategy = RSYNC_ROOT; + rpki_config.rsync.retry.count = 1; + rpki_config.rsync.retry.interval = 3; rpki_config.rsync.program = strdup("rsync"); if (rpki_config.rsync.program == NULL) { error = pr_enomem(); @@ -705,6 +753,8 @@ set_default_values(void) rpki_config.rrdp.enabled = true; rpki_config.rrdp.priority = 50; + rpki_config.rrdp.retry.count = 1; + rpki_config.rrdp.retry.interval = 3; rpki_config.http.user_agent = strdup(PACKAGE_NAME "/" PACKAGE_VERSION); if (rpki_config.http.user_agent == NULL) { @@ -1006,6 +1056,18 @@ config_get_rsync_strategy(void) return rpki_config.rsync.strategy; } +unsigned int +config_get_rsync_retry_count(void) +{ + return rpki_config.rsync.retry.count; +} + +unsigned int +config_get_rsync_retry_interval(void) +{ + return rpki_config.rsync.retry.interval; +} + char * config_get_rsync_program(void) { @@ -1043,6 +1105,18 @@ config_get_rrdp_priority(void) return rpki_config.rrdp.priority; } +unsigned int +config_get_rrdp_retry_count(void) +{ + return rpki_config.rrdp.retry.count; +} + +unsigned int +config_get_rrdp_retry_interval(void) +{ + return rpki_config.rrdp.retry.interval; +} + char const * config_get_http_user_agent(void) { diff --git a/src/config.h b/src/config.h index 07926cd0..31ca9cd0 100644 --- a/src/config.h +++ b/src/config.h @@ -42,10 +42,14 @@ enum log_output config_get_log_output(void); bool config_get_rsync_enabled(void); unsigned int config_get_rsync_priority(void); enum rsync_strategy config_get_rsync_strategy(void); +unsigned int config_get_rsync_retry_count(void); +unsigned int config_get_rsync_retry_interval(void); char *config_get_rsync_program(void); struct string_array const *config_get_rsync_args(bool); bool config_get_rrdp_enabled(void); unsigned int config_get_rrdp_priority(void); +unsigned int config_get_rrdp_retry_count(void); +unsigned int config_get_rrdp_retry_interval(void); char const *config_get_output_roa(void); char const *config_get_output_bgpsec(void); unsigned int config_get_asn1_decode_max_stack(void); diff --git a/src/http/http.c b/src/http/http.c index adafa976..a55faf08 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -142,11 +142,11 @@ __http_download_file(struct rpki_uri *uri, http_write_cb cb, error = create_dir_recursive(uri_get_local(uri)); if (error) - return error; + return ENSURE_NEGATIVE(error); error = file_write(uri_get_local(uri), &out, &stat); if (error) - return error; + return ENSURE_NEGATIVE(error); error = http_easy_init(&handler); if (error) @@ -165,11 +165,11 @@ __http_download_file(struct rpki_uri *uri, http_write_cb cb, file_close(out); /* Error 0 it's ok */ - return error; + return ENSURE_NEGATIVE(error); close_file: file_close(out); delete_dir_recursive_bottom_up(uri_get_local(uri)); - return error; + return ENSURE_NEGATIVE(error); } /* diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c index 661412d8..861ad929 100644 --- a/src/rrdp/rrdp_parser.c +++ b/src/rrdp/rrdp_parser.c @@ -111,6 +111,37 @@ write_local(unsigned char *content, size_t size, size_t nmemb, void *arg) return read; } +static int +download_file(struct rpki_uri *uri, long last_update) +{ + unsigned int retries; + int error; + + retries = 0; + do { + if (last_update > 0) + error = http_download_file_with_ims(uri, write_local, + last_update); + else + error = http_download_file(uri, write_local); + + /* Remember: positive values are expected */ + if (error >= 0) + return error; + + if (retries == config_get_rrdp_retry_count()) { + pr_info("Max RRDP retries (%u) reached, won't retry again.", + retries); + return error; + } + pr_info("Retrying RRDP file download in %u seconds, %u attempts remaining.", + config_get_rrdp_retry_interval(), + config_get_rrdp_retry_count() - retries); + retries++; + sleep(config_get_rrdp_retry_interval()); + } while (true); +} + /* Trim @from, setting the result at @result pointer */ static int trim(char *from, char **result, size_t *result_size) @@ -1042,7 +1073,7 @@ process_delta(struct delta_head *delta_head, void *arg) if (error) return error; - error = http_download_file(uri, write_local); + error = download_file(uri, 0); if (error) goto release_uri; @@ -1079,7 +1110,7 @@ rrdp_parse_notification(struct rpki_uri *uri, if (error && error != -ENOENT) return error; - error = http_download_file_with_ims(uri, write_local, last_update); + error = download_file(uri, last_update); if (error < 0) return error; @@ -1128,7 +1159,7 @@ rrdp_parse_snapshot(struct update_notification *parent, if (error) return error; - error = http_download_file(uri, write_local); + error = download_file(uri, 0); if (error) goto release_uri; diff --git a/src/rsync/rsync.c b/src/rsync/rsync.c index 1e4d8143..a6bddc4c 100644 --- a/src/rsync/rsync.c +++ b/src/rsync/rsync.c @@ -331,48 +331,65 @@ do_rsync(struct rpki_uri *uri, bool is_ta) /* Descriptors to pipe stderr (first element) and stdout (second) */ int fork_fds[2][2]; pid_t child_pid; + unsigned int retries; int child_status; int error; - child_status = 0; - error = create_dir_recursive(uri_get_local(uri)); - if (error) - return error; - - error = create_pipes(fork_fds); - if (error) - return error; + retries = 0; + do { + child_status = 0; + error = create_dir_recursive(uri_get_local(uri)); + if (error) + return error; - /* We need to fork because execvp() magics the thread away. */ - child_pid = fork(); - if (child_pid == 0) { - /* This code is run by the child. */ - handle_child_thread(uri, is_ta, fork_fds); - } + error = create_pipes(fork_fds); + if (error) + return error; - /* This code is run by us. */ - error = read_pipes(fork_fds); - if (error) - return error; + /* We need to fork because execvp() magics the thread away. */ + child_pid = fork(); + if (child_pid == 0) { + /* This code is run by the child. */ + handle_child_thread(uri, is_ta, fork_fds); + } - error = waitpid(child_pid, &child_status, 0); - do { - if (error == -1) { - error = errno; - pr_err("The rsync sub-process returned error %d (%s)", - error, strerror(error)); - if (child_status > 0) - break; + /* This code is run by us. */ + error = read_pipes(fork_fds); + if (error) return error; - } - } while (0); - if (WIFEXITED(child_status)) { - /* Happy path (but also sad path sometimes). */ - error = WEXITSTATUS(child_status); - pr_debug("Child terminated with error code %d.", error); - return error; - } + error = waitpid(child_pid, &child_status, 0); + do { + if (error == -1) { + error = errno; + pr_err("The rsync sub-process returned error %d (%s)", + error, strerror(error)); + if (child_status > 0) + break; + return error; + } + } while (0); + + if (WIFEXITED(child_status)) { + /* Happy path (but also sad path sometimes). */ + error = WEXITSTATUS(child_status); + pr_debug("Child terminated with error code %d.", error); + if (!error) + return 0; + if (retries == config_get_rsync_retry_count()) { + pr_info("Max RSYNC retries (%u) reached, won't retry again.", + retries); + return error; + } + pr_info("Retrying RSYNC in %u seconds, %u attempts remaining.", + config_get_rsync_retry_interval(), + config_get_rsync_retry_count() - retries); + retries++; + sleep(config_get_rsync_retry_interval()); + continue; + } + break; + } while (true); if (WIFSIGNALED(child_status)) { switch (WTERMSIG(child_status)) { -- 2.47.2