From: pcarana Date: Wed, 13 May 2020 23:43:43 +0000 (-0500) Subject: Allow to work with cache on requests errors, common func to get date X-Git-Tag: v1.3.0~21 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=db73dac49fbb556cc01fd59db891c5d8e23ae73e;p=thirdparty%2FFORT-validator.git Allow to work with cache on requests errors, common func to get date + Create common function to get the current date and time. + Identify request errors, specifically after trying to fetch data via http/rsync without success. This helps to identify if a whole repository can't be downloaded after a considerable time (it can be configured). + Allow to work with local files even when there was a download error. + Add 'stale-repository-period' argument to set the time period that must lapse to warn about stale repositories (this will be logged to the operation log). --- diff --git a/src/Makefile.am b/src/Makefile.am index 6a10b990..103f2790 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -24,6 +24,7 @@ fort_SOURCES += nid.h nid.c fort_SOURCES += notify.c notify.h fort_SOURCES += output_printer.h output_printer.c fort_SOURCES += random.h random.c +fort_SOURCES += reqs_errors.h reqs_errors.c fort_SOURCES += resource.h resource.c fort_SOURCES += rpp.h rpp.c fort_SOURCES += sorted_array.h sorted_array.c diff --git a/src/common.c b/src/common.c index 0888e12e..bf25d89c 100644 --- a/src/common.c +++ b/src/common.c @@ -399,3 +399,16 @@ release_str: free(work_loc); return error; } + +int +get_current_time(time_t *result) +{ + time_t now; + + now = time(NULL); + if (now == ((time_t) -1)) + return pr_errno(errno, "Error getting the current time"); + + *result = now; + return 0; +} diff --git a/src/common.h b/src/common.h index 7c07ac64..0606651f 100644 --- a/src/common.h +++ b/src/common.h @@ -4,6 +4,7 @@ #include #include #include +#include #include /* "I think that this is not supposed to be implemented." */ @@ -23,6 +24,12 @@ * still coexist in most scenarios. */ #define ENOTHTTPS 3175 + +/* + * A request made to a server (eg. rsync, http) has failed, even after retrying + */ +#define EREQFAILED 3176 + /* * If you're wondering why I'm not using -abs(error), it's because abs(INT_MIN) * overflows, so gcc complains sometimes. @@ -54,4 +61,6 @@ char const *addr2str6(struct in6_addr const *, char *); int create_dir_recursive(char const *); int delete_dir_recursive_bottom_up(char const *); +int get_current_time(time_t *); + #endif /* SRC_RTR_COMMON_H_ */ diff --git a/src/config.c b/src/config.c index e9b95dc4..e1396038 100644 --- a/src/config.c +++ b/src/config.c @@ -150,6 +150,9 @@ struct rpki_config { /* ASN1 decoder max stack size allowed */ unsigned int asn1_decode_max_stack; + + /* Time period that must lapse to warn about a stale repository */ + unsigned int stale_repository_period; }; static void print_usage(FILE *, bool); @@ -560,6 +563,15 @@ static const struct option_field options[] = { .min = 1, .max = UINT_MAX, }, + { + .id = 8001, + .name = "stale-repository-period", + .type = >_uint, + .offset = offsetof(struct rpki_config, stale_repository_period), + .doc = "Time period that must lapse to warn about stale repositories", + .min = 0, + .max = UINT_MAX, + }, { 0 }, }; @@ -788,6 +800,7 @@ set_default_values(void) rpki_config.output.bgpsec = NULL; rpki_config.asn1_decode_max_stack = 4096; /* 4kB */ + rpki_config.stale_repository_period = 43200; /* 12 hours */ return 0; revert_flat_array: @@ -1191,6 +1204,12 @@ config_get_asn1_decode_max_stack(void) return rpki_config.asn1_decode_max_stack; } +unsigned int +config_get_stale_repository_period(void) +{ + return rpki_config.stale_repository_period; +} + void config_set_rsync_enabled(bool value) { diff --git a/src/config.h b/src/config.h index a15f1774..a60c57be 100644 --- a/src/config.h +++ b/src/config.h @@ -54,6 +54,7 @@ unsigned int config_get_rrdp_retry_interval(void); char const *config_get_output_roa(void); char const *config_get_output_bgpsec(void); unsigned int config_get_asn1_decode_max_stack(void); +unsigned int config_get_stale_repository_period(void); /* * Public, so that work-offline can set them, or (to be deprecated) diff --git a/src/crypto/base64.c b/src/crypto/base64.c index 3366e72b..cb3f0c5f 100644 --- a/src/crypto/base64.c +++ b/src/crypto/base64.c @@ -274,5 +274,5 @@ base64url_encode(unsigned char const *in, int in_len, char **result) return 0; free_mem: BIO_free_all(b64); - return error ? error_ul2i(error) : -ENOMEM;; + return error ? error_ul2i(error) : -ENOMEM; } diff --git a/src/http/http.c b/src/http/http.c index ee179e53..9f15a8b7 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -107,7 +107,7 @@ curl_err_string(struct http_handler *handler, CURLcode res) */ static int http_fetch(struct http_handler *handler, char const *uri, long *response_code, - long *cond_met, http_write_cb cb, void *arg) + long *cond_met, bool log_operation, http_write_cb cb, void *arg) { CURLcode res; long unmet = 0; @@ -143,8 +143,15 @@ http_fetch(struct http_handler *handler, char const *uri, long *response_code, return pr_err("Error requesting URL %s (received HTTP code %ld): %s", uri, *response_code, curl_err_string(handler, res)); - return pr_err("Error requesting URL %s: %s", uri, + /* FIXME (NOW) Always log to validation log */ + pr_err("[VALIDATION] Error requesting URL %s: %s", uri, curl_err_string(handler, res)); + /* FIXME (NOW) and send to operation log when requested */ + if (log_operation) + pr_err("[OPERATION] Error requesting URL %s: %s", uri, + curl_err_string(handler, res)); + + return EREQFAILED; } static void @@ -155,7 +162,7 @@ http_easy_cleanup(struct http_handler *handler) static int __http_download_file(struct rpki_uri *uri, http_write_cb cb, - long *response_code, long ims_value, long *cond_met) + long *response_code, long ims_value, long *cond_met, bool log_operation) { struct http_handler handler; struct stat stat; @@ -188,7 +195,7 @@ __http_download_file(struct rpki_uri *uri, http_write_cb cb, } error = http_fetch(&handler, uri_get_global(uri), response_code, - cond_met, cb, out); + cond_met, log_operation, cb, out); http_easy_cleanup(&handler); file_close(out); @@ -208,14 +215,16 @@ delete_dir: * from local @uri. The @cb should be utilized to write into a file; the file * will be sent to @cb as the last argument (its a FILE reference). * - * Regular return value: 0 on success, any other value is an error. + * Return values: 0 on success, negative value on error, EREQFAILED if the + * request to the server failed. */ int -http_download_file(struct rpki_uri *uri, http_write_cb cb) +http_download_file(struct rpki_uri *uri, http_write_cb cb, bool log_operation) { long response; long cond_met; - return __http_download_file(uri, cb, &response, 0, &cond_met); + return __http_download_file(uri, cb, &response, 0, &cond_met, + log_operation); } /* @@ -225,19 +234,22 @@ http_download_file(struct rpki_uri *uri, http_write_cb cb) * of @value (if @value is 0, the header isn't set). * * Returns: + * EREQFAILED the request to the server has failed. * > 0 file was requested but wasn't downloaded since the server didn't sent * a response due to its policy using the header 'If-Modified-Since'. * = 0 file successfully downloaded. * < 0 an actual error happened. */ int -http_download_file_with_ims(struct rpki_uri *uri, http_write_cb cb, long value) +http_download_file_with_ims(struct rpki_uri *uri, http_write_cb cb, long value, + bool log_operation) { long response; long cond_met; int error; - error = __http_download_file(uri, cb, &response, value, &cond_met); + error = __http_download_file(uri, cb, &response, value, &cond_met, + log_operation); if (error) return error; @@ -256,6 +268,7 @@ http_download_file_with_ims(struct rpki_uri *uri, http_write_cb cb, long value) if (cond_met) return 0; - return __http_download_file(uri, cb, &response, 0, &cond_met); + return __http_download_file(uri, cb, &response, 0, &cond_met, + log_operation); } diff --git a/src/http/http.h b/src/http/http.h index 073de4ea..3ec2d305 100644 --- a/src/http/http.h +++ b/src/http/http.h @@ -1,6 +1,7 @@ #ifndef SRC_HTTP_HTTP_H_ #define SRC_HTTP_HTTP_H_ +#include #include #include "uri.h" @@ -9,7 +10,7 @@ int http_init(void); void http_cleanup(void); typedef size_t (http_write_cb)(unsigned char *, size_t, size_t, void *); -int http_download_file(struct rpki_uri *, http_write_cb); -int http_download_file_with_ims(struct rpki_uri *, http_write_cb, long); +int http_download_file(struct rpki_uri *, http_write_cb, bool); +int http_download_file_with_ims(struct rpki_uri *, http_write_cb, long, bool); #endif /* SRC_HTTP_HTTP_H_ */ diff --git a/src/main.c b/src/main.c index 87da64f4..fd91f86b 100644 --- a/src/main.c +++ b/src/main.c @@ -3,6 +3,7 @@ #include "debug.h" #include "extension.h" #include "nid.h" +#include "reqs_errors.h" #include "thread_var.h" #include "http/http.h" #include "rtr/rtr.h" @@ -23,8 +24,14 @@ start_rtr_server(void) if (error) goto vrps_cleanup; + error = reqs_errors_init(); + if (error) + goto db_rrdp_cleanup; + error = rtr_listen(); + reqs_errors_cleanup(); +db_rrdp_cleanup: db_rrdp_cleanup(); vrps_cleanup: vrps_destroy(); diff --git a/src/object/certificate.c b/src/object/certificate.c index c7d5c20b..35a79771 100644 --- a/src/object/certificate.c +++ b/src/object/certificate.c @@ -1950,7 +1950,7 @@ use_access_method(struct sia_ca_uris *sia_uris, * children CAs being the same as the parent CA. * * Two possible scenarios arise: - * 1) CA Parent didn't utilized (or didn't had) and RRDP update + * 1) CA Parent didn't utilized (or didn't had) an RRDP update * notification URI. * 2) CA Parent successfully utilized an RRDP update notification URI. * @@ -1985,6 +1985,7 @@ use_access_method(struct sia_ca_uris *sia_uris, } if (primary_rrdp) { + working_repo_push(uri_get_global(sia_uris->rpkiNotify.uri)); if (error != -EPERM) pr_info("Couldn't fetch data from RRDP repository '%s', trying to fetch data now from '%s'.", uri_get_global(sia_uris->rpkiNotify.uri), @@ -1994,13 +1995,18 @@ use_access_method(struct sia_ca_uris *sia_uris, uri_get_global(sia_uris->rpkiNotify.uri), uri_get_global(sia_uris->caRepository.uri)); } else { + working_repo_push(uri_get_global(sia_uris->caRepository.uri)); pr_info("Couldn't fetch data from repository '%s', trying to fetch data now from RRDP '%s'.", uri_get_global(sia_uris->caRepository.uri), uri_get_global(sia_uris->rpkiNotify.uri)); } (*rsync_utilized) = primary_rrdp; - return cb_secondary(sia_uris); + error = cb_secondary(sia_uris); + /* No need to remember the working repository anymore */ + working_repo_pop(); + + return error; } /** Boilerplate code for CA certificate validation and recursive traversal. */ @@ -2143,6 +2149,13 @@ certificate_traverse(struct rpp *rpp_parent, struct rpki_uri *cert_uri) if (error == 0 || !mft_retry) break; + /* + * Don't reach here if: + * - Manifest is valid. + * - Working with local files due to a download error. + * - RRDP was utilized to fetch the manifest. + * - There was a previous attempt to re-fetch the repository. + */ pr_info("Retrying repository download to discard 'transient inconsistency' manifest issue (see RFC 6481 section 5) '%s'", uri_get_printable(sia_uris.caRepository.uri)); error = download_files(sia_uris.caRepository.uri, false, true); diff --git a/src/object/manifest.c b/src/object/manifest.c index bacbfce4..d1e160b5 100644 --- a/src/object/manifest.c +++ b/src/object/manifest.c @@ -3,6 +3,7 @@ #include #include "algorithm.h" +#include "common.h" #include "log.h" #include "thread_var.h" #include "asn1/decode.h" @@ -40,6 +41,7 @@ validate_dates(GeneralizedTime_t *this, GeneralizedTime_t *next) time_t now; struct tm thisUpdate_tm; struct tm nextUpdate_tm; + int error; /* * BTW: We only need the tm variables for error messages, which are @@ -58,9 +60,10 @@ validate_dates(GeneralizedTime_t *this, GeneralizedTime_t *next) TM_ARGS(nextUpdate_tm)); } - now = time(NULL); - if (now == ((time_t) -1)) - return pr_errno(errno, "Error getting the current time"); + now = 0; + error = get_current_time(&now); + if (error) + return error; if (difftime(now, thisUpdate) < 0) { return pr_err( diff --git a/src/object/tal.c b/src/object/tal.c index 6e1433c6..39029a05 100644 --- a/src/object/tal.c +++ b/src/object/tal.c @@ -18,6 +18,7 @@ #include "line_file.h" #include "log.h" #include "random.h" +#include "reqs_errors.h" #include "state.h" #include "thread_var.h" #include "validation_handler.h" @@ -450,7 +451,8 @@ write_http_cer(unsigned char *content, size_t size, size_t nmemb, void *arg) static int handle_https_uri(struct rpki_uri *uri) { - return http_download_file(uri, write_http_cer); + return http_download_file(uri, write_http_cer, + reqs_errors_log_uri(uri_get_global(uri))); } /** @@ -494,10 +496,11 @@ handle_tal_uri(struct tal *tal, struct rpki_uri *uri, void *arg) else error = handle_https_uri(uri); + /* FIXME (NOW) Try to work with local data on the first run? */ if (error) { validation_destroy(state); return pr_warn("TAL '%s' could not be downloaded.", - uri_get_printable(uri));; + uri_get_printable(uri)); } pr_debug("TAL URI '%s' {", uri_get_printable(uri)); @@ -578,6 +581,8 @@ do_file_validation(void *thread_arg) fnstack_init(); fnstack_push(thread->tal_file); + working_repo_init(); + error = tal_load(thread->tal_file, &tal); if (error) goto end; @@ -593,6 +598,7 @@ do_file_validation(void *thread_arg) tal_destroy(tal); end: + working_repo_cleanup(); fnstack_cleanup(); thread->exit_status = error; return NULL; @@ -704,6 +710,9 @@ perform_standalone_validation(struct db_table *table) /* The parameter isn't needed anymore */ free(param); + /* FIXME (NOW) Clarify if this really belongs here */ + reqs_errors_log_summary(); + /* One thread has errors, validation can't keep the resulting table */ if (t_error) return t_error; diff --git a/src/reqs_errors.c b/src/reqs_errors.c new file mode 100644 index 00000000..42b2f4bd --- /dev/null +++ b/src/reqs_errors.c @@ -0,0 +1,227 @@ +#include "reqs_errors.h" + +#include +#include +#include "data_structure/uthash_nonfatal.h" +#include "common.h" +#include "config.h" +#include "log.h" +#include "thread_var.h" + +struct error_uri { + /* Key */ + char *uri; + /* Date when the first attempt was made */ + time_t first_attempt; + /* Related URI (points to a key of another element) */ + char *uri_related; + UT_hash_handle hh; +}; + +static struct error_uri *err_uris_db; + +/* Prepare for multithreading. */ +static pthread_rwlock_t db_lock; + +static int +error_uri_create(char const *uri, struct error_uri **err_uri) +{ + struct error_uri *tmp; + int error; + + tmp = malloc(sizeof(struct error_uri)); + if (tmp == NULL) + return pr_enomem(); + + /* Needed by uthash */ + memset(tmp, 0, sizeof(struct error_uri)); + + tmp->uri = strdup(uri); + if (tmp->uri == NULL) { + error = pr_enomem(); + goto release_tmp; + } + + error = get_current_time(&tmp->first_attempt); + if (error) + goto release_uri; + + *err_uri = tmp; + return 0; +release_uri: + free(tmp->uri); +release_tmp: + free(tmp); + return error; +} + +static void +error_uri_destroy(struct error_uri *err_uri) +{ + free(err_uri->uri); + free(err_uri); +} + +int +reqs_errors_init(void) +{ + int error; + + error = pthread_rwlock_init(&db_lock, NULL); + if (error) + return pr_errno(error, "pthread_rwlock_init() errored"); + + err_uris_db = NULL; + + return 0; +} + +void +reqs_errors_cleanup(void) +{ + /* Remove all the uris */ + struct error_uri *node, *tmp; + + HASH_ITER(hh, err_uris_db, node, tmp) { + HASH_DEL(err_uris_db, node); + error_uri_destroy(node); + } + + pthread_rwlock_destroy(&db_lock); /* Nothing to do with error code */ +} + +static struct error_uri * +find_error_uri(char const *search) +{ + struct error_uri *found; + + rwlock_read_lock(&db_lock); + HASH_FIND_STR(err_uris_db, search, found); + rwlock_unlock(&db_lock); + + return found; +} + +static void +set_working_repo(struct error_uri *err_uri) +{ + struct error_uri *ref; + char const *work_uri; + + err_uri->uri_related = NULL; + work_uri = working_repo_peek(); + if (work_uri == NULL) + return; + + ref = find_error_uri(work_uri); + if (ref == NULL) + return; + + err_uri->uri_related = ref->uri; +} + +int +reqs_errors_add_uri(char const *uri) +{ + struct error_uri *new_uri, *found_uri; + int error; + + /* Don't overwrite if it already exists */ + found_uri = find_error_uri(uri); + if (found_uri != NULL) + return 0; + + new_uri = NULL; + error = error_uri_create(uri, &new_uri); + if (error) + return error; + + set_working_repo(new_uri); + + rwlock_write_lock(&db_lock); + HASH_ADD_KEYPTR(hh, err_uris_db, new_uri->uri, strlen(new_uri->uri), + new_uri); + rwlock_unlock(&db_lock); + + return 0; +} + +void +reqs_errors_rem_uri(char const *uri) +{ + struct error_uri *found_uri; + + found_uri = find_error_uri(uri); + if (found_uri == NULL) + return; + + /* Remove also its related repository */ + if (found_uri->uri_related != NULL) + reqs_errors_rem_uri(found_uri->uri_related); + + rwlock_write_lock(&db_lock); + HASH_DELETE(hh, err_uris_db, found_uri); + error_uri_destroy(found_uri); + rwlock_unlock(&db_lock); +} + +bool +reqs_errors_log_uri(char const *uri) +{ + struct error_uri *node; + time_t now; + int error; + + now = 0; + error = get_current_time(&now); + if (error) + return false; + + node = find_error_uri(uri); + if (node == NULL) + return false; + + return difftime(now, node->first_attempt) >= + (double)config_get_stale_repository_period(); +} + +/* + * Logs the repository errors and return the number of current errors. + */ +void +reqs_errors_log_summary(void) +{ + /* Remove all the uris */ + struct error_uri *node, *tmp; + time_t now; + bool first; + int error; + + first = true; + now = 0; + error = get_current_time(&now); + if (error) + return; + + /* + * FIXME (NOW) Log a friendly warning, listing the URIs that error'd. + * The time diff (difftime) must be from the same date when + * reqs_errors_log_uri was called. + */ + rwlock_read_lock(&db_lock); + HASH_ITER(hh, err_uris_db, node, tmp) { + if (difftime(now, node->first_attempt) < + (double)config_get_stale_repository_period()) + continue; + if (first) { + /* FIXME (NOW) Send to operation log */ + pr_warn("The following repositories URIs couldn't be fetched (it can be a local issue or a server issue), please review previous log messages related to such URIs/servers:"); + first = false; + } + /* FIXME (NOW) Send to operation log */ + pr_warn("- '%s': can't be downloaded since %s", node->uri, + asctime(localtime(&node->first_attempt))); + } + + rwlock_unlock(&db_lock); +} diff --git a/src/reqs_errors.h b/src/reqs_errors.h new file mode 100644 index 00000000..a7ec970d --- /dev/null +++ b/src/reqs_errors.h @@ -0,0 +1,16 @@ +#ifndef SRC_REQS_ERRORS_H_ +#define SRC_REQS_ERRORS_H_ + +#include + +int reqs_errors_init(void); +void reqs_errors_cleanup(void); + +int reqs_errors_add_uri(char const *); +void reqs_errors_rem_uri(char const *); + +bool reqs_errors_log_uri(char const *); + +void reqs_errors_log_summary(void); + +#endif /* SRC_REQS_ERRORS_H_ */ diff --git a/src/rrdp/db/db_rrdp_uris.c b/src/rrdp/db/db_rrdp_uris.c index e7ef8fce..c67f2c85 100644 --- a/src/rrdp/db/db_rrdp_uris.c +++ b/src/rrdp/db/db_rrdp_uris.c @@ -4,9 +4,9 @@ #include #include #include "data_structure/uthash_nonfatal.h" +#include "common.h" #include "log.h" #include "thread_var.h" -#include "visited_uris.h" struct uris_table { /* Key */ @@ -236,25 +236,13 @@ db_rrdp_uris_get_last_update(char const *uri, long *date) return 0; } -static int -get_current_time(long *result) -{ - time_t now; - - now = time(NULL); - if (now == ((time_t) -1)) - return pr_errno(errno, "Error getting the current time"); - - *result = now; - return 0; -} - /* Set the last update to now */ int db_rrdp_uris_set_last_update(char const *uri) { struct db_rrdp_uri *uris; struct uris_table *found; + time_t now; int error; uris = NULL; @@ -263,7 +251,14 @@ db_rrdp_uris_set_last_update(char const *uri) return error; RET_NOT_FOUND_URI(uris, uri, found) - return get_current_time(&found->last_update); + + now = 0; + error = get_current_time(&now); + if (error) + return error; + + found->last_update = (long)now; + return 0; } int diff --git a/src/rrdp/rrdp_loader.c b/src/rrdp/rrdp_loader.c index 304218de..e5ae490d 100644 --- a/src/rrdp/rrdp_loader.c +++ b/src/rrdp/rrdp_loader.c @@ -4,15 +4,17 @@ #include "rrdp/rrdp_objects.h" #include "rrdp/rrdp_parser.h" #include "rsync/rsync.h" +#include "common.h" #include "config.h" #include "log.h" +#include "reqs_errors.h" #include "thread_var.h" #include "visited_uris.h" /* Fetch and process the deltas from the @notification */ static int process_diff_serial(struct update_notification *notification, - struct visited_uris **visited) + bool log_operation, struct visited_uris **visited) { unsigned long serial; int error; @@ -26,12 +28,13 @@ process_diff_serial(struct update_notification *notification, if (error) return error; - return rrdp_process_deltas(notification, serial, *visited); + return rrdp_process_deltas(notification, serial, *visited, + log_operation); } /* Fetch and process the snapshot from the @notification */ static int -process_snapshot(struct update_notification *notification, +process_snapshot(struct update_notification *notification, bool log_operation, struct visited_uris **visited) { struct visited_uris *tmp; @@ -42,7 +45,7 @@ process_snapshot(struct update_notification *notification, if (error) return error; - error = rrdp_parse_snapshot(notification, tmp); + error = rrdp_parse_snapshot(notification, tmp, log_operation); if (error) { visited_uris_refput(tmp); return error; @@ -109,6 +112,7 @@ rrdp_load(struct rpki_uri *uri) struct visited_uris *visited; rrdp_req_status_t requested; rrdp_uri_cmp_result_t res; + bool log_operation; int error, upd_error; if (!config_get_rrdp_enabled()) @@ -116,7 +120,8 @@ rrdp_load(struct rpki_uri *uri) /* Avoid multiple requests on the same run */ requested = RRDP_URI_REQ_UNVISITED; - error = db_rrdp_uris_get_request_status(uri_get_global(uri), &requested); + error = db_rrdp_uris_get_request_status(uri_get_global(uri), + &requested); if (error && error != -ENOENT) return error; @@ -130,14 +135,15 @@ rrdp_load(struct rpki_uri *uri) return -EPERM; } - error = rrdp_parse_notification(uri, &upd_notification); + log_operation = reqs_errors_log_uri(uri_get_global(uri)); + error = rrdp_parse_notification(uri, log_operation, &upd_notification); if (error) - goto upd_error; + goto upd_end; /* No updates at the file (yet), didn't pushed to fnstack */ if (upd_notification == NULL) { pr_debug("No updates yet at '%s'.", uri_get_global(uri)); - return 0; + goto upd_end; } error = db_rrdp_uris_cmp(uri_get_global(uri), @@ -155,12 +161,14 @@ rrdp_load(struct rpki_uri *uri) error = remove_rrdp_uri_files(upd_notification->uri); if (error) goto upd_destroy; - error = process_snapshot(upd_notification, &visited); + error = process_snapshot(upd_notification, log_operation, + &visited); if (error) goto upd_destroy; break; case RRDP_URI_DIFF_SERIAL: - error = process_diff_serial(upd_notification, &visited); + error = process_diff_serial(upd_notification, log_operation, + &visited); if (!error) { visited_uris_refget(visited); break; @@ -168,7 +176,8 @@ rrdp_load(struct rpki_uri *uri) /* Something went wrong, use snapshot */ pr_info("There was an error processing RRDP deltas, using the snapshot instead."); case RRDP_URI_NOTFOUND: - error = process_snapshot(upd_notification, &visited); + error = process_snapshot(upd_notification, log_operation, + &visited); if (error) goto upd_destroy; break; @@ -198,14 +207,26 @@ upd_destroy: update_notification_destroy(upd_notification); fnstack_pop(); /* Pop from rrdp_parse_notification */ } -upd_error: - /* Don't fall here on success */ - if (error) { - /* Reset RSYNC visited URIs, this may force the update */ - reset_downloaded(); - upd_error = mark_rrdp_uri_request_err(uri_get_global(uri)); +upd_end: + /* Just return on success */ + if (!error) { + /* The repository URI is the notification file URI */ + reqs_errors_rem_uri(uri_get_global(uri)); + return 0; + } + + /* Request failed, store the repository URI */ + if (error == EREQFAILED) { + upd_error = reqs_errors_add_uri(uri_get_global(uri)); if (upd_error) return upd_error; } + + /* Reset RSYNC visited URIs, this may force the update */ + reset_downloaded(); + upd_error = mark_rrdp_uri_request_err(uri_get_global(uri)); + if (upd_error) + return upd_error; + return error; } diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c index adb4c8d4..c01cfdbf 100644 --- a/src/rrdp/rrdp_parser.c +++ b/src/rrdp/rrdp_parser.c @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -75,6 +74,7 @@ struct rdr_delta_ctx { struct proc_upd_args { struct update_notification *parent; struct visited_uris *visited_uris; + bool log_operation; }; static int @@ -110,7 +110,7 @@ write_local(unsigned char *content, size_t size, size_t nmemb, void *arg) } static int -download_file(struct rpki_uri *uri, long last_update) +download_file(struct rpki_uri *uri, long last_update, bool log_operation) { unsigned int retries; int error; @@ -119,9 +119,10 @@ download_file(struct rpki_uri *uri, long last_update) do { if (last_update > 0) error = http_download_file_with_ims(uri, write_local, - last_update); + last_update, log_operation); else - error = http_download_file(uri, write_local); + error = http_download_file(uri, write_local, + log_operation); /* Remember: positive values are expected */ if (error >= 0) @@ -130,7 +131,13 @@ download_file(struct rpki_uri *uri, long last_update) if (retries == config_get_rrdp_retry_count()) { pr_info("Max RRDP retries (%u) reached fetching '%s', won't retry again.", retries, uri_get_global(uri)); - return error; + /* + * Since distinct files can be downloaded (notification, + * snapshot, delta) just return the error and let the + * caller to add only the update notification URI to + * the request errors DB. + */ + return EREQFAILED; } pr_info("Retrying RRDP file download '%s' in %u seconds, %u attempts remaining.", uri_get_global(uri), @@ -996,9 +1003,9 @@ xml_read_delta(xmlTextReaderPtr reader, void *arg) } static int -parse_delta(struct rpki_uri *uri, struct delta_head *parents_data, void *arg) +parse_delta(struct rpki_uri *uri, struct delta_head *parents_data, + struct proc_upd_args *args) { - struct proc_upd_args *args = arg; struct rdr_delta_ctx ctx; struct delta *delta; struct doc_data *expected_data; @@ -1032,6 +1039,7 @@ pop_fnstack: static int process_delta(struct delta_head *delta_head, void *arg) { + struct proc_upd_args *args = arg; struct rpki_uri *uri; struct doc_data *head_data; int error; @@ -1044,11 +1052,11 @@ process_delta(struct delta_head *delta_head, void *arg) if (error) return error; - error = download_file(uri, 0); + error = download_file(uri, 0, args->log_operation); if (error) goto release_uri; - error = parse_delta(uri, delta_head, arg); + error = parse_delta(uri, delta_head, args); delete_from_uri(uri, NULL); /* Error 0 its ok */ @@ -1066,7 +1074,7 @@ release_uri: * 'If-Modified-Since' header, return 0 and set @result to NULL. */ int -rrdp_parse_notification(struct rpki_uri *uri, +rrdp_parse_notification(struct rpki_uri *uri, bool log_operation, struct update_notification **result) { long last_update; @@ -1081,10 +1089,14 @@ rrdp_parse_notification(struct rpki_uri *uri, if (error && error != -ENOENT) return error; - error = download_file(uri, last_update); + error = download_file(uri, last_update, log_operation); if (error < 0) return error; + /* Request error, stop processing to handle as such */ + if (error == EREQFAILED) + return error; + /* * Mark as visited, if it doesn't exists yet, there's no problem since * this is probably the first time is visited (first run), so it will @@ -1116,7 +1128,7 @@ rrdp_parse_notification(struct rpki_uri *uri, int rrdp_parse_snapshot(struct update_notification *parent, - struct visited_uris *visited_uris) + struct visited_uris *visited_uris, bool log_operation) { struct proc_upd_args args; struct rpki_uri *uri; @@ -1131,7 +1143,7 @@ rrdp_parse_snapshot(struct update_notification *parent, if (error) return error; - error = download_file(uri, 0); + error = download_file(uri, 0, log_operation); if (error) goto release_uri; @@ -1146,12 +1158,14 @@ release_uri: int rrdp_process_deltas(struct update_notification *parent, - unsigned long cur_serial, struct visited_uris *visited_uris) + unsigned long cur_serial, struct visited_uris *visited_uris, + bool log_operation) { struct proc_upd_args args; args.parent = parent; args.visited_uris = visited_uris; + args.log_operation = log_operation; return deltas_head_for_each(parent->deltas_list, parent->global_data.serial, cur_serial, process_delta, &args); diff --git a/src/rrdp/rrdp_parser.h b/src/rrdp/rrdp_parser.h index 945695ce..c5bd60ad 100644 --- a/src/rrdp/rrdp_parser.h +++ b/src/rrdp/rrdp_parser.h @@ -1,14 +1,17 @@ #ifndef SRC_RRDP_RRDP_PARSER_H_ #define SRC_RRDP_RRDP_PARSER_H_ +#include #include "rrdp/rrdp_objects.h" #include "uri.h" #include "visited_uris.h" -int rrdp_parse_notification(struct rpki_uri *, struct update_notification **); -int rrdp_parse_snapshot(struct update_notification *, struct visited_uris *); +int rrdp_parse_notification(struct rpki_uri *, bool, + struct update_notification **); +int rrdp_parse_snapshot(struct update_notification *, struct visited_uris *, + bool); int rrdp_process_deltas(struct update_notification *, - unsigned long serial, struct visited_uris *); + unsigned long serial, struct visited_uris *, bool); #endif /* SRC_RRDP_RRDP_PARSER_H_ */ diff --git a/src/rsync/rsync.c b/src/rsync/rsync.c index f3dcc570..52c94481 100644 --- a/src/rsync/rsync.c +++ b/src/rsync/rsync.c @@ -11,6 +11,7 @@ #include "common.h" #include "config.h" #include "log.h" +#include "reqs_errors.h" #include "str.h" #include "thread_var.h" @@ -251,7 +252,7 @@ create_pipes(int fds[2][2]) } static void -log_buffer(char const *buffer, ssize_t read, int type) +log_buffer(char const *buffer, ssize_t read, int type, bool log_operation) { #define PRE_RSYNC "[RSYNC exec]: " char *cpy, *cur, *tmp; @@ -272,10 +273,19 @@ log_buffer(char const *buffer, ssize_t read, int type) cur = tmp + 1; continue; } - if (type == 0) - pr_err(PRE_RSYNC "%s", cur); - else - pr_info(PRE_RSYNC "%s", cur); + if (type == 0) { + /* FIXME (NOW) Send to operation log if requested */ + if (log_operation) + pr_err(PRE_RSYNC " [OPERATION]: %s", cur); + /* FIXME (NOW) Always send to validation log */ + pr_err(PRE_RSYNC " [VALIDATION]: %s", cur); + } else { + /* FIXME (NOW) Send to operation log if requested */ + if (log_operation) + pr_info(PRE_RSYNC " [OPERATION]: %s", cur); + /* FIXME (NOW) Always send to validation log */ + pr_info(PRE_RSYNC " [VALIDATION]: %s", cur); + } cur = tmp + 1; } free(cpy); @@ -283,7 +293,7 @@ log_buffer(char const *buffer, ssize_t read, int type) } static int -read_pipe(int fd_pipe[2][2], int type) +read_pipe(int fd_pipe[2][2], int type, bool log_operation) { char buffer[4096]; ssize_t count; @@ -298,14 +308,14 @@ read_pipe(int fd_pipe[2][2], int type) if (count == 0) break; - log_buffer(buffer, count, type); + log_buffer(buffer, count, type, log_operation); } close(fd_pipe[type][0]); return 0; } static int -read_pipes(int fds[2][2]) +read_pipes(int fds[2][2], bool log_operation) { int error; @@ -314,19 +324,19 @@ read_pipes(int fds[2][2]) close(fds[1][1]); /* stderr pipe */ - error = read_pipe(fds, 0); + error = read_pipe(fds, 0, log_operation); if (error) return error; /* stdout pipe */ - return read_pipe(fds, 1); + return read_pipe(fds, 1, log_operation); } /* * Downloads the @uri->global file into the @uri->local path. */ static int -do_rsync(struct rpki_uri *uri, bool is_ta) +do_rsync(struct rpki_uri *uri, bool is_ta, bool log_operation) { /* Descriptors to pipe stderr (first element) and stdout (second) */ int fork_fds[2][2]; @@ -354,7 +364,7 @@ do_rsync(struct rpki_uri *uri, bool is_ta) } /* This code is run by us. */ - error = read_pipes(fork_fds); + error = read_pipes(fork_fds, log_operation); if (error) return error; @@ -374,12 +384,15 @@ do_rsync(struct rpki_uri *uri, bool is_ta) /* Happy path (but also sad path sometimes). */ error = WEXITSTATUS(child_status); pr_debug("Child terminated with error code %d.", error); - if (!error) + if (!error) { + reqs_errors_rem_uri(uri_get_global(uri)); return 0; + } if (retries == config_get_rsync_retry_count()) { pr_info("Max RSYNC retries (%u) reached on '%s', won't retry again.", retries, uri_get_global(uri)); - return error; + + return EREQFAILED; } pr_info("Retrying RSYNC '%s' in %u seconds, %u attempts remaining.", uri_get_global(uri), @@ -434,6 +447,7 @@ download_files(struct rpki_uri *requested_uri, bool is_ta, bool force) struct validation *state; struct uri_list *visited_uris; struct rpki_uri *rsync_uri; + bool to_op_log; int error; if (!config_get_rsync_enabled()) @@ -461,11 +475,24 @@ download_files(struct rpki_uri *requested_uri, bool is_ta, bool force) pr_debug("Going to RSYNC '%s'.", uri_get_printable(rsync_uri)); - /* Don't store when "force" and if its already downloaded */ - error = do_rsync(rsync_uri, is_ta); - if (!error && - !(force && is_already_downloaded(rsync_uri, visited_uris))) + to_op_log = reqs_errors_log_uri(uri_get_global(requested_uri)); + error = do_rsync(rsync_uri, is_ta, to_op_log); + switch(error) { + case 0: + /* Don't store when "force" and if its already downloaded */ + if (!(force && is_already_downloaded(rsync_uri, visited_uris))) + error = mark_as_downloaded(rsync_uri, visited_uris); + break; + case EREQFAILED: + /* All attempts failed, avoid future requests */ + error = reqs_errors_add_uri(uri_get_global(requested_uri)); + if (error) + break; error = mark_as_downloaded(rsync_uri, visited_uris); + break; + default: + break; + } uri_refput(rsync_uri); return error; diff --git a/src/rtr/db/vrps.c b/src/rtr/db/vrps.c index aaf4ef85..5dce2fa5 100644 --- a/src/rtr/db/vrps.c +++ b/src/rtr/db/vrps.c @@ -79,6 +79,7 @@ deltagroup_cleanup(struct delta_group *group) int vrps_init(void) { + time_t now; int error; state.base = NULL; @@ -93,7 +94,12 @@ vrps_init(void) state.next_serial = START_SERIAL; /* Get the bits that'll fit in session_id */ - state.v0_session_id = time(NULL) & 0xFFFF; + now = 0; + error = get_current_time(&now); + if (error) + goto release_deltas; + state.v0_session_id = now & 0xFFFF; + /* Minus 1 to prevent same ID */ state.v1_session_id = (state.v0_session_id != 0) ? (state.v0_session_id - 1) diff --git a/src/thread_var.c b/src/thread_var.c index 077c3138..e1f18a90 100644 --- a/src/thread_var.c +++ b/src/thread_var.c @@ -12,6 +12,7 @@ static pthread_key_t state_key; static pthread_key_t filenames_key; +static pthread_key_t repository_key; struct filename_stack { /* This can be NULL. Abort all operations if this is the case. */ @@ -20,6 +21,10 @@ struct filename_stack { unsigned int size; }; +struct working_repo { + char const *uri; +}; + static void fnstack_discard(void *arg) { @@ -28,6 +33,13 @@ fnstack_discard(void *arg) free(files); } +static void +working_repo_discard(void *arg) +{ + struct working_repo *repo = arg; + free(repo); +} + /** Initializes this entire module. Call once per runtime lifetime. */ int thvar_init(void) @@ -56,6 +68,14 @@ thvar_init(void) return error; } + error = pthread_key_create(&repository_key, working_repo_discard); + if (error) { + pr_err( + "Fatal: Errcode %d while initializing the 'working repository' thread variable.", + error); + return error; + } + return 0; } @@ -197,6 +217,83 @@ fnstack_pop(void) files->len--; } +/** Initializes the current thread's working repo. Call once per thread. */ +void +working_repo_init(void) +{ + struct working_repo *repo; + int error; + + repo = malloc(sizeof(struct working_repo)); + if (repo == NULL) + return; + + repo->uri = NULL; + + error = pthread_setspecific(repository_key, repo); + if (error) + pr_err("pthread_setspecific() returned %d.", error); +} + +void +working_repo_cleanup(void) +{ + struct working_repo *repo; + int error; + + repo = pthread_getspecific(repository_key); + if (repo == NULL) + return; + + working_repo_discard(repo); + + error = pthread_setspecific(repository_key, NULL); + if (error) + pr_err("pthread_setspecific() returned %d.", error); +} + +/* + * Call whenever a certificate has more than one repository where its childs + * live (rsync or RRDP). + */ +void +working_repo_push(char const *location) +{ + struct working_repo *repo; + + repo = pthread_getspecific(repository_key); + if (repo == NULL) + return; + + repo->uri = location; +} + +char const * +working_repo_peek(void) +{ + struct working_repo *repo; + + repo = pthread_getspecific(repository_key); + + return repo == NULL ? NULL : repo->uri; +} + +/* + * Call once the certificate's repositories were downloaded (either successful + * or erroneously). + */ +void +working_repo_pop(void) +{ + struct working_repo *repo; + + repo = pthread_getspecific(repository_key); + if (repo == NULL) + return; + + repo->uri = NULL; +} + static char const * addr2str(int af, void const *addr, char *(*buffer_cb)(struct validation *)) { diff --git a/src/thread_var.h b/src/thread_var.h index 15769964..0c19e2ca 100644 --- a/src/thread_var.h +++ b/src/thread_var.h @@ -16,6 +16,13 @@ void fnstack_push_uri(struct rpki_uri *); char const *fnstack_peek(void); void fnstack_pop(void); +void working_repo_init(void); +void working_repo_cleanup(void); + +void working_repo_push(char const *); +char const *working_repo_peek(void); +void working_repo_pop(void); + /* Please remember that these functions can only be used during validations. */ char const *v4addr2str(struct in_addr const *); char const *v4addr2str2(struct in_addr const *); diff --git a/test/http_test.c b/test/http_test.c index 4487fc57..5519c4db 100644 --- a/test/http_test.c +++ b/test/http_test.c @@ -52,7 +52,8 @@ local_download(char const *url, long *response_code, struct response *resp) if (error) return error; - error = http_fetch(&handler, url, response_code, &cond, write_cb, resp); + error = http_fetch(&handler, url, response_code, &cond, false, + write_cb, resp); http_easy_cleanup(&handler); return error; } diff --git a/test/impersonator.c b/test/impersonator.c index 94fac4f5..fc944275 100644 --- a/test/impersonator.c +++ b/test/impersonator.c @@ -40,6 +40,12 @@ fnstack_peek(void) return NULL; } +void +reqs_errors_log_summary(void) +{ + /* Nothing here */ +} + char const * config_get_tal(void) {