From: pcarana Date: Fri, 13 Dec 2019 17:50:46 +0000 (-0600) Subject: Refactor RRDP URIs storage, implement session ID update. X-Git-Tag: v1.2.0~37 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=3092d5d5c17f6b119da2c8c490dbce6b0a31da6b;p=thirdparty%2FFORT-validator.git Refactor RRDP URIs storage, implement session ID update. +Delete dir daemon: detach thread, renames the directory that's going to be deleted. +Update logic (structs and relations) to remember RRDP URIs: each TAL thread will hold its own RRDP URIs, and each URI (update notification URI) will have its own visited uris struct; the main thread holds each TALs information, so that it can be accesed during every validation run. This way we know who owns what, and in case of a session ID update it's easier to remove the whole file system directory tree related to an RRDP URI. +Rename 'visited_uris' of rsync to 'rsync_visited_uris', in validation state struct. +Assure that update notification files are requested only once per cycle (in case they're found as the prefered access method). +Implement session ID update, remove all files related to the previous session ID. --- diff --git a/src/Makefile.am b/src/Makefile.am index 1debbcf4..953a6ae5 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -81,11 +81,13 @@ fort_SOURCES += resource/ip4.h resource/ip4.c fort_SOURCES += resource/ip6.h resource/ip6.c fort_SOURCES += resource/asn.h resource/asn.c -fort_SOURCES += rrdp/db_rrdp.h rrdp/db_rrdp.c fort_SOURCES += rrdp/rrdp_loader.h rrdp/rrdp_loader.c fort_SOURCES += rrdp/rrdp_objects.h rrdp/rrdp_objects.c fort_SOURCES += rrdp/rrdp_parser.h rrdp/rrdp_parser.c +fort_SOURCES += rrdp/db/db_rrdp.h rrdp/db/db_rrdp.c +fort_SOURCES += rrdp/db/db_rrdp_uris.h rrdp/db/db_rrdp_uris.c + fort_SOURCES += rsync/rsync.h rsync/rsync.c fort_SOURCES += rtr/err_pdu.c rtr/err_pdu.h diff --git a/src/delete_dir_daemon.c b/src/delete_dir_daemon.c index cb066eec..5df32cda 100644 --- a/src/delete_dir_daemon.c +++ b/src/delete_dir_daemon.c @@ -5,9 +5,13 @@ #include #include #include +#include +#include +#include #include -#include "common.h" #include "log.h" +#include "random.h" +#include "uri.h" #define MAX_FD_ALLOWED 20 @@ -60,18 +64,20 @@ traverse(char const *path, struct stat const *sb, int flag, struct FTW *ftwbuf) static void * remove_from_root(void *arg) { - char const *root = arg; - struct stat attr; + char *root_arg = arg; + char *root; int error; - error = stat(root, &attr); - if (error) { - pr_errno(errno, "Error reading path '%s'", root); - return NULL; - } + pr_debug("Trying to remove dir '%s'.", root_arg); + root = strdup(root_arg); - if (!S_ISDIR(attr.st_mode)) { - pr_err("Path '%s' exists and is not a directory.", root); + /* Release received arg, and detach thread */ + free(root_arg); + pthread_detach(pthread_self()); + + if (root == NULL) { + pr_err("Couldn't allocate memory for a string, the directory '%s' won't be deleted, please delete it manually.", + root); return NULL; } @@ -79,30 +85,138 @@ remove_from_root(void *arg) FTW_DEPTH|FTW_MOUNT|FTW_PHYS); if (error) { if (errno) - pr_errno(errno, "Error deleting directory '%s'", root); + pr_errno(errno, "Error deleting directory '%s', please delete it manually.", + root); else - pr_err("Couldn't delete directory '%s'", root); + pr_err("Couldn't delete directory '%s', please delete it manually", + root); } + + pr_debug("Done removing dir '%s'.", root); + free(root); return NULL; } +static int +get_local_path(char const *rcvd, char **result) +{ + struct stat attr; + struct rpki_uri *uri; + char *tmp, *local_path; + size_t tmp_size; + int error; + + error = uri_create_mixed_str(&uri, rcvd, strlen(rcvd)); + if (error) + return error; + + local_path = strdup(uri_get_local(uri)); + if (local_path == NULL) { + error = pr_enomem(); + goto release_uri; + } + + error = stat(local_path, &attr); + if (error) { + error = -pr_errno(errno, "Error reading path '%s'", local_path); + goto release_local; + } + + if (!S_ISDIR(attr.st_mode)) { + error = pr_err("Path '%s' exists but is not a directory.", + local_path); + goto release_local; + } + + /* Assure that root dir ends without '/' */ + tmp_size = strlen(local_path); + if (strrchr(local_path, '/') == local_path + strlen(local_path) - 1) + tmp_size--; + + tmp = malloc(tmp_size + 1); + if (tmp == NULL) { + error = pr_enomem(); + goto release_local; + } + strncpy(tmp, local_path, tmp_size); + tmp[tmp_size] = '\0'; + + free(local_path); + uri_refput(uri); + + *result = tmp; + return 0; +release_local: + free(local_path); +release_uri: + uri_refput(uri); + return error; +} + +static int +rename_local_path(char const *rcvd, char **result) +{ + char *tmp; + long random_sfx; + size_t rcvd_size, tmp_size; + int error; + + rcvd_size = strlen(rcvd); + /* original size + one underscore + hex random val (8 chars) */ + tmp_size = rcvd_size + 1 + (sizeof(RAND_MAX) * 2); + tmp = malloc(tmp_size + 1); + if (tmp == NULL) + return pr_enomem(); + + /* Rename the path with a random suffix */ + random_init(); + random_sfx = random_at_most(RAND_MAX); + + snprintf(tmp, tmp_size + 1, "%s_%08lX", rcvd, random_sfx); + + error = rename(rcvd, tmp); + if (error) { + free(tmp); + return -pr_errno(errno, "Couldn't rename '%s' to delete it.", + rcvd); + } + + *result = tmp; + return 0; +} + /* - * Start the @thread that will delete every file under @path, @thread must be - * joined. + * Start the @thread that will delete every file under @path, @thread must not + * be joined, since it's detached. @path will be renamed at the file system. */ int -delete_dir_daemon_start(pthread_t *thread, char const *path) +delete_dir_daemon_start(char const *path) { - errno = pthread_create(thread, NULL, remove_from_root, (void *) path); - if (errno) + pthread_t thread; + char *local_path, *delete_path; + int error; + + error = get_local_path(path, &local_path); + if (error) + return error; + + delete_path = NULL; + error = rename_local_path(local_path, &delete_path); + if (error) { + free(local_path); + return error; + } + + /* Thread arg is released at thread before being detached */ + errno = pthread_create(&thread, NULL, remove_from_root, + (void *) delete_path); + if (errno) { + free(delete_path); + free(local_path); return -pr_errno(errno, - "Could not spawn the update daemon thread"); + "Could not spawn the delete dir daemon thread"); + } + free(local_path); return 0; } - -void -delete_dir_daemon_destroy(pthread_t thread) -{ - close_thread(thread, "Delete dir"); -} diff --git a/src/delete_dir_daemon.h b/src/delete_dir_daemon.h index fd93ba5e..ca3ff323 100644 --- a/src/delete_dir_daemon.h +++ b/src/delete_dir_daemon.h @@ -1,9 +1,6 @@ #ifndef SRC_DELETE_DIR_DAEMON_H_ #define SRC_DELETE_DIR_DAEMON_H_ -#include - -int delete_dir_daemon_start(pthread_t *, char const *); -void delete_dir_daemon_destroy(pthread_t); +int delete_dir_daemon_start(char const *); #endif /* SRC_DELETE_DIR_DAEMON_H_ */ diff --git a/src/main.c b/src/main.c index 722f753b..96d1f4df 100644 --- a/src/main.c +++ b/src/main.c @@ -4,11 +4,11 @@ #include "extension.h" #include "nid.h" #include "thread_var.h" -#include "visited_uris.h" #include "http/http.h" #include "rtr/rtr.h" #include "rtr/db/vrps.h" #include "xml/relax_ng.h" +#include "rrdp/db/db_rrdp.h" static int start_rtr_server(void) @@ -19,13 +19,13 @@ start_rtr_server(void) if (error) goto just_quit; - error = visited_uris_init(); + error = db_rrdp_init(); if (error) goto vrps_cleanup; error = rtr_listen(); - visited_uris_destroy(); + db_rrdp_cleanup(); vrps_cleanup: vrps_destroy(); just_quit: diff --git a/src/object/certificate.c b/src/object/certificate.c index 85ac1aef..94944bbb 100644 --- a/src/object/certificate.c +++ b/src/object/certificate.c @@ -11,7 +11,6 @@ #include "nid.h" #include "str.h" #include "thread_var.h" -#include "visited_uris.h" #include "asn1/decode.h" #include "asn1/oid.h" #include "asn1/asn1c/IPAddrBlocks.h" @@ -2077,7 +2076,9 @@ certificate_traverse(struct rpp *rpp_parent, struct rpki_uri *cert_uri) return error; break; case CA: - if (!visited_uris_exists(uri_get_global(sia_uris.mft.uri))) { + if (!db_rrdp_uris_visited_exists( + validation_get_rrdp_uris(state), + uri_get_global(sia_uris.mft.uri))) { error = use_access_method(&sia_uris, exec_rsync_method, exec_rrdp_method); if (error) diff --git a/src/object/tal.c b/src/object/tal.c index f81f9b4b..80312160 100644 --- a/src/object/tal.c +++ b/src/object/tal.c @@ -26,6 +26,7 @@ #include "object/certificate.h" #include "rsync/rsync.h" #include "rtr/db/vrps.h" +#include "rrdp/db/db_rrdp.h" #define TAL_FILE_EXTENSION ".tal" @@ -503,6 +504,14 @@ handle_tal_uri(struct tal *tal, struct rpki_uri *uri, void *arg) goto fail; } + /* + * Set all RRDPs URIs to non-requested, this way we will force the + * request on every cycle (to check if there are updates). + */ + error = db_rrdp_uris_set_all_nonrequested(); + if (error) + goto end; + /* Handle root certificate. */ error = certificate_traverse(NULL, uri); if (error) { @@ -519,8 +528,6 @@ handle_tal_uri(struct tal *tal, struct rpki_uri *uri, void *arg) validation_pubkey_state(state)); } - /* FIXME (now) Consider RRDP found scenario */ - /* * From now on, the tree should be considered valid, even if subsequent * certificates fail. @@ -606,9 +613,15 @@ __do_file_validation(char const *tal_file, void *arg) static pthread_t pid; int error; + error = db_rrdp_add_tal(tal_file); + if (error) + return error; + param = malloc(sizeof(struct fv_param)); - if (param == NULL) - return pr_enomem(); + if (param == NULL) { + error = pr_enomem(); + goto free_db_rrdp; + } param->tal_file = strdup(tal_file); param->arg = arg; @@ -635,6 +648,8 @@ __do_file_validation(char const *tal_file, void *arg) free_param: free(param->tal_file); free(param); +free_db_rrdp: + db_rrdp_rem_tal(tal_file); return error; } @@ -661,6 +676,8 @@ perform_standalone_validation(struct db_table *table) thread_destroy(thread); } + /* FIXME (now) Remove non-visited rrdps URIS by tal */ + return error; } diff --git a/src/rrdp/db/db_rrdp.c b/src/rrdp/db/db_rrdp.c new file mode 100644 index 00000000..c1a688c2 --- /dev/null +++ b/src/rrdp/db/db_rrdp.c @@ -0,0 +1,154 @@ +#include "rrdp/db/db_rrdp.h" + +#include +#include +#include +#include +#include +#include "common.h" +#include "log.h" + +struct tal_elem { + char *file_name; + struct db_rrdp_uri *uris; + bool visited; + SLIST_ENTRY(tal_elem) next; +}; + +SLIST_HEAD(tal_list, tal_elem); + +struct db_rrdp { + struct tal_list tals; +} db; + +/** Read/write lock, which protects @db. */ +static pthread_rwlock_t lock; + +static int +tal_elem_create(struct tal_elem **elem, char const *name) +{ + struct tal_elem *tmp; + struct db_rrdp_uri *tmp_uris; + int error; + + tmp = malloc(sizeof(struct tal_elem)); + if (tmp == NULL) + return pr_enomem(); + + tmp_uris = NULL; + error = db_rrdp_uris_create(&tmp_uris); + if (error) { + free(tmp); + return error; + } + tmp->uris = tmp_uris; + + tmp->visited = true; + tmp->file_name = strdup(name); + if (tmp->file_name == NULL) { + db_rrdp_uris_destroy(tmp->uris); + free(tmp); + return pr_enomem(); + } + + *elem = tmp; + return 0; +} + +static void +tal_elem_destroy(struct tal_elem *elem) +{ + db_rrdp_uris_destroy(elem->uris); + free(elem->file_name); + free(elem); +} + +int +db_rrdp_init(void) +{ + int error; + + error = pthread_rwlock_init(&lock, NULL); + if (error) + return pr_errno(error, "DB RRDP pthread_rwlock_init() errored"); + + SLIST_INIT(&db.tals); + return 0; +} + +void +db_rrdp_cleanup(void) +{ + struct tal_elem *elem; + + while (!SLIST_EMPTY(&db.tals)) { + elem = db.tals.slh_first; + SLIST_REMOVE_HEAD(&db.tals, next); + tal_elem_destroy(elem); + } + pthread_rwlock_destroy(&lock); +} + +static struct tal_elem * +db_rrdp_find_tal(char const *tal_name) +{ + struct tal_elem *found; + + rwlock_read_lock(&lock); + SLIST_FOREACH(found, &db.tals, next) { + if (strcmp(tal_name, found->file_name) == 0) { + rwlock_unlock(&lock); + return found; + } + } + rwlock_unlock(&lock); + + return NULL; +} + +int +db_rrdp_add_tal(char const *tal_name) +{ + struct tal_elem *elem; + int error; + + /* Element exists, no need to create it again */ + if (db_rrdp_find_tal(tal_name) != NULL) + return 0; + + error = tal_elem_create(&elem, tal_name); + if (error) + return error; + + rwlock_write_lock(&lock); + SLIST_INSERT_HEAD(&db.tals, elem, next); + rwlock_unlock(&lock); + + return 0; +} + +void +db_rrdp_rem_tal(char const *tal_name) +{ + struct tal_elem *found; + + found = db_rrdp_find_tal(tal_name); + if (found == NULL) + return; + + rwlock_write_lock(&lock); + SLIST_REMOVE(&db.tals, found, tal_elem, next); + rwlock_unlock(&lock); + + tal_elem_destroy(found); +} + +/* Returns the reference to RRDP URIs of a TAL */ +struct db_rrdp_uri * +db_rrdp_get_uris(char const *tal_name) +{ + struct tal_elem *found; + + found = db_rrdp_find_tal(tal_name); + return found != NULL ? found->uris : NULL; +} diff --git a/src/rrdp/db/db_rrdp.h b/src/rrdp/db/db_rrdp.h new file mode 100644 index 00000000..dc7e368f --- /dev/null +++ b/src/rrdp/db/db_rrdp.h @@ -0,0 +1,13 @@ +#ifndef SRC_RRDP_DB_DB_RRDP_H_ +#define SRC_RRDP_DB_DB_RRDP_H_ + +#include "rrdp/db/db_rrdp_uris.h" + +int db_rrdp_init(void); +void db_rrdp_cleanup(void); + +int db_rrdp_add_tal(char const *); +void db_rrdp_rem_tal(char const *); +struct db_rrdp_uri *db_rrdp_get_uris(char const *); + +#endif /* SRC_RRDP_DB_DB_RRDP_H_ */ diff --git a/src/rrdp/db/db_rrdp_uris.c b/src/rrdp/db/db_rrdp_uris.c new file mode 100644 index 00000000..752ac9fb --- /dev/null +++ b/src/rrdp/db/db_rrdp_uris.c @@ -0,0 +1,348 @@ +#include "rrdp/db/db_rrdp_uris.h" + +#include +#include +#include +#include "data_structure/uthash_nonfatal.h" +#include "log.h" +#include "thread_var.h" +#include "visited_uris.h" + +struct uris_table { + /* Key */ + char *uri; + /* Last session ID and serial from the URI */ + struct global_data data; + /* Last local update of the URI (after a successful processing) */ + long last_update; + /* The URI has been requested (HTTPS) at this cycle? */ + bool requested; + /* MFT URIs loaded from the @uri */ + struct visited_uris *visited_uris; + UT_hash_handle hh; +}; + +struct db_rrdp_uri { + struct uris_table *table; +}; + +static int +uris_table_create(char const *uri, char const *session_id, + unsigned long serial, struct uris_table **result) +{ + struct uris_table *tmp; + int error; + + tmp = malloc(sizeof(struct uris_table)); + if (tmp == NULL) + return pr_enomem(); + /* Needed by uthash */ + memset(tmp, 0, sizeof(struct uris_table)); + + tmp->uri = strdup(uri); + if (tmp->uri == NULL) { + error = pr_enomem(); + goto release_tmp; + } + + tmp->data.session_id = strdup(session_id); + if (tmp->data.session_id == NULL) { + error = pr_enomem(); + goto release_uri; + } + + tmp->data.serial = serial; + tmp->last_update = 0; + tmp->requested = true; + tmp->visited_uris = NULL; + + *result = tmp; + return 0; +release_uri: + free(tmp->uri); +release_tmp: + free(tmp); + return error; +} + +static void +uris_table_destroy(struct uris_table *uri) +{ + visited_uris_refput(uri->visited_uris); + free(uri->data.session_id); + free(uri->uri); + free(uri); +} + +static struct uris_table * +find_rrdp_uri(struct db_rrdp_uri *uris, const char *search) +{ + struct uris_table *found; + HASH_FIND_STR(uris->table, search, found); + return found; +} + +#define RET_NOT_FOUND_URI(uris, search, found, retval) \ + found = find_rrdp_uri(uris, search); \ + if (found == NULL) \ + return retval; + +static void +add_rrdp_uri(struct db_rrdp_uri *uris, struct uris_table *new_uri) +{ + struct uris_table *old_uri; + + old_uri = find_rrdp_uri(uris, new_uri->uri); + if (old_uri != NULL) { + HASH_DELETE(hh, uris->table, old_uri); + uris_table_destroy(old_uri); + } + HASH_ADD_KEYPTR(hh, uris->table, new_uri->uri, strlen(new_uri->uri), + new_uri); +} + +static int +get_thread_rrdp_uris(struct db_rrdp_uri **result) +{ + struct validation *state; + + state = state_retrieve(); + if (state == NULL) + return pr_err("No state related to this thread"); + + *result = validation_get_rrdp_uris(state); + return 0; +} + +int +db_rrdp_uris_create(struct db_rrdp_uri **uris) +{ + struct db_rrdp_uri *tmp; + + tmp = malloc(sizeof(struct db_rrdp_uri)); + if (tmp == NULL) + return pr_enomem(); + + tmp->table = NULL; + + *uris = tmp; + return 0; +} + +void +db_rrdp_uris_destroy(struct db_rrdp_uri *uris) +{ + struct uris_table *uri_node, *uri_tmp; + + HASH_ITER(hh, uris->table, uri_node, uri_tmp) { + HASH_DEL(uris->table, uri_node); + uris_table_destroy(uri_node); + } + free(uris); +} + +int +db_rrdp_uris_cmp(char const *uri, char const *session_id, unsigned long serial, + rrdp_uri_cmp_result_t *result) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + found = find_rrdp_uri(uris, uri); + if (found == NULL) { + *result = RRDP_URI_NOTFOUND; + return 0; + } + + if (strcmp(session_id, found->data.session_id) != 0) { + *result = RRDP_URI_DIFF_SESSION; + return 0; + } + + if (serial != found->data.serial) { + *result = RRDP_URI_DIFF_SERIAL; + return 0; + } + + *result = RRDP_URI_EQUAL; + return 0; +} + +int +db_rrdp_uris_update(char const *uri, char const *session_id, + unsigned long serial, struct visited_uris *visited_uris) +{ + struct db_rrdp_uri *uris; + struct uris_table *db_uri; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + db_uri = NULL; + error = uris_table_create(uri, session_id, serial, &db_uri); + if (error) + return error; + + /* Ownership transfered */ + db_uri->visited_uris = visited_uris; + + add_rrdp_uri(uris, db_uri); + + return 0; +} + +int +db_rrdp_uris_get_serial(char const *uri, unsigned long *serial) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + *serial = found->data.serial; + return 0; +} + +int +db_rrdp_uris_get_last_update(char const *uri, long *date) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + *date = found->last_update; + return 0; +} + +static int +get_current_time(long *result) +{ + time_t now; + + now = time(NULL); + if (now == ((time_t) -1)) + return pr_errno(errno, "Error getting the current time"); + + *result = now; + return 0; +} + +/* Set the last update to now */ +int +db_rrdp_uris_set_last_update(char const *uri) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + return get_current_time(&found->last_update); +} + +int +db_rrdp_uris_get_requested(char const *uri, bool *result) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + *result = found->requested; + return 0; +} + +int +db_rrdp_uris_set_requested(char const *uri, bool value) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + found->requested = value; + return 0; +} + +int +db_rrdp_uris_set_all_nonrequested(void) +{ + struct db_rrdp_uri *uris; + struct uris_table *uri_node, *uri_tmp; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + HASH_ITER(hh, uris->table, uri_node, uri_tmp) + uri_node->requested = false; + + return 0; +} + +/* + * Returns a pointer (set in @result) to the visited_uris of the current + * thread. + */ +int +db_rrdp_uris_get_visited_uris(char const *uri, struct visited_uris **result) +{ + struct db_rrdp_uri *uris; + struct uris_table *found; + int error; + + error = get_thread_rrdp_uris(&uris); + if (error) + return error; + + RET_NOT_FOUND_URI(uris, uri, found, -ENOENT) + *result = found->visited_uris; + return 0; +} + +/* + * Check if the @uri exists at the @uris struct. + * + * Get at another level the @uris related to a thread, since this call may be + * frequently done. + */ +bool +db_rrdp_uris_visited_exists(struct db_rrdp_uri *uris, char const *uri) +{ + struct uris_table *uri_node, *uri_tmp; + + /* Search at each 'visited_uris' from all the table */ + HASH_ITER(hh, uris->table, uri_node, uri_tmp) + if (visited_uris_exists(uri_node->visited_uris, uri)) + return true; + + return false; +} diff --git a/src/rrdp/db/db_rrdp_uris.h b/src/rrdp/db/db_rrdp_uris.h new file mode 100644 index 00000000..ec316803 --- /dev/null +++ b/src/rrdp/db/db_rrdp_uris.h @@ -0,0 +1,35 @@ +#ifndef SRC_RRDP_DB_DB_RRDP_URIS_H_ +#define SRC_RRDP_DB_DB_RRDP_URIS_H_ + +#include +#include "rrdp/rrdp_objects.h" +#include "visited_uris.h" + +/* + * RRDP URI fetched from 'rpkiNotify' OID at a CA certificate, each TAL thread + * may have a reference to one of these (it holds information such as update + * notification URI, session ID, serial, visited mft uris). + */ +struct db_rrdp_uri; + +int db_rrdp_uris_create(struct db_rrdp_uri **); +void db_rrdp_uris_destroy(struct db_rrdp_uri *); + +int db_rrdp_uris_cmp(char const *, char const *, unsigned long, + rrdp_uri_cmp_result_t *); +int db_rrdp_uris_update(char const *, char const *session_id, unsigned long, + struct visited_uris *); +int db_rrdp_uris_get_serial(char const *, unsigned long *); + +int db_rrdp_uris_get_last_update(char const *, long *); +int db_rrdp_uris_set_last_update(char const *); + +int db_rrdp_uris_get_requested(char const *, bool *); +int db_rrdp_uris_set_requested(char const *, bool); +int db_rrdp_uris_set_all_nonrequested(void); + +int db_rrdp_uris_get_visited_uris(char const *, struct visited_uris **); + +bool db_rrdp_uris_visited_exists(struct db_rrdp_uri *, char const *); + +#endif /* SRC_RRDP_DB_DB_RRDP_URIS_H_ */ diff --git a/src/rrdp/db_rrdp.c b/src/rrdp/db_rrdp.c deleted file mode 100644 index 82053576..00000000 --- a/src/rrdp/db_rrdp.c +++ /dev/null @@ -1,247 +0,0 @@ -#include "db_rrdp.h" - -#include -#include -#include -#include -#include "data_structure/uthash_nonfatal.h" -#include "rrdp/rrdp_objects.h" -#include "log.h" - -struct db_rrdp_uri { - /* Key */ - char *uri; - struct global_data data; - long last_update; - bool visited; - UT_hash_handle hh; -}; - -struct db_rrdp { - struct db_rrdp_uri *uris; -}; - -static int -get_current_time(long *result) -{ - time_t now; - - now = time(NULL); - if (now == ((time_t) -1)) - return pr_errno(errno, "Error getting the current time"); - - *result = now; - return 0; -} - -static int -db_rrdp_uri_create(char const *uri, char const *session_id, - unsigned long serial, struct db_rrdp_uri **result) -{ - struct db_rrdp_uri *tmp; - int error; - - tmp = malloc(sizeof(struct db_rrdp_uri)); - if (tmp == NULL) - return pr_enomem(); - /* Needed by uthash */ - memset(tmp, 0, sizeof(struct db_rrdp_uri)); - - tmp->uri = strdup(uri); - if (tmp->uri == NULL) { - error = pr_enomem(); - goto release_tmp; - } - - tmp->data.session_id = strdup(session_id); - if (tmp->data.session_id == NULL) { - error = pr_enomem(); - goto release_uri; - } - - tmp->data.serial = serial; - tmp->last_update = 0; - tmp->visited = true; - - *result = tmp; - return 0; -release_uri: - free(tmp->uri); -release_tmp: - free(tmp); - return error; -} - -static void -db_rrdp_uri_destroy(struct db_rrdp_uri *uri) -{ - free(uri->data.session_id); - free(uri->uri); - free(uri); -} - -static int -add_rrdp_uri(struct db_rrdp *db, struct db_rrdp_uri *new_uri) -{ - struct db_rrdp_uri *old_uri; - - errno = 0; - HASH_FIND_STR(db->uris, new_uri->uri, old_uri); - if (errno) - return pr_errno(errno, - "RRDP URI couldn't be added to hash table"); - - if (old_uri != NULL) { - HASH_DELETE(hh, db->uris, old_uri); - db_rrdp_uri_destroy(old_uri); - } - HASH_ADD_KEYPTR(hh, db->uris, new_uri->uri, strlen(new_uri->uri), - new_uri); - - return 0; -} - -rrdp_uri_cmp_result_t -db_rrdp_cmp_uri(struct db_rrdp *db, char const *uri, char const *session_id, - unsigned long serial) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return RRDP_URI_NOTFOUND; - - if (strcmp(session_id, found->data.session_id) != 0) - return RRDP_URI_DIFF_SESSION; - - if (serial != found->data.serial) - return RRDP_URI_DIFF_SERIAL; - - return RRDP_URI_EQUAL; -} - -int -db_rrdp_add_uri(struct db_rrdp *db, char const *uri, char const *session_id, - unsigned long serial) -{ - struct db_rrdp_uri *db_uri; - int error; - - db_uri = NULL; - error = db_rrdp_uri_create(uri, session_id, serial, &db_uri); - if (error) - return error; - - error = add_rrdp_uri(db, db_uri); - if (error) { - db_rrdp_uri_destroy(db_uri); - return error; - } - - return 0; -} - -int -db_rrdp_get_serial(struct db_rrdp *db, char const *uri, unsigned long *serial) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return -ENOENT; - - *serial = found->data.serial; - - return 0; -} - -int -db_rrdp_get_last_update(struct db_rrdp *db, char const *uri, long *date) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return -ENOENT; - - *date = found->last_update; - - return 0; -} - -/* Set the last update to now */ -int -db_rrdp_set_last_update(struct db_rrdp *db, char const *uri) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return -ENOENT; - - return get_current_time(&found->last_update); -} - -bool -db_rrdp_get_visited(struct db_rrdp *db, char const *uri) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return false; - - return found->visited; -} - -int -db_rrdp_set_visited(struct db_rrdp *db, char const *uri, bool value) -{ - struct db_rrdp_uri *found; - - HASH_FIND_STR(db->uris, uri, found); - if (found == NULL) - return -ENOENT; - - found->visited = value; - return 0; -} - -int -db_rrdp_set_all_nonvisited(struct db_rrdp *db) -{ - struct db_rrdp_uri *uri_node, *uri_tmp; - - HASH_ITER(hh, db->uris, uri_node, uri_tmp) - uri_node->visited = false; - - return 0; -} - -int -db_rrdp_create(struct db_rrdp **result) -{ - struct db_rrdp *tmp; - - tmp = malloc(sizeof(struct db_rrdp)); - if (tmp == NULL) - return pr_enomem(); - - tmp->uris = NULL; - - *result = tmp; - return 0; -} - -void -db_rddp_destroy(struct db_rrdp *db) -{ - struct db_rrdp_uri *uri_node, *uri_tmp; - - HASH_ITER(hh, db->uris, uri_node, uri_tmp) { - HASH_DEL(db->uris, uri_node); - db_rrdp_uri_destroy(uri_node); - } - - free(db); -} diff --git a/src/rrdp/db_rrdp.h b/src/rrdp/db_rrdp.h deleted file mode 100644 index 0339bf8a..00000000 --- a/src/rrdp/db_rrdp.h +++ /dev/null @@ -1,28 +0,0 @@ -#ifndef SRC_RRDP_DB_RRDP_H_ -#define SRC_RRDP_DB_RRDP_H_ - -#include "rrdp/rrdp_objects.h" - -/* - * Struct and methods to persist RRDP information, such as URIs, session ID per - * URI and current serial number. - */ -struct db_rrdp; - -int db_rrdp_create(struct db_rrdp **); -void db_rddp_destroy(struct db_rrdp *); - -rrdp_uri_cmp_result_t db_rrdp_cmp_uri(struct db_rrdp *, char const *, - char const *, unsigned long); -int db_rrdp_add_uri(struct db_rrdp *, char const *, char const *, - unsigned long); -int db_rrdp_get_serial(struct db_rrdp *, char const *, unsigned long *); -int db_rrdp_get_last_update(struct db_rrdp *, char const *, long *); - -int db_rrdp_set_last_update(struct db_rrdp *, char const *); - -bool db_rrdp_get_visited(struct db_rrdp *, char const *); -int db_rrdp_set_visited(struct db_rrdp *, char const *, bool); -int db_rrdp_set_all_nonvisited(struct db_rrdp *); - -#endif /* SRC_RRDP_DB_RRDP_H_ */ diff --git a/src/rrdp/rrdp_loader.c b/src/rrdp/rrdp_loader.c index ce7523ce..8a139252 100644 --- a/src/rrdp/rrdp_loader.c +++ b/src/rrdp/rrdp_loader.c @@ -1,41 +1,98 @@ #include "rrdp_loader.h" +#include "rrdp/db/db_rrdp_uris.h" #include "rrdp/rrdp_objects.h" #include "rrdp/rrdp_parser.h" -#include "rtr/db/vrps.h" +#include "delete_dir_daemon.h" #include "log.h" #include "thread_var.h" +#include "visited_uris.h" -/* Fetch and process the deltas from the @notification located at @uri */ +/* Fetch and process the deltas from the @notification */ static int -process_diff_serial(struct update_notification *notification, char const *uri) +process_diff_serial(struct update_notification *notification, + struct visited_uris **visited) { unsigned long serial; int error; - error = rrdp_uri_get_serial(uri, &serial); + error = db_rrdp_uris_get_serial(notification->uri, &serial); if (error) return error; - return rrdp_process_deltas(notification, serial); + /* Work with the existent visited uris */ + error = db_rrdp_uris_get_visited_uris(notification->uri, visited); + if (error) + return error; + + return rrdp_process_deltas(notification, serial, *visited); } -/* Fetch and process the snapshot from the @notification located at @uri */ +/* Fetch and process the snapshot from the @notification */ static int -process_snapshot(struct update_notification *notification, char const *uri) +process_snapshot(struct update_notification *notification, + struct visited_uris **visited) { - return rrdp_parse_snapshot(notification); + struct visited_uris *tmp; + int error; + + /* Use a new allocated visited_uris struct */ + error = visited_uris_create(&tmp); + if (error) + return error; + + error = rrdp_parse_snapshot(notification, tmp); + if (error) { + free(tmp); + return error; + } + + *visited = tmp; + return 0; +} + +static int +remove_rrdp_uri_files(struct update_notification *notification) +{ + struct visited_uris *tmp; + char *root_path; + int error; + + /* Work with the existent visited uris */ + error = db_rrdp_uris_get_visited_uris(notification->uri, &tmp); + if (error) + return error; + + error = visited_uris_get_root(tmp, &root_path); + if (error) + return error; + + error = delete_dir_daemon_start(root_path); + if (error) { + free(root_path); + return error; + } + + free(root_path); + return 0; } int rrdp_load(struct rpki_uri *uri) { struct update_notification *upd_notification; + struct visited_uris *visited; rrdp_uri_cmp_result_t res; int error; + bool requested; /* Avoid multiple requests on the same run */ - if (rrdp_uri_visited(uri_get_global(uri))) + requested = false; + error = db_rrdp_uris_get_requested(uri_get_global(uri), &requested); + if (error && error != -ENOENT) + return error; + + if (requested) return 0; error = rrdp_parse_notification(uri, &upd_notification); @@ -46,25 +103,31 @@ rrdp_load(struct rpki_uri *uri) if (upd_notification == NULL) return 0; - res = rrdp_uri_cmp(uri_get_global(uri), + error = db_rrdp_uris_cmp(uri_get_global(uri), upd_notification->global_data.session_id, - upd_notification->global_data.serial); + upd_notification->global_data.serial, + &res); + if (error) + goto end; + switch (res) { case RRDP_URI_EQUAL: goto set_update; case RRDP_URI_DIFF_SERIAL: - error = process_diff_serial(upd_notification, - uri_get_global(uri)); + error = process_diff_serial(upd_notification, &visited); /* Something went wrong, use snapshot */ - if (!error) + if (!error) { + visited_uris_refget(visited); break; + } pr_warn("There was an error processing RRDP deltas, using the snapshot instead."); - error = process_snapshot(upd_notification, uri_get_global(uri)); - break; case RRDP_URI_DIFF_SESSION: - /* FIXME (now) delete the old session files */ + /* Delete the old session files */ + error = remove_rrdp_uri_files(upd_notification); + if (error) + break; case RRDP_URI_NOTFOUND: - error = process_snapshot(upd_notification, uri_get_global(uri)); + error = process_snapshot(upd_notification, &visited); break; default: pr_crit("Unexpected RRDP URI comparison result"); @@ -72,16 +135,17 @@ rrdp_load(struct rpki_uri *uri) /* Any change, and no error during the process, update db */ if (!error) { - error = rrdp_uri_update(uri_get_global(uri), + error = db_rrdp_uris_update(uri_get_global(uri), upd_notification->global_data.session_id, - upd_notification->global_data.serial); + upd_notification->global_data.serial, + visited); if (error) goto end; } set_update: /* Set the last update to now */ - error = rrdp_uri_set_last_update(uri_get_global(uri)); + error = db_rrdp_uris_set_last_update(uri_get_global(uri)); end: if (upd_notification != NULL) { update_notification_destroy(upd_notification); diff --git a/src/rrdp/rrdp_objects.c b/src/rrdp/rrdp_objects.c index 1192cf40..7a2d8aa4 100644 --- a/src/rrdp/rrdp_objects.c +++ b/src/rrdp/rrdp_objects.c @@ -247,6 +247,7 @@ update_notification_create(struct update_notification **file) return pr_enomem(); } tmp->deltas_list = list; + tmp->uri = NULL; global_data_init(&tmp->global_data); doc_data_init(&tmp->snapshot); @@ -261,6 +262,7 @@ update_notification_destroy(struct update_notification *file) doc_data_cleanup(&file->snapshot); global_data_cleanup(&file->global_data); deltas_head_destroy(file->deltas_list); + free(file->uri); free(file); } diff --git a/src/rrdp/rrdp_objects.h b/src/rrdp/rrdp_objects.h index b6275d25..a3048aa8 100644 --- a/src/rrdp/rrdp_objects.h +++ b/src/rrdp/rrdp_objects.h @@ -69,11 +69,12 @@ struct delta_head { /* List of deltas inside an update notification file */ struct deltas_head; -/* Update notification file content */ +/* Update notification file content and location URI */ struct update_notification { struct global_data global_data; struct doc_data snapshot; struct deltas_head *deltas_list; + char *uri; }; void global_data_init(struct global_data *); diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c index f8839769..12b2107f 100644 --- a/src/rrdp/rrdp_parser.c +++ b/src/rrdp/rrdp_parser.c @@ -10,16 +10,15 @@ #include #include +#include "rrdp/db/db_rrdp_uris.h" #include "crypto/base64.h" #include "crypto/hash.h" #include "http/http.h" -#include "rtr/db/vrps.h" #include "xml/relax_ng.h" #include "common.h" #include "file.h" #include "log.h" #include "thread_var.h" -#include "visited_uris.h" /* XML Common Namespace of files */ #define RRDP_NAMESPACE "http://www.ripe.net/rpki/rrdp" @@ -46,8 +45,6 @@ DEFINE_ARRAY_LIST_FUNCTIONS(deltas_parsed, struct delta_head *, static) struct rdr_notification_ctx { /* Data being parsed */ struct update_notification *notification; - /* Global URI of the notification */ - char const *uri; /* The snapshot must be allocated? */ bool create_snapshot; /* Unordered list of deltas */ @@ -60,6 +57,8 @@ struct rdr_snapshot_ctx { struct snapshot *snapshot; /* Parent data to validate session ID and serial */ struct update_notification *parent; + /* Visited URIs related to this thread */ + struct visited_uris *visited_uris; }; /* Context while reading a delta */ @@ -70,24 +69,32 @@ struct rdr_delta_ctx { struct update_notification *parent; /* Current serial loaded from update notification deltas list */ unsigned long expected_serial; + /* Visited URIs related to this thread */ + struct visited_uris *visited_uris; +}; + +/* Args to send on update (snapshot/delta) files parsing */ +struct proc_upd_args { + struct update_notification *parent; + struct visited_uris *visited_uris; }; static int -add_mft_to_list(char const *uri) +add_mft_to_list(struct visited_uris *visited_uris, char const *uri) { if (strcmp(".mft", strrchr(uri, '.')) != 0) return 0; - return visited_uris_add(uri); + return visited_uris_add(visited_uris, uri); } static int -rem_mft_from_list(char const *uri) +rem_mft_from_list(struct visited_uris *visited_uris, char const *uri) { if (strcmp(".mft", strrchr(uri, '.')) != 0) return 0; - return visited_uris_remove(uri); + return visited_uris_remove(visited_uris, uri); } static size_t @@ -561,7 +568,8 @@ release_tmp: } static int -write_from_uri(char const *location, unsigned char *content, size_t content_len) +write_from_uri(char const *location, unsigned char *content, size_t content_len, + struct visited_uris *visited_uris) { struct rpki_uri *uri; struct stat stat; @@ -589,11 +597,11 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len) if (written != content_len) { uri_refput(uri); file_close(out); - return pr_err("Coudln't write bytes to file %s", + return pr_err("Couldn't write bytes to file %s", uri_get_local(uri)); } - error = add_mft_to_list(uri_get_global(uri)); + error = add_mft_to_list(visited_uris, uri_get_global(uri)); if (error) { uri_refput(uri); file_close(out); @@ -606,7 +614,7 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len) } static int -delete_from_uri(char const *location) +delete_from_uri(char const *location, struct visited_uris *visited_uris) { struct rpki_uri *uri; char *local_uri, *work_loc, *tmp; @@ -629,6 +637,10 @@ delete_from_uri(char const *location) goto release_str; } + error = rem_mft_from_list(visited_uris, uri_get_global(uri)); + if (error) + goto release_str; + /* * Delete parent dirs only if empty. * @@ -658,10 +670,6 @@ delete_from_uri(char const *location) goto release_str; } while (true); - error = rem_mft_from_list(uri_get_global(uri)); - if (error) - goto release_str; - uri_refput(uri); free(local_uri); return 0; @@ -677,7 +685,8 @@ release_uri: * other type at the caller. */ static int -parse_publish_elem(xmlTextReaderPtr reader, bool parse_hash, bool hash_required) +parse_publish_elem(xmlTextReaderPtr reader, bool parse_hash, bool hash_required, + struct visited_uris *visited_uris) { struct publish *tmp; int error; @@ -688,7 +697,7 @@ parse_publish_elem(xmlTextReaderPtr reader, bool parse_hash, bool hash_required) return error; error = write_from_uri(tmp->doc_data.uri, tmp->content, - tmp->content_len); + tmp->content_len, visited_uris); publish_destroy(tmp); if (error) return error; @@ -701,7 +710,7 @@ parse_publish_elem(xmlTextReaderPtr reader, bool parse_hash, bool hash_required) * other type at the caller. */ static int -parse_withdraw_elem(xmlTextReaderPtr reader) +parse_withdraw_elem(xmlTextReaderPtr reader, struct visited_uris *visited_uris) { struct withdraw *tmp; int error; @@ -710,7 +719,7 @@ parse_withdraw_elem(xmlTextReaderPtr reader) if (error) return error; - error = delete_from_uri(tmp->doc_data.uri); + error = delete_from_uri(tmp->doc_data.uri, visited_uris); withdraw_destroy(tmp); if (error) return error; @@ -722,12 +731,17 @@ static int rdr_notification_ctx_init(struct rdr_notification_ctx *ctx) { rrdp_uri_cmp_result_t res; + int error; ctx->create_snapshot = false; - res = rrdp_uri_cmp(ctx->uri, + error = db_rrdp_uris_cmp(ctx->notification->uri, ctx->notification->global_data.session_id, - ctx->notification->global_data.serial); + ctx->notification->global_data.serial, + &res); + if (error) + return error; + switch (res) { case RRDP_URI_EQUAL: /* Just validate content */ @@ -892,14 +906,20 @@ parse_notification(struct rpki_uri *uri, struct update_notification **file) { struct rdr_notification_ctx ctx; struct update_notification *tmp; + char *dup; int error; + dup = strdup(uri_get_global(uri)); + if (dup == NULL) + return pr_enomem(); + error = update_notification_create(&tmp); if (error) return error; + tmp->uri = dup; + ctx.notification = tmp; - ctx.uri = uri_get_global(uri); error = relax_ng_parse(uri_get_local(uri), xml_read_notification, &ctx); if (error) { @@ -924,7 +944,8 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *arg) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = parse_publish_elem(reader, false, false); + error = parse_publish_elem(reader, false, false, + ctx->visited_uris); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_SNAPSHOT)) error = parse_global_data(reader, &ctx->snapshot->global_data, @@ -944,7 +965,7 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *arg) } static int -parse_snapshot(struct rpki_uri *uri, struct update_notification *parent) +parse_snapshot(struct rpki_uri *uri, struct proc_upd_args *args) { struct rdr_snapshot_ctx ctx; struct snapshot *snapshot; @@ -952,8 +973,8 @@ parse_snapshot(struct rpki_uri *uri, struct update_notification *parent) fnstack_push_uri(uri); /* Hash validation */ - error = hash_validate_file("sha256", uri, parent->snapshot.hash, - parent->snapshot.hash_len); + error = hash_validate_file("sha256", uri, args->parent->snapshot.hash, + args->parent->snapshot.hash_len); if (error) goto pop; @@ -962,7 +983,8 @@ parse_snapshot(struct rpki_uri *uri, struct update_notification *parent) goto pop; ctx.snapshot = snapshot; - ctx.parent = parent; + ctx.parent = args->parent; + ctx.visited_uris = args->visited_uris; error = relax_ng_parse(uri_get_local(uri), xml_read_snapshot, &ctx); /* Error 0 is ok */ @@ -985,9 +1007,10 @@ xml_read_delta(xmlTextReaderPtr reader, void *arg) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = parse_publish_elem(reader, true, false); + error = parse_publish_elem(reader, true, false, + ctx->visited_uris); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_WITHDRAW)) - error = parse_withdraw_elem(reader); + error = parse_withdraw_elem(reader, ctx->visited_uris); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_DELTA)) error = parse_global_data(reader, &ctx->delta->global_data, @@ -1007,9 +1030,9 @@ xml_read_delta(xmlTextReaderPtr reader, void *arg) } static int -parse_delta(struct rpki_uri *uri, struct update_notification *parent, - struct delta_head *parents_data) +parse_delta(struct rpki_uri *uri, struct delta_head *parents_data, void *arg) { + struct proc_upd_args *args = arg; struct rdr_delta_ctx ctx; struct delta *delta; struct doc_data *expected_data; @@ -1028,7 +1051,8 @@ parse_delta(struct rpki_uri *uri, struct update_notification *parent, goto pop_fnstack; ctx.delta = delta; - ctx.parent = parent; + ctx.parent = args->parent; + ctx.visited_uris = args->visited_uris; ctx.expected_serial = parents_data->serial; error = relax_ng_parse(uri_get_local(uri), xml_read_delta, &ctx); @@ -1042,13 +1066,13 @@ pop_fnstack: static int process_delta(struct delta_head *delta_head, void *arg) { - struct update_notification *parent = arg; struct rpki_uri *uri; struct doc_data *head_data; int error; head_data = &delta_head->doc_data; + pr_debug("Processing delta '%s'.", delta_head->doc_data.uri); error = uri_create_https_str(&uri, head_data->uri, strlen(head_data->uri)); if (error) @@ -1058,7 +1082,7 @@ process_delta(struct delta_head *delta_head, void *arg) if (error) goto release_uri; - error = parse_delta(uri, parent, delta_head); + error = parse_delta(uri, delta_head, arg); /* Error 0 its ok */ release_uri: @@ -1084,8 +1108,9 @@ rrdp_parse_notification(struct rpki_uri *uri, if (uri == NULL || uri_is_rsync(uri)) pr_crit("Wrong call, trying to parse a non HTTPS URI"); + pr_debug("Processing notification '%s'.", uri_get_global(uri)); last_update = 0; - error = rrdp_uri_get_last_update(uri_get_global(uri), &last_update); + error = db_rrdp_uris_get_last_update(uri_get_global(uri), &last_update); if (error && error != -ENOENT) return error; @@ -1098,8 +1123,8 @@ rrdp_parse_notification(struct rpki_uri *uri, * this is probably the first time is visited (first run), so it will * be marked as visited when the URI is stored at DB. */ - vis_err = rrdp_uri_mark_visited(uri_get_global(uri)); - if (vis_err && vis_err !=-ENOENT) + vis_err = db_rrdp_uris_set_requested(uri_get_global(uri), true); + if (vis_err && vis_err != -ENOENT) return pr_err("Coudln't mark '%s' as visited", uri_get_global(uri)); @@ -1120,11 +1145,17 @@ rrdp_parse_notification(struct rpki_uri *uri, } int -rrdp_parse_snapshot(struct update_notification *parent) +rrdp_parse_snapshot(struct update_notification *parent, + struct visited_uris *visited_uris) { + struct proc_upd_args args; struct rpki_uri *uri; int error; + args.parent = parent; + args.visited_uris = visited_uris; + + pr_debug("Processing snapshot '%s'.", parent->snapshot.uri); error = uri_create_https_str(&uri, parent->snapshot.uri, strlen(parent->snapshot.uri)); if (error) @@ -1134,7 +1165,7 @@ rrdp_parse_snapshot(struct update_notification *parent) if (error) goto release_uri; - error = parse_snapshot(uri, parent); + error = parse_snapshot(uri, &args); /* Error 0 is ok */ release_uri: @@ -1144,8 +1175,13 @@ release_uri: int rrdp_process_deltas(struct update_notification *parent, - unsigned long cur_serial) + unsigned long cur_serial, struct visited_uris *visited_uris) { + struct proc_upd_args args; + + args.parent = parent; + args.visited_uris = visited_uris; + return deltas_head_for_each(parent->deltas_list, - parent->global_data.serial, cur_serial, process_delta, parent); + parent->global_data.serial, cur_serial, process_delta, &args); } diff --git a/src/rrdp/rrdp_parser.h b/src/rrdp/rrdp_parser.h index e4c26735..945695ce 100644 --- a/src/rrdp/rrdp_parser.h +++ b/src/rrdp/rrdp_parser.h @@ -3,10 +3,12 @@ #include "rrdp/rrdp_objects.h" #include "uri.h" +#include "visited_uris.h" int rrdp_parse_notification(struct rpki_uri *, struct update_notification **); -int rrdp_parse_snapshot(struct update_notification *); +int rrdp_parse_snapshot(struct update_notification *, struct visited_uris *); -int rrdp_process_deltas(struct update_notification *, unsigned long serial); +int rrdp_process_deltas(struct update_notification *, + unsigned long serial, struct visited_uris *); #endif /* SRC_RRDP_RRDP_PARSER_H_ */ diff --git a/src/rsync/rsync.c b/src/rsync/rsync.c index 652a7780..9ba58c7e 100644 --- a/src/rsync/rsync.c +++ b/src/rsync/rsync.c @@ -311,7 +311,7 @@ download_files(struct rpki_uri *requested_uri, bool is_ta, bool force) if (state == NULL) return -EINVAL; - visited_uris = validation_visited_uris(state); + visited_uris = validation_rsync_visited_uris(state); if (!force && is_already_downloaded(requested_uri, visited_uris)) { pr_debug("No need to redownload '%s'.", diff --git a/src/rtr/db/vrps.c b/src/rtr/db/vrps.c index 386ae8da..8350d9d3 100644 --- a/src/rtr/db/vrps.c +++ b/src/rtr/db/vrps.c @@ -11,7 +11,6 @@ #include "data_structure/array_list.h" #include "object/router_key.h" #include "object/tal.h" -#include "rrdp/db_rrdp.h" #include "rtr/db/db_table.h" #include "slurm/slurm_loader.h" @@ -61,10 +60,6 @@ struct state { serial_t next_serial; uint16_t v0_session_id; uint16_t v1_session_id; - - /* RRDP URIs known from certificates */ - struct db_rrdp *rrdp_uris; - } state; /** Read/write lock, which protects @state and its inhabitants. */ @@ -103,14 +98,11 @@ vrps_init(void) : (0xFFFFu); state.slurm = NULL; - error = db_rrdp_create(&state.rrdp_uris); - if (error) - goto release_deltas; error = pthread_rwlock_init(&state_lock, NULL); if (error) { error = pr_errno(error, "state pthread_rwlock_init() errored"); - goto release_db_rrdp; + goto release_deltas; } error = pthread_rwlock_init(&table_lock, NULL); @@ -122,8 +114,6 @@ vrps_init(void) return 0; release_state_lock: pthread_rwlock_destroy(&state_lock); -release_db_rrdp: - db_rddp_destroy(state.rrdp_uris); release_deltas: deltas_db_cleanup(&state.deltas, deltagroup_cleanup); return error; @@ -137,7 +127,6 @@ vrps_destroy(void) if (state.slurm != NULL) db_slurm_destroy(state.slurm); deltas_db_cleanup(&state.deltas, deltagroup_cleanup); - db_rddp_destroy(state.rrdp_uris); /* Nothing to do with error codes from now on */ pthread_rwlock_destroy(&state_lock); pthread_rwlock_destroy(&table_lock); @@ -181,67 +170,6 @@ handle_router_key(unsigned char const *ski, uint32_t as, rtrhandler_handle_router_key(arg, ski, as, spk)) } -rrdp_uri_cmp_result_t -rrdp_uri_cmp(char const *uri, char const *session_id, unsigned long serial) -{ - RLOCK_HANDLER(&state_lock, - db_rrdp_cmp_uri(state.rrdp_uris, uri, session_id, serial)) -} - -int -rrdp_uri_update(char const *uri, char const *session_id, unsigned long serial) -{ - WLOCK_HANDLER(&state_lock, - db_rrdp_add_uri(state.rrdp_uris, uri, session_id, serial)) -} - -int -rrdp_uri_get_serial(char const *uri, unsigned long *serial) -{ - RLOCK_HANDLER(&state_lock, - db_rrdp_get_serial(state.rrdp_uris, uri, serial)) -} - -int -rrdp_uri_get_last_update(char const *uri, long *last_update) -{ - RLOCK_HANDLER(&state_lock, - db_rrdp_get_last_update(state.rrdp_uris, uri, last_update)) -} - -int -rrdp_uri_set_last_update(char const *uri) -{ - WLOCK_HANDLER(&state_lock, - db_rrdp_set_last_update(state.rrdp_uris, uri)) -} - -bool -rrdp_uri_visited(char const *uri) -{ - bool result; - - rwlock_read_lock(&state_lock); - result = db_rrdp_get_visited(state.rrdp_uris, uri); - rwlock_unlock(&state_lock); - - return result; -} - -int -rrdp_uri_mark_visited(char const *uri) -{ - WLOCK_HANDLER(&state_lock, - db_rrdp_set_visited(state.rrdp_uris, uri, true)) -} - -static int -rrdp_uri_reset_visited(void) -{ - WLOCK_HANDLER(&state_lock, - db_rrdp_set_all_nonvisited(state.rrdp_uris)) -} - static int __perform_standalone_validation(struct db_table **result) { @@ -252,17 +180,6 @@ __perform_standalone_validation(struct db_table **result) if (db == NULL) return pr_enomem(); - /* - * Reset all RDDP URIs to 'non visited' every cycle, this way we can - * assure that the update notification file will be requested when - * needed and one time per cycle. - */ - error = rrdp_uri_reset_visited(); - if (error) { - db_table_destroy(db); - return error; - } - error = perform_standalone_validation(db); if (error) { terminate_standalone_validation(); @@ -368,7 +285,6 @@ __vrps_update(bool *changed) old_base = NULL; new_base = NULL; - /* FIXME (now) identify if RRDP is being utilized? */ error = __perform_standalone_validation(&new_base); if (error) return error; diff --git a/src/rtr/db/vrps.h b/src/rtr/db/vrps.h index 2c44b0e9..78770849 100644 --- a/src/rtr/db/vrps.h +++ b/src/rtr/db/vrps.h @@ -3,7 +3,6 @@ #include #include "data_structure/array_list.h" -#include "rrdp/rrdp_objects.h" #include "rtr/db/delta.h" /* @@ -42,15 +41,6 @@ int handle_roa_v6(uint32_t, struct ipv6_prefix const *, uint8_t, void *); int handle_router_key(unsigned char const *, uint32_t, unsigned char const *, void *); -rrdp_uri_cmp_result_t rrdp_uri_cmp(char const *, char const *, - unsigned long); -int rrdp_uri_update(char const *, char const *, unsigned long); -int rrdp_uri_get_serial(char const *, unsigned long *); -int rrdp_uri_get_last_update(char const *, long *); -int rrdp_uri_set_last_update(char const *); -bool rrdp_uri_visited(char const *); -int rrdp_uri_mark_visited(char const *); - uint16_t get_current_session_id(uint8_t); #endif /* SRC_VRPS_H_ */ diff --git a/src/state.c b/src/state.c index 343f8a24..d8974d13 100644 --- a/src/state.c +++ b/src/state.c @@ -1,6 +1,7 @@ #include "state.h" #include +#include "rrdp/db/db_rrdp.h" #include "log.h" #include "thread_var.h" @@ -22,7 +23,10 @@ struct validation { struct cert_stack *certstack; - struct uri_list *visited_uris; + struct uri_list *rsync_visited_uris; + + /* Shallow copy of RRDP URIs and its corresponding visited uris */ + struct db_rrdp_uri *rrdp_uris; /* Did the TAL's public key match the root certificate's public key? */ enum pubkey_state pubkey_state; @@ -84,6 +88,7 @@ validation_prepare(struct validation **out, struct tal *tal, struct validation_handler *validation_handler) { struct validation *result; + struct db_rrdp_uri *uris_table; X509_VERIFY_PARAM *params; int error; @@ -117,10 +122,15 @@ validation_prepare(struct validation **out, struct tal *tal, if (error) goto abort3; - error = rsync_create(&result->visited_uris); + error = rsync_create(&result->rsync_visited_uris); if (error) goto abort4; + uris_table = db_rrdp_get_uris(tal_get_file_name(tal)); + if (uris_table == NULL) + pr_crit("db_rrdp_get_uris() returned NULL, means it hasn't been initialized"); + result->rrdp_uris = uris_table; + result->pubkey_state = PKS_UNTESTED; result->validation_handler = *validation_handler; result->x509_data.params = params; /* Ownership transfered */ @@ -144,7 +154,7 @@ validation_destroy(struct validation *state) X509_VERIFY_PARAM_free(state->x509_data.params); X509_STORE_free(state->x509_data.store); certstack_destroy(state->certstack); - rsync_destroy(state->visited_uris); + rsync_destroy(state->rsync_visited_uris); free(state); } @@ -167,9 +177,9 @@ validation_certstack(struct validation *state) } struct uri_list * -validation_visited_uris(struct validation *state) +validation_rsync_visited_uris(struct validation *state) { - return state->visited_uris; + return state->rsync_visited_uris; } void @@ -207,3 +217,9 @@ validation_get_validation_handler(struct validation *state) { return &state->validation_handler; } + +struct db_rrdp_uri * +validation_get_rrdp_uris(struct validation *state) +{ + return state->rrdp_uris; +} diff --git a/src/state.h b/src/state.h index 495ae873..190a5719 100644 --- a/src/state.h +++ b/src/state.h @@ -6,6 +6,7 @@ #include "validation_handler.h" #include "object/tal.h" #include "rsync/rsync.h" +#include "rrdp/db/db_rrdp_uris.h" struct validation; @@ -16,7 +17,7 @@ void validation_destroy(struct validation *); struct tal *validation_tal(struct validation *); X509_STORE *validation_store(struct validation *); struct cert_stack *validation_certstack(struct validation *); -struct uri_list *validation_visited_uris(struct validation *); +struct uri_list *validation_rsync_visited_uris(struct validation *); enum pubkey_state { PKS_VALID, @@ -34,4 +35,6 @@ char *validation_get_ip_buffer2(struct validation *); struct validation_handler const * validation_get_validation_handler(struct validation *); +struct db_rrdp_uri *validation_get_rrdp_uris(struct validation *); + #endif /* SRC_STATE_H_ */ diff --git a/src/visited_uris.c b/src/visited_uris.c index 51e0cc3f..d4ad0c6d 100644 --- a/src/visited_uris.c +++ b/src/visited_uris.c @@ -1,10 +1,8 @@ #include "visited_uris.h" #include -#include #include #include -#include "common.h" #include "log.h" /* @@ -15,10 +13,12 @@ struct visited_elem { SLIST_ENTRY(visited_elem) next; }; -SLIST_HEAD(visited_uris, visited_elem) uris_db; +SLIST_HEAD(visited_list, visited_elem); -/** Read/write lock, which protects @uris_db. */ -static pthread_rwlock_t lock; +struct visited_uris { + struct visited_list *list; + unsigned int refs; +}; static int visited_elem_create(struct visited_elem **elem, char const *uri) @@ -47,86 +47,127 @@ visited_elem_destroy(struct visited_elem *elem) } int -visited_uris_init(void) +visited_uris_create(struct visited_uris **uris) { - int error; + struct visited_uris *tmp; + struct visited_list *tmp_list; - error = pthread_rwlock_init(&lock, NULL); - if (error) - return pr_errno(error, "Visited uris pthread_rwlock_init() errored"); + tmp = malloc(sizeof(struct visited_uris)); + if (tmp == NULL) + return pr_enomem(); + + tmp_list = malloc(sizeof(struct visited_list)); + if (tmp_list == NULL) { + free(tmp); + return pr_enomem(); + } + + SLIST_INIT(tmp_list); + tmp->list = tmp_list; + tmp->refs = 1; - SLIST_INIT(&uris_db); + *uris = tmp; return 0; } void -visited_uris_destroy(void) +visited_uris_refget(struct visited_uris *uris) +{ + uris->refs++; +} + +void +visited_uris_refput(struct visited_uris *uris) { struct visited_elem *elem; - while (!SLIST_EMPTY(&uris_db)) { - elem = uris_db.slh_first; - SLIST_REMOVE_HEAD(&uris_db, next); - visited_elem_destroy(elem); + uris->refs--; + if (uris->refs == 0) { + while (!SLIST_EMPTY(uris->list)) { + elem = uris->list->slh_first; + SLIST_REMOVE_HEAD(uris->list, next); + visited_elem_destroy(elem); + } + free(uris->list); + free(uris); } - pthread_rwlock_destroy(&lock); } static struct visited_elem * -elem_find(struct visited_uris *list, char const *uri) +elem_find(struct visited_list *list, char const *uri) { struct visited_elem *found; - rwlock_read_lock(&lock); - SLIST_FOREACH(found, list, next) { - if (strcmp(uri, found->uri) == 0) { - rwlock_unlock(&lock); + SLIST_FOREACH(found, list, next) + if (strcmp(uri, found->uri) == 0) return found; - } - } - rwlock_unlock(&lock); + return NULL; } int -visited_uris_add(char const *uri) +visited_uris_add(struct visited_uris *uris, char const *uri) { struct visited_elem *elem; int error; - if (elem_find(&uris_db, uri) != NULL) + if (elem_find(uris->list, uri) != NULL) return 0; /* Exists, don't add again */ error = visited_elem_create(&elem, uri); if (error) return error; - rwlock_write_lock(&lock); - SLIST_INSERT_HEAD(&uris_db, elem, next); - rwlock_unlock(&lock); + SLIST_INSERT_HEAD(uris->list, elem, next); return 0; } int -visited_uris_remove(char const *uri) +visited_uris_remove(struct visited_uris *uris, char const *uri) { struct visited_elem *elem; - elem = elem_find(&uris_db, uri); + elem = elem_find(uris->list, uri); if (elem == NULL) return pr_err("Trying to remove a nonexistent URI '%s'", uri); - rwlock_write_lock(&lock); - SLIST_REMOVE(&uris_db, elem, visited_elem, next); + SLIST_REMOVE(uris->list, elem, visited_elem, next); visited_elem_destroy(elem); - rwlock_unlock(&lock); return 0; } bool -visited_uris_exists(char const *uri) +visited_uris_exists(struct visited_uris *uris, char const *uri) +{ + return elem_find(uris->list, uri) != NULL; +} + +int +visited_uris_get_root(struct visited_uris *uris, char **result) { - return elem_find(&uris_db, uri) != NULL; + struct visited_elem *elem; + char *tmp, *ptr; + size_t size; + int i; + + elem = SLIST_FIRST(uris->list); + + i = 0; + ptr = strchr(elem->uri, '/'); + while(i < 2) { + ptr = strchr(ptr + 1, '/'); + i++; + } + size = ptr - elem->uri; + tmp = malloc(size + 1); + if (tmp == NULL) + return pr_enomem(); + + strncpy(tmp, elem->uri, size); + tmp[size + 1] = '\0'; + + *result = tmp; + return 0; } diff --git a/src/visited_uris.h b/src/visited_uris.h index 1bf77950..eff91b32 100644 --- a/src/visited_uris.h +++ b/src/visited_uris.h @@ -3,11 +3,15 @@ #include -int visited_uris_init(void); -void visited_uris_destroy(void); +struct visited_uris; -int visited_uris_add(char const *); -int visited_uris_remove(char const *); -bool visited_uris_exists(char const *); +int visited_uris_create(struct visited_uris **); +void visited_uris_refput(struct visited_uris *); +void visited_uris_refget(struct visited_uris *); + +int visited_uris_add(struct visited_uris *, char const *); +int visited_uris_remove(struct visited_uris *, char const *); +bool visited_uris_exists(struct visited_uris *, char const *); +int visited_uris_get_root(struct visited_uris *, char **); #endif /* SRC_VISITED_URIS_H_ */ diff --git a/test/rtr/db/vrps_test.c b/test/rtr/db/vrps_test.c index 0802761b..962cc447 100644 --- a/test/rtr/db/vrps_test.c +++ b/test/rtr/db/vrps_test.c @@ -11,7 +11,6 @@ #include "log.c" #include "output_printer.c" #include "object/router_key.c" -#include "rrdp/db_rrdp.c" #include "rtr/db/delta.c" #include "rtr/db/db_table.c" #include "rtr/db/rtr_db_impersonator.c" diff --git a/test/rtr/pdu_handler_test.c b/test/rtr/pdu_handler_test.c index 67d28207..efe1e358 100644 --- a/test/rtr/pdu_handler_test.c +++ b/test/rtr/pdu_handler_test.c @@ -12,7 +12,6 @@ #include "output_printer.c" #include "crypto/base64.c" #include "object/router_key.c" -#include "rrdp/db_rrdp.c" #include "rtr/pdu.c" #include "rtr/pdu_handler.c" #include "rtr/primitive_reader.c"