From: Alberto Leiva Popper Date: Sat, 6 Jul 2024 00:26:02 +0000 (-0600) Subject: Cache refactor 2 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=bbd1111a0fb78750f1e044a2906b4ab0f2c82eda;p=thirdparty%2FFORT-validator.git Cache refactor 2 Dirty commit; doesn't compile. I'll probably squash it later. --- diff --git a/src/Makefile.am b/src/Makefile.am index 9bf92579..49644d31 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,7 +4,10 @@ LDFLAGS_DEBUG = -rdynamic bin_PROGRAMS = fort +fort_SOURCES = alloc.c alloc.h +fort_SOURCES += cache/cachent.c cache/cachent.h fort_SOURCES += cache/local_cache.c cache/local_cache.h +fort_SOURCES += log.c log.h fort_CFLAGS = -Wall -Wpedantic -Werror #fort_CFLAGS += $(GCC_WARNS) diff --git a/src/cache/cachent.c b/src/cache/cachent.c new file mode 100644 index 00000000..272bb1b4 --- /dev/null +++ b/src/cache/cachent.c @@ -0,0 +1,284 @@ +#include "cache/cachent.h" + +#include "alloc.h" +#include "config.h" +#include "data_structure/common.h" +#include "data_structure/path_builder.h" + +/* Preorder. @cb returns whether the children should be traversed. */ +int +cachent_traverse(struct cache_node *root, + bool (*cb)(struct cache_node *, char const *)) +{ + struct cache_node *iter_start; + struct cache_node *parent, *child; + struct cache_node *tmp; + struct path_builder pb; + int error; + + if (!root) + return 0; + + pb_init(&pb); + + error = pb_append(&pb, config_get_local_repository()); + if (error) + goto end; + + error = pb_append(&pb, root->name); + if (error) + goto end; + if (!cb(root, pb.string)) + goto end; + + parent = root; + iter_start = parent->children; + if (iter_start == NULL) + goto end; + +reloop: /* iter_start must not be NULL */ + HASH_ITER(hh, iter_start, child, tmp) { + error = pb_append(&pb, child->name); + if (error) + goto end; + + if (cb(child, pb.string) && (child->children != NULL)) { + parent = child; + iter_start = parent->children; + goto reloop; + } + + pb_pop(&pb, true); + } + + parent = iter_start->parent; + do { + if (parent == NULL) + goto end; + pb_pop(&pb, true); + iter_start = parent->hh.next; + parent = parent->parent; + } while (iter_start == NULL); + + goto reloop; + +end: pb_cleanup(&pb); + return error; +} + +struct tokenizer { + char const *str; + size_t len; +}; + +static bool +is_delimiter(char chara) +{ + return chara == '/' || chara == '\0'; +} + +static void +token_init(struct tokenizer *tkn, char const *str) +{ + tkn->str = str; + tkn->len = 0; +} + +/* Like strtok_r(), but doesn't corrupt the string. */ +static bool +token_next(struct tokenizer *tkn) +{ + tkn->str += tkn->len; + while (tkn->str[0] == '/') + tkn->str++; + if (tkn->str[0] == '\0') + return false; + for (tkn->len = 1; !is_delimiter(tkn->str[tkn->len]); tkn->len++) + ; + return true; +} + +static char * +path_rewind(char const *root, char *cursor) +{ + for (cursor -= 2; root <= cursor; cursor--) + if (*cursor == '/') + return cursor + 1; + return NULL; +} + +/* Collapses '//' (after the schema), '.' and '..'. */ +static char * +normalize(char const *url) +{ + char *normal, *dst, *root; + struct tokenizer tkn; + + if (strncmp(url, "rsync://", RPKI_SCHEMA_LEN) && + strncmp(url, "https://", RPKI_SCHEMA_LEN)) + return NULL; + + normal = pstrdup(url); + dst = normal + RPKI_SCHEMA_LEN; + root = dst - 1; + token_init(&tkn, url + RPKI_SCHEMA_LEN); + + while (token_next(&tkn)) { + if (tkn.len == 1 && tkn.str[0] == '.') + continue; + if (tkn.len == 2 && tkn.str[0] == '.' && tkn.str[1] == '.') { + dst = path_rewind(root, dst); + if (!dst) + goto fail; + continue; + } + strncpy(dst, tkn.str, tkn.len); + dst[tkn.len] = '/'; + dst += tkn.len + 1; + } + + /* Reject URL if there's nothing after the schema. Maybe unnecessary. */ + if (dst == normal + RPKI_SCHEMA_LEN) + goto fail; + + dst[-1] = '\0'; + return normal; + +fail: free(normal); + return NULL; +} + +static struct cache_node * +provide(struct cache_node *parent, char const *url, + char const *name, size_t namelen) +{ + struct cache_node *child; + + if (parent != NULL) { + HASH_FIND(hh, parent->children, name, namelen, child); + if (child != NULL) + return child; + } + + child = pzalloc(sizeof(struct cache_node)); + child->url = pstrndup(url, name - url + namelen); + child->name = child->url + (name - url); + child->parent = parent; + if (parent != NULL) + HASH_ADD_KEYPTR(hh, parent->children, child->name, + namelen, child); + + return child; +} + +/* + * Find and return. If not found, create and return. + * + * Suppose @url is "rsync://a.b.c/d/e/f.cer": @ancestor has to be either + * NULL, "rsync", "rsync://a.b.c", "rsync://a.b.c/d", "rsync://a.b.c/d/e" or + * "rsync://a.b.c/d/e/f.cer". + * + * Returns NULL if @ancestor doesn't match @url. + * + * The point of @ancestor is caging. @url will not be allowed to point to + * anything that is not @ancestor or one of its descendants. (ie. dot-dotting is + * allowed, but the end result must not land outside of @ancestor.) + * + * XXX review callers; can now return NULL. + */ +struct cache_node * +cachent_provide(struct cache_node *ancestor, char const *url) +{ + char *normal; + array_index i = 0; + struct tokenizer tkn; + + normal = normalize(url); + if (!normal) + return NULL; + + if (ancestor != NULL) { + for (; ancestor->url[i] != 0; i++) + if (ancestor->url[i] != normal[i]) + goto fail; + if (!is_delimiter(normal[i])) + goto fail; + } + + token_init(&tkn, normal + i); + while (token_next(&tkn)) + ancestor = provide(ancestor, normal, tkn.str, tkn.len); + free(normal); + return ancestor; + +fail: free(normal); + return NULL; +} + +#ifdef UNIT_TESTING +static void __delete_node_cb(struct cache_node const *); +#endif + +static void +__delete_node(struct cache_node *node) +{ +#ifdef UNIT_TESTING + __delete_node_cb(node); +#endif + + if (node->parent != NULL) + HASH_DEL(node->parent->children, node); + free(node->url); + free(node->tmpdir); + free(node); +} + +void +cachent_delete(struct cache_node *node) +{ + struct cache_node *parent; + + if (!node) + return; + + parent = node->parent; + if (parent != NULL) { + HASH_DEL(parent->children, node); + node->parent = NULL; + } + + do { + while (node->children) + node = node->children; + + parent = node->parent; + __delete_node(node); + node = parent; + } while (node != NULL); +} + +void +print_node(struct cache_node *node, unsigned int tabs) +{ + unsigned int i; + struct cache_node *child, *tmp; + + for (i = 0; i < tabs; i++) + printf("\t"); + + printf("%s ", node->name); + printf("%s", (node->flags & CNF_RSYNC) ? "RSYNC " : ""); + printf("%s", (node->flags & CNF_DOWNLOADED) ? "DL " : ""); + printf("%s", (node->flags & CNF_TOUCHED) ? "Touched " : ""); + printf("%s", (node->flags & CNF_VALIDATED) ? "Valid " : ""); + printf("%s\n", (node->flags & CNF_WITHDRAWN) ? "Withdrawn " : ""); + + HASH_ITER(hh, node->children, child, tmp) + print_node(child, tabs + 1); +} + +void +cachent_print(struct cache_node *node) +{ + print_node(node, 0); +} diff --git a/src/cache/cachent.h b/src/cache/cachent.h new file mode 100644 index 00000000..2676a57f --- /dev/null +++ b/src/cache/cachent.h @@ -0,0 +1,69 @@ +#ifndef SRC_CACHE_CACHENT_H_ +#define SRC_CACHE_CACHENT_H_ + +/* CACHE ENTity, CACHE elemENT, CACHE componENT */ + +#include "data_structure/uthash.h" + +#define RPKI_SCHEMA_LEN 8 /* strlen("rsync://"), strlen("https://") */ + +/* XXX rename "touched" and "validated" into "preserve"? */ + +#define CNF_RSYNC (1 << 0) +/* Do we have a copy in the cache? */ +#define CNF_CACHED (1 << 1) +/* Was it downloaded during the current cycle? XXX Probably rename to "FRESH" */ +#define CNF_DOWNLOADED (1 << 2) +/* Did it change between the previous cycle and the current one? */ +#define CNF_CHANGED (1 << 3) +/* Was it read during the current cycle? */ +#define CNF_TOUCHED (1 << 4) +/* + * Did it validate successfully (at least once) during the current cycle? + * (It's technically possible for two different repositories to map to the same + * cache node. One of them is likely going to fail validation.) + */ +#define CNF_VALIDATED (1 << 5) +/* Is the node an RRDP Update Notification? */ +#define CNF_NOTIFICATION (1 << 6) +/* Withdrawn by RRDP? */ +#define CNF_WITHDRAWN (1 << 7) + +// XXX rename to cache_entity or cachent +struct cache_node { + char const *name; /* Points to the last component of @url */ + char *url; + int flags; + /* Last successful download time, or zero */ + time_t mtim; + /* + * If flags & CNF_DOWNLOADED, path to the temporal directory where we + * downloaded the latest refresh. + * (See --compare-dest at rsync(1). RRDP is basically the same.) + * Otherwise undefined. + * + * XXX this is not always a directory; rename to "tmppath" + */ + char *tmpdir; + + /* Only if flags & CNF_NOTIFICATION. */ +// struct cachefile_notification notif; + + /* Tree parent. Only defined during cleanup. */ + struct cache_node *parent; + /* Tree children. */ + struct cache_node *children; + + UT_hash_handle hh; /* Hash table hook */ +}; + +int cachent_traverse(struct cache_node *, + bool (*cb)(struct cache_node *, char const *)); + +struct cache_node *cachent_provide(struct cache_node *, char const *); +void cachent_delete(struct cache_node *); + +/* Recursive; tests only. */ +void cachent_print(struct cache_node *); + +#endif /* SRC_CACHE_CACHENT_H_ */ diff --git a/src/cache/local_cache.c b/src/cache/local_cache.c index c47cf242..faef9be4 100644 --- a/src/cache/local_cache.c +++ b/src/cache/local_cache.c @@ -38,44 +38,9 @@ struct cached_rpp { struct cached_file *ht; }; -#define CNF_RSYNC (1 << 0) -/* Was it downloaded during the current cycle? */ -#define CNF_DOWNLOADED (1 << 1) -/* Was it read during the current cycle? */ -#define CNF_TOUCHED (1 << 2) -/* - * Did it validate successfully (at least once) during the current cycle? - * (It's technically possible for two different repositories to map to the same - * cache node. One of them is likely going to fail validation.) - */ -#define CNF_VALIDATED (1 << 3) -/* Withdrawn by RRDP? */ -#define CNF_WITHDRAWN (1 << 4) - -struct cache_node { - char const *name; /* Points to the last component of @url */ - char *url; - int flags; - /* Last successful download time, or zero */ - time_t mtim; - /* - * If flags & CNF_DOWNLOADED, path to the temporal directory where we - * downloaded the latest refresh. - * (See --compare-dest at rsync(1). RRDP is basically the same.) - * Otherwise undefined. - */ - char *tmpdir; - - /* Tree parent. Only defined during cleanup. */ - struct cache_node *parent; - /* Tree children. */ - struct cache_node *children; - - UT_hash_handle hh; /* Hash table hook */ -}; - static struct rpki_cache { - struct cache_node root; /* It's a tree. */ + struct cache_node *https; + struct cache_node *rsync; // time_t startup_ts; /* When we started the last validation */ } cache; @@ -516,60 +481,6 @@ end: json_decref(json); free(filename); } -/* - * Returns perfect match. (Even if it needs to create it.) - * Always consumes @path. - * - * Unit Test (perfect match): - * root - * a - * b - * c - * - Find c - * - Find a - * - Find a/b - * - Find a/b/c - * - Find a/b/c/d - */ -static struct cache_node * -find_node(char *path, int flags) -{ - struct cache_node *node, *child; - char *nm, *sp; /* name, saveptr */ - size_t keylen; - - node = &cache.root; - nm = strtok_r(path + RPKI_SCHEMA_LEN, "/", &sp); // XXX - - for (; nm; nm = strtok_r(NULL, "/", &sp)) { - keylen = strlen(nm); - HASH_FIND(hh, node->children, nm, keylen, child); - if (child == NULL) - goto create_children; - node = child; - sp[-1] = '/'; /* XXX this will need a compliance unit test */ - } - - goto end; - -create_children: - for (; nm; nm = strtok_r(NULL, "/", &sp)) { - child = pmalloc(sizeof(struct cache_node)); - child->url = pstrdup(path); - child->name = strrchr(child->url, '/') + 1; // XXX - child->flags = flags; - - keylen = strlen(nm); - HASH_ADD_KEYPTR(hh, node->children, child->name, keylen, child); - - node = child; - sp[-1] = '/'; - } - -end: free(path); - return node; -} - /* * Returns perfect match or NULL. @msm will point to the Most Specific Match. * Always consumes @path. @@ -582,7 +493,7 @@ find_msm(char *path, struct cache_node **msm) size_t keylen; *msm = NULL; - node = &cache.root; + node = cache.root; nm = strtok_r(path + RPKI_SCHEMA_LEN, "/", &sp); // XXX for (; nm; nm = strtok_r(NULL, "/", &sp)) { @@ -601,86 +512,10 @@ find_msm(char *path, struct cache_node **msm) return node; } -static char * -get_rsync_module(char const *url) -{ - char const *c; - char *dup; - unsigned int slashes; - - /* - * Careful with this code. rsync(1): - * - * > A trailing slash on the source changes this behavior to avoid - * > creating an additional directory level at the destination. You can - * > think of a trailing / on a source as meaning "copy the contents of - * > this directory" as opposed to "copy the directory by name", but in - * > both cases the attributes of the containing directory are - * > transferred to the containing directory on the destination. In - * > other words, each of the following commands copies the files in the - * > same way, including their setting of the attributes of /dest/foo: - * > - * > rsync -av /src/foo /dest - * > rsync -av /src/foo/ /dest/foo - * - * This quirk does not behave consistently. In practice, if you rsync - * at the module level, rsync servers behave as if the trailing slash - * always existed. - * - * ie. the two following rsyncs behave identically: - * - * rsync -rtz rsync://repository.lacnic.net/rpki potatoes - * (Copies the content of rpki to potatoes.) - * rsync -rtz rsync://repository.lacnic.net/rpki/ potatoes - * (Copies the content of rpki to potatoes.) - * - * Even though the following do not: - * - * rsync -rtz rsync://repository.lacnic.net/rpki/lacnic potatoes - * (Copies lacnic to potatoes.) - * rsync -rtz rsync://repository.lacnic.net/rpki/lacnic/ potatoes - * (Copies the content of lacnic to potatoes.) - * - * This is important to us, because an inconsistent missing directory - * component will screw our URLs-to-cache mappings. - * - * My solution is to add the slash myself. That's all I can do to force - * it to behave consistently, it seems. - * - * But note: This only works if we're synchronizing a directory. - * But this is fine, because this hack stacks with the minimum common - * path performance hack. - * - * Minimum common path performance hack: rsync the rsync module root, - * not every RPP separately. The former is much faster. - */ - - slashes = 0; - for (c = url; *c != '\0'; c++) { - if (*c == '/') { - slashes++; - if (slashes == 4) - /* XXX test the if I rm'd here */ - return pstrndup(url, c - url + 1); - } - } - - if (slashes == 3 && c[-1] != '/') { - dup = pmalloc(c - url + 2); - memcpy(dup, url, c - url); - dup[c - url] = '/'; - dup[c - url + 1] = '\0'; - return dup; - } - - pr_val_err("Can't rsync URL '%s': The URL seems to be missing a domain or rsync module.", - url); - return NULL; -} - static int -dl_rsync(struct cache_node *node) +dl_rsync(char const *uri, struct cache_node *node) { + struct cache_node *module; char *path; int error; @@ -689,6 +524,11 @@ dl_rsync(struct cache_node *node) return 1; } + /* XXX this is probably wrong; get the real module. */ + module = node; + while (module->parent != NULL) + module = module->parent; + error = cache_tmpfile(&path); if (error) return error; @@ -702,13 +542,20 @@ dl_rsync(struct cache_node *node) goto cancel; // XXX looks like the third argument is redundant now. - error = rsync_download(node->url, path, true); + error = rsync_download(module->url, path, true); if (error) goto cancel; - node->flags |= CNF_DOWNLOADED; - node->mtim = time(NULL); // XXX catch -1 - node->tmpdir = path; + module->flags |= CNF_DOWNLOADED; + module->mtim = time(NULL); // XXX catch -1 + module->tmpdir = path; + + while (node != NULL) { + node->flags |= CNF_DOWNLOADED; + node->mtim = module->mtim; + node = node->parent; + } + return 0; cancel: free(path); @@ -718,72 +565,41 @@ cancel: free(path); static int dl_http(struct cache_node *node) { - char *path; - bool changed; - int error; - if (!config_get_http_enabled()) { pr_val_debug("HTTP is disabled."); return 1; } - error = cache_tmpfile(&path); - if (error) - return error; - - error = http_download(node->url, path, node->mtim, &changed); - if (error) { - free(path); - return error; - } - - node->flags |= CNF_DOWNLOADED; - if (changed) - node->mtim = time(NULL); // XXX catch -1 - node->tmpdir = path; - return 0; + return http_download_cache_node(node); } static int -dl_rrdp(struct cache_node *node) +dl_rrdp(char const *notif_url, struct cache_node *mft) { - char *path; - int error; - if (!config_get_http_enabled()) { pr_val_debug("HTTP is disabled."); return 1; } - error = cache_tmpfile(&path); - if (error) - return error; - - // XXX needs to add all files to node. - // Probably also update node itself. - error = rrdp_update(path, node); - if (error) { - free(path); - return error; - } + // XXX needs to add all files to node. Probably also update node itself. + // XXX maybe pr_crit() on !mft->parent? + return rrdp_update(cachent_provide(cache.https, notif_url), mft->parent); - node->flags |= CNF_DOWNLOADED; - node->mtim = time(NULL); // XXX catch -1 - node->tmpdir = path; - return 0; +// node->flags |= CNF_DOWNLOADED; +// node->mtim = time(NULL); // XXX catch -1 +// node->tmpdir = path; } static int -download(struct cache_mapping *map, struct cache_node *node) +download(char const *uri, enum map_type type, struct cache_node *mft) { - switch (map_get_type(map)) { + switch (type) { case MAP_RSYNC: - return dl_rsync(node); + return dl_rsync(uri, mft); case MAP_HTTP: - case MAP_TMP: - return dl_http(node); + return dl_http(mft); case MAP_NOTIF: - return dl_rrdp(node); + return dl_rrdp(uri, mft); } pr_crit("Unreachable."); @@ -794,36 +610,22 @@ download(struct cache_mapping *map, struct cache_node *node) * XXX review result sign */ static int -try_url(struct cache_mapping *map, bool online, maps_dl_cb cb, void *arg) +try_uri(struct cache_node *mft, char const *uri, enum map_type type, + bool online, maps_dl_cb cb, void *arg) { - bool is_rsync; - char *url; - struct cache_node *node; int error; - // XXX if RRDP, @map needs to be unwrapped... - - is_rsync = map_get_type(map) == MAP_RSYNC; - url = is_rsync - ? get_rsync_module(map_get_url(map)) - : pstrdup(map_get_url(map)); - if (!url) - return -EINVAL; + pr_val_debug("Trying %s...", uri); - pr_val_debug("Trying RPP URL %s...", url); - - /* XXX mutex */ - node = find_node(url, is_rsync ? CNF_RSYNC : 0); - - if (online && !(node->flags & CNF_DOWNLOADED)) { - error = download(map, node); + if (online) { + error = download(uri, type, mft); if (error) { pr_val_debug("RPP refresh failed."); return error; } } - error = cb(node, arg); + error = cb(mft, arg); if (error) { pr_val_debug("RPP validation failed."); return error; @@ -834,56 +636,6 @@ try_url(struct cache_mapping *map, bool online, maps_dl_cb cb, void *arg) return 0; } -static int -download_maps(struct map_list *maps, bool online, enum map_type type, - maps_dl_cb cb, void *arg) -{ - struct cache_mapping **_map, *map; - int error; - - ARRAYLIST_FOREACH(maps, _map) { - map = *_map; - - if ((map_get_type(map) & type) != type) - continue; - - error = try_url(map, online, cb, arg); - if (error <= 0) - return error; - } - - return 1; -} - -static int -try_alts(struct map_list *maps, bool online, maps_dl_cb cb, void *arg) -{ - struct cache_mapping **cursor; - int error; - - /* XXX during cleanup, always preserve only one? */ - if (config_get_http_priority() > config_get_rsync_priority()) { - error = download_maps(maps, online, MAP_HTTP, cb, arg); - if (error <= 0) - return error; - return download_maps(maps, online, MAP_RSYNC, cb, arg); - - } else if (config_get_http_priority() < config_get_rsync_priority()) { - error = download_maps(maps, online, MAP_RSYNC, cb, arg); - if (error <= 0) - return error; - return download_maps(maps, online, MAP_HTTP, cb, arg); - - } else { - ARRAYLIST_FOREACH(maps, cursor) { - error = try_url(*cursor, online, cb, arg); - if (error <= 0) - return error; - } - return 1; - } -} - /** * Assumes the URIs represent different ways to access the same content. * @@ -896,140 +648,33 @@ try_alts(struct map_list *maps, bool online, maps_dl_cb cb, void *arg) * that's already cached, and callbacks it. */ int -cache_download_alt(struct map_list *maps, maps_dl_cb cb, void *arg) -{ - int error; - - error = try_alts(maps, true, cb, arg); - if (error) - error = try_alts(maps, false, cb, arg); - - return error; -} - -static void -print_node(struct cache_node *node, unsigned int tabs) -{ - struct cache_node *child, *tmp; - unsigned int i; - - for (i = 0; i < tabs; i++) - printf("\t"); - - printf("%s ", node->name); - printf("%s", (node->flags & CNF_RSYNC) ? "RSYNC " : ""); - printf("%s", (node->flags & CNF_DOWNLOADED) ? "DL " : ""); - printf("%s", (node->flags & CNF_TOUCHED) ? "Touched " : ""); - printf("%s", (node->flags & CNF_VALIDATED) ? "Valid " : ""); - printf("%s\n", (node->flags & CNF_WITHDRAWN) ? "Withdrawn " : ""); - - HASH_ITER(hh, node->children, child, tmp) - print_node(child, tabs + 1); -} - -/* Recursive; tests only. */ -void -cache_print(struct rpki_cache *cache) +cache_download_alt(struct sia_uris *uris, maps_dl_cb cb, void *arg) { - print_node(&cache->root, 0); -} - -#ifdef UNIT_TESTING -static void __delete_node_cb(struct cache_node const *); -#endif + struct cache_node *mft; + int online; + char **uri; + int error = 0; -static void -__delete_node(struct cache_node *node) -{ -#ifdef UNIT_TESTING - __delete_node_cb(node); -#endif - - if (node->parent != NULL) - HASH_DEL(node->parent->children, node); - free(node->url); - free(node->tmpdir); - free(node); -} - -/* - * Caveats: - * - * - node->parent has to be set. - * - Don't use this on the root. - */ -static void -delete_node(struct cache_node *node) -{ - struct cache_node *parent; + /* XXX mutex */ + /* XXX if parent is downloaded, child is downloaded. */ + mft = cachent_provide(cache.rsync, uris->rpkiManifest); - parent = node->parent; - if (parent != NULL) { - HASH_DEL(parent->children, node); - node->parent = NULL; - } + if (mft->flags & CNF_DOWNLOADED) + return cb(mft, arg); - do { - while (node->children) { - node->children->parent = node; - node = node->children; + for (online = 1; online >= 0; online--) { + ARRAYLIST_FOREACH(&uris->rpkiNotify, uri) { + error = try_uri(mft, *uri, MAP_NOTIF, online, cb, arg); + if (error <= 0) + return error; } - - parent = node->parent; - __delete_node(node); - node = parent; - } while (node != NULL); -} - -/* Preorder. @cb returns whether the children should be traversed. */ -static int -traverse_cache(bool (*cb)(struct cache_node *, char const *)) -{ - struct cache_node *iter_start; - struct cache_node *parent, *child; - struct cache_node *tmp; - struct path_builder pb; - int error; - - pb_init(&pb); - - error = pb_append(&pb, cache.root.name); - if (error) - goto end; - - parent = &cache.root; - iter_start = parent->children; - if (iter_start == NULL) - goto end; - -reloop: /* iter_start must not be NULL */ - HASH_ITER(hh, iter_start, child, tmp) { - error = pb_append(&pb, child->name); - if (error) - goto end; - - child->parent = parent; - if (cb(child, pb.string) && (child->children != NULL)) { - parent = child; - iter_start = parent->children; - goto reloop; + ARRAYLIST_FOREACH(&uris->caRepository, uri) { + error = try_uri(mft, *uri, MAP_RSYNC, online, cb, arg); + if (error <= 0) + return error; } - - pb_pop(&pb, true); } - parent = iter_start->parent; - do { - if (parent == NULL) - goto end; - pb_pop(&pb, true); - iter_start = parent->hh.next; - parent = parent->parent; - } while (iter_start == NULL); - - goto reloop; - -end: pb_cleanup(&pb); return error; } @@ -1219,9 +864,9 @@ __remove_abandoned(const char *path, const struct stat *st, int typeflag, * This will happen most of the time. */ if (rmdir(path) == 0) - delete_node(pm); + cachent_delete(pm); else if (errno == ENOENT) - delete_node(pm); + cachent_delete(pm); } else if (S_ISREG(st->st_mode)) { if (pm->flags & (CNF_RSYNC | CNF_WITHDRAWN)) { @@ -1238,7 +883,7 @@ __remove_abandoned(const char *path, const struct stat *st, int typeflag, abandoned: if (pm) - delete_node(pm); + cachent_delete(pm); unknown: remove(path); // XXX return 0; @@ -1261,13 +906,14 @@ remove_abandoned(void) * Deletes unknown and old untraversed cached files, writes metadata into XML. */ static void -cache_cleanup(struct rpki_cache *cache) +cache_cleanup(void) { // struct cache_node *node, *tmp; // time_t last_week; pr_op_debug("Committing successful RPPs."); - traverse_cache(commit_rpp_delta); + cachent_traverse(cache.rsync, commit_rpp_delta); + cachent_traverse(cache.https, commit_rpp_delta); // pr_op_debug("Cleaning up temporal files."); // HASH_ITER(hh, cache->ht, node, tmp) @@ -1280,11 +926,9 @@ cache_cleanup(struct rpki_cache *cache) } //void -//cache_destroy(struct rpki_cache *cache) +//cache_destroy(void) //{ -// struct cache_node *node, *tmp; -// -// cache_cleanup(cache); +// cache_cleanup(); // write_tal_json(cache); // // HASH_ITER(hh, cache->ht, node, tmp) diff --git a/src/cache/local_cache.h b/src/cache/local_cache.h index d0e6d179..3c6a66b7 100644 --- a/src/cache/local_cache.h +++ b/src/cache/local_cache.h @@ -3,8 +3,7 @@ #include "types/map.h" #include "types/str.h" - -#define RPKI_SCHEMA_LEN 8 /* strlen("rsync://"), strlen("https://") */ +#include "cache/cachent.h" struct rpki_cache; struct cache_node; @@ -16,7 +15,13 @@ int cache_tmpfile(char **); struct rpki_cache *cache_create(void); /* Will destroy the cache object, but not the cache directory itself, obv. */ -void cache_destroy(struct rpki_cache *); +void cache_destroy(void); + +struct sia_uris { + struct strlist caRepository; /* rsync RPPs */ + struct strlist rpkiNotify; /* RRDP Notifications */ + char *rpkiManifest; +}; /* * The callback should return @@ -28,7 +33,7 @@ void cache_destroy(struct rpki_cache *); * XXX rename */ typedef int (*maps_dl_cb)(struct cache_node *, void *); -int cache_download_alt(struct map_list *, maps_dl_cb, void *); +int cache_download_alt(struct sia_uris *, maps_dl_cb, void *); /* Prints the cache in standard output. */ void cache_print(struct rpki_cache *); diff --git a/src/config.c b/src/config.c index fafaab24..eb4d5e2d 100644 --- a/src/config.c +++ b/src/config.c @@ -451,6 +451,7 @@ static const struct option_field options[] = { .doc = "rsync's priority for repository file fetching. Higher value means higher priority.", .min = 0, .max = 100, + /* XXX deprecated? */ }, { .id = 3002, .name = "rsync.strategy", @@ -520,6 +521,7 @@ static const struct option_field options[] = { .doc = "HTTP's priority for repository file fetching. Higher value means higher priority.", .min = 0, .max = 100, + /* XXX deprecated? */ }, { .id = 9002, .name = "http.retry.count", diff --git a/src/http/http.c b/src/http/http.c index d4b8cd57..1314c91f 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -1,6 +1,7 @@ #include "http/http.h" #include "alloc.h" +#include "cache/cache_entity.h" #include "cache/local_cache.h" #include "common.h" #include "config.h" @@ -431,3 +432,40 @@ http_download_direct(char const *src, char const *dst) pr_val_info("HTTP GET: %s -> %s", src, dst); return http_fetch(src, dst, 0, NULL); } + +int +http_download_tmp(char const *url, char const **path, + curl_off_t ims, bool *changed) +{ + int error; + + error = cache_tmpfile(path); + if (error) + return error; + + error = http_download(url, *path, ims, changed); + if (error) + free(*path); + + return error; +} + +int +http_download_cache_node(struct cache_node *node) +{ + char *path; + bool changed; + int error; + + error = http_download_tmp(node->url, &path, node->mtim, &changed); + if (error) + return error; + + node->flags |= CNF_DOWNLOADED; // XXX on notification, preserve node but not file + if (changed) { + node->flags |= CNF_CHANGED; + node->mtim = time(NULL); // XXX catch -1 + } + node->tmpdir = path; + return 0; +} diff --git a/src/http/http.h b/src/http/http.h index a7b3ac5e..55b63f87 100644 --- a/src/http/http.h +++ b/src/http/http.h @@ -8,6 +8,7 @@ void http_cleanup(void); int http_download(char const *, char const *, curl_off_t, bool *); int http_download_direct(char const *, char const *); -int http_download_tmp(char const *, char **, bool *, void *); +int http_download_tmp(char const *, char const **, curl_off_t, bool *); +int http_download_cache_node(struct cache_node *); #endif /* SRC_HTTP_HTTP_H_ */ diff --git a/src/object/certificate.c b/src/object/certificate.c index 7d0b3df2..e44d5038 100644 --- a/src/object/certificate.c +++ b/src/object/certificate.c @@ -46,11 +46,6 @@ struct ski_arguments { OCTET_STRING_t *sid; }; -struct sia_uris { - struct strlist rpp; - char *mft; -}; - struct bgpsec_ski { X509 *cert; unsigned char **ski_data; @@ -104,15 +99,17 @@ static const struct ad_metadata RPKI_MANIFEST = { static void sia_uris_init(struct sia_uris *uris) { - strlist_init(&uris->rpp); - uris->mft = NULL; + strlist_init(&uris->caRepository); + strlist_init(&uris->rpkiNotify); + uris->rpkiManifest = NULL; } static void sia_uris_cleanup(struct sia_uris *uris) { - strlist_cleanup(&uris->rpp); - free(uris->mft); + strlist_cleanup(&uris->caRepository); + strlist_cleanup(&uris->rpkiNotify); + free(uris->rpkiManifest); } static void @@ -1196,7 +1193,7 @@ static void handle_rpkiManifest(char *uri, void *arg) { struct sia_uris *uris = arg; - uris->mft = uri; + uris->rpkiManifest = uri; } static void @@ -1204,7 +1201,7 @@ handle_caRepository(char *uri, void *arg) { struct sia_uris *uris = arg; pr_val_debug("caRepository: %s", uri); - strlist_add(&uris->rpp, uri); + strlist_add(&uris->caRepository, uri); } static void @@ -1212,7 +1209,7 @@ handle_rpkiNotify(char *uri, void *arg) { struct sia_uris *uris = arg; pr_val_debug("rpkiNotify: %s", uri); - strlist_add(&uris->rpp, uri); + strlist_add(&uris->rpkiNotify, uri); } static void @@ -1805,11 +1802,10 @@ certificate_validate_aia(char const *caIssuers, X509 *cert) static int download_rpp(struct sia_uris *uris) { - if (uris->rpp.len == 0) + if (uris->caRepository.len == 0 && uris->rpkiNotify.len == 0) return pr_val_err("SIA lacks both caRepository and rpkiNotify."); - return cache_download_alt(validation_cache(state_retrieve()), - &uris->rpp, MAP_NOTIF, NULL, NULL); + return cache_download_alt(uris, MAP_NOTIF, NULL, NULL); } /** Boilerplate code for CA certificate validation and recursive traversal. */ @@ -1897,7 +1893,7 @@ certificate_traverse(struct rpp *rpp_parent, struct cache_mapping *cert_map) goto revert_uris; cert = NULL; /* Ownership stolen */ - error = handle_manifest(sia_uris.mft, &pp); + error = handle_manifest(sia_uris.rpkiManifest, &pp); if (error) { x509stack_cancel(validation_certstack(state)); goto revert_uris; diff --git a/src/object/tal.c b/src/object/tal.c index 9c02149f..35ab767b 100644 --- a/src/object/tal.c +++ b/src/object/tal.c @@ -27,8 +27,6 @@ struct tal { struct strlist urls; unsigned char *spki; /* Decoded; not base64. */ size_t spki_len; - - struct rpki_cache *cache; }; struct validation_thread { @@ -137,14 +135,9 @@ tal_init(struct tal *tal, char const *file_path) strlist_init(&tal->urls); error = read_content((char *)file.buffer, tal); - if (error) { + if (error) strlist_cleanup(&tal->urls); - goto end; - } - tal->cache = cache_create(); - -end: file_free(&file); return error; } @@ -152,7 +145,6 @@ end: static void tal_cleanup(struct tal *tal) { - cache_destroy(tal->cache); free(tal->spki); strlist_cleanup(&tal->urls); } @@ -170,12 +162,6 @@ tal_get_spki(struct tal *tal, unsigned char const **buffer, size_t *len) *len = tal->spki_len; } -struct rpki_cache * -tal_get_cache(struct tal *tal) -{ - return tal->cache; -} - /** * Performs the whole validation walkthrough on the @map mapping, which is * assumed to have been extracted from TAL @tal. @@ -281,8 +267,8 @@ do_file_validation(void *arg) goto end; args.db = db_table_create(); - thread->error = cache_download_alt(args.tal.cache, &args.tal.urls, - MAP_HTTP, __handle_tal_map, &args); + thread->error = cache_download_alt(&args.tal.urls, MAP_HTTP, + __handle_tal_map, &args); if (thread->error) { pr_op_err("None of the URIs of the TAL '%s' yielded a successful traversal.", thread->tal_file); diff --git a/src/object/tal.h b/src/object/tal.h index 38becd0c..46eaabda 100644 --- a/src/object/tal.h +++ b/src/object/tal.h @@ -10,7 +10,6 @@ struct tal; char const *tal_get_file_name(struct tal *); void tal_get_spki(struct tal *, unsigned char const **, size_t *); -struct rpki_cache *tal_get_cache(struct tal *); struct db_table *perform_standalone_validation(void); diff --git a/src/rrdp.c b/src/rrdp.c index 8c4740d1..c0f87151 100644 --- a/src/rrdp.c +++ b/src/rrdp.c @@ -13,6 +13,7 @@ #include "log.h" #include "thread_var.h" #include "http/http.h" +#include "cache/cache_entity.h" #include "crypto/base64.h" #include "crypto/hash.h" #include "xml/relax_ng.h" @@ -73,8 +74,6 @@ struct publish { struct file_metadata meta; unsigned char *content; size_t content_len; - - char *path; }; /* A deserialized tag, from a delta. */ @@ -84,6 +83,7 @@ struct withdraw { char *path; }; +// XXX delete? typedef enum { HR_MANDATORY, HR_OPTIONAL, @@ -111,6 +111,11 @@ struct cachefile_notification { STAILQ_HEAD(, rrdp_hash) delta_hashes; }; +struct parser_args { + struct rrdp_session *session; + struct cache_node *rpp; +}; + static BIGNUM * BN_create(void) { @@ -312,25 +317,18 @@ fail: } static int -parse_hash(xmlTextReaderPtr reader, hash_requirement hr, - unsigned char **result, size_t *result_len) +parse_hash(xmlTextReaderPtr reader, unsigned char **result, size_t *result_len) { xmlChar *xmlattr; int error; - if (hr == HR_IGNORE) - return 0; - xmlattr = xmlTextReaderGetAttribute(reader, BAD_CAST RRDP_ATTR_HASH); if (xmlattr == NULL) - return (hr == HR_MANDATORY) - ? pr_val_err("Tag is missing the '" RRDP_ATTR_HASH "' attribute.") - : 0; + return 0; error = hexstr2sha256(xmlattr, result, result_len); xmlFree(xmlattr); - if (error) return pr_val_err("The '" RRDP_ATTR_HASH "' xml attribute does not appear to be a SHA-256 hash."); return 0; @@ -449,8 +447,7 @@ end: * 2. "hash" (optional, depending on @hr) */ static int -parse_file_metadata(xmlTextReaderPtr reader, hash_requirement hr, - struct file_metadata *meta) +parse_file_metadata(xmlTextReaderPtr reader, struct file_metadata *meta) { int error; @@ -460,7 +457,7 @@ parse_file_metadata(xmlTextReaderPtr reader, hash_requirement hr, if (meta->uri == NULL) return -EINVAL; - error = parse_hash(reader, hr, &meta->hash, &meta->hash_len); + error = parse_hash(reader, &meta->hash, &meta->hash_len); if (error) { free(meta->uri); meta->uri = NULL; @@ -471,12 +468,12 @@ parse_file_metadata(xmlTextReaderPtr reader, hash_requirement hr, } static int -parse_publish(xmlTextReaderPtr reader, hash_requirement hr, struct publish *tag) +parse_publish(xmlTextReaderPtr reader, struct publish *tag) { xmlChar *base64_str; int error; - error = parse_file_metadata(reader, hr, &tag->meta); + error = parse_file_metadata(reader, &tag->meta); if (error) return error; @@ -494,17 +491,8 @@ parse_publish(xmlTextReaderPtr reader, hash_requirement hr, struct publish *tag) if (!base64_decode((char *)base64_str, 0, &tag->content, &tag->content_len)) error = pr_val_err("Cannot decode publish tag's base64."); xmlFree(base64_str); - if (error) - return error; - - tag->path = url2path(tag->meta.uri); - if (tag->path == NULL) - return -EINVAL; - /* rfc8181#section-2.2 but considering optional hash */ - return (tag->meta.hash != NULL) - ? validate_hash(&tag->meta, tag->path) - : 0; + return error; } static int @@ -559,20 +547,47 @@ delete_file(char const *path) } static int -handle_publish(xmlTextReaderPtr reader, hash_requirement hr) +handle_publish(xmlTextReaderPtr reader, struct cache_node *rpp) { struct publish tag = { 0 }; + struct cache_node *node; int error; - error = parse_publish(reader, hr, &tag); + error = parse_publish(reader, &tag); if (error) return error; - error = write_file(tag.path, tag.content, tag.content_len); + node = cachent_provide(rpp, tag->meta.uri); + if (!node) { + error = pr_val_err("Malicious RRDP: is attempting to create file '%s' outside of its publication point '%s'.", + tag->meta.uri, rpp->url); + goto end; + } + + /* rfc8181#section-2.2 */ + if (node->flags & CNF_CACHED) { + if (tag->meta.hash == NULL) { + // XXX watch out for this in the log before release + error = pr_val_err("RRDP desync: is attempting to create '%s', but the file is already cached.", + tag->meta.uri); + goto end; + } + + error = validate_hash(&tag->meta, cachent_path(node)); + if (error) + goto end; - metadata_cleanup(&tag.meta); + } else if (tag->meta.hash != NULL) { + // XXX watch out for this in the log before release + error = pr_val_err("RRDP desync: is attempting to overwrite '%s', but the file is absent in the cache.", + tag->meta.uri); + goto end; + } + + error = write_file(node->tmpdir, tag.content, tag.content_len); + +end: metadata_cleanup(&tag.meta); free(tag.content); - free(tag.path); return error; } @@ -753,14 +768,13 @@ xml_read_notif(xmlTextReaderPtr reader, void *arg) } static int -parse_notification(char const *url, char const *path, - struct update_notification *result) +parse_notification(struct cache_node *node, struct update_notification *result) { int error; - update_notification_init(result, url); + update_notification_init(result, node->url); - error = relax_ng_parse(path, xml_read_notif, result); + error = relax_ng_parse(node->tmpdir, xml_read_notif, result); if (error) update_notification_cleanup(result); @@ -768,8 +782,9 @@ parse_notification(char const *url, char const *path, } static int -xml_read_snapshot(xmlTextReaderPtr reader, void *session) +xml_read_snapshot(xmlTextReaderPtr reader, void *_args) { + struct parser_args *args = _args; xmlReaderTypes type; xmlChar const *name; int error; @@ -779,9 +794,9 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *session) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = handle_publish(reader, HR_IGNORE); + error = handle_publish(reader, args->rpp); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_SNAPSHOT)) - error = validate_session(reader, session); + error = validate_session(reader, args->session); else return pr_val_err("Unexpected '%s' element", name); if (error) @@ -795,9 +810,11 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *session) } static int -parse_snapshot(struct update_notification *notif, char const *path) +parse_snapshot(struct update_notification *notif, char const *path, + struct cache_node *rpp) { - return relax_ng_parse(path, xml_read_snapshot, ¬if->session); + struct parser_args args = { .session = ¬if->session, .rpp = rpp }; + return relax_ng_parse(path, xml_read_snapshot, &args); } static int @@ -836,7 +853,7 @@ validate_session_desync(struct cachefile_notification *old_notif, } static int -handle_snapshot(struct update_notification *notif) +handle_snapshot(struct update_notification *notif, struct cache_node *rpp) { char const *url = notif->snapshot.uri; char *path; @@ -851,15 +868,15 @@ handle_snapshot(struct update_notification *notif) * TODO (performance) Is there a point in caching the snapshot? * Especially considering we delete it 4 lines afterwards. * Maybe stream it instead. - * Same for deltas. + * Same for the notification and deltas. */ - error = http_download_tmp(url, &path, NULL, NULL); + error = http_download_tmp(url, &path, 0, NULL); if (error) goto end1; error = validate_hash(¬if->snapshot, path); if (error) goto end2; - error = parse_snapshot(notif, path); + error = parse_snapshot(notif, path, rpp); delete_file(path); end2: free(path); @@ -879,7 +896,7 @@ xml_read_delta(xmlTextReaderPtr reader, void *session) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = handle_publish(reader, HR_OPTIONAL); + error = handle_publish(reader); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_WITHDRAW)) error = handle_withdraw(reader); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_DELTA)) @@ -1070,51 +1087,51 @@ update_notif(struct cachefile_notification *old, struct update_notification *new } /* - * Downloads the Update Notification pointed by @url, and updates the cache - * accordingly. + * Downloads the Update Notification @notif, and updates the cache accordingly. * * "Updates the cache accordingly" means it downloads the missing deltas or - * snapshot, and explodes them into the corresponding RPP's local directory. + * snapshot, and explodes them into @rpp's tmp directory. */ int -rrdp_update(char const *notif_url, struct cache_node *node) +rrdp_update(struct cache_node *notif, struct cache_node *rpp) { char *path = NULL; - struct cachefile_notification **cached, *old; + struct cachefile_notification *old; struct update_notification new; - bool changed; int serial_cmp; int error; - fnstack_push(notif_url); + fnstack_push(notif->url); pr_val_debug("Processing notification."); - error = http_download_tmp(notif_url, &path, &changed, &cached); + error = http_download_cache_node(notif); if (error) goto end; - if (!changed) { + + if (!(notif->flags & CNF_CHANGED)) { pr_val_debug("The Notification has not changed."); + rpp->flags |= CNF_DOWNLOADED; /* Success */ goto end; } - error = parse_notification(notif_url, path, &new); + error = parse_notification(notif, &new); if (error) goto end; pr_val_debug("New session/serial: %s/%s", new.session.session_id, new.session.serial.str); - old = *cached; - if (old == NULL) { + if (!(notif->flags & CNF_NOTIFICATION)) { pr_val_debug("This is a new Notification."); - error = handle_snapshot(&new); + error = handle_snapshot(&new, rpp); if (error) goto clean_notif; - *cached = pmalloc(sizeof(struct cachefile_notification)); - init_notif(*cached, &new); + notif->flags |= CNF_NOTIFICATION; + init_notif(¬if->notif, &new); goto end; } + old = ¬if->notif; serial_cmp = BN_cmp(old->session.serial.num, new.session.serial.num); if (serial_cmp < 0) { pr_val_debug("The Notification's serial changed."); diff --git a/src/rrdp.h b/src/rrdp.h index d7e74689..89950010 100644 --- a/src/rrdp.h +++ b/src/rrdp.h @@ -6,7 +6,7 @@ struct cachefile_notification; struct cache_node; -int rrdp_update(char const *, struct cache_node *); +int rrdp_update(struct cache_node *, struct cache_node *); json_t *rrdp_notif2json(struct cachefile_notification *); int rrdp_json2notif(json_t *, struct cachefile_notification **); diff --git a/src/state.c b/src/state.c index 5b1d30f0..c36a67c7 100644 --- a/src/state.c +++ b/src/state.c @@ -143,12 +143,6 @@ validation_tal(struct validation *state) return (state != NULL) ? state->tal : NULL; } -struct rpki_cache * -validation_cache(struct validation *state) -{ - return tal_get_cache(state->tal); -} - X509_STORE * validation_store(struct validation *state) { diff --git a/src/state.h b/src/state.h index c11828d1..19d160c9 100644 --- a/src/state.h +++ b/src/state.h @@ -11,7 +11,6 @@ int validation_prepare(struct validation **, struct tal *, void validation_destroy(struct validation *); struct tal *validation_tal(struct validation *); -struct rpki_cache *validation_cache(struct validation *); X509_STORE *validation_store(struct validation *); struct cert_stack *validation_certstack(struct validation *); diff --git a/src/types/map.c b/src/types/map.c index 58638cef..84c2b9a8 100644 --- a/src/types/map.c +++ b/src/types/map.c @@ -111,7 +111,6 @@ init_url(struct cache_mapping *map, char const *str) break; case MAP_HTTP: case MAP_NOTIF: - case MAP_TMP: pfx = "https://"; error = ENOTHTTPS; break; @@ -248,12 +247,9 @@ init_path(struct cache_mapping *map) switch (map->type) { case MAP_RSYNC: return map_simple(map, "rsync"); - case MAP_HTTP: return map_simple(map, "https"); - case MAP_NOTIF: - case MAP_TMP: return cache_tmpfile(&map->path); } diff --git a/src/types/map.h b/src/types/map.h index c4dfd1ef..7a4b0f61 100644 --- a/src/types/map.h +++ b/src/types/map.h @@ -24,13 +24,6 @@ enum map_type { * The metadata is cached until it's untraversed for a "long" time. */ MAP_NOTIF = (MAP_HTTP | (1 << 2)), - - /* - * RRDP Snapshot or Delta; downloaded via HTTP. - * The file itself is not cached, but we preserve some small metadata. - * The metadata is destroyed once the iteration finishes. - */ - MAP_TMP = (MAP_HTTP | (1 << 3)), }; struct cache_mapping; diff --git a/test/Makefile.am b/test/Makefile.am index 95d97794..ee26a1db 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -23,14 +23,18 @@ AM_CFLAGS += -I../src -DUNIT_TESTING ${CHECK_CFLAGS} ${XML2_CFLAGS} ${JANSSON_CF MY_LDADD = ${CHECK_LIBS} ${JANSSON_LIBS} #check_PROGRAMS = file.test -check_PROGRAMS = cache.test +#check_PROGRAMS = cache.test +check_PROGRAMS = cachent.test TESTS = ${check_PROGRAMS} #file_test_SOURCES = file_test.c #file_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} -cache_test_SOURCES = cache/local_cache_test.c -cache_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} +#cache_test_SOURCES = cache/local_cache_test.c +#cache_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} + +cachent_test_SOURCES = cache/cachent_test.c +cachent_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} EXTRA_DIST = mock.c mock.h EXTRA_DIST += resources/lorem-ipsum.txt diff --git a/test/cache/cachent_test.c b/test/cache/cachent_test.c new file mode 100644 index 00000000..a4c6b571 --- /dev/null +++ b/test/cache/cachent_test.c @@ -0,0 +1,388 @@ +#include + +#include "alloc.c" +#include "cache/cachent.c" +#include "data_structure/path_builder.c" +#include "mock.c" + +static char deleted[16][5]; +static unsigned int dn; + +static void +__delete_node_cb(struct cache_node const *node) +{ + strcpy(deleted[dn++], node->name); +} + +static struct cache_node * +node(char const *name, int flags, ...) +{ + struct cache_node *result; + struct cache_node *child; + va_list args; + + result = pzalloc(sizeof(struct cache_node)); + result->name = pstrdup(name); + result->flags = flags; + + va_start(args, flags); + while ((child = va_arg(args, struct cache_node *)) != NULL) { + HASH_ADD_KEYPTR(hh, result->children, child->name, + strlen(child->name), child); + child->parent = result; + } + va_end(args); + + return result; +} + +START_TEST(test_delete) +{ + struct cache_node *root, *a, *b; + + a = node("a", 0, NULL); + dn = 0; + cachent_delete(a); + ck_assert_uint_eq(1, dn); + ck_assert_str_eq("a", deleted[0]); + + a = node("a", 0, NULL); + root = node("root", 0, a, NULL); + dn = 0; + cachent_delete(a); + ck_assert_ptr_eq(NULL, root->children); + ck_assert_uint_eq(1, dn); + ck_assert_str_eq("a", deleted[0]); + + dn = 0; + cachent_delete(root); + ck_assert_uint_eq(1, dn); + ck_assert_str_eq("root", deleted[0]); + + b = node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), + node("e", 0, NULL), + node("f", 0, NULL), NULL); + a = node("a", 0, + b, + node("g", 0, + node("h", 0, + node("i", 0, NULL), NULL), + node("j", 0, + node("k", 0, NULL), NULL), + node("l", 0, + node("m", 0, NULL), NULL), + node("n", 0, + node("o", 0, NULL), NULL), NULL), NULL); + root = node("root", 0, a, NULL); + + dn = 0; + cachent_delete(b); + ck_assert_int_eq(1, HASH_COUNT(a->children)); + ck_assert_str_eq("c", deleted[0]); + ck_assert_str_eq("d", deleted[1]); + ck_assert_str_eq("e", deleted[2]); + ck_assert_str_eq("f", deleted[3]); + ck_assert_str_eq("b", deleted[4]); + + dn = 0; + cachent_delete(a); + ck_assert_int_eq(0, HASH_COUNT(root->children)); + ck_assert_str_eq("i", deleted[0]); + ck_assert_str_eq("h", deleted[1]); + ck_assert_str_eq("k", deleted[2]); + ck_assert_str_eq("j", deleted[3]); + ck_assert_str_eq("m", deleted[4]); + ck_assert_str_eq("l", deleted[5]); + ck_assert_str_eq("o", deleted[6]); + ck_assert_str_eq("n", deleted[7]); + ck_assert_str_eq("g", deleted[8]); + ck_assert_str_eq("a", deleted[9]); + + dn = 0; + cachent_delete(root); + ck_assert_uint_eq(1, dn); + ck_assert_str_eq("root", deleted[0]); +} +END_TEST + +static char const *expected[32]; +static unsigned int e; + +static bool +ck_traverse_cb(struct cache_node *node, char const *path) +{ + ck_assert_str_eq(expected[e++], path); + return true; +} + +static void +ck_traverse(struct cache_node *root, ...) +{ + char const *path; + unsigned int p = 0; + va_list args; + + va_start(args, root); + while ((path = va_arg(args, char const *)) != NULL) + expected[p++] = path; + va_end(args); + expected[p] = NULL; + + e = 0; + ck_assert_int_eq(0, cachent_traverse(root, ck_traverse_cb)); + ck_assert_uint_eq(p, e); + + cachent_delete(root); +} + +START_TEST(test_traverse) +{ + struct cache_node *root; + + root = NULL; + ck_traverse(root, NULL); + + root = node("a", 0, NULL); + ck_traverse(root, "tmp/a", NULL); + + root = node("a", 0, + node("b", 0, NULL), NULL); + ck_traverse(root, "tmp/a", "tmp/a/b", NULL); + + root = node("a", 0, + node("b", 0, + node("c", 0, NULL), NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", NULL); + + root = node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", NULL); + + root = node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), + node("e", 0, NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/e", NULL); + + root = node("a", 0, + node("b", 0, NULL), + node("c", 0, + node("d", 0, NULL), + node("e", 0, NULL), NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/c", + "tmp/a/c/d", + "tmp/a/c/e", NULL); + + root = node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), + node("e", 0, + node("f", 0, NULL), + node("g", 0, NULL), NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/e", + "tmp/a/e/f", + "tmp/a/e/g", NULL); + + root = node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), + node("e", 0, NULL), + node("f", 0, NULL), NULL), + node("g", 0, + node("h", 0, + node("i", 0, NULL), NULL), + node("j", 0, + node("k", 0, NULL), NULL), + node("l", 0, + node("m", 0, NULL), NULL), + node("n", 0, + node("o", 0, NULL), NULL), NULL), NULL); + ck_traverse(root, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/b/e", + "tmp/a/b/f", + "tmp/a/g", + "tmp/a/g/h", + "tmp/a/g/h/i", + "tmp/a/g/j", + "tmp/a/g/j/k", + "tmp/a/g/l", + "tmp/a/g/l/m", + "tmp/a/g/n", + "tmp/a/g/n/o", NULL); +} +END_TEST + +#define TEST_NORMALIZE(dirty, clean) \ + normal = normalize(dirty); \ + ck_assert_str_eq(clean, normal); \ + free(normal) + +START_TEST(test_normalize) +{ + char *normal; + + TEST_NORMALIZE("rsync://a.b.c", "rsync://a.b.c"); + TEST_NORMALIZE("rsync://a.b.c/", "rsync://a.b.c"); + TEST_NORMALIZE("rsync://a.b.c//////", "rsync://a.b.c"); + TEST_NORMALIZE("rsync://a.b.c/d/e", "rsync://a.b.c/d/e"); + TEST_NORMALIZE("rsync://a.b.c/d/e/.", "rsync://a.b.c/d/e"); + TEST_NORMALIZE("rsync://a.b.c/d/e/.", "rsync://a.b.c/d/e"); + TEST_NORMALIZE("rsync://a.b.c/d/./e/.", "rsync://a.b.c/d/e"); + TEST_NORMALIZE("rsync://a.b.c/d/../d/../d/e/", "rsync://a.b.c/d/e"); + TEST_NORMALIZE("rsync://a.b.c/../x/y/z", "rsync://x/y/z"); + TEST_NORMALIZE("rsync://x//y/z/../../../m/./n/o", "rsync://m/n/o"); + ck_assert_ptr_eq(NULL, normalize("rsync://")); + ck_assert_ptr_eq(NULL, normalize("rsync://..")); + ck_assert_ptr_eq(NULL, normalize("rsync://a.b.c/..")); + ck_assert_ptr_eq(NULL, normalize("rsync://a.b.c/d/e/../../..")); + ck_assert_ptr_eq(NULL, normalize("abcde://a.b.c/d")); +} +END_TEST + +START_TEST(test_provide) +{ + struct cache_node *rsync, *abc, *d, *e, *f, *g, *h, *ee; + + /* Create tree from nothing */ + e = cachent_provide(NULL, "rsync://a.b.c/d/e"); + ck_assert_ptr_ne(NULL, e); + ck_assert_str_eq("rsync://a.b.c/d/e", e->url); + ck_assert_str_eq("e", e->name); + + d = e->parent; + ck_assert_ptr_ne(NULL, d); + ck_assert_str_eq("rsync://a.b.c/d", d->url); + ck_assert_str_eq("d", d->name); + + abc = d->parent; + ck_assert_ptr_ne(NULL, abc); + ck_assert_str_eq("rsync://a.b.c", abc->url); + ck_assert_str_eq("a.b.c", abc->name); + + rsync = abc->parent; + ck_assert_ptr_ne(NULL, rsync); + ck_assert_ptr_eq(NULL, rsync->parent); + ck_assert_str_eq("rsync:", rsync->url); + ck_assert_str_eq("rsync:", rsync->name); + + /* Find leaf from root */ + ck_assert_ptr_eq(e, cachent_provide(rsync, "rsync://a.b.c/d/e")); + /* Find branch from root */ + ck_assert_ptr_eq(d, cachent_provide(rsync, "rsync://a.b.c/d")); + /* Find leaf from non-root ancestor */ + ck_assert_ptr_eq(e, cachent_provide(abc, "rsync://a.b.c/d/e")); + /* Find branch from non-root ancestor */ + ck_assert_ptr_eq(d, cachent_provide(abc, "rsync://a.b.c/d")); + /* Find selves */ + ck_assert_ptr_eq(NULL, cachent_provide(rsync, "rsync://")); /* Illegal */ + ck_assert_ptr_eq(abc, cachent_provide(abc, "rsync://a.b.c")); + ck_assert_ptr_eq(e, cachent_provide(e, "rsync://a.b.c/d/e")); + + /* Some not normalized noise */ + ck_assert_ptr_eq(e, cachent_provide(e, "rsync://a.b.c/d/e////")); + ck_assert_ptr_eq(e, cachent_provide(e, "rsync://a.b.c///d/./e//")); + ck_assert_ptr_eq(e, cachent_provide(e, "rsync://a/../z/../a.b.c/d/e/")); + + /* Create sibling from root */ + f = cachent_provide(rsync, "rsync://a.b.c/f"); + ck_assert_ptr_ne(NULL, f); + ck_assert_ptr_eq(abc, f->parent); + ck_assert_str_eq("rsync://a.b.c/f", f->url); + ck_assert_str_eq("f", f->name); + + /* Create more than one descendant from root */ + h = cachent_provide(rsync, "rsync://a.b.c/f/g/h"); + ck_assert_ptr_ne(NULL, h); + ck_assert_str_eq("rsync://a.b.c/f/g/h", h->url); + ck_assert_str_eq("h", h->name); + + g = h->parent; + ck_assert_ptr_ne(NULL, g); + ck_assert_ptr_eq(f, g->parent); + ck_assert_str_eq("rsync://a.b.c/f/g", g->url); + ck_assert_str_eq("g", g->name); + + /* Try to create a conflict by prefix */ + ee = cachent_provide(rsync, "rsync://a.b.c/d/ee"); + ck_assert_ptr_ne(e, ee); + ck_assert_ptr_eq(d, ee->parent); + ck_assert_str_eq("rsync://a.b.c/d/ee", ee->url); + ck_assert_str_eq("ee", ee->name); + ck_assert_ptr_eq(e, cachent_provide(abc, "rsync://a.b.c/d/e")); + ck_assert_ptr_eq(ee, cachent_provide(abc, "rsync://a.b.c/d/ee")); + + /* Prefixes don't match */ + ck_assert_ptr_eq(NULL, cachent_provide(d, "rsync://a.b.c/dd")); + ck_assert_ptr_eq(NULL, cachent_provide(d, "rsync://a.b.c/f")); + ck_assert_ptr_eq(NULL, cachent_provide(d, "rsync://a.b.c/d/../f")); + + cachent_delete(rsync); +} +END_TEST + +static Suite *thread_pool_suite(void) +{ + Suite *suite; + TCase *traverses, *provide; + + traverses = tcase_create("traverses"); + tcase_add_test(traverses, test_delete); + tcase_add_test(traverses, test_traverse); + + provide = tcase_create("provide"); + tcase_add_test(provide, test_normalize); + tcase_add_test(provide, test_provide); + + suite = suite_create("cachent"); + suite_add_tcase(suite, traverses); + suite_add_tcase(suite, provide); + + return suite; +} + +int main(void) +{ + Suite *suite; + SRunner *runner; + int tests_failed; + + suite = thread_pool_suite(); + + runner = srunner_create(suite); + srunner_run_all(runner, CK_NORMAL); + tests_failed = srunner_ntests_failed(runner); + srunner_free(runner); + + return (tests_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/cache/local_cache_test.c b/test/cache/local_cache_test.c index 7e726522..2b0fc0fc 100644 --- a/test/cache/local_cache_test.c +++ b/test/cache/local_cache_test.c @@ -12,13 +12,10 @@ //#include "json_util.c" #include "mock.c" #include "cache/local_cache.c" -#include "data_structure/path_builder.c" //#include "types/map.c" /* Mocks */ -static struct rpki_cache *cache; - static bool dl_error; /* Download should return error? */ struct downloaded_path { @@ -101,15 +98,6 @@ http_download(struct cache_mapping *map, curl_off_t ims, bool *changed) return error; } -static char deleted[16][4]; -static unsigned int dn; - -static void -__delete_node_cb(struct cache_node const *node) -{ - strcpy(deleted[dn++], node->name); -} - MOCK_ABORT_INT(rrdp_update, struct cache_mapping *map) __MOCK_ABORT(rrdp_notif2json, json_t *, NULL, struct cachefile_notification *notif) MOCK_VOID(rrdp_notif_free, struct cachefile_notification *notif) @@ -919,266 +907,12 @@ START_TEST(test_recover) } END_TEST -static void -add_children(struct cache_node *parent, va_list children) -{ - struct cache_node *child; - - while ((child = va_arg(children, struct cache_node *)) != NULL) - HASH_ADD_KEYPTR(hh, parent->children, child->name, - strlen(child->name), child); -} - -static void -tree(struct rpki_cache *cache, ...) -{ - va_list args; - va_start(args, cache); - add_children(&cache->root, args); - va_end(args); -} - -static struct cache_node * -node(char const *name, int flags, ...) -{ - struct cache_node *result; - va_list args; - - result = pzalloc(sizeof(struct cache_node)); - result->name = pstrdup(name); - result->flags = flags; - - va_start(args, flags); - add_children(result, args); - va_end(args); - - return result; -} - -static char const *expected[32]; -static unsigned int e; - -static bool -ck_traverse_cb(struct cache_node *node, char const *path) -{ - ck_assert_str_eq(expected[e++], path); - return true; -} - -static void -cleanup_cache_nodes(void) -{ - struct cache_node *node, *tmp; - - HASH_ITER(hh, cache.root.children, node, tmp) { - node->parent = &cache.root; - delete_node(node); - } -} - -static void -ck_traverse(struct rpki_cache *cache, ...) -{ - char const *path; - unsigned int p = 0; - va_list args; - - va_start(args, cache); - while ((path = va_arg(args, char const *)) != NULL) - expected[p++] = path; - va_end(args); - expected[p] = NULL; - - e = 0; - ck_assert_int_eq(0, traverse_cache(ck_traverse_cb)); - ck_assert_uint_eq(p, e); - - cleanup_cache_nodes(); -} - -START_TEST(test_delete_node) -{ - struct rpki_cache cache = { - .root.name = "tmp" - }; - struct cache_node *a, *b; - - a = node("a", 0, NULL); - tree(&cache, a, NULL); - a->parent = &cache.root; - dn = 0; - - delete_node(a); - ck_assert_ptr_eq(NULL, cache.root.children); - ck_assert_uint_eq(1, dn); - ck_assert_str_eq("a", deleted[0]); - - b = node("b", 0, - node("c", 0, NULL), - node("d", 0, NULL), - node("e", 0, NULL), - node("f", 0, NULL), NULL); - a = node("a", 0, - b, - node("g", 0, - node("h", 0, - node("i", 0, NULL), NULL), - node("j", 0, - node("k", 0, NULL), NULL), - node("l", 0, - node("m", 0, NULL), NULL), - node("n", 0, - node("o", 0, NULL), NULL), NULL), NULL); - tree(&cache, a, NULL); - b->parent = a; - a->parent = &cache.root; - - dn = 0; - delete_node(b); - ck_assert_int_eq(1, HASH_COUNT(a->children)); - ck_assert_str_eq("c", deleted[0]); - ck_assert_str_eq("d", deleted[1]); - ck_assert_str_eq("e", deleted[2]); - ck_assert_str_eq("f", deleted[3]); - ck_assert_str_eq("b", deleted[4]); - - dn = 0; - delete_node(a); - ck_assert_ptr_eq(NULL, cache.root.children); - ck_assert_str_eq("i", deleted[0]); - ck_assert_str_eq("h", deleted[1]); - ck_assert_str_eq("k", deleted[2]); - ck_assert_str_eq("j", deleted[3]); - ck_assert_str_eq("m", deleted[4]); - ck_assert_str_eq("l", deleted[5]); - ck_assert_str_eq("o", deleted[6]); - ck_assert_str_eq("n", deleted[7]); - ck_assert_str_eq("g", deleted[8]); - ck_assert_str_eq("a", deleted[9]); -} -END_TEST - -START_TEST(test_traverse) -{ - struct rpki_cache cache = { - .root.name = "tmp" - }; - - tree(&cache, NULL); - ck_traverse(&cache, NULL); - - tree(&cache, node("a", 0, NULL), NULL); - ck_traverse(&cache, "tmp/a", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, NULL), NULL), NULL); - ck_traverse(&cache, "tmp/a", "tmp/a/b", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, - node("c", 0, NULL), NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/b/c", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, - node("c", 0, NULL), - node("d", 0, NULL), NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/b/c", - "tmp/a/b/d", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, - node("c", 0, NULL), - node("d", 0, NULL), NULL), - node("e", 0, NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/b/c", - "tmp/a/b/d", - "tmp/a/e", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, NULL), - node("c", 0, - node("d", 0, NULL), - node("e", 0, NULL), NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/c", - "tmp/a/c/d", - "tmp/a/c/e", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, - node("c", 0, NULL), - node("d", 0, NULL), NULL), - node("e", 0, - node("f", 0, NULL), - node("g", 0, NULL), NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/b/c", - "tmp/a/b/d", - "tmp/a/e", - "tmp/a/e/f", - "tmp/a/e/g", NULL); - - tree(&cache, - node("a", 0, - node("b", 0, - node("c", 0, NULL), - node("d", 0, NULL), - node("e", 0, NULL), - node("f", 0, NULL), NULL), - node("g", 0, - node("h", 0, - node("i", 0, NULL), NULL), - node("j", 0, - node("k", 0, NULL), NULL), - node("l", 0, - node("m", 0, NULL), NULL), - node("n", 0, - node("o", 0, NULL), NULL), NULL), NULL), NULL); - ck_traverse(&cache, - "tmp/a", - "tmp/a/b", - "tmp/a/b/c", - "tmp/a/b/d", - "tmp/a/b/e", - "tmp/a/b/f", - "tmp/a/g", - "tmp/a/g/h", - "tmp/a/g/h/i", - "tmp/a/g/j", - "tmp/a/g/j/k", - "tmp/a/g/l", - "tmp/a/g/l/m", - "tmp/a/g/n", - "tmp/a/g/n/o", NULL); -} -END_TEST - /* Boilerplate */ static Suite *thread_pool_suite(void) { Suite *suite; - TCase *rsync , *https, *dot, *meta, *recover, *traverse; + TCase *rsync, *https, *dot, *meta, *recover; rsync = tcase_create("rsync"); tcase_add_test(rsync, test_cache_download_rsync); @@ -1201,17 +935,12 @@ static Suite *thread_pool_suite(void) recover = tcase_create("recover"); tcase_add_test(recover, test_recover); - traverse = tcase_create("traverse"); - tcase_add_test(traverse, test_delete_node); - tcase_add_test(traverse, test_traverse); - suite = suite_create("local-cache"); -// suite_add_tcase(suite, rsync); -// suite_add_tcase(suite, https); -// suite_add_tcase(suite, dot); -// suite_add_tcase(suite, meta); -// suite_add_tcase(suite, recover); - suite_add_tcase(suite, traverse); + suite_add_tcase(suite, rsync); + suite_add_tcase(suite, https); + suite_add_tcase(suite, dot); + suite_add_tcase(suite, meta); + suite_add_tcase(suite, recover); return suite; }