From: Alberto Leiva Popper Date: Wed, 12 Jun 2024 15:50:50 +0000 (-0600) Subject: Cache refactor startup X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=762767ccc3e9699b1121379512996d391f6087d2;p=thirdparty%2FFORT-validator.git Cache refactor startup I've been asked to upload commits more often. Because #82 is a handful, some of these will have to be WIPs. This is most of June's progress. I'm in the middle of converting the cache code to the new design. Dirty commit; doesn't compile. I'll probably squash it later. --- diff --git a/src/Makefile.am b/src/Makefile.am index 7bbc61b3..9bf92579 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -4,117 +4,9 @@ LDFLAGS_DEBUG = -rdynamic bin_PROGRAMS = fort -fort_SOURCES = main.c - -fort_SOURCES += as_number.h -fort_SOURCES += algorithm.h algorithm.c -fort_SOURCES += alloc.h alloc.c -fort_SOURCES += certificate_refs.h certificate_refs.c -fort_SOURCES += cert_stack.h cert_stack.c -fort_SOURCES += common.c common.h -fort_SOURCES += config.h config.c -fort_SOURCES += daemon.h daemon.c -fort_SOURCES += extension.h extension.c -fort_SOURCES += file.h file.c -fort_SOURCES += init.h init.c -fort_SOURCES += json_util.c json_util.h -fort_SOURCES += libcrypto_util.h libcrypto_util.c -fort_SOURCES += log.h log.c -fort_SOURCES += nid.h nid.c -fort_SOURCES += output_printer.h output_printer.c -fort_SOURCES += print_file.h print_file.c -fort_SOURCES += resource.h resource.c -fort_SOURCES += rpp.h rpp.c -fort_SOURCES += rrdp.h rrdp.c -fort_SOURCES += sorted_array.h sorted_array.c -fort_SOURCES += state.h state.c -fort_SOURCES += str_token.h str_token.c -fort_SOURCES += thread_var.h thread_var.c -fort_SOURCES += json_handler.h json_handler.c -fort_SOURCES += validation_handler.h validation_handler.c - -fort_SOURCES += asn1/content_info.h asn1/content_info.c -fort_SOURCES += asn1/decode.h asn1/decode.c -fort_SOURCES += asn1/oid.h asn1/oid.c -fort_SOURCES += asn1/signed_data.h asn1/signed_data.c - -fort_SOURCES += types/address.h types/address.c -fort_SOURCES += types/bio_seq.c types/bio_seq.h -fort_SOURCES += types/delta.c types/delta.h -fort_SOURCES += types/router_key.c types/router_key.h -fort_SOURCES += types/serial.h types/serial.c -fort_SOURCES += types/map.h types/map.c -fort_SOURCES += types/vrp.c types/vrp.h - fort_SOURCES += cache/local_cache.c cache/local_cache.h -fort_SOURCES += config/boolean.c config/boolean.h -fort_SOURCES += config/file_type.h config/file_type.c -fort_SOURCES += config/filename_format.h config/filename_format.c -fort_SOURCES += config/log_conf.h config/log_conf.c -fort_SOURCES += config/mode.c config/mode.h -fort_SOURCES += config/incidences.h config/incidences.c -fort_SOURCES += config/output_format.h config/output_format.c -fort_SOURCES += config/str.c config/str.h -fort_SOURCES += config/string_array.h config/string_array.c -fort_SOURCES += config/types.h -fort_SOURCES += config/uint.c config/uint.h -fort_SOURCES += config/work_offline.c config/work_offline.h - -fort_SOURCES += crypto/base64.h crypto/base64.c -fort_SOURCES += crypto/hash.h crypto/hash.c - -fort_SOURCES += data_structure/array_list.h -fort_SOURCES += data_structure/common.h -fort_SOURCES += data_structure/path_builder.h data_structure/path_builder.c -fort_SOURCES += data_structure/uthash.h - -fort_SOURCES += http/http.h http/http.c - -fort_SOURCES += incidence/incidence.h incidence/incidence.c - -fort_SOURCES += object/bgpsec.h object/bgpsec.c -fort_SOURCES += object/certificate.h object/certificate.c -fort_SOURCES += object/crl.h object/crl.c -fort_SOURCES += object/ghostbusters.h object/ghostbusters.c -fort_SOURCES += object/manifest.h object/manifest.c -fort_SOURCES += object/name.h object/name.c -fort_SOURCES += object/roa.h object/roa.c -fort_SOURCES += object/signed_object.h object/signed_object.c -fort_SOURCES += object/tal.h object/tal.c -fort_SOURCES += object/vcard.h object/vcard.c - -fort_SOURCES += resource/ip4.h resource/ip4.c -fort_SOURCES += resource/ip6.h resource/ip6.c -fort_SOURCES += resource/asn.h resource/asn.c - -fort_SOURCES += rsync/rsync.h rsync/rsync.c - -fort_SOURCES += rtr/pdu_stream.c rtr/pdu_stream.h -fort_SOURCES += rtr/err_pdu.c rtr/err_pdu.h -fort_SOURCES += rtr/pdu_handler.c rtr/pdu_handler.h -fort_SOURCES += rtr/pdu_sender.c rtr/pdu_sender.h -fort_SOURCES += rtr/pdu.c rtr/pdu.h -fort_SOURCES += rtr/primitive_writer.c rtr/primitive_writer.h -fort_SOURCES += rtr/rtr.c rtr/rtr.h - -fort_SOURCES += rtr/db/db_table.c rtr/db/db_table.h -fort_SOURCES += rtr/db/delta.c rtr/db/delta.h -fort_SOURCES += rtr/db/deltas_array.c rtr/db/deltas_array.h -fort_SOURCES += rtr/db/vrps.c rtr/db/vrps.h - -fort_SOURCES += slurm/db_slurm.c slurm/db_slurm.h -fort_SOURCES += slurm/slurm_loader.c slurm/slurm_loader.h -fort_SOURCES += slurm/slurm_parser.c slurm/slurm_parser.h - -fort_SOURCES += thread/thread_pool.c thread/thread_pool.h - -fort_SOURCES += xml/relax_ng.c xml/relax_ng.h - -include asn1/asn1c/Makefile.include -fort_SOURCES += $(ASN_MODULE_SRCS) $(ASN_MODULE_HDRS) - -fort_CFLAGS = -Wall -Wpedantic +fort_CFLAGS = -Wall -Wpedantic -Werror #fort_CFLAGS += $(GCC_WARNS) fort_CFLAGS += -std=c99 -D_DEFAULT_SOURCE=1 -D_XOPEN_SOURCE=700 -D_BSD_SOURCE=1 fort_CFLAGS += -O2 -g $(FORT_FLAGS) ${XML2_CFLAGS} diff --git a/src/cache/local_cache.c b/src/cache/local_cache.c index 6df1c5a8..c47cf242 100644 --- a/src/cache/local_cache.c +++ b/src/cache/local_cache.c @@ -1,3 +1,10 @@ +/* + * Current design notes: + * + * - We only need to keep nodes for the rsync root. + * - The tree traverse only needs to touch files. + */ + #include "cache/local_cache.h" #include @@ -17,31 +24,62 @@ #include "data_structure/uthash.h" #include "http/http.h" #include "rsync/rsync.h" +#include "types/str.h" -struct cache_node { - struct cache_mapping *map; +/* XXX force RRDP if one RPP fails to validate by rsync? */ + +struct cached_file { + char *url; + char *path; + UT_hash_handle hh; /* Hash table hook */ +}; + +struct cached_rpp { + struct cached_file *ht; +}; - struct { - time_t ts; /* Last download attempt's timestamp */ - int result; /* Last download attempt's result status code */ - } attempt; +#define CNF_RSYNC (1 << 0) +/* Was it downloaded during the current cycle? */ +#define CNF_DOWNLOADED (1 << 1) +/* Was it read during the current cycle? */ +#define CNF_TOUCHED (1 << 2) +/* + * Did it validate successfully (at least once) during the current cycle? + * (It's technically possible for two different repositories to map to the same + * cache node. One of them is likely going to fail validation.) + */ +#define CNF_VALIDATED (1 << 3) +/* Withdrawn by RRDP? */ +#define CNF_WITHDRAWN (1 << 4) - struct { - /* Has a download attempt ever been successful? */ - bool happened; - /* Last successful download timestamp. (Only if @happened.) */ - time_t ts; - } success; +struct cache_node { + char const *name; /* Points to the last component of @url */ + char *url; + int flags; + /* Last successful download time, or zero */ + time_t mtim; + /* + * If flags & CNF_DOWNLOADED, path to the temporal directory where we + * downloaded the latest refresh. + * (See --compare-dest at rsync(1). RRDP is basically the same.) + * Otherwise undefined. + */ + char *tmpdir; - struct cachefile_notification *notif; + /* Tree parent. Only defined during cleanup. */ + struct cache_node *parent; + /* Tree children. */ + struct cache_node *children; UT_hash_handle hh; /* Hash table hook */ }; -struct rpki_cache { - struct cache_node *ht; - time_t startup_ts; /* When we started the last validation */ -}; +static struct rpki_cache { + struct cache_node root; /* It's a tree. */ +// time_t startup_ts; /* When we started the last validation */ +} cache; + +static atomic_uint file_counter; #define CACHE_METAFILE "cache.json" #define TAGNAME_VERSION "fort-version" @@ -57,13 +95,10 @@ struct rpki_cache { #define TAGNAME_SUCCESS_TS "success-timestamp" #define TAGNAME_NOTIF "notification" -#define TYPEVALUE_TA_RSYNC "TA (rsync)" #define TYPEVALUE_TA_HTTP "TA (HTTP)" #define TYPEVALUE_RPP "RPP" #define TYPEVALUE_NOTIF "RRDP Notification" -static atomic_uint file_counter; - static char * get_cache_filename(char const *name, bool fatal) { @@ -210,7 +245,8 @@ cache_teardown(void) } /* - * Returns a unique temporary file name in the local cache. + * Returns a unique temporary file name in the local cache. Note, it's a name, + * and it's pretty much reserved. The file itself will not be created. * * The file will not be automatically deleted when it is closed or the program * terminates. @@ -249,110 +285,88 @@ get_tal_json_filename(void) static struct cache_node * json2node(json_t *json) { - struct cache_node *node; - char const *type_str; - enum map_type type; - char const *url; - json_t *notif; - int error; - - node = pzalloc(sizeof(struct cache_node)); - - error = json_get_str(json, TAGNAME_TYPE, &type_str); - if (error) { - if (error > 0) - pr_op_err("Node is missing the '" TAGNAME_TYPE "' tag."); - goto fail; - } - - if (strcmp(type_str, TYPEVALUE_TA_RSYNC) == 0) - type = MAP_TA_RSYNC; - else if (strcmp(type_str, TYPEVALUE_TA_HTTP) == 0) - type = MAP_TA_HTTP; - else if (strcmp(type_str, TYPEVALUE_RPP) == 0) - type = MAP_RPP; - else if (strcmp(type_str, TYPEVALUE_NOTIF) == 0) - type = MAP_NOTIF; - else { - pr_op_err("Unknown node type: %s", type_str); - goto fail; - } - - error = json_get_str(json, TAGNAME_URL, &url); - if (error) { - if (error > 0) - pr_op_err("Node is missing the '" TAGNAME_URL "' tag."); - goto fail; - } - - if (type == MAP_NOTIF) { - error = json_get_object(json, TAGNAME_NOTIF, ¬if); - switch (error) { - case 0: - error = rrdp_json2notif(notif, &node->notif); - if (error) - goto fail; - break; - case ENOENT: - node->notif = NULL; - break; - default: - goto fail; - } - } - - error = map_create(&node->map, type, NULL, url); - if (error) { - pr_op_err("Cannot parse '%s' into a URI.", url); - goto fail; - } - - error = json_get_ts(json, TAGNAME_ATTEMPT_TS, &node->attempt.ts); - if (error) { - if (error > 0) - pr_op_err("Node '%s' is missing the '" - TAGNAME_ATTEMPT_TS "' tag.", url); - goto fail; - } - - if (json_get_int(json, TAGNAME_ATTEMPT_ERR, &node->attempt.result) < 0) - goto fail; - - error = json_get_ts(json, TAGNAME_SUCCESS_TS, &node->success.ts); - if (error < 0) - goto fail; - node->success.happened = (error == 0); - - pr_op_debug("Node '%s' loaded successfully.", url); - return node; - -fail: - map_refput(node->map); - rrdp_notif_free(node->notif); - free(node); +// struct cache_node *node; +// char const *type_str; +// enum map_type type; +// char const *url; +// json_t *notif; +// int error; +// +// node = pzalloc(sizeof(struct cache_node)); +// +// error = json_get_str(json, TAGNAME_TYPE, &type_str); +// if (error) { +// if (error > 0) +// pr_op_err("Node is missing the '" TAGNAME_TYPE "' tag."); +// goto fail; +// } +// +// if (strcmp(type_str, TYPEVALUE_TA_HTTP) == 0) +// type = MAP_HTTP; +// else if (strcmp(type_str, TYPEVALUE_RPP) == 0) +// type = MAP_RSYNC; +// else if (strcmp(type_str, TYPEVALUE_NOTIF) == 0) +// type = MAP_NOTIF; +// else { +// pr_op_err("Unknown node type: %s", type_str); +// goto fail; +// } +// +// error = json_get_str(json, TAGNAME_URL, &url); +// if (error) { +// if (error > 0) +// pr_op_err("Node is missing the '" TAGNAME_URL "' tag."); +// goto fail; +// } +// +// if (type == MAP_NOTIF) { +// error = json_get_object(json, TAGNAME_NOTIF, ¬if); +// switch (error) { +// case 0: +// error = rrdp_json2notif(notif, &node->notif); +// if (error) +// goto fail; +// break; +// case ENOENT: +// node->notif = NULL; +// break; +// default: +// goto fail; +// } +// } +// +// error = map_create(&node->map, type, url); +// if (error) { +// pr_op_err("Cannot parse '%s' into a URI.", url); +// goto fail; +// } +// +// error = json_get_ts(json, TAGNAME_ATTEMPT_TS, &node->attempt.ts); +// if (error) { +// if (error > 0) +// pr_op_err("Node '%s' is missing the '" +// TAGNAME_ATTEMPT_TS "' tag.", url); +// goto fail; +// } +// +// if (json_get_int(json, TAGNAME_ATTEMPT_ERR, &node->attempt.result) < 0) +// goto fail; +// +// error = json_get_ts(json, TAGNAME_SUCCESS_TS, &node->success.ts); +// if (error < 0) +// goto fail; +// node->success.happened = (error == 0); +// +// pr_op_debug("Node '%s' loaded successfully.", url); +// return node; +// +//fail: +// map_refput(node->map); +// rrdp_notif_free(node->notif); +// free(node); return NULL; } -static struct cache_node * -find_node(struct rpki_cache *cache, struct cache_mapping *map) -{ - char const *key; - struct cache_node *result; - - key = map_get_url(map); - HASH_FIND_STR(cache->ht, key, result); - - return result; -} - -static void -add_node(struct rpki_cache *cache, struct cache_node *node) -{ - char const *key = map_get_url(node->map); - size_t keylen = strlen(key); - HASH_ADD_KEYPTR(hh, cache->ht, key, keylen, node); -} - static void load_tal_json(struct rpki_cache *cache) { @@ -413,75 +427,72 @@ cache_create(void) static json_t * node2json(struct cache_node *node) { - json_t *json; - char const *type; - json_t *notification; - - json = json_obj_new(); - if (json == NULL) - return NULL; - - switch (map_get_type(node->map)) { - case MAP_TA_RSYNC: - type = TYPEVALUE_TA_RSYNC; - break; - case MAP_TA_HTTP: - type = TYPEVALUE_TA_HTTP; - break; - case MAP_RPP: - type = TYPEVALUE_RPP; - break; - case MAP_NOTIF: - type = TYPEVALUE_NOTIF; - break; - default: - goto cancel; - } - - if (json_add_str(json, TAGNAME_TYPE, type)) - goto cancel; - if (json_add_str(json, TAGNAME_URL, map_get_url(node->map))) - goto cancel; - if (node->notif != NULL) { - notification = rrdp_notif2json(node->notif); - if (json_object_add(json, TAGNAME_NOTIF, notification)) - goto cancel; - } - if (json_add_ts(json, TAGNAME_ATTEMPT_TS, node->attempt.ts)) - goto cancel; - if (json_add_int(json, TAGNAME_ATTEMPT_ERR, node->attempt.result)) - goto cancel; - if (node->success.happened) - if (json_add_ts(json, TAGNAME_SUCCESS_TS, node->success.ts)) - goto cancel; - - return json; - -cancel: - json_decref(json); +// json_t *json; +// char const *type; +// json_t *notification; +// +// json = json_obj_new(); +// if (json == NULL) +// return NULL; +// +// switch (map_get_type(node->map)) { +// case MAP_HTTP: +// type = TYPEVALUE_TA_HTTP; +// break; +// case MAP_RSYNC: +// type = TYPEVALUE_RPP; +// break; +// case MAP_NOTIF: +// type = TYPEVALUE_NOTIF; +// break; +// default: +// goto cancel; +// } +// +// if (json_add_str(json, TAGNAME_TYPE, type)) +// goto cancel; +// if (json_add_str(json, TAGNAME_URL, map_get_url(node->map))) +// goto cancel; +// if (node->notif != NULL) { +// notification = rrdp_notif2json(node->notif); +// if (json_object_add(json, TAGNAME_NOTIF, notification)) +// goto cancel; +// } +// if (json_add_ts(json, TAGNAME_ATTEMPT_TS, node->attempt.ts)) +// goto cancel; +// if (json_add_int(json, TAGNAME_ATTEMPT_ERR, node->attempt.result)) +// goto cancel; +// if (node->success.happened) +// if (json_add_ts(json, TAGNAME_SUCCESS_TS, node->success.ts)) +// goto cancel; +// +// return json; +// +//cancel: +// json_decref(json); return NULL; } static json_t * build_tal_json(struct rpki_cache *cache) { - struct cache_node *node, *tmp; - json_t *root, *child; - - root = json_array_new(); - if (root == NULL) +// struct cache_node *node, *tmp; +// json_t *root, *child; +// +// root = json_array_new(); +// if (root == NULL) return NULL; - HASH_ITER(hh, cache->ht, node, tmp) { - child = node2json(node); - if (child != NULL && json_array_append_new(root, child)) { - pr_op_err("Cannot push %s json node into json root; unknown cause.", - map_op_get_printable(node->map)); - continue; - } - } - - return root; +// HASH_ITER(hh, cache->ht, node, tmp) { +// child = node2json(node); +// if (child != NULL && json_array_append_new(root, child)) { +// pr_op_err("Cannot push %s json node into json root; unknown cause.", +// map_op_get_printable(node->map)); +// continue; +// } +// } +// +// return root; } static void @@ -505,16 +516,97 @@ end: json_decref(json); free(filename); } -static int -fix_url(struct cache_mapping *map, struct cache_mapping **result) +/* + * Returns perfect match. (Even if it needs to create it.) + * Always consumes @path. + * + * Unit Test (perfect match): + * root + * a + * b + * c + * - Find c + * - Find a + * - Find a/b + * - Find a/b/c + * - Find a/b/c/d + */ +static struct cache_node * +find_node(char *path, int flags) +{ + struct cache_node *node, *child; + char *nm, *sp; /* name, saveptr */ + size_t keylen; + + node = &cache.root; + nm = strtok_r(path + RPKI_SCHEMA_LEN, "/", &sp); // XXX + + for (; nm; nm = strtok_r(NULL, "/", &sp)) { + keylen = strlen(nm); + HASH_FIND(hh, node->children, nm, keylen, child); + if (child == NULL) + goto create_children; + node = child; + sp[-1] = '/'; /* XXX this will need a compliance unit test */ + } + + goto end; + +create_children: + for (; nm; nm = strtok_r(NULL, "/", &sp)) { + child = pmalloc(sizeof(struct cache_node)); + child->url = pstrdup(path); + child->name = strrchr(child->url, '/') + 1; // XXX + child->flags = flags; + + keylen = strlen(nm); + HASH_ADD_KEYPTR(hh, node->children, child->name, keylen, child); + + node = child; + sp[-1] = '/'; + } + +end: free(path); + return node; +} + +/* + * Returns perfect match or NULL. @msm will point to the Most Specific Match. + * Always consumes @path. + */ +static struct cache_node * +find_msm(char *path, struct cache_node **msm) +{ + struct cache_node *node, *child; + char *nm, *sp; /* name, saveptr */ + size_t keylen; + + *msm = NULL; + node = &cache.root; + nm = strtok_r(path + RPKI_SCHEMA_LEN, "/", &sp); // XXX + + for (; nm; nm = strtok_r(NULL, "/", &sp)) { + keylen = strlen(nm); + HASH_FIND(hh, node->children, nm, keylen, child); + if (child == NULL) { + free(path); + *msm = node; + return NULL; + } + node = child; + } + + free(path); + *msm = node; + return node; +} + +static char * +get_rsync_module(char const *url) { - char const *url, *c; + char const *c; char *dup; unsigned int slashes; - int error; - - if (map_get_type(map) != MAP_RPP) - goto reuse_mapping; /* * Careful with this code. rsync(1): @@ -563,17 +655,13 @@ fix_url(struct cache_mapping *map, struct cache_mapping **result) * not every RPP separately. The former is much faster. */ - url = map_get_url(map); slashes = 0; for (c = url; *c != '\0'; c++) { if (*c == '/') { slashes++; - if (slashes == 4) { - if (c[1] == '\0') - goto reuse_mapping; - dup = pstrndup(url, c - url + 1); - goto dup2url; - } + if (slashes == 4) + /* XXX test the if I rm'd here */ + return pstrndup(url, c - url + 1); } } @@ -582,496 +670,591 @@ fix_url(struct cache_mapping *map, struct cache_mapping **result) memcpy(dup, url, c - url); dup[c - url] = '/'; dup[c - url + 1] = '\0'; - goto dup2url; + return dup; } - return pr_val_err("Can't rsync URL '%s': The URL seems to be missing a domain or rsync module.", + pr_val_err("Can't rsync URL '%s': The URL seems to be missing a domain or rsync module.", url); + return NULL; +} + +static int +dl_rsync(struct cache_node *node) +{ + char *path; + int error; + + if (!config_get_rsync_enabled()) { + pr_val_debug("rsync is disabled."); + return 1; + } + + error = cache_tmpfile(&path); + if (error) + return error; + + /* + * XXX the slow (-p) version is unlikely to be necessary. + * Maybe this function should also short-circuit by parent. + */ + error = mkdir_p(path, true); + if (error) + goto cancel; + + // XXX looks like the third argument is redundant now. + error = rsync_download(node->url, path, true); + if (error) + goto cancel; -reuse_mapping: - map_refget(map); - *result = map; + node->flags |= CNF_DOWNLOADED; + node->mtim = time(NULL); // XXX catch -1 + node->tmpdir = path; return 0; -dup2url: - error = map_create(result, MAP_RPP, NULL, dup); - free(dup); +cancel: free(path); return error; } -static bool -was_recently_downloaded(struct rpki_cache *cache, struct cache_node *node) -{ - return difftime(cache->startup_ts, node->attempt.ts) <= 0; -} - static int -cache_check(struct cache_mapping *url) +dl_http(struct cache_node *node) { + char *path; + bool changed; int error; - error = file_exists(map_get_path(url)); - switch (error) { - case 0: - pr_val_debug("Offline mode, file is cached."); - break; - case ENOENT: - pr_val_debug("Offline mode, file is not cached."); - break; - default: - pr_val_debug("Offline mode, unknown result %d (%s)", - error, strerror(error)); + if (!config_get_http_enabled()) { + pr_val_debug("HTTP is disabled."); + return 1; } - return error; + error = cache_tmpfile(&path); + if (error) + return error; + + error = http_download(node->url, path, node->mtim, &changed); + if (error) { + free(path); + return error; + } + + node->flags |= CNF_DOWNLOADED; + if (changed) + node->mtim = time(NULL); // XXX catch -1 + node->tmpdir = path; + return 0; } -/** - * @ims and @changed only on HTTP. - * @ims can be zero, which means "no IMS." - * @changed can be NULL. - */ -int -cache_download(struct rpki_cache *cache, struct cache_mapping *map, - bool *changed, struct cachefile_notification ***notif) +static int +dl_rrdp(struct cache_node *node) { - struct cache_mapping *map2; - struct cache_node *node; + char *path; int error; - if (changed != NULL) - *changed = false; + if (!config_get_http_enabled()) { + pr_val_debug("HTTP is disabled."); + return 1; + } - error = fix_url(map, &map2); + error = cache_tmpfile(&path); if (error) return error; - node = find_node(cache, map2); - if (node != NULL) { - if (was_recently_downloaded(cache, node)) { - error = node->attempt.result; - goto end; - } - } else { - node = pzalloc(sizeof(struct cache_node)); - node->map = map2; - map_refget(map2); - add_node(cache, node); + // XXX needs to add all files to node. + // Probably also update node itself. + error = rrdp_update(path, node); + if (error) { + free(path); + return error; } - switch (map_get_type(map2)) { - case MAP_TA_HTTP: - case MAP_NOTIF: - case MAP_TMP: - error = config_get_http_enabled() - ? http_download(map2, node->success.ts, changed) - : cache_check(map2); - break; - case MAP_TA_RSYNC: - case MAP_RPP: - error = config_get_rsync_enabled() - ? rsync_download(map_get_url(map2), map_get_path(map2), true) - : cache_check(map2); - break; - default: - pr_crit("Mapping type not downloadable: %d", map_get_type(map2)); - } + node->flags |= CNF_DOWNLOADED; + node->mtim = time(NULL); // XXX catch -1 + node->tmpdir = path; + return 0; +} - node->attempt.ts = time(NULL); - if (node->attempt.ts == (time_t) -1) - pr_crit("time(NULL) returned (time_t) -1"); - node->attempt.result = error; - if (!error) { - node->success.happened = true; - node->success.ts = node->attempt.ts; +static int +download(struct cache_mapping *map, struct cache_node *node) +{ + switch (map_get_type(map)) { + case MAP_RSYNC: + return dl_rsync(node); + case MAP_HTTP: + case MAP_TMP: + return dl_http(node); + case MAP_NOTIF: + return dl_rrdp(node); } -end: - map_refput(map2); - if (!error && (notif != NULL)) - *notif = &node->notif; - return error; + pr_crit("Unreachable."); + return -EINVAL; /* Warning shutupper */ } +/* + * XXX review result sign + */ static int -download(struct rpki_cache *cache, struct cache_mapping *map, maps_dl_cb cb, - void *arg) +try_url(struct cache_mapping *map, bool online, maps_dl_cb cb, void *arg) { + bool is_rsync; + char *url; + struct cache_node *node; int error; - pr_val_debug("Trying URL %s...", map_get_url(map)); + // XXX if RRDP, @map needs to be unwrapped... - switch (map_get_type(map)) { - case MAP_TA_HTTP: - case MAP_TA_RSYNC: - case MAP_RPP: - error = cache_download(cache, map, NULL, NULL); - break; - case MAP_NOTIF: - error = rrdp_update(map); - break; - default: - pr_crit("Mapping type is not a legal alt candidate: %u", - map_get_type(map)); + is_rsync = map_get_type(map) == MAP_RSYNC; + url = is_rsync + ? get_rsync_module(map_get_url(map)) + : pstrdup(map_get_url(map)); + if (!url) + return -EINVAL; + + pr_val_debug("Trying RPP URL %s...", url); + + /* XXX mutex */ + node = find_node(url, is_rsync ? CNF_RSYNC : 0); + + if (online && !(node->flags & CNF_DOWNLOADED)) { + error = download(map, node); + if (error) { + pr_val_debug("RPP refresh failed."); + return error; + } } - return error ? 1 : cb(map, arg); + error = cb(node, arg); + if (error) { + pr_val_debug("RPP validation failed."); + return error; + } + + /* XXX commit the files (later, during cleanup) */ + pr_val_debug("RPP downloaded and validated successfully."); + return 0; } static int -download_maps(struct rpki_cache *cache, struct map_list *maps, - enum map_type type, maps_dl_cb cb, void *arg) +download_maps(struct map_list *maps, bool online, enum map_type type, + maps_dl_cb cb, void *arg) { - struct cache_mapping **map; + struct cache_mapping **_map, *map; int error; - ARRAYLIST_FOREACH(maps, map) { - if (map_get_type(*map) == type) { - error = download(cache, *map, cb, arg); - if (error <= 0) - return error; - } + ARRAYLIST_FOREACH(maps, _map) { + map = *_map; + + if ((map_get_type(map) & type) != type) + continue; + + error = try_url(map, online, cb, arg); + if (error <= 0) + return error; } return 1; } -/** - * Assumes the mappings represent different ways to access the same content. - * - * Sequentially (in the order dictated by their priorities) attempts to update - * (in the cache) the content pointed by each mapping's URL. - * If a download succeeds, calls cb on it. If cb succeeds, returns without - * trying more URLs. - * - * If none of the URLs download and callback properly, attempts to find one - * that's already cached, and callbacks it. - */ -int -cache_download_alt(struct rpki_cache *cache, struct map_list *maps, - enum map_type http_type, enum map_type rsync_type, maps_dl_cb cb, void *arg) +static int +try_alts(struct map_list *maps, bool online, maps_dl_cb cb, void *arg) { - struct cache_mapping **cursor, *map; + struct cache_mapping **cursor; int error; + /* XXX during cleanup, always preserve only one? */ if (config_get_http_priority() > config_get_rsync_priority()) { - error = download_maps(cache, maps, http_type, cb, arg); - if (error <= 0) - return error; - error = download_maps(cache, maps, rsync_type, cb, arg); + error = download_maps(maps, online, MAP_HTTP, cb, arg); if (error <= 0) return error; + return download_maps(maps, online, MAP_RSYNC, cb, arg); } else if (config_get_http_priority() < config_get_rsync_priority()) { - error = download_maps(cache, maps, rsync_type, cb, arg); - if (error <= 0) - return error; - error = download_maps(cache, maps, http_type, cb, arg); + error = download_maps(maps, online, MAP_RSYNC, cb, arg); if (error <= 0) return error; + return download_maps(maps, online, MAP_HTTP, cb, arg); } else { ARRAYLIST_FOREACH(maps, cursor) { - error = download(cache, *cursor, cb, arg); + error = try_url(*cursor, online, cb, arg); if (error <= 0) return error; } + return 1; } - - map = cache_recover(cache, maps); - return (map != NULL) ? cb(map, arg) : ESRCH; } -/* - * Highest to lowest priority: +/** + * Assumes the URIs represent different ways to access the same content. + * + * Sequentially (in the order dictated by their priorities) attempts to update + * (in the cache) the content pointed by each URL. + * If a download succeeds, calls cb on it. If cb succeeds, returns without + * trying more URLs. * - * 1. Recent Success: !error, CNF_SUCCESS, high ts_success. - * 2. Old Success: !error, CNF_SUCCESS, low ts_success. - * 3. Previous Recent Success: error, CNF_SUCCESS, high ts_success. - * 4. Previous Old Success: error, CNF_SUCCESS, old ts_success. - * 5. No Success: !CNF_SUCCESS (completely unviable) + * If none of the URLs download and callback properly, attempts to find one + * that's already cached, and callbacks it. */ -static struct cache_node * -choose_better(struct cache_node *old, struct cache_node *new) +int +cache_download_alt(struct map_list *maps, maps_dl_cb cb, void *arg) { - if (!new->success.happened) - return old; - if (old == NULL) - return new; + int error; - /* - * We're gonna have to get subjective here. - * Should we prioritize a candidate that was successfully downloaded a - * long time ago (with no retries since), or one that failed recently? - * Both are terrible, but returning something is still better than - * returning nothing, because the validator might manage to salvage - * remnant cached ROAs that haven't expired yet. - */ + error = try_alts(maps, true, cb, arg); + if (error) + error = try_alts(maps, false, cb, arg); - if (old->attempt.result && !new->attempt.result) - return new; - if (!old->attempt.result && new->attempt.result) - return old; - return (difftime(old->success.ts, new->success.ts) < 0) ? new : old; + return error; } -struct map_and_node { - struct cache_mapping *map; - struct cache_node *node; -}; - -/* Separated because of unit tests. */ static void -__cache_recover(struct rpki_cache *cache, struct map_list *maps, - struct map_and_node *best) +print_node(struct cache_node *node, unsigned int tabs) { - struct cache_mapping **map; - struct cache_mapping *fixed; - struct map_and_node cursor; - - ARRAYLIST_FOREACH(maps, map) { - cursor.map = *map; + struct cache_node *child, *tmp; + unsigned int i; - if (fix_url(cursor.map, &fixed) != 0) - continue; - cursor.node = find_node(cache, fixed); - map_refput(fixed); - if (cursor.node == NULL) - continue; + for (i = 0; i < tabs; i++) + printf("\t"); - if (choose_better(best->node, cursor.node) == cursor.node) - *best = cursor; - } -} + printf("%s ", node->name); + printf("%s", (node->flags & CNF_RSYNC) ? "RSYNC " : ""); + printf("%s", (node->flags & CNF_DOWNLOADED) ? "DL " : ""); + printf("%s", (node->flags & CNF_TOUCHED) ? "Touched " : ""); + printf("%s", (node->flags & CNF_VALIDATED) ? "Valid " : ""); + printf("%s\n", (node->flags & CNF_WITHDRAWN) ? "Withdrawn " : ""); -struct cache_mapping * -cache_recover(struct rpki_cache *cache, struct map_list *maps) -{ - struct map_and_node best = { 0 }; - __cache_recover(cache, maps, &best); - return best.map; + HASH_ITER(hh, node->children, child, tmp) + print_node(child, tabs + 1); } +/* Recursive; tests only. */ void cache_print(struct rpki_cache *cache) { - struct cache_node *node, *tmp; - - HASH_ITER(hh, cache->ht, node, tmp) - printf("- %s (%s): %ssuccess error:%d\n", - map_get_path(node->map), - map_get_url(node->map), - node->success.happened ? "" : "!", - node->attempt.result); + print_node(&cache->root, 0); } -static bool -is_node_fresh(struct cache_node *node, time_t epoch) -{ - /* TODO This is a startup; probably complicate this. */ - return difftime(epoch, node->attempt.ts) < 0; -} +#ifdef UNIT_TESTING +static void __delete_node_cb(struct cache_node const *); +#endif static void -delete_node(struct rpki_cache *cache, struct cache_node *node) +__delete_node(struct cache_node *node) { - HASH_DEL(cache->ht, node); - map_refput(node->map); - rrdp_notif_free(node->notif); +#ifdef UNIT_TESTING + __delete_node_cb(node); +#endif + + if (node->parent != NULL) + HASH_DEL(node->parent->children, node); + free(node->url); + free(node->tmpdir); free(node); } +/* + * Caveats: + * + * - node->parent has to be set. + * - Don't use this on the root. + */ static void -delete_node_and_cage(struct rpki_cache *cache, struct cache_node *node) +delete_node(struct cache_node *node) { - struct cache_mapping *cage; + struct cache_node *parent; - if (map_get_type(node->map) == MAP_NOTIF) { - if (map_create_cage(&cage, node->map) == 0) { - pr_op_debug("Deleting cage %s.", map_get_path(cage)); - file_rm_rf(map_get_path(cage)); - map_refput(cage); - } + parent = node->parent; + if (parent != NULL) { + HASH_DEL(parent->children, node); + node->parent = NULL; } - delete_node(cache, node); -} - -static time_t -get_days_ago(int days) -{ - time_t tt_now, last_week; - struct tm tm; - int error; - - tt_now = time(NULL); - if (tt_now == (time_t) -1) - pr_crit("time(NULL) returned (time_t) -1."); - if (localtime_r(&tt_now, &tm) == NULL) { - error = errno; - pr_crit("localtime_r(tt, &tm) returned error: %s", - strerror(error)); - } - tm.tm_mday -= days; - last_week = mktime(&tm); - if (last_week == (time_t) -1) - pr_crit("mktime(tm) returned (time_t) -1."); + do { + while (node->children) { + node->children->parent = node; + node = node->children; + } - return last_week; + parent = node->parent; + __delete_node(node); + node = parent; + } while (node != NULL); } -static void -cleanup_tmp(struct rpki_cache *cache, struct cache_node *node) +/* Preorder. @cb returns whether the children should be traversed. */ +static int +traverse_cache(bool (*cb)(struct cache_node *, char const *)) { - enum map_type type; - char const *path; + struct cache_node *iter_start; + struct cache_node *parent, *child; + struct cache_node *tmp; + struct path_builder pb; int error; - type = map_get_type(node->map); - if (type != MAP_NOTIF && type != MAP_TMP) - return; + pb_init(&pb); - path = map_get_path(node->map); - pr_op_debug("Deleting temporal file '%s'.", path); - error = file_rm_f(path); + error = pb_append(&pb, cache.root.name); if (error) - pr_op_err("Could not delete '%s': %s", path, strerror(error)); + goto end; - if (type != MAP_NOTIF) - delete_node(cache, node); -} + parent = &cache.root; + iter_start = parent->children; + if (iter_start == NULL) + goto end; -static void -cleanup_node(struct rpki_cache *cache, struct cache_node *node, - time_t last_week) -{ - char const *path; - int error; +reloop: /* iter_start must not be NULL */ + HASH_ITER(hh, iter_start, child, tmp) { + error = pb_append(&pb, child->name); + if (error) + goto end; - path = map_get_path(node->map); - if (map_get_type(node->map) == MAP_NOTIF) - goto skip_file; - - error = file_exists(path); - switch (error) { - case 0: - break; - case ENOENT: - /* Node exists but file doesn't: Delete node */ - pr_op_debug("Node exists but file doesn't: %s", path); - delete_node_and_cage(cache, node); - return; - default: - pr_op_err("Trouble cleaning '%s'; stat() returned errno %d: %s", - map_op_get_printable(node->map), error, strerror(error)); - } + child->parent = parent; + if (cb(child, pb.string) && (child->children != NULL)) { + parent = child; + iter_start = parent->children; + goto reloop; + } -skip_file: - if (!is_node_fresh(node, last_week)) { - pr_op_debug("Deleting expired cache element %s.", path); - file_rm_rf(path); - delete_node_and_cage(cache, node); + pb_pop(&pb, true); } + + parent = iter_start->parent; + do { + if (parent == NULL) + goto end; + pb_pop(&pb, true); + iter_start = parent->hh.next; + parent = parent->parent; + } while (iter_start == NULL); + + goto reloop; + +end: pb_cleanup(&pb); + return error; } /* - * "Do not clean." List of mappings that should not be deleted from the cache. - * Global because nftw doesn't have a generic argument. + * XXX this needs to be hit only by files now + * XXX result is redundant */ -static struct map_list dnc; -static pthread_mutex_t dnc_lock = PTHREAD_MUTEX_INITIALIZER; - static bool -is_cached(char const *_fpath) +commit_rpp_delta(struct cache_node *node, char const *path) { - struct cache_mapping **node; - char const *fpath, *npath; - size_t c; - - /* - * This relies on paths being normalized, which is currently done by the - * struct cache_mapping constructors. - */ - - ARRAYLIST_FOREACH(&dnc, node) { - fpath = _fpath; - npath = map_get_path(*node); - - for (c = 0; fpath[c] == npath[c]; c++) - if (fpath[c] == '\0') - return true; - if (fpath[c] == '\0' && npath[c] == '/') - return true; - if (npath[c] == '\0' && fpath[c] == '/') - return true; - } - - return false; + if (node->tmpdir == NULL) + return true; /* Not updated */ + + if (node->flags & CNF_VALIDATED) + /* XXX nftw() no longer needed; rename() is enough */ + file_merge_into(node->tmpdir, path); + else + /* XXX same; just do remove(). */ + /* XXX and rename "tmpdir" into "tmp". */ + file_rm_f(node->tmpdir); + + free(node->tmpdir); + node->tmpdir = NULL; + return true; } +//static bool +//is_node_fresh(struct cache_node *node, time_t epoch) +//{ +// /* TODO This is a startup; probably complicate this. */ +// return difftime(epoch, node->attempt.ts) < 0; +//} +// +//static void +//delete_node(struct rpki_cache *cache, struct cache_node *node) +//{ +// HASH_DEL(cache->ht, node); +// map_refput(node->map); +// rrdp_notif_free(node->notif); +// free(node); +//} +// +//static void +//delete_node_and_cage(struct rpki_cache *cache, struct cache_node *node) +//{ +// struct cache_mapping *cage; +// +// if (map_get_type(node->map) == MAP_NOTIF) { +// if (map_create_cage(&cage, node->map) == 0) { +// pr_op_debug("Deleting cage %s.", map_get_path(cage)); +// file_rm_rf(map_get_path(cage)); +// map_refput(cage); +// } +// } +// +// delete_node(cache, node); +//} +// +//static time_t +//get_days_ago(int days) +//{ +// time_t tt_now, last_week; +// struct tm tm; +// int error; +// +// tt_now = time(NULL); +// if (tt_now == (time_t) -1) +// pr_crit("time(NULL) returned (time_t) -1."); +// if (localtime_r(&tt_now, &tm) == NULL) { +// error = errno; +// pr_crit("localtime_r(tt, &tm) returned error: %s", +// strerror(error)); +// } +// tm.tm_mday -= days; +// last_week = mktime(&tm); +// if (last_week == (time_t) -1) +// pr_crit("mktime(tm) returned (time_t) -1."); +// +// return last_week; +//} +// +//static void +//cleanup_tmp(struct rpki_cache *cache, struct cache_node *node) +//{ +// enum map_type type; +// char const *path; +// int error; +// +// type = map_get_type(node->map); +// if (type != MAP_NOTIF && type != MAP_TMP) +// return; +// +// path = map_get_path(node->map); +// pr_op_debug("Deleting temporal file '%s'.", path); +// error = file_rm_f(path); +// if (error) +// pr_op_err("Could not delete '%s': %s", path, strerror(error)); +// +// if (type != MAP_NOTIF) +// delete_node(cache, node); +//} +// +//static void +//cleanup_node(struct rpki_cache *cache, struct cache_node *node, +// time_t last_week) +//{ +// char const *path; +// int error; +// +// path = map_get_path(node->map); +// if (map_get_type(node->map) == MAP_NOTIF) +// goto skip_file; +// +// error = file_exists(path); +// switch (error) { +// case 0: +// break; +// case ENOENT: +// /* Node exists but file doesn't: Delete node */ +// pr_op_debug("Node exists but file doesn't: %s", path); +// delete_node_and_cage(cache, node); +// return; +// default: +// pr_op_err("Trouble cleaning '%s'; stat() returned errno %d: %s", +// map_op_get_printable(node->map), error, strerror(error)); +// } +// +//skip_file: +// if (!is_node_fresh(node, last_week)) { +// pr_op_debug("Deleting expired cache element %s.", path); +// file_rm_rf(path); +// delete_node_and_cage(cache, node); +// } +//} +// +///* +// * "Do not clean." List of mappings that should not be deleted from the cache. +// * Global because nftw doesn't have a generic argument. +// */ +//static struct map_list dnc; +//static pthread_mutex_t dnc_lock = PTHREAD_MUTEX_INITIALIZER; +// +//static bool +//is_cached(char const *_fpath) +//{ +// struct cache_mapping **node; +// char const *fpath, *npath; +// size_t c; +// +// /* +// * This relies on paths being normalized, which is currently done by the +// * struct cache_mapping constructors. +// */ +// +// ARRAYLIST_FOREACH(&dnc, node) { +// fpath = _fpath; +// npath = map_get_path(*node); +// +// for (c = 0; fpath[c] == npath[c]; c++) +// if (fpath[c] == '\0') +// return true; +// if (fpath[c] == '\0' && npath[c] == '/') +// return true; +// if (npath[c] == '\0' && fpath[c] == '/') +// return true; +// } +// +// return false; +//} + static int -delete_if_unknown(const char *fpath, const struct stat *sb, int typeflag, +__remove_abandoned(const char *path, const struct stat *st, int typeflag, struct FTW *ftw) { - if (!is_cached(fpath)) { - pr_op_debug("Deleting untracked file or directory %s.", fpath); - errno = 0; - if (remove(fpath) != 0) - pr_op_err("Cannot delete '%s': %s", fpath, strerror(errno)); + struct cache_node *pm; /* Perfect Match */ + struct cache_node *msm; /* Most Specific Match */ + struct timespec now; + + /* XXX node->parent has to be set */ + pm = find_msm(pstrdup(path), &msm); + if (!pm && !(msm->flags & CNF_RSYNC)) + goto unknown; /* The traversal is depth-first */ + + if (S_ISDIR(st->st_mode)) { + /* + * rmdir() fails if the directory is not empty. + * This will happen most of the time. + */ + if (rmdir(path) == 0) + delete_node(pm); + else if (errno == ENOENT) + delete_node(pm); + + } else if (S_ISREG(st->st_mode)) { + if (pm->flags & (CNF_RSYNC | CNF_WITHDRAWN)) { + clock_gettime(CLOCK_REALTIME, &now); // XXX + if (now.tv_sec - st->st_atim.tv_sec > cfg_cache_threshold()) + goto abandoned; + } + + } else { + goto abandoned; } return 0; + +abandoned: + if (pm) + delete_node(pm); +unknown: + remove(path); // XXX + return 0; } /* - * FIXME this needs to account I'm merging the TAL directories. - * It might already work. + * Note: It'll probably be healthy if touched nodes also touch their parents. + * You don't always need to go up all the way to the root. + * But I'm afraid this will hit the mutexes. */ static void -delete_unknown_files(struct rpki_cache *cache) +remove_abandoned(void) { - struct cache_node *node, *tmp; - struct cache_mapping *cage; - struct path_builder pb; - int error; - - error = pb_init_cache(&pb, TAL_METAFILE); - if (error) { - pr_op_err("Cannot delete unknown files from the cache: %s", - strerror(error)); - return; - } - - mutex_lock(&dnc_lock); - maps_init(&dnc); - - maps_add(&dnc, map_create_cache(pb.string)); - HASH_ITER(hh, cache->ht, node, tmp) { - map_refget(node->map); - maps_add(&dnc, node->map); - - if (map_get_type(node->map) != MAP_NOTIF) - continue; - - if (map_create_cage(&cage, node->map) != 0) { - pr_op_err("Cannot generate %s's cage. I'm probably going to end up deleting it from the cache.", - map_op_get_printable(node->map)); - continue; - } - maps_add(&dnc, cage); - } - - pb_pop(&pb, true); - /* TODO (performance) optimize that 32 */ - error = nftw(pb.string, delete_if_unknown, 32, FTW_PHYS); - if (error) - pr_op_warn("The cache cleanup ended prematurely with error code %d (%s)", - error, strerror(error)); - - maps_cleanup(&dnc); - mutex_unlock(&dnc_lock); - - pb_cleanup(&pb); + char *root = join_paths(config_get_local_repository(), "rsync"); + nftw(root, __remove_abandoned, 32, FTW_DEPTH | FTW_PHYS); // XXX + free(root); } /* @@ -1080,31 +1263,31 @@ delete_unknown_files(struct rpki_cache *cache) static void cache_cleanup(struct rpki_cache *cache) { - struct cache_node *node, *tmp; - time_t last_week; +// struct cache_node *node, *tmp; +// time_t last_week; - pr_op_debug("Cleaning up temporal files."); - HASH_ITER(hh, cache->ht, node, tmp) - cleanup_tmp(cache, node); + pr_op_debug("Committing successful RPPs."); + traverse_cache(commit_rpp_delta); - pr_op_debug("Cleaning up old abandoned cache files."); - last_week = get_days_ago(7); - HASH_ITER(hh, cache->ht, node, tmp) - cleanup_node(cache, node, last_week); +// pr_op_debug("Cleaning up temporal files."); +// HASH_ITER(hh, cache->ht, node, tmp) +// cleanup_tmp(cache, node); - pr_op_debug("Cleaning up unknown cache files."); - delete_unknown_files(cache); -} + pr_op_debug("Cleaning up old abandoned and unknown cache files."); + remove_abandoned(); -void -cache_destroy(struct rpki_cache *cache) -{ - struct cache_node *node, *tmp; - - cache_cleanup(cache); - write_tal_json(cache); - - HASH_ITER(hh, cache->ht, node, tmp) - delete_node(cache, node); - free(cache); + /* XXX delete nodes for which no file exists? */ } + +//void +//cache_destroy(struct rpki_cache *cache) +//{ +// struct cache_node *node, *tmp; +// +// cache_cleanup(cache); +// write_tal_json(cache); +// +// HASH_ITER(hh, cache->ht, node, tmp) +// delete_node(cache, node); +// free(cache); +//} diff --git a/src/cache/local_cache.h b/src/cache/local_cache.h index 35d18cb2..d0e6d179 100644 --- a/src/cache/local_cache.h +++ b/src/cache/local_cache.h @@ -2,8 +2,12 @@ #define SRC_CACHE_LOCAL_CACHE_H_ #include "types/map.h" +#include "types/str.h" + +#define RPKI_SCHEMA_LEN 8 /* strlen("rsync://"), strlen("https://") */ struct rpki_cache; +struct cache_node; void cache_setup(void); void cache_teardown(void); @@ -14,25 +18,18 @@ struct rpki_cache *cache_create(void); /* Will destroy the cache object, but not the cache directory itself, obv. */ void cache_destroy(struct rpki_cache *); -struct cachefile_notification; /* FIXME */ - -/* Downloads @map into the cache */ -int cache_download(struct rpki_cache *, struct cache_mapping *map, bool *, - struct cachefile_notification ***); - /* * The callback should return * * - 0 on success ("Mapping handled successfully") * - > 0 on soft errors ("Try another mapping") * - < 0 on hard errors ("Abandon foreach") + * + * XXX rename */ -typedef int (*maps_dl_cb)(struct cache_mapping *, void *); -int cache_download_alt(struct rpki_cache *, struct map_list *, enum map_type, - enum map_type, maps_dl_cb, void *); +typedef int (*maps_dl_cb)(struct cache_node *, void *); +int cache_download_alt(struct map_list *, maps_dl_cb, void *); -/* Returns the most recent successfully cached mapping of the list */ -struct cache_mapping *cache_recover(struct rpki_cache *, struct map_list *); /* Prints the cache in standard output. */ void cache_print(struct rpki_cache *); diff --git a/src/certificate_refs.c b/src/certificate_refs.c index cac3313e..c63cf8e5 100644 --- a/src/certificate_refs.c +++ b/src/certificate_refs.c @@ -14,9 +14,9 @@ refs_cleanup(struct certificate_refs *refs) { free(refs->crldp); if (refs->caIssuers != NULL) - map_refput(refs->caIssuers); + free(refs->caIssuers); if (refs->signedObject != NULL) - map_refput(refs->signedObject); + free(refs->signedObject); } static int @@ -46,10 +46,10 @@ validate_signedObject(struct certificate_refs *refs, if (refs->signedObject == NULL) pr_crit("Certificate's signedObject was not recorded."); - if (!map_equals(refs->signedObject, signedObject_map)) { + /* XXX the left one is no longer normalized */ + if (strcmp(refs->signedObject, map_get_url(signedObject_map)) != 0) { return pr_val_err("Certificate's signedObject ('%s') does not match the URI of its own signed object (%s).", - map_val_get_printable(refs->signedObject), - map_val_get_printable(signedObject_map)); + refs->signedObject, map_get_url(signedObject_map)); } return 0; @@ -73,7 +73,7 @@ refs_validate_ca(struct certificate_refs *refs, struct rpp const *pp) if (refs->signedObject != NULL) pr_crit("CA summary has a signedObject ('%s').", - map_op_get_printable(refs->signedObject)); + refs->signedObject); return 0; } diff --git a/src/certificate_refs.h b/src/certificate_refs.h index 01794d12..009f4f91 100644 --- a/src/certificate_refs.h +++ b/src/certificate_refs.h @@ -28,12 +28,12 @@ struct certificate_refs { * AIA's caIssuers. Non-TA certificates only. * RFC 6487, section 4.8.7. */ - struct cache_mapping *caIssuers; + char *caIssuers; /** * SIA's signedObject. EE certificates only. * RFC 6487, section 4.8.8.2. */ - struct cache_mapping *signedObject; + char *signedObject; }; void refs_init(struct certificate_refs *); diff --git a/src/config.h b/src/config.h index 8467eaea..6f14fc40 100644 --- a/src/config.h +++ b/src/config.h @@ -31,6 +31,7 @@ char const *config_get_slurm(void); char const *config_get_tal(void); char const *config_get_local_repository(void); +time_t cfg_cache_threshold(void); unsigned int config_get_max_cert_depth(void); enum mode config_get_mode(void); char const *config_get_http_user_agent(void); diff --git a/src/crypto/hash.c b/src/crypto/hash.c index 1b608f1d..b6cdbc8a 100644 --- a/src/crypto/hash.c +++ b/src/crypto/hash.c @@ -156,15 +156,14 @@ end: } int -hash_validate_file(struct hash_algorithm const *algorithm, - struct cache_mapping *map, unsigned char const *expected, - size_t expected_len) +hash_validate_file(struct hash_algorithm const *algorithm, char const *path, + unsigned char const *expected, size_t expected_len) { unsigned char actual[EVP_MAX_MD_SIZE]; size_t actual_len; int error; - error = hash_file(algorithm, map_get_path(map), actual, &actual_len); + error = hash_file(algorithm, path, actual, &actual_len); if (error) return error; @@ -176,8 +175,7 @@ hash_validate_file(struct hash_algorithm const *algorithm, return 0; fail: - return pr_val_err("File '%s' does not match its expected hash.", - map_val_get_printable(map)); + return pr_val_err("File '%s' does not match its expected hash.", path); } static int diff --git a/src/crypto/hash.h b/src/crypto/hash.h index 8d4c6c2e..c99c9469 100644 --- a/src/crypto/hash.h +++ b/src/crypto/hash.h @@ -2,7 +2,6 @@ #define SRC_HASH_H_ #include -#include "types/map.h" struct hash_algorithm; @@ -15,7 +14,7 @@ struct hash_algorithm const *hash_get_sha256(void); int hash_file(struct hash_algorithm const *, char const *, unsigned char *, size_t *); -int hash_validate_file(struct hash_algorithm const *, struct cache_mapping *, +int hash_validate_file(struct hash_algorithm const *, char const *, unsigned char const *, size_t); int hash_validate(struct hash_algorithm const *, unsigned char const *, size_t, unsigned char const *, size_t); diff --git a/src/data_structure/uthash.h b/src/data_structure/uthash.h index 0013ed37..3db49db0 100644 --- a/src/data_structure/uthash.h +++ b/src/data_structure/uthash.h @@ -1133,6 +1133,8 @@ for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL)); (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL))) #endif +/* #define HASH_ITER_UNSAFE(head, el) for ((el)=(head); (el)!=NULL; (el)=(el)->hh.next) */ + /* obtain a count of items in the hash */ #define HASH_COUNT(head) HASH_CNT(hh,head) #define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U) diff --git a/src/file.c b/src/file.c index b23e90e5..838dae30 100644 --- a/src/file.c +++ b/src/file.c @@ -1,6 +1,8 @@ #include "file.h" +#include #include +#include #include "alloc.h" #include "log.h" @@ -133,6 +135,53 @@ file_exists(char const *path) return (stat(path, &meta) == 0) ? 0 : errno; } +/* strlen("cache/tmp/123"), ie. 13 */ +static size_t src_offset; +/* cache/rsync/a.b.c/d/e */ +static char const *merge_dst; + +/* Moves cache/tmp/123/z into cache/rsync/a.b.c/d/e/z. */ +static int +merge_into(const char *src, const struct stat *st, int typeflag, + struct FTW *ftw) +{ + char *dst; + struct timespec times[2]; + + dst = join_paths(merge_dst, &src[src_offset]); + + if (S_ISDIR(st->st_mode)) { + mkdir(dst, st->st_mode); /* XXX catch error */ + + times[0] = st->st_atim; + times[1] = st->st_mtim; + utimensat(AT_FDCWD, dst, times, AT_SYMLINK_NOFOLLOW); /* XXX catch error */ + } else { + rename(src, dst); /* XXX catch error */ + } + + free(dst); + return 0; +} + +/* + * Move all the files contained in @src to @dst, overwriting when necessary, + * not touching files that exist in @dst but not in @src. + * + * Both directories have to already exist. + * + * @src: cache/tmp/123 + * @dst: cache/rsync/a.b.c/d/e + */ +int +file_merge_into(char const *src, char const *dst) +{ + src_offset = strlen(src); + merge_dst = dst; + /* TODO (performance) optimize that 32 */ + return nftw(src, merge_into, 32, FTW_PHYS); +} + /* * Like remove(), but don't care if the file is already deleted. */ @@ -166,3 +215,21 @@ file_rm_rf(char const *path) /* TODO (performance) optimize that 32 */ return nftw(path, rm, 32, FTW_DEPTH | FTW_PHYS); } + +/* Cannot return NULL. */ +char * +join_paths(char const *path1, char const *path2) +{ + size_t n; + char *result; + int written; + + n = strlen(path1) + strlen(path2) + 2; + result = pmalloc(n); + + written = snprintf(result, n, "%s/%s", path1, path2); + if (written != n - 1) + pr_crit("join_paths: %zu %d %s %s", n, written, path1, path2); + + return result; +} diff --git a/src/file.h b/src/file.h index db3e9623..8f6f959b 100644 --- a/src/file.h +++ b/src/file.h @@ -32,9 +32,12 @@ void file_free(struct file_contents *); int file_exists(char const *); +int file_merge_into(char const *, char const *); int file_rm_f(char const *); int file_rm_rf(char const *); +char *join_paths(char const *, char const *); + /* * Remember that this API is awkward: * diff --git a/src/http/http.c b/src/http/http.c index c58e7bc0..d4b8cd57 100644 --- a/src/http/http.c +++ b/src/http/http.c @@ -367,7 +367,7 @@ end: http_easy_cleanup(&handler); } /* - * Download @map->url into @map->path; HTTP assumed. + * Download @url into @path; HTTP assumed. * * If @changed returns true, the file was downloaded normally. * If @changed returns false, the file has not been modified since @ims. @@ -377,22 +377,22 @@ end: http_easy_cleanup(&handler); * If @changed is not NULL, initialize it to false. */ int -http_download(struct cache_mapping *map, curl_off_t ims, bool *changed) +http_download(char const *url, char const *path, curl_off_t ims, bool *changed) { unsigned int r; int error; - pr_val_info("HTTP GET: %s -> %s", map_get_url(map), map_get_path(map)); + pr_val_info("HTTP GET: %s -> %s", url, path); - error = mkdir_p(map_get_path(map), false); + /* XXX might not be needed anymore */ + error = mkdir_p(path, false); if (error) return error; for (r = 0; true; r++) { pr_val_debug("Download attempt #%u...", r + 1); - error = http_fetch(map_get_url(map), map_get_path(map), - ims, changed); + error = http_fetch(url, path, ims, changed); switch (error) { case 0: pr_val_debug("Download successful."); diff --git a/src/http/http.h b/src/http/http.h index c1a0a21e..a7b3ac5e 100644 --- a/src/http/http.h +++ b/src/http/http.h @@ -2,12 +2,12 @@ #define SRC_HTTP_HTTP_H_ #include -#include "types/map.h" int http_init(void); void http_cleanup(void); -int http_download(struct cache_mapping *, curl_off_t, bool *); +int http_download(char const *, char const *, curl_off_t, bool *); int http_download_direct(char const *, char const *); +int http_download_tmp(char const *, char **, bool *, void *); #endif /* SRC_HTTP_HTTP_H_ */ diff --git a/src/object/certificate.c b/src/object/certificate.c index cf7294c9..7d0b3df2 100644 --- a/src/object/certificate.c +++ b/src/object/certificate.c @@ -21,7 +21,7 @@ #include "cert_stack.h" #include "config.h" #include "crypto/hash.h" -#include "data_structure/array_list.h" +#include "types/str.h" #include "extension.h" #include "incidence/incidence.h" #include "log.h" @@ -47,8 +47,8 @@ struct ski_arguments { }; struct sia_uris { - struct map_list rpp; - struct cache_mapping *mft; + struct strlist rpp; + char *mft; }; struct bgpsec_ski { @@ -62,63 +62,57 @@ typedef int (access_method_exec)(struct sia_uris *); struct ad_metadata { char const *name; char const *ia_name; - enum map_type type; - char const *type_str; + char const *type; bool required; }; static const struct ad_metadata CA_ISSUERS = { .name = "caIssuers", .ia_name = "AIA", - .type = MAP_AIA, - .type_str = "rsync", + .type = "rsync", .required = true, }; static const struct ad_metadata SIGNED_OBJECT = { .name = "signedObject", .ia_name = "SIA", - .type = MAP_SO, - .type_str = "rsync", + .type = "rsync", .required = true, }; static const struct ad_metadata CA_REPOSITORY = { .name = "caRepository", .ia_name = "SIA", - .type = MAP_RPP, - .type_str = "rsync", + .type = "rsync", .required = false, }; static const struct ad_metadata RPKI_NOTIFY = { .name = "rpkiNotify", .ia_name = "SIA", - .type = MAP_NOTIF, - .type_str = "HTTPS", + .type = "HTTPS", .required = false, }; static const struct ad_metadata RPKI_MANIFEST = { .name = "rpkiManifest", .ia_name = "SIA", - .type = MAP_MFT, - .type_str = "rsync", + .type = "rsync", .required = true, }; static void sia_uris_init(struct sia_uris *uris) { - maps_init(&uris->rpp); + strlist_init(&uris->rpp); uris->mft = NULL; } static void sia_uris_cleanup(struct sia_uris *uris) { - maps_cleanup(&uris->rpp); - map_refput(uris->mft); + strlist_cleanup(&uris->rpp); + free(uris->mft); } static void @@ -1198,41 +1192,35 @@ is_rsync(ASN1_IA5STRING *uri) : false; } -static int -handle_rpkiManifest(struct cache_mapping *map, void *arg) +static void +handle_rpkiManifest(char *uri, void *arg) { struct sia_uris *uris = arg; - uris->mft = map_refget(map); - return 0; + uris->mft = uri; } -static int -handle_caRepository(struct cache_mapping *map, void *arg) +static void +handle_caRepository(char *uri, void *arg) { struct sia_uris *uris = arg; - pr_val_debug("caRepository: %s", map_val_get_printable(map)); - maps_add(&uris->rpp, map); - map_refget(map); - return 0; + pr_val_debug("caRepository: %s", uri); + strlist_add(&uris->rpp, uri); } -static int -handle_rpkiNotify(struct cache_mapping *map, void *arg) +static void +handle_rpkiNotify(char *uri, void *arg) { struct sia_uris *uris = arg; - pr_val_debug("rpkiNotify: %s", map_val_get_printable(map)); - maps_add(&uris->rpp, map); - map_refget(map); - return 0; + pr_val_debug("rpkiNotify: %s", uri); + strlist_add(&uris->rpp, uri); } -static int -handle_signedObject(struct cache_mapping *map, void *arg) +static void +handle_signedObject(char *uri, void *arg) { struct certificate_refs *refs = arg; - pr_val_debug("signedObject: %s", map_val_get_printable(map)); - refs->signedObject = map_refget(map); - return 0; + pr_val_debug("signedObject: %s", uri); + refs->signedObject = uri; } static int @@ -1431,13 +1419,10 @@ dist_point_error: * Create @map from the @ad */ static int -map_create_ad(struct cache_mapping **map, ACCESS_DESCRIPTION *ad, - enum map_type type) +ad2uri(char **uri, ACCESS_DESCRIPTION *ad) { ASN1_STRING *asn1str; - char *str; int ptype; - int error; asn1str = GENERAL_NAME_get0_value(ad->location, &ptype); @@ -1474,14 +1459,16 @@ map_create_ad(struct cache_mapping **map, ACCESS_DESCRIPTION *ad, * conversion, so we should be looking at precisely the IA5String * directory our g2l version of @asn1_string should contain. * But ask the testers to keep an eye on it anyway. + * + * XXX There used to be a map_create() here. Make sure validations are + * restored somewhere: + * 1. ascii + * 2. "rsync://" or "https://" prefix (ENOTRSYNC, ENOTHTTPS) + * 3. URL normalization */ - str = pstrndup((char const *)ASN1_STRING_get0_data(asn1str), + *uri = pstrndup((char const *)ASN1_STRING_get0_data(asn1str), ASN1_STRING_length(asn1str)); - - error = map_create(map, type, NULL, str); - - free(str); - return error; + return 0; } /** @@ -1500,13 +1487,17 @@ map_create_ad(struct cache_mapping **map, ACCESS_DESCRIPTION *ad, * supposed to be ignored. * 5. Other access descriptions that match the NID but do not have recognized * URLs are also allowed, and also supposed to be ignored. + * + * cb() always steals ownership of the URL string. + * + * TODO (test) is this tested somewhere? */ static int handle_ad(int nid, struct ad_metadata const *meta, SIGNATURE_INFO_ACCESS *ia, - int (*cb)(struct cache_mapping *, void *), void *arg) + void (*cb)(char *, void *), void *arg) { ACCESS_DESCRIPTION *ad; - struct cache_mapping *map; + char *uri; bool found; unsigned int i; int error; @@ -1515,12 +1506,10 @@ handle_ad(int nid, struct ad_metadata const *meta, SIGNATURE_INFO_ACCESS *ia, for (i = 0; i < sk_ACCESS_DESCRIPTION_num(ia); i++) { ad = sk_ACCESS_DESCRIPTION_value(ia, i); if (OBJ_obj2nid(ad->method) == nid) { - error = map_create_ad(&map, ad, meta->type); + error = ad2uri(&uri, ad); switch (error) { case 0: break; - case ENOTRSYNC: - case ENOTHTTPS: case ENOTSUPPORTED: continue; default: @@ -1528,33 +1517,27 @@ handle_ad(int nid, struct ad_metadata const *meta, SIGNATURE_INFO_ACCESS *ia, } if (found) { - map_refput(map); + free(uri); return pr_val_err("Extension '%s' has multiple '%s' %s URIs.", - meta->ia_name, meta->name, meta->type_str); - } - - error = cb(map, arg); - if (error) { - map_refput(map); - return error; + meta->ia_name, meta->name, meta->type); } - map_refput(map); + cb(uri, arg); /* Ownership of uri stolen */ found = true; } } if (meta->required && !found) { pr_val_err("Extension '%s' lacks a '%s' valid %s URI.", - meta->ia_name, meta->name, meta->type_str); + meta->ia_name, meta->name, meta->type); return -ESRCH; } return 0; } -static int -handle_caIssuers(struct cache_mapping *map, void *arg) +static void +handle_caIssuers(char *uri, void *arg) { struct certificate_refs *refs = arg; /* @@ -1562,8 +1545,7 @@ handle_caIssuers(struct cache_mapping *map, void *arg) * over here is too much trouble, so do the handle_cdp() * hack. */ - refs->caIssuers = map_refget(map); - return 0; + refs->caIssuers = uri; } static int @@ -1810,7 +1792,7 @@ err: } int -certificate_validate_aia(struct cache_mapping *caIssuers, X509 *cert) +certificate_validate_aia(char const *caIssuers, X509 *cert) { /* * FIXME Compare the AIA to the parent's URI. @@ -1821,27 +1803,13 @@ certificate_validate_aia(struct cache_mapping *caIssuers, X509 *cert) } static int -retrieve_mapping(struct cache_mapping *map, void *arg) -{ - struct cache_mapping **result = arg; - *result = map; - return 0; -} - -static struct cache_mapping * download_rpp(struct sia_uris *uris) { - struct cache_mapping *map; - int error; + if (uris->rpp.len == 0) + return pr_val_err("SIA lacks both caRepository and rpkiNotify."); - if (uris->rpp.len == 0) { - pr_val_err("SIA lacks both caRepository and rpkiNotify."); - return NULL; - } - - error = cache_download_alt(validation_cache(state_retrieve()), - &uris->rpp, MAP_NOTIF, MAP_RPP, retrieve_mapping, &map); - return error ? NULL : map; + return cache_download_alt(validation_cache(state_retrieve()), + &uris->rpp, MAP_NOTIF, NULL, NULL); } /** Boilerplate code for CA certificate validation and recursive traversal. */ @@ -1853,7 +1821,6 @@ certificate_traverse(struct rpp *rpp_parent, struct cache_mapping *cert_map) STACK_OF(X509_CRL) *rpp_parent_crl; X509 *cert; struct sia_uris sia_uris; - struct cache_mapping *downloaded; enum rpki_policy policy; enum cert_type certype; struct rpp *pp; @@ -1920,11 +1887,9 @@ certificate_traverse(struct rpp *rpp_parent, struct cache_mapping *cert_map) if (error) goto revert_uris; - downloaded = download_rpp(&sia_uris); - if (downloaded == NULL) { - error = EINVAL; + error = download_rpp(&sia_uris); + if (error) goto revert_uris; - } error = x509stack_push(validation_certstack(state), cert_map, cert, policy, certype); @@ -1932,9 +1897,7 @@ certificate_traverse(struct rpp *rpp_parent, struct cache_mapping *cert_map) goto revert_uris; cert = NULL; /* Ownership stolen */ - error = handle_manifest(sia_uris.mft, - (map_get_type(downloaded) == MAP_NOTIF) ? downloaded : NULL, - &pp); + error = handle_manifest(sia_uris.mft, &pp); if (error) { x509stack_cancel(validation_certstack(state)); goto revert_uris; diff --git a/src/object/certificate.h b/src/object/certificate.h index b2b68ace..40d6fc25 100644 --- a/src/object/certificate.h +++ b/src/object/certificate.h @@ -55,7 +55,7 @@ int certificate_validate_extensions_bgpsec(X509 *, unsigned char **, * Specific validation of AIA (rfc6487#section-4.8.7) extension, public so that * CAs and EEs can access it. */ -int certificate_validate_aia(struct cache_mapping *, X509 *); +int certificate_validate_aia(char const *, X509 *); int certificate_traverse(struct rpp *, struct cache_mapping *); diff --git a/src/object/manifest.c b/src/object/manifest.c index e1ed8575..a898a5e0 100644 --- a/src/object/manifest.c +++ b/src/object/manifest.c @@ -1,5 +1,6 @@ #include "object/manifest.h" +#include "alloc.h" #include "algorithm.h" #include "asn1/asn1c/GeneralizedTime.h" #include "asn1/asn1c/Manifest.h" @@ -14,18 +15,6 @@ #include "object/signed_object.h" #include "thread_var.h" -static int -cage(struct cache_mapping **map, struct cache_mapping *notif) -{ - if (notif == NULL) { - /* No need to cage */ - map_refget(*map); - return 0; - } - - return map_create_caged(map, notif, map_get_url(*map)); -} - static int decode_manifest(struct signed_object *sobj, struct Manifest **result) { @@ -185,6 +174,43 @@ validate_manifest(struct Manifest *manifest) return 0; } +static bool +is_valid_mft_file_chara(uint8_t chara) +{ + return ('a' <= chara && chara <= 'z') + || ('A' <= chara && chara <= 'Z') + || ('0' <= chara && chara <= '9') + || (chara == '-') + || (chara == '_'); +} + +/* RFC 6486bis, section 4.2.2 */ +static int +validate_mft_file(IA5String_t *ia5) +{ + size_t dot; + size_t i; + + if (ia5->size < 5) + return pr_val_err("File name is too short (%zu < 5).", ia5->size); + dot = ia5->size - 4; + if (ia5->buf[dot] != '.') + return pr_val_err("File name seems to lack a three-letter extension."); + + for (i = 0; i < ia5->size; i++) { + if (i != dot && !is_valid_mft_file_chara(ia5->buf[i])) { + return pr_val_err("File name contains illegal character #%u", + ia5->buf[i]); + } + } + + /* + * Actual extension doesn't matter; if there's no handler, + * we'll naturally ignore the file. + */ + return 0; +} + /** * Computes the hash of the file @map, and compares it to @expected (The * "expected" hash). @@ -243,29 +269,53 @@ hash_validate_mft_file(struct cache_mapping *map, BIT_STRING_t const *expected) } static int -build_rpp(struct Manifest *mft, struct cache_mapping *notif, - struct cache_mapping *mft_map, struct rpp **pp) +build_rpp(struct Manifest *mft, char const *mft_url, struct rpp **pp) { + char *path, *slash; + size_t path_len; int i; struct FileAndHash *fah; + char *file; + size_t file_len; + int written; struct cache_mapping *map; int error; *pp = rpp_create(); + slash = strrchr(mft_url, '/'); + if (!slash) + ; // XXX + path_len = slash - mft_url; + path = pstrndup(mft_url, path_len); + for (i = 0; i < mft->fileList.list.count; i++) { fah = mft->fileList.list.array[i]; - error = map_create_mft(&map, notif, mft_map, &fah->file); /* - * Not handling ENOTRSYNC is fine because the manifest URL - * should have been RSYNC. Something went wrong if an RSYNC URL - * plus a relative path is not RSYNC. + * IA5String is a subset of ASCII. However, IA5String_t doesn't + * seem to be guaranteed to be NULL-terminated. + * `(char *) ia5->buf` is fair, but `strlen(ia5->buf)` is not. */ + + error = validate_mft_file(&fah->file); if (error) - goto fail; + ; // XXX + + file_len = path_len + fah->file.size + 2; + file = pmalloc(file_len); + written = snprintf(file, file_len, "%s/%.*s", path, + (int)fah->file.size, fah->file.buf); + if (written >= file_len) + ; // XXX + + map = create_map(file); /* Needs to swallow @file. */ + if (!map) + ; // XXX /* + * XXX I think this should be moved somewhere else. + * * Expect: * - Negative value: an error not to be ignored, the whole * manifest will be discarded. @@ -316,13 +366,13 @@ fail: } /** - * Validates the manifest pointed by @map, returns the RPP described by it in + * Validates the manifest pointed by @uri, returns the RPP described by it in * @pp. */ int -handle_manifest(struct cache_mapping *map, struct cache_mapping *notif, - struct rpp **pp) +handle_manifest(char const *uri, struct rpp **pp) { + struct cache_mapping *map; static OID oid = OID_MANIFEST; struct oid_arcs arcs = OID2ARCS("manifest", oid); struct signed_object sobj; @@ -331,10 +381,11 @@ handle_manifest(struct cache_mapping *map, struct cache_mapping *notif, STACK_OF(X509_CRL) *crl; int error; + map = create_map(uri); + if (!map) + return -EINVAL; /* XXX msg */ + /* Prepare */ - error = cage(&map, notif); /* ref++ */ - if (error) - return error; pr_val_debug("Manifest '%s' {", map_val_get_printable(map)); fnstack_push_map(map); @@ -347,7 +398,7 @@ handle_manifest(struct cache_mapping *map, struct cache_mapping *notif, goto revert_sobj; /* Initialize out parameter (@pp) */ - error = build_rpp(mft, notif, map, pp); + error = build_rpp(mft, uri, pp); if (error) goto revert_manifest; @@ -383,6 +434,6 @@ revert_sobj: revert_log: pr_val_debug("}"); fnstack_pop(); - map_refput(map); /* ref-- */ + map_refput(map); return error; } diff --git a/src/object/manifest.h b/src/object/manifest.h index 6ec44118..24d7c17d 100644 --- a/src/object/manifest.h +++ b/src/object/manifest.h @@ -3,6 +3,6 @@ #include "rpp.h" -int handle_manifest(struct cache_mapping *, struct cache_mapping *, struct rpp **); +int handle_manifest(char const *, struct rpp **); #endif /* SRC_OBJECT_MANIFEST_H_ */ diff --git a/src/object/tal.c b/src/object/tal.c index ee072b93..9c02149f 100644 --- a/src/object/tal.c +++ b/src/object/tal.c @@ -24,7 +24,7 @@ typedef int (*foreach_map_cb)(struct tal *, struct cache_mapping *, void *); struct tal { char const *file_name; - struct map_list maps; + struct strlist urls; unsigned char *spki; /* Decoded; not base64. */ size_t spki_len; @@ -70,31 +70,11 @@ is_blank(char const *str) return true; } -static int -add_url(struct tal *tal, char *url) -{ - struct cache_mapping *new = NULL; - int error; - - if (str_starts_with(url, "rsync://")) - error = map_create(&new, MAP_TA_RSYNC, NULL, url); - else if (str_starts_with(url, "https://")) - error = map_create(&new, MAP_TA_HTTP, NULL, url); - else - return pr_op_err("TAL has non-rsync/HTTPS URI: %s", url); - if (error) - return error; - - maps_add(&tal->maps, new); - return 0; -} - static int read_content(char *fc /* File Content */, struct tal *tal) { char *nl; /* New Line */ bool cr; /* Carriage return */ - int error; /* Comment section */ while (fc[0] == '#') { @@ -115,16 +95,15 @@ read_content(char *fc /* File Content */, struct tal *tal) if (is_blank(fc)) break; - error = add_url(tal, fc); - if (error) - return error; + // XXX no longer validating schema + strlist_add(&tal->urls, pstrdup(fc)); fc = nl + cr + 1; if (*fc == '\0') return pr_op_err("The TAL seems to be missing the public key."); } while (true); - if (tal->maps.len == 0) + if (tal->urls.len == 0) return pr_op_err("There seems to be an empty/blank line before the end of the URI section."); /* subjectPublicKeyInfo section */ @@ -156,10 +135,10 @@ tal_init(struct tal *tal, char const *file_path) file_name = (file_name != NULL) ? (file_name + 1) : file_path; tal->file_name = file_name; - maps_init(&tal->maps); + strlist_init(&tal->urls); error = read_content((char *)file.buffer, tal); if (error) { - maps_cleanup(&tal->maps); + strlist_cleanup(&tal->urls); goto end; } @@ -175,7 +154,7 @@ tal_cleanup(struct tal *tal) { cache_destroy(tal->cache); free(tal->spki); - maps_cleanup(&tal->maps); + strlist_cleanup(&tal->urls); } char const * @@ -221,7 +200,7 @@ handle_tal_map(struct tal *tal, struct cache_mapping *map, struct db_table *db) if (error) return ENSURE_NEGATIVE(error); - if (!map_is_certificate(map)) { + if (!map_has_extension(map, ".cer")) { pr_op_err("TAL URI does not point to a certificate. (Expected .cer, got '%s')", map_op_get_printable(map)); error = EINVAL; @@ -302,8 +281,8 @@ do_file_validation(void *arg) goto end; args.db = db_table_create(); - thread->error = cache_download_alt(args.tal.cache, &args.tal.maps, - MAP_TA_HTTP, MAP_TA_RSYNC, __handle_tal_map, &args); + thread->error = cache_download_alt(args.tal.cache, &args.tal.urls, + MAP_HTTP, __handle_tal_map, &args); if (thread->error) { pr_op_err("None of the URIs of the TAL '%s' yielded a successful traversal.", thread->tal_file); diff --git a/src/print_file.c b/src/print_file.c index 6db8eaae..8e8e6e0a 100644 --- a/src/print_file.c +++ b/src/print_file.c @@ -65,23 +65,18 @@ end: pb_cleanup(&pb); static BIO * rsync2bio_cache(char const *src) { - struct cache_mapping *map = NULL; + char *dst; BIO *bio; - int error; - /* - * TODO (#82) maybe rename MAP_TA_RSYNC into single rsync. - * If applies and it's going to survive. - */ - error = map_create(&map, MAP_TA_RSYNC, NULL, src); - if (error) { - pr_op_err("Unparseable rsync URI: %s", strerror(abs(error))); + dst = url2path(src); + if (!dst) { + pr_op_err("Unparseable rsync URI."); return NULL; } - bio = __rsync2bio(map_get_url(map), map_get_path(map)); + bio = __rsync2bio(src, dst); - map_refput(map); + free(dst); return bio; } diff --git a/src/rrdp.c b/src/rrdp.c index dc6a7a94..8c4740d1 100644 --- a/src/rrdp.c +++ b/src/rrdp.c @@ -12,7 +12,7 @@ #include "json_util.h" #include "log.h" #include "thread_var.h" -#include "cache/local_cache.h" +#include "http/http.h" #include "crypto/base64.h" #include "crypto/hash.h" #include "xml/relax_ng.h" @@ -46,7 +46,7 @@ struct rrdp_session { }; struct file_metadata { - struct cache_mapping *uri; + char *uri; unsigned char *hash; /* Array. Sometimes omitted. */ size_t hash_len; }; @@ -65,7 +65,7 @@ struct update_notification { struct rrdp_session session; struct file_metadata snapshot; struct notification_deltas deltas; - struct cache_mapping *map; + char const *url; }; /* A deserialized tag, from a snapshot or delta. */ @@ -73,17 +73,15 @@ struct publish { struct file_metadata meta; unsigned char *content; size_t content_len; + + char *path; }; /* A deserialized tag, from a delta. */ struct withdraw { struct file_metadata meta; -}; -/* Helpful context while reading a snapshot or delta. */ -struct rrdp_ctx { - struct cache_mapping *notif; - struct rrdp_session session; + char *path; }; typedef enum { @@ -143,7 +141,7 @@ static void metadata_cleanup(struct file_metadata *meta) { free(meta->hash); - map_refput(meta->uri); + free(meta->uri); } static void @@ -154,13 +152,12 @@ notification_delta_cleanup(struct notification_delta *delta) } static void -update_notification_init(struct update_notification *notif, - struct cache_mapping *map) +update_notification_init(struct update_notification *notif, char const *url) { memset(¬if->session, 0, sizeof(notif->session)); memset(¬if->snapshot, 0, sizeof(notif->snapshot)); notification_deltas_init(¬if->deltas); - notif->map = map_refget(map); + notif->url = url; } static void @@ -168,7 +165,6 @@ __update_notification_cleanup(struct update_notification *notif) { metadata_cleanup(¬if->snapshot); notification_deltas_cleanup(¬if->deltas, notification_delta_cleanup); - map_refput(notif->map); } static void @@ -179,10 +175,10 @@ update_notification_cleanup(struct update_notification *notif) } static int -validate_hash(struct file_metadata *meta) +validate_hash(struct file_metadata *meta, char const *path) { - return hash_validate_file(hash_get_sha256(), meta->uri, meta->hash, - meta->hash_len); + return hash_validate_file(hash_get_sha256(), path, + meta->hash, meta->hash_len); } static int @@ -254,22 +250,20 @@ parse_string(xmlTextReaderPtr reader, char const *attr) return result; } -static int -parse_uri(xmlTextReaderPtr reader, struct cache_mapping *notif, - struct cache_mapping **result) +static char * +parse_uri(xmlTextReaderPtr reader) { xmlChar *xmlattr; - int error; + char *result; xmlattr = parse_string(reader, RRDP_ATTR_URI); if (xmlattr == NULL) - return -EINVAL; + return NULL; - error = map_create(result, (notif != NULL) ? MAP_CAGED : MAP_TMP, notif, - (char const *)xmlattr); + result = pstrdup((char const *)xmlattr); xmlFree(xmlattr); - return error; + return result; } static unsigned int @@ -455,20 +449,20 @@ end: * 2. "hash" (optional, depending on @hr) */ static int -parse_file_metadata(xmlTextReaderPtr reader, struct cache_mapping *notif, - hash_requirement hr, struct file_metadata *meta) +parse_file_metadata(xmlTextReaderPtr reader, hash_requirement hr, + struct file_metadata *meta) { int error; memset(meta, 0, sizeof(*meta)); - error = parse_uri(reader, notif, &meta->uri); - if (error) - return error; + meta->uri = parse_uri(reader); + if (meta->uri == NULL) + return -EINVAL; error = parse_hash(reader, hr, &meta->hash, &meta->hash_len); if (error) { - map_refput(meta->uri); + free(meta->uri); meta->uri = NULL; return error; } @@ -477,13 +471,12 @@ parse_file_metadata(xmlTextReaderPtr reader, struct cache_mapping *notif, } static int -parse_publish(xmlTextReaderPtr reader, struct cache_mapping *notif, - hash_requirement hr, struct publish *tag) +parse_publish(xmlTextReaderPtr reader, hash_requirement hr, struct publish *tag) { xmlChar *base64_str; int error; - error = parse_file_metadata(reader, notif, hr, &tag->meta); + error = parse_file_metadata(reader, hr, &tag->meta); if (error) return error; @@ -491,7 +484,7 @@ parse_publish(xmlTextReaderPtr reader, struct cache_mapping *notif, if (xmlTextReaderRead(reader) != 1) { return pr_val_err( "Couldn't read publish content of element '%s'", - map_get_url(tag->meta.uri) + tag->meta.uri ); } @@ -504,86 +497,99 @@ parse_publish(xmlTextReaderPtr reader, struct cache_mapping *notif, if (error) return error; + tag->path = url2path(tag->meta.uri); + if (tag->path == NULL) + return -EINVAL; + /* rfc8181#section-2.2 but considering optional hash */ - return (tag->meta.hash != NULL) ? validate_hash(&tag->meta) : 0; + return (tag->meta.hash != NULL) + ? validate_hash(&tag->meta, tag->path) + : 0; } static int -parse_withdraw(xmlTextReaderPtr reader, struct cache_mapping *notif, - struct withdraw *tag) +parse_withdraw(xmlTextReaderPtr reader, struct withdraw *tag) { int error; - error = parse_file_metadata(reader, notif, HR_MANDATORY, &tag->meta); + error = parse_file_metadata(reader, HR_MANDATORY, &tag->meta); if (error) return error; - return validate_hash(&tag->meta); + tag->path = url2path(tag->meta.uri); + if (tag->path == NULL) + return -EINVAL; + + return validate_hash(&tag->meta, tag->path); } static int -write_file(struct cache_mapping *map, unsigned char *content, size_t content_len) +write_file(char const *path, unsigned char *content, size_t content_len) { FILE *out; size_t written; int error; - error = mkdir_p(map_get_path(map), false); + error = mkdir_p(path, false); if (error) return error; - error = file_write(map_get_path(map), "wb", &out); + error = file_write(path, "wb", &out); if (error) return error; written = fwrite(content, sizeof(unsigned char), content_len, out); file_close(out); - if (written != content_len) { + if (written != content_len) return pr_val_err( "Couldn't write file '%s' (error code not available)", - map_get_path(map) + path ); - } return 0; } /* Remove a local file and its directory tree (if empty) */ static int -delete_file(struct cache_mapping *map) +delete_file(char const *path) { /* Delete parent dirs only if empty. */ - return delete_dir_recursive_bottom_up(map_get_path(map)); + return delete_dir_recursive_bottom_up(path); } static int -handle_publish(xmlTextReaderPtr reader, struct cache_mapping *notif, - hash_requirement hr) +handle_publish(xmlTextReaderPtr reader, hash_requirement hr) { struct publish tag = { 0 }; int error; - error = parse_publish(reader, notif, hr, &tag); - if (!error) - error = write_file(tag.meta.uri, tag.content, tag.content_len); + error = parse_publish(reader, hr, &tag); + if (error) + return error; + + error = write_file(tag.path, tag.content, tag.content_len); metadata_cleanup(&tag.meta); free(tag.content); + free(tag.path); return error; } static int -handle_withdraw(xmlTextReaderPtr reader, struct cache_mapping *notif) +handle_withdraw(xmlTextReaderPtr reader) { struct withdraw tag = { 0 }; int error; - error = parse_withdraw(reader, notif, &tag); - if (!error) - error = delete_file(tag.meta.uri); + error = parse_withdraw(reader, &tag); + if (error) + return error; + + error = delete_file(tag.path); metadata_cleanup(&tag.meta); + free(tag.path); return error; } @@ -593,13 +599,13 @@ parse_notification_snapshot(xmlTextReaderPtr reader, { int error; - error = parse_file_metadata(reader, NULL, HR_MANDATORY, ¬if->snapshot); + error = parse_file_metadata(reader, HR_MANDATORY, ¬if->snapshot); if (error) return error; - if (!map_same_origin(notif->map, notif->snapshot.uri)) + if (!str_same_origin(notif->url, notif->snapshot.uri)) return pr_val_err("Notification %s and Snapshot %s are not hosted by the same origin.", - map_get_url(notif->map), map_get_url(notif->snapshot.uri)); + notif->url, notif->snapshot.uri); return 0; } @@ -615,15 +621,15 @@ parse_notification_delta(xmlTextReaderPtr reader, if (error) return error; - error = parse_file_metadata(reader, NULL, HR_MANDATORY, &delta.meta); + error = parse_file_metadata(reader, HR_MANDATORY, &delta.meta); if (error) { serial_cleanup(&delta.serial); return error; } - if (!map_same_origin(notif->map, delta.meta.uri)) + if (!str_same_origin(notif->url, delta.meta.uri)) return pr_val_err("Notification %s and Delta %s are not hosted by the same origin.", - map_get_url(notif->map), map_get_url(delta.meta.uri)); + notif->url, delta.meta.uri); notification_deltas_add(¬if->deltas, &delta); return 0; @@ -747,32 +753,23 @@ xml_read_notif(xmlTextReaderPtr reader, void *arg) } static int -parse_notification(struct cache_mapping *map, struct update_notification *result) +parse_notification(char const *url, char const *path, + struct update_notification *result) { int error; - update_notification_init(result, map); + update_notification_init(result, url); - error = relax_ng_parse(map_get_path(map), xml_read_notif, result); + error = relax_ng_parse(path, xml_read_notif, result); if (error) update_notification_cleanup(result); return error; } -static void -delete_rpp(struct cache_mapping *notif) -{ - char *path = map_get_rrdp_workspace(notif); - pr_val_debug("Snapshot: Deleting cached RPP '%s'.", path); - file_rm_rf(path); - free(path); -} - static int -xml_read_snapshot(xmlTextReaderPtr reader, void *arg) +xml_read_snapshot(xmlTextReaderPtr reader, void *session) { - struct rrdp_ctx *ctx = arg; xmlReaderTypes type; xmlChar const *name; int error; @@ -782,9 +779,9 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *arg) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = handle_publish(reader, ctx->notif, HR_IGNORE); + error = handle_publish(reader, HR_IGNORE); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_SNAPSHOT)) - error = validate_session(reader, &ctx->session); + error = validate_session(reader, session); else return pr_val_err("Unexpected '%s' element", name); if (error) @@ -798,15 +795,9 @@ xml_read_snapshot(xmlTextReaderPtr reader, void *arg) } static int -parse_snapshot(struct update_notification *notif) +parse_snapshot(struct update_notification *notif, char const *path) { - struct rrdp_ctx ctx; - - ctx.notif = notif->map; - ctx.session = notif->session; - - return relax_ng_parse(map_get_path(notif->snapshot.uri), - xml_read_snapshot, &ctx); + return relax_ng_parse(path, xml_read_snapshot, ¬if->session); } static int @@ -847,15 +838,14 @@ validate_session_desync(struct cachefile_notification *old_notif, static int handle_snapshot(struct update_notification *notif) { - struct cache_mapping *map; + char const *url = notif->snapshot.uri; + char *path; int error; - delete_rpp(notif->map); - - map = notif->snapshot.uri; +// delete_rpp(notif->map); XXX - pr_val_debug("Processing snapshot '%s'.", map_val_get_printable(map)); - fnstack_push_map(map); + pr_val_debug("Processing snapshot '%s'.", url); + fnstack_push(url); /* * TODO (performance) Is there a point in caching the snapshot? @@ -863,25 +853,23 @@ handle_snapshot(struct update_notification *notif) * Maybe stream it instead. * Same for deltas. */ - error = cache_download(validation_cache(state_retrieve()), map, NULL, - NULL); + error = http_download_tmp(url, &path, NULL, NULL); if (error) - goto end; - error = validate_hash(¬if->snapshot); + goto end1; + error = validate_hash(¬if->snapshot, path); if (error) - goto end; - error = parse_snapshot(notif); - delete_file(map); + goto end2; + error = parse_snapshot(notif, path); + delete_file(path); -end: - fnstack_pop(); +end2: free(path); +end1: fnstack_pop(); return error; } static int -xml_read_delta(xmlTextReaderPtr reader, void *arg) +xml_read_delta(xmlTextReaderPtr reader, void *session) { - struct rrdp_ctx *ctx = arg; xmlReaderTypes type; xmlChar const *name; int error; @@ -891,11 +879,11 @@ xml_read_delta(xmlTextReaderPtr reader, void *arg) switch (type) { case XML_READER_TYPE_ELEMENT: if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_PUBLISH)) - error = handle_publish(reader, ctx->notif, HR_OPTIONAL); + error = handle_publish(reader, HR_OPTIONAL); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_WITHDRAW)) - error = handle_withdraw(reader, ctx->notif); + error = handle_withdraw(reader); else if (xmlStrEqual(name, BAD_CAST RRDP_ELEM_DELTA)) - error = validate_session(reader, &ctx->session); + error = validate_session(reader, session); else return pr_val_err("Unexpected '%s' element", name); if (error) @@ -909,42 +897,40 @@ xml_read_delta(xmlTextReaderPtr reader, void *arg) } static int -parse_delta(struct update_notification *notif, struct notification_delta *delta) +parse_delta(struct update_notification *notif, struct notification_delta *delta, + char const *path) { - struct rrdp_ctx ctx; + struct rrdp_session session; int error; - error = validate_hash(&delta->meta); + error = validate_hash(&delta->meta, path); if (error) return error; - ctx.notif = notif->map; - ctx.session.session_id = notif->session.session_id; - ctx.session.serial = delta->serial; + session.session_id = notif->session.session_id; + session.serial = delta->serial; - return relax_ng_parse(map_get_path(delta->meta.uri), xml_read_delta, - &ctx); + return relax_ng_parse(path, xml_read_delta, &session); } static int handle_delta(struct update_notification *notif, struct notification_delta *delta) { - struct cache_mapping *map; + char const *url = delta->meta.uri; + char *path; int error; - map = delta->meta.uri; + pr_val_debug("Processing delta '%s'.", url); + fnstack_push(url); - pr_val_debug("Processing delta '%s'.", map_val_get_printable(map)); - fnstack_push_map(map); - - error = cache_download(validation_cache(state_retrieve()), map, NULL, NULL); + error = http_download_tmp(url, &path, NULL, NULL); if (error) goto end; - error = parse_delta(notif, delta); - delete_file(map); + error = parse_delta(notif, delta, path); + delete_file(path); -end: - fnstack_pop(); + free(path); +end: fnstack_pop(); return error; } @@ -1084,26 +1070,26 @@ update_notif(struct cachefile_notification *old, struct update_notification *new } /* - * Downloads the Update Notification pointed by @map, and updates the cache + * Downloads the Update Notification pointed by @url, and updates the cache * accordingly. * * "Updates the cache accordingly" means it downloads the missing deltas or * snapshot, and explodes them into the corresponding RPP's local directory. */ int -rrdp_update(struct cache_mapping *map) +rrdp_update(char const *notif_url, struct cache_node *node) { + char *path = NULL; struct cachefile_notification **cached, *old; struct update_notification new; bool changed; int serial_cmp; int error; - fnstack_push_map(map); + fnstack_push(notif_url); pr_val_debug("Processing notification."); - error = cache_download(validation_cache(state_retrieve()), map, - &changed, &cached); + error = http_download_tmp(notif_url, &path, &changed, &cached); if (error) goto end; if (!changed) { @@ -1111,7 +1097,7 @@ rrdp_update(struct cache_mapping *map) goto end; } - error = parse_notification(map, &new); + error = parse_notification(notif_url, path, &new); if (error) goto end; pr_val_debug("New session/serial: %s/%s", new.session.session_id, @@ -1175,6 +1161,7 @@ clean_notif: update_notification_cleanup(&new); end: + free(path); fnstack_pop(); return error; } diff --git a/src/rrdp.h b/src/rrdp.h index fa7877ff..d7e74689 100644 --- a/src/rrdp.h +++ b/src/rrdp.h @@ -1,11 +1,12 @@ #ifndef SRC_RRDP_H_ #define SRC_RRDP_H_ -#include "types/map.h" +#include struct cachefile_notification; +struct cache_node; -int rrdp_update(struct cache_mapping *); +int rrdp_update(char const *, struct cache_node *); json_t *rrdp_notif2json(struct cachefile_notification *); int rrdp_json2notif(json_t *, struct cachefile_notification **); diff --git a/src/types/map.c b/src/types/map.c index 503087a7..58638cef 100644 --- a/src/types/map.c +++ b/src/types/map.c @@ -1,5 +1,7 @@ #include "types/map.h" +#include + #include "alloc.h" #include "common.h" #include "config.h" @@ -83,8 +85,6 @@ static int normalize_url(struct path_builder *, char const *, char const *, int) static int init_url(struct cache_mapping *map, char const *str) { -#define SCHEMA_LEN 8 /* strlen("rsync://"), strlen("https://") */ - char const *s; char const *pfx; int error; @@ -105,16 +105,11 @@ init_url(struct cache_mapping *map, char const *str) error = 0; switch (map->type) { - case MAP_TA_RSYNC: - case MAP_RPP: - case MAP_CAGED: - case MAP_AIA: - case MAP_SO: - case MAP_MFT: + case MAP_RSYNC: pfx = "rsync://"; error = ENOTRSYNC; break; - case MAP_TA_HTTP: + case MAP_HTTP: case MAP_NOTIF: case MAP_TMP: pfx = "https://"; @@ -125,92 +120,14 @@ init_url(struct cache_mapping *map, char const *str) if (pfx == NULL) pr_crit("Unknown mapping type: %u", map->type); - __pb_init(&pb, SCHEMA_LEN - 1); + __pb_init(&pb, RPKI_SCHEMA_LEN - 1); error = normalize_url(&pb, str, pfx, error); if (error) { pb_cleanup(&pb); return error; } - map->url = strncpy(pb.string, str, SCHEMA_LEN); - return 0; -} - -static bool -is_valid_mft_file_chara(uint8_t chara) -{ - return ('a' <= chara && chara <= 'z') - || ('A' <= chara && chara <= 'Z') - || ('0' <= chara && chara <= '9') - || (chara == '-') - || (chara == '_'); -} - -/* RFC 6486bis, section 4.2.2 */ -static int -validate_mft_file(IA5String_t *ia5) -{ - size_t dot; - size_t i; - - if (ia5->size < 5) - return pr_val_err("File name is too short (%zu < 5).", ia5->size); - dot = ia5->size - 4; - if (ia5->buf[dot] != '.') - return pr_val_err("File name seems to lack a three-letter extension."); - - for (i = 0; i < ia5->size; i++) { - if (i != dot && !is_valid_mft_file_chara(ia5->buf[i])) { - return pr_val_err("File name contains illegal character #%u", - ia5->buf[i]); - } - } - - /* - * Actual extension doesn't matter; if there's no handler, - * we'll naturally ignore the file. - */ - return 0; -} - -/** - * Initializes @map->url given manifest path @mft and its referenced file @ia5. - * - * ie. if @mft is "rsync://a/b/c.mft" and @ia5 is "d.cer", @map->url will be - * "rsync://a/b/d.cer". - * - * Assumes @mft is already normalized. - */ -static int -ia5str2url(struct cache_mapping *map, char const *mft, IA5String_t *ia5) -{ - char *joined; - char const *slash_pos; - int dir_len; - int error; - - /* - * IA5String is a subset of ASCII. However, IA5String_t doesn't seem to - * be guaranteed to be NULL-terminated. - * `(char *) ia5->buf` is fair, but `strlen(ia5->buf)` is not. - */ - - error = validate_mft_file(ia5); - if (error) - return error; - - slash_pos = strrchr(mft, '/'); - if (slash_pos == NULL) - return pr_val_err("Manifest URL '%s' contains no slashes.", mft); - - dir_len = (slash_pos + 1) - mft; - joined = pmalloc(dir_len + ia5->size + 1); - - strncpy(joined, mft, dir_len); - strncpy(joined + dir_len, (char *) ia5->buf, ia5->size); - joined[dir_len + ia5->size] = '\0'; - - map->url = joined; + map->url = strncpy(pb.string, str, RPKI_SCHEMA_LEN); return 0; } @@ -302,22 +219,6 @@ normalize_url(struct path_builder *pb, char const *url, char const *pfx, return 0; } -static int -get_rrdp_workspace(struct path_builder *pb, struct cache_mapping *notif) -{ - int error; - - error = pb_init_cache(pb, "rrdp"); - if (error) - return error; - - error = pb_append(pb, ¬if->url[SCHEMA_LEN]); - if (error) - pb_cleanup(pb); - - return error; -} - /* * Maps "rsync://a.b.c/d/e.cer" into "/rsync/a.b.c/d/e.cer". */ @@ -341,65 +242,27 @@ map_simple(struct cache_mapping *map, char const *subdir) return 0; } -/* - * Maps "rsync://a.b.c/d/e.cer" into - * "/rrdp//a.b.c/d/e.cer". - */ -static int -map_caged(struct cache_mapping *map, struct cache_mapping *notif) -{ - struct path_builder pb; - int error; - - error = get_rrdp_workspace(&pb, notif); - if (error) - return error; - - if (map->url == NULL) - goto success; /* Caller is only interested in the cage. */ - - error = pb_append(&pb, &map->url[SCHEMA_LEN]); - if (error) { - pb_cleanup(&pb); - return error; - } - -success: - map->path = pb.string; - return 0; -} - static int -init_path(struct cache_mapping *map, struct cache_mapping *notif) +init_path(struct cache_mapping *map) { switch (map->type) { - case MAP_TA_RSYNC: - case MAP_RPP: - case MAP_MFT: + case MAP_RSYNC: return map_simple(map, "rsync"); - case MAP_TA_HTTP: + case MAP_HTTP: return map_simple(map, "https"); case MAP_NOTIF: case MAP_TMP: return cache_tmpfile(&map->path); - - case MAP_CAGED: - return map_caged(map, notif); - - case MAP_AIA: - case MAP_SO: - map->path = NULL; - return 0; } pr_crit("Unknown URL type: %u", map->type); + return -EINVAL; /* Unreachable */ } int -map_create(struct cache_mapping **result, enum map_type type, - struct cache_mapping *notif, char const *url) +map_create(struct cache_mapping **result, enum map_type type, char const *url) { struct cache_mapping *map; int error; @@ -414,39 +277,7 @@ map_create(struct cache_mapping **result, enum map_type type, return error; } - error = init_path(map, notif); - if (error) { - free(map->url); - free(map); - return error; - } - - *result = map; - return 0; -} - -/* - * Manifest fileList entries are a little special in that they're just file - * names. This function will infer the rest of the URL. - */ -int -map_create_mft(struct cache_mapping **result, struct cache_mapping *notif, - struct cache_mapping *mft, IA5String_t *ia5) -{ - struct cache_mapping *map; - int error; - - map = pmalloc(sizeof(struct cache_mapping)); - map->type = (notif == NULL) ? MAP_RPP : MAP_CAGED; - map->references = 1; - - error = ia5str2url(map, mft->url, ia5); - if (error) { - free(map); - return error; - } - - error = init_path(map, notif); + error = init_path(map); if (error) { free(map->url); free(map); @@ -535,12 +366,6 @@ str_same_origin(char const *url1, char const *url2) return false; } -bool -map_same_origin(struct cache_mapping *m1, struct cache_mapping *m2) -{ - return str_same_origin(m1->url, m2->url); -} - /* @ext must include the period. */ bool map_has_extension(struct cache_mapping *map, char const *ext) @@ -548,12 +373,6 @@ map_has_extension(struct cache_mapping *map, char const *ext) return str_ends_with(map->url, ext); } -bool -map_is_certificate(struct cache_mapping *map) -{ - return map_has_extension(map, ".cer"); -} - enum map_type map_get_type(struct cache_mapping *map) { @@ -601,13 +420,6 @@ map_op_get_printable(struct cache_mapping *map) return map_get_printable(map, format); } -char * -map_get_rrdp_workspace(struct cache_mapping *notif) -{ - struct path_builder pb; - return (get_rrdp_workspace(&pb, notif) == 0) ? pb.string : NULL; -} - DEFINE_ARRAY_LIST_FUNCTIONS(map_list, struct cache_mapping *, static) void diff --git a/src/types/map.h b/src/types/map.h index 16443a58..c4dfd1ef 100644 --- a/src/types/map.h +++ b/src/types/map.h @@ -9,18 +9,13 @@ * Currently hardcoded, but queued for tweakability. */ enum map_type { - /* - * TAL's TA URL. - * The file is cached until it's untraversed for a "long" time. - */ - MAP_TA_RSYNC, - MAP_TA_HTTP, - /* * (rsync) Repository Publication Point. RFC 6481. * The directory is cached until it's untraversed for a "long" time. */ - MAP_RPP, + MAP_RSYNC = (1 << 0), + + MAP_HTTP = (1 << 1), /* * An RRDP notification file; downloaded via HTTP. @@ -28,63 +23,42 @@ enum map_type { * that is needed in subsequent iterations. * The metadata is cached until it's untraversed for a "long" time. */ - MAP_NOTIF, + MAP_NOTIF = (MAP_HTTP | (1 << 2)), /* * RRDP Snapshot or Delta; downloaded via HTTP. * The file itself is not cached, but we preserve some small metadata. * The metadata is destroyed once the iteration finishes. */ - MAP_TMP, - - /* - * Endangered species; bound to be removed once RFC 9286 is implemented. - */ - MAP_CAGED, - - MAP_AIA, /* caIssuers. Not directly downloaded. */ - MAP_SO, /* signedObject. Not directly downloaded. */ - MAP_MFT, /* rpkiManifest. Not directly downloaded. */ + MAP_TMP = (MAP_HTTP | (1 << 3)), }; struct cache_mapping; -int map_create(struct cache_mapping **, enum map_type, struct cache_mapping *, - char const *); -int map_create_mft(struct cache_mapping **, struct cache_mapping *, struct cache_mapping *, - IA5String_t *); -struct cache_mapping *map_create_cache(char const *); - -#define map_create_caged(map, notif, url) \ - map_create(map, MAP_CAGED, notif, url) -#define map_create_cage(map, notif) \ - map_create_caged(map, notif, NULL) +char *url2path(char const *); +struct cache_mapping *create_map(char const *); struct cache_mapping *map_refget(struct cache_mapping *); void map_refput(struct cache_mapping *); /* * Note that, if you intend to print some mapping, you're likely supposed to use - * map_get_printable() instead. + * map_*_get_printable() instead. */ char const *map_get_url(struct cache_mapping *); char const *map_get_path(struct cache_mapping *); -bool map_equals(struct cache_mapping *, struct cache_mapping *); bool str_same_origin(char const *, char const *); -bool map_same_origin(struct cache_mapping *, struct cache_mapping *); bool map_has_extension(struct cache_mapping *, char const *); -bool map_is_certificate(struct cache_mapping *); enum map_type map_get_type(struct cache_mapping *); char const *map_val_get_printable(struct cache_mapping *); char const *map_op_get_printable(struct cache_mapping *); -char *map_get_rrdp_workspace(struct cache_mapping *); - /* Plural */ +/* XXX still used? */ DEFINE_ARRAY_LIST_STRUCT(map_list, struct cache_mapping *); void maps_init(struct map_list *); diff --git a/src/types/str.h b/src/types/str.h new file mode 100644 index 00000000..4ca8572a --- /dev/null +++ b/src/types/str.h @@ -0,0 +1,12 @@ +#ifndef SRC_TYPES_STR_H_ +#define SRC_TYPES_STR_H_ + +#include "data_structure/array_list.h" + +DEFINE_ARRAY_LIST_STRUCT(strlist, char *); + +void strlist_init(struct strlist *); +void strlist_cleanup(struct strlist *); +void strlist_add(struct strlist *list, char *elem); + +#endif /* SRC_TYPES_STR_H_ */ diff --git a/test/Makefile.am b/test/Makefile.am index 98d84c93..95d97794 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -22,84 +22,16 @@ AM_CFLAGS += -I../src -DUNIT_TESTING ${CHECK_CFLAGS} ${XML2_CFLAGS} ${JANSSON_CF # target. MY_LDADD = ${CHECK_LIBS} ${JANSSON_LIBS} -check_PROGRAMS = address.test -check_PROGRAMS += base64.test -check_PROGRAMS += cache.test -check_PROGRAMS += db_table.test -check_PROGRAMS += deltas_array.test -check_PROGRAMS += hash.test -check_PROGRAMS += json.test -check_PROGRAMS += pb.test -check_PROGRAMS += pdu_handler.test -check_PROGRAMS += pdu_stream.test -check_PROGRAMS += rrdp.test -check_PROGRAMS += serial.test -check_PROGRAMS += tal.test -check_PROGRAMS += thread_pool.test -check_PROGRAMS += map.test -check_PROGRAMS += uthash.test -check_PROGRAMS += vcard.test -check_PROGRAMS += vrps.test -check_PROGRAMS += xml.test +#check_PROGRAMS = file.test +check_PROGRAMS = cache.test TESTS = ${check_PROGRAMS} -address_test_SOURCES = types/address_test.c -address_test_LDADD = ${MY_LDADD} - -base64_test_SOURCES = crypto/base64_test.c -base64_test_LDADD = ${MY_LDADD} +#file_test_SOURCES = file_test.c +#file_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} cache_test_SOURCES = cache/local_cache_test.c cache_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} -db_table_test_SOURCES = rtr/db/db_table_test.c -db_table_test_LDADD = ${MY_LDADD} - -deltas_array_test_SOURCES = rtr/db/deltas_array_test.c -deltas_array_test_LDADD = ${MY_LDADD} - -hash_test_SOURCES = crypto/hash_test.c -hash_test_LDADD = ${MY_LDADD} - -json_test_SOURCES = json_util_test.c -json_test_LDADD = ${MY_LDADD} - -pb_test_SOURCES = data_structure/path_builder_test.c -pb_test_LDADD = ${MY_LDADD} - -pdu_handler_test_SOURCES = rtr/pdu_handler_test.c -pdu_handler_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} - -pdu_stream_test_SOURCES = rtr/pdu_stream_test.c -pdu_stream_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} - -rrdp_test_SOURCES = rrdp_test.c -rrdp_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} ${XML2_LIBS} - -serial_test_SOURCES = types/serial_test.c -serial_test_LDADD = ${MY_LDADD} - -tal_test_SOURCES = tal_test.c -tal_test_LDADD = ${MY_LDADD} - -thread_pool_test_SOURCES = thread_pool_test.c -thread_pool_test_LDADD = ${MY_LDADD} - -map_test_SOURCES = types/map_test.c -map_test_LDADD = ${MY_LDADD} - -uthash_test_SOURCES = data_structure/uthash_test.c -uthash_test_LDADD = ${MY_LDADD} - -vcard_test_SOURCES = vcard_test.c -vcard_test_LDADD = ${MY_LDADD} - -vrps_test_SOURCES = rtr/db/vrps_test.c -vrps_test_LDADD = ${MY_LDADD} ${JANSSON_LIBS} - -xml_test_SOURCES = xml_test.c -xml_test_LDADD = ${MY_LDADD} ${XML2_LIBS} - EXTRA_DIST = mock.c mock.h EXTRA_DIST += resources/lorem-ipsum.txt EXTRA_DIST += rtr/db/rtr_db_mock.c diff --git a/test/cache/local_cache_test.c b/test/cache/local_cache_test.c index 71ea7252..7e726522 100644 --- a/test/cache/local_cache_test.c +++ b/test/cache/local_cache_test.c @@ -1,17 +1,19 @@ -/* This test will create temporal directory "tmp/". Needs permissions. */ - -#include "cache/local_cache.c" +/* + * This test will create some temporal directories on "/tmp". + * Needs permissions. + */ #include -#include -#include - +//#include +//#include +// #include "alloc.c" -#include "common.c" -#include "json_util.c" +//#include "common.c" +//#include "json_util.c" #include "mock.c" +#include "cache/local_cache.c" #include "data_structure/path_builder.c" -#include "types/map.c" +//#include "types/map.c" /* Mocks */ @@ -99,6 +101,15 @@ http_download(struct cache_mapping *map, curl_off_t ims, bool *changed) return error; } +static char deleted[16][4]; +static unsigned int dn; + +static void +__delete_node_cb(struct cache_node const *node) +{ + strcpy(deleted[dn++], node->name); +} + MOCK_ABORT_INT(rrdp_update, struct cache_mapping *map) __MOCK_ABORT(rrdp_notif2json, json_t *, NULL, struct cachefile_notification *notif) MOCK_VOID(rrdp_notif_free, struct cachefile_notification *notif) @@ -125,16 +136,16 @@ run_cache_download(char const *url, int expected_error, enum map_type type; if (str_starts_with(url, "https://")) - type = MAP_TA_HTTP; + type = MAP_HTTP; else if (str_starts_with(url, "rsync://")) - type = MAP_RPP; + type = MAP_RSYNC; else ck_abort_msg("Bad protocol: %s", url); rsync_counter = 0; https_counter = 0; - ck_assert_int_eq(0, map_create(&map, type, NULL, url)); + ck_assert_int_eq(0, map_create(&map, type, url)); ck_assert_int_eq(expected_error, cache_download(cache, map, NULL, NULL)); ck_assert_uint_eq(rsync_calls, rsync_counter); ck_assert_uint_eq(https_calls, https_counter); @@ -150,14 +161,14 @@ node(char const *url, time_t attempt, int err, bool succeeded, time_t success, struct cache_node *result; if (str_starts_with(url, "https://")) - type = is_notif ? MAP_NOTIF : MAP_TA_HTTP; + type = is_notif ? MAP_NOTIF : MAP_HTTP; else if (str_starts_with(url, "rsync://")) - type = MAP_RPP; + type = MAP_RSYNC; else ck_abort_msg("Bad protocol: %s", url); result = pzalloc(sizeof(struct cache_node)); - ck_assert_int_eq(0, map_create(&result->map, type, NULL, url)); + ck_assert_int_eq(0, map_create(&result->map, type, url)); result->attempt.ts = attempt; result->attempt.result = err; result->success.happened = succeeded; @@ -166,8 +177,7 @@ node(char const *url, time_t attempt, int err, bool succeeded, time_t success, return result; } -#define NODE(url, err, succeeded, has_file) \ - node(url, has_file, err, succeeded, 0, 0) +#define NODE(url, err, succeeded, has_file) node(url, has_file, err, succeeded, 0, 0) static void reset_visiteds(void) @@ -782,12 +792,12 @@ prepare_map_list(struct map_list *maps, ...) va_start(args, maps); while ((str = va_arg(args, char const *)) != NULL) { if (str_starts_with(str, "https://")) - type = MAP_TA_HTTP; + type = MAP_HTTP; else if (str_starts_with(str, "rsync://")) - type = MAP_RPP; + type = MAP_RSYNC; else ck_abort_msg("Bad protocol: %s", str); - ck_assert_int_eq(0, map_create(&map, type, NULL, str)); + ck_assert_int_eq(0, map_create(&map, type, str)); maps_add(maps, map); } va_end(args); @@ -909,12 +919,266 @@ START_TEST(test_recover) } END_TEST +static void +add_children(struct cache_node *parent, va_list children) +{ + struct cache_node *child; + + while ((child = va_arg(children, struct cache_node *)) != NULL) + HASH_ADD_KEYPTR(hh, parent->children, child->name, + strlen(child->name), child); +} + +static void +tree(struct rpki_cache *cache, ...) +{ + va_list args; + va_start(args, cache); + add_children(&cache->root, args); + va_end(args); +} + +static struct cache_node * +node(char const *name, int flags, ...) +{ + struct cache_node *result; + va_list args; + + result = pzalloc(sizeof(struct cache_node)); + result->name = pstrdup(name); + result->flags = flags; + + va_start(args, flags); + add_children(result, args); + va_end(args); + + return result; +} + +static char const *expected[32]; +static unsigned int e; + +static bool +ck_traverse_cb(struct cache_node *node, char const *path) +{ + ck_assert_str_eq(expected[e++], path); + return true; +} + +static void +cleanup_cache_nodes(void) +{ + struct cache_node *node, *tmp; + + HASH_ITER(hh, cache.root.children, node, tmp) { + node->parent = &cache.root; + delete_node(node); + } +} + +static void +ck_traverse(struct rpki_cache *cache, ...) +{ + char const *path; + unsigned int p = 0; + va_list args; + + va_start(args, cache); + while ((path = va_arg(args, char const *)) != NULL) + expected[p++] = path; + va_end(args); + expected[p] = NULL; + + e = 0; + ck_assert_int_eq(0, traverse_cache(ck_traverse_cb)); + ck_assert_uint_eq(p, e); + + cleanup_cache_nodes(); +} + +START_TEST(test_delete_node) +{ + struct rpki_cache cache = { + .root.name = "tmp" + }; + struct cache_node *a, *b; + + a = node("a", 0, NULL); + tree(&cache, a, NULL); + a->parent = &cache.root; + dn = 0; + + delete_node(a); + ck_assert_ptr_eq(NULL, cache.root.children); + ck_assert_uint_eq(1, dn); + ck_assert_str_eq("a", deleted[0]); + + b = node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), + node("e", 0, NULL), + node("f", 0, NULL), NULL); + a = node("a", 0, + b, + node("g", 0, + node("h", 0, + node("i", 0, NULL), NULL), + node("j", 0, + node("k", 0, NULL), NULL), + node("l", 0, + node("m", 0, NULL), NULL), + node("n", 0, + node("o", 0, NULL), NULL), NULL), NULL); + tree(&cache, a, NULL); + b->parent = a; + a->parent = &cache.root; + + dn = 0; + delete_node(b); + ck_assert_int_eq(1, HASH_COUNT(a->children)); + ck_assert_str_eq("c", deleted[0]); + ck_assert_str_eq("d", deleted[1]); + ck_assert_str_eq("e", deleted[2]); + ck_assert_str_eq("f", deleted[3]); + ck_assert_str_eq("b", deleted[4]); + + dn = 0; + delete_node(a); + ck_assert_ptr_eq(NULL, cache.root.children); + ck_assert_str_eq("i", deleted[0]); + ck_assert_str_eq("h", deleted[1]); + ck_assert_str_eq("k", deleted[2]); + ck_assert_str_eq("j", deleted[3]); + ck_assert_str_eq("m", deleted[4]); + ck_assert_str_eq("l", deleted[5]); + ck_assert_str_eq("o", deleted[6]); + ck_assert_str_eq("n", deleted[7]); + ck_assert_str_eq("g", deleted[8]); + ck_assert_str_eq("a", deleted[9]); +} +END_TEST + +START_TEST(test_traverse) +{ + struct rpki_cache cache = { + .root.name = "tmp" + }; + + tree(&cache, NULL); + ck_traverse(&cache, NULL); + + tree(&cache, node("a", 0, NULL), NULL); + ck_traverse(&cache, "tmp/a", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, NULL), NULL), NULL); + ck_traverse(&cache, "tmp/a", "tmp/a/b", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, + node("c", 0, NULL), NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), + node("e", 0, NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/e", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, NULL), + node("c", 0, + node("d", 0, NULL), + node("e", 0, NULL), NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/c", + "tmp/a/c/d", + "tmp/a/c/e", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), NULL), + node("e", 0, + node("f", 0, NULL), + node("g", 0, NULL), NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/e", + "tmp/a/e/f", + "tmp/a/e/g", NULL); + + tree(&cache, + node("a", 0, + node("b", 0, + node("c", 0, NULL), + node("d", 0, NULL), + node("e", 0, NULL), + node("f", 0, NULL), NULL), + node("g", 0, + node("h", 0, + node("i", 0, NULL), NULL), + node("j", 0, + node("k", 0, NULL), NULL), + node("l", 0, + node("m", 0, NULL), NULL), + node("n", 0, + node("o", 0, NULL), NULL), NULL), NULL), NULL); + ck_traverse(&cache, + "tmp/a", + "tmp/a/b", + "tmp/a/b/c", + "tmp/a/b/d", + "tmp/a/b/e", + "tmp/a/b/f", + "tmp/a/g", + "tmp/a/g/h", + "tmp/a/g/h/i", + "tmp/a/g/j", + "tmp/a/g/j/k", + "tmp/a/g/l", + "tmp/a/g/l/m", + "tmp/a/g/n", + "tmp/a/g/n/o", NULL); +} +END_TEST + /* Boilerplate */ static Suite *thread_pool_suite(void) { Suite *suite; - TCase *rsync , *https, *dot, *meta, *recover; + TCase *rsync , *https, *dot, *meta, *recover, *traverse; rsync = tcase_create("rsync"); tcase_add_test(rsync, test_cache_download_rsync); @@ -937,12 +1201,17 @@ static Suite *thread_pool_suite(void) recover = tcase_create("recover"); tcase_add_test(recover, test_recover); + traverse = tcase_create("traverse"); + tcase_add_test(traverse, test_delete_node); + tcase_add_test(traverse, test_traverse); + suite = suite_create("local-cache"); - suite_add_tcase(suite, rsync); - suite_add_tcase(suite, https); - suite_add_tcase(suite, dot); - suite_add_tcase(suite, meta); - suite_add_tcase(suite, recover); +// suite_add_tcase(suite, rsync); +// suite_add_tcase(suite, https); +// suite_add_tcase(suite, dot); +// suite_add_tcase(suite, meta); +// suite_add_tcase(suite, recover); + suite_add_tcase(suite, traverse); return suite; } diff --git a/test/crypto/hash_test.c b/test/crypto/hash_test.c index ec979f57..63455439 100644 --- a/test/crypto/hash_test.c +++ b/test/crypto/hash_test.c @@ -46,7 +46,7 @@ START_TEST(test_hash) map.url = "https://example.com/resources/lorem-ipsum.txt"; map.path = "resources/lorem-ipsum.txt"; - map.type = MAP_TA_HTTP; + map.type = MAP_HTTP; map.references = 1; ha = hash_get_sha1(); diff --git a/test/file_test.c b/test/file_test.c new file mode 100644 index 00000000..31cb1ef7 --- /dev/null +++ b/test/file_test.c @@ -0,0 +1,247 @@ +#include +#include +#include + +#include "alloc.c" +#include "file.c" +#include "mock.c" + +static void +create_file(char const *prefix, char const *suffix) +{ + char *full_path; + FILE *file; + + full_path = join_paths(prefix, suffix); + ck_assert_ptr_ne(NULL, full_path); + + file = fopen(full_path, "w"); + ck_assert_ptr_ne(NULL, file); + fclose(file); + + free(full_path); +} + +static void +create_dir(char const *prefix, char const *suffix) +{ + char *full_path; + + full_path = join_paths(prefix, suffix); + ck_assert_ptr_ne(NULL, full_path); + + ck_assert_int_eq(0, mkdir(full_path, S_IRWXU)); + + free(full_path); +} + +static void +__file_merge_into(char const *root, char const *src, char const *dst) +{ + char *full_src; + char *full_dst; + + full_src = join_paths(root, src); + ck_assert_ptr_ne(NULL, full_src); + full_dst = join_paths(root, dst); + ck_assert_ptr_ne(NULL, full_dst); + + ck_assert_int_eq(0, file_merge_into(full_src, full_dst)); + + free(full_src); + free(full_dst); +} + +static bool +is_dots(char const *str) +{ + return (strcmp(".", str) == 0) || (strcmp("..", str) == 0); +} + +static void +check_empty_dir(char const *prefix, char const *suffix) +{ + char *full_path; + DIR *dir; + struct dirent *child; + + full_path = join_paths(prefix, suffix); + ck_assert_ptr_ne(NULL, full_path); + + dir = opendir(full_path); + ck_assert_ptr_ne(NULL, dir); + child = readdir(dir); + ck_assert(is_dots(child->d_name)); + child = readdir(dir); + ck_assert(is_dots(child->d_name)); + errno = 0; + ck_assert_ptr_eq(NULL, readdir(dir)); + ck_assert_int_eq(0, errno); + closedir(dir); + + free(full_path); +} + +static void +check_file(char const *prefix, char const *suffix) +{ + char *full_path; + struct stat st; + + full_path = join_paths(prefix, suffix); + ck_assert_ptr_ne(NULL, full_path); + + ck_assert_int_eq(0, stat(full_path, &st)); + ck_assert_int_ne(0, S_ISREG(st.st_mode)); + + free(full_path); +} + +START_TEST(test_merge_empty) +{ + char *root; + + root = mkdtemp(pstrdup("/tmp/fort_test_XXXXXX")); + ck_assert_ptr_ne(NULL, root); + + create_dir(root, "src"); + create_dir(root, "dst"); + + __file_merge_into(root, "src", "dst"); + + check_empty_dir(root, "dst"); + + file_rm_rf(root); + free(root); +} +END_TEST + +START_TEST(test_merge_simple) +{ + char *root; + + root = mkdtemp(pstrdup("/tmp/fort_test_XXXXXX")); + ck_assert_ptr_ne(NULL, root); + + create_dir(root, "src"); + create_file(root, "src/a"); + create_dir(root, "dst"); + + __file_merge_into(root, "src", "dst"); + + check_file(root, "dst/a"); + + file_rm_rf(root); + free(root); +} +END_TEST + +START_TEST(test_merge_no_override) +{ + char *root; + + root = mkdtemp(pstrdup("/tmp/fort_test_XXXXXX")); + ck_assert_ptr_ne(NULL, root); + + create_dir(root, "src"); + create_dir(root, "dst"); + create_file(root, "dst/a"); + + __file_merge_into(root, "src", "dst"); + + check_file(root, "dst/a"); + + file_rm_rf(root); + free(root); +} +END_TEST + +START_TEST(test_merge_override) +{ + char *root; + + root = mkdtemp(pstrdup("/tmp/fort_test_XXXXXX")); + ck_assert_ptr_ne(NULL, root); + + create_dir(root, "src"); + create_file(root, "src/a"); + create_dir(root, "dst"); + create_file(root, "dst/a"); + + __file_merge_into(root, "src", "dst"); + + check_file(root, "dst/a"); + + file_rm_rf(root); + free(root); +} +END_TEST + +START_TEST(test_merge_dirs) +{ + char *root; + + root = mkdtemp(pstrdup("/tmp/fort_test_XXXXXX")); + ck_assert_ptr_ne(NULL, root); + + create_dir(root, "src"); + create_file(root, "src/a"); + create_dir(root, "src/c"); + create_file(root, "src/c/m"); + create_file(root, "src/c/n"); + create_file(root, "src/e"); + + create_dir(root, "dst"); + create_file(root, "dst/b"); + create_dir(root, "dst/d"); + create_file(root, "dst/d/o"); + create_file(root, "dst/d/p"); + + __file_merge_into(root, "src", "dst"); + + check_file(root, "dst/a"); + check_file(root, "dst/b"); + check_file(root, "dst/c/m"); + check_file(root, "dst/c/n"); + check_file(root, "dst/d/o"); + check_file(root, "dst/d/p"); + check_file(root, "dst/e"); + + file_rm_rf(root); + free(root); +} +END_TEST + +static Suite *xml_load_suite(void) +{ + Suite *suite; + TCase *todo; + + todo = tcase_create("misc"); + tcase_add_test(todo, test_merge_empty); + tcase_add_test(todo, test_merge_simple); + tcase_add_test(todo, test_merge_no_override); + tcase_add_test(todo, test_merge_override); + tcase_add_test(todo, test_merge_dirs); + + suite = suite_create("file"); + suite_add_tcase(suite, todo); + + return suite; +} + +int main(void) +{ + Suite *suite; + SRunner *runner; + int tests_failed; + + suite = xml_load_suite(); + + runner = srunner_create(suite); + srunner_run_all(runner, CK_NORMAL); + tests_failed = srunner_ntests_failed(runner); + srunner_free(runner); + + return (tests_failed == 0) ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/test/tal_test.c b/test/tal_test.c index b1e639d8..c98c5aeb 100644 --- a/test/tal_test.c +++ b/test/tal_test.c @@ -100,8 +100,8 @@ test_1url(char const *file) ck_assert_int_eq(0, tal_init(&tal, file)); - ck_assert_uint_eq(1, tal.maps.len); - ck_assert_str_eq("rsync://example.com/rpki/ta.cer", tal.maps.array[0]->url); + ck_assert_uint_eq(1, tal.urls.len); + ck_assert_str_eq("rsync://example.com/rpki/ta.cer", tal.urls.array[0]->url); check_spki(&tal); tal_cleanup(&tal); @@ -121,11 +121,11 @@ test_4urls(char const *file) ck_assert_int_eq(0, tal_init(&tal, file)); - ck_assert_uint_eq(4, tal.maps.len); - ck_assert_str_eq("rsync://example.com/rpki/ta.cer", tal.maps.array[0]->url); - ck_assert_str_eq("https://example.com/rpki/ta.cer", tal.maps.array[1]->url); - ck_assert_str_eq("rsync://www.example.com/potato/ta.cer", tal.maps.array[2]->url); - ck_assert_str_eq("https://wx3.example.com/tomato/ta.cer", tal.maps.array[3]->url); + ck_assert_uint_eq(4, tal.urls.len); + ck_assert_str_eq("rsync://example.com/rpki/ta.cer", tal.urls.array[0]); + ck_assert_str_eq("https://example.com/rpki/ta.cer", tal.urls.array[1]); + ck_assert_str_eq("rsync://www.example.com/potato/ta.cer", tal.urls.array[2]); + ck_assert_str_eq("https://wx3.example.com/tomato/ta.cer", tal.urls.array[3]); check_spki(&tal); diff --git a/test/types/map_test.c b/test/types/map_test.c index 037cf54f..86ba479a 100644 --- a/test/types/map_test.c +++ b/test/types/map_test.c @@ -35,8 +35,8 @@ cache_tmpfile(char **filename) /* Tests */ -#define MAP_CREATE_HTTP(map, str) map_create(&map, MAP_TA_HTTP, NULL, str) -#define MAP_CREATE(map, type, str) map_create(&map, type, NULL, str) +#define MAP_CREATE_HTTP(map, str) map_create(&map, MAP_TA_HTTP, str) +#define MAP_CREATE(map, type, str) map_create(&map, type, str) START_TEST(test_constructor) { @@ -102,14 +102,14 @@ START_TEST(test_constructor) ck_assert_int_eq(ENOTHTTPS, MAP_CREATE_HTTP(map, "rsync://a.b.c/d")); ck_assert_int_eq(ENOTHTTPS, MAP_CREATE_HTTP(map, "http://a.b.c/d")); - ck_assert_int_eq(ENOTRSYNC, MAP_CREATE(map, MAP_RPP, "https://a.b.c/d")); + ck_assert_int_eq(ENOTRSYNC, MAP_CREATE(map, MAP_RSYNC, "https://a.b.c/d")); - ck_assert_int_eq(0, MAP_CREATE(map, MAP_RPP, "rsync://a.b.c/d")); + ck_assert_int_eq(0, MAP_CREATE(map, MAP_RSYNC, "rsync://a.b.c/d")); ck_assert_str_eq("rsync://a.b.c/d", map_get_url(map)); ck_assert_str_eq("tmp/rsync/a.b.c/d", map_get_path(map)); map_refput(map); - ck_assert_int_eq(0, MAP_CREATE(map, MAP_TA_RSYNC, "rsync://a.b.c/d.cer")); + ck_assert_int_eq(0, MAP_CREATE(map, MAP_RSYNC, "rsync://a.b.c/d.cer")); ck_assert_str_eq("rsync://a.b.c/d.cer", map_get_url(map)); ck_assert_str_eq("tmp/rsync/a.b.c/d.cer", map_get_path(map)); map_refput(map); @@ -182,13 +182,13 @@ START_TEST(check_caged) { struct cache_mapping *map; - ck_assert_int_eq(0, map_create(¬if, MAP_NOTIF, NULL, "https://a.b.c/d/e.xml")); + ck_assert_int_eq(0, map_create(¬if, MAP_NOTIF, "https://a.b.c/d/e.xml")); ck_assert_int_eq(0, map_create(&map, MAP_CAGED, notif, "rsync://x.y.z/v/w.cer")); ck_assert_str_eq("tmp/rrdp/a.b.c/d/e.xml/x.y.z/v/w.cer", map_get_path(map)); map_refput(map); map_refput(notif); - ck_assert_int_eq(0, map_create(¬if, MAP_NOTIF, NULL, "https://a.b.c")); + ck_assert_int_eq(0, map_create(¬if, MAP_NOTIF, "https://a.b.c")); ck_assert_int_eq(0, map_create(&map, MAP_CAGED, notif, "rsync://w")); ck_assert_str_eq("tmp/rrdp/a.b.c/w", map_get_path(map)); map_refput(map);