+Remember all manifests URIs that were processed from a snapshot or delta file, this will aid to avoid unnecessary rsync's on child CAs.
+Create 'visited_uris' struct and methods to remember URIs from RRDP snapshot/delta file(s). This should be updated to use another struct more efficient than an SLIST.
+Remove 'rrdp_handler' and do its calls directly where needed.
+Add warning message whenever an access method fails and the secondary access method is utilized.
+Assure that RRDP Update Notification URIs are visited only once per validation run.
+In case there's a manifest error, don't retry the repository download if the accessMethod to get the manifest was RRDP.
fort_SOURCES += uri.h uri.c
fort_SOURCES += json_handler.h json_handler.c
fort_SOURCES += validation_handler.h validation_handler.c
+fort_SOURCES += visited_uris.h visited_uris.c
fort_SOURCES += asn1/content_info.h asn1/content_info.c
fort_SOURCES += asn1/decode.h asn1/decode.c
fort_SOURCES += resource/asn.h resource/asn.c
fort_SOURCES += rrdp/db_rrdp.h rrdp/db_rrdp.c
-fort_SOURCES += rrdp/rrdp_handler.h rrdp/rrdp_handler.c
fort_SOURCES += rrdp/rrdp_loader.h rrdp/rrdp_loader.c
fort_SOURCES += rrdp/rrdp_objects.h rrdp/rrdp_objects.c
fort_SOURCES += rrdp/rrdp_parser.h rrdp/rrdp_parser.c
* Fetch the file from @uri, write it using the @cb.
*
* The HTTP request is made using the header 'If-Modified-Since' with a value
- * of @value.
+ * of @value (if @value is 0, the header isn't set).
*
* Returns:
* > 0 file was requested but wasn't downloaded since the server didn't sent
#include "extension.h"
#include "nid.h"
#include "thread_var.h"
+#include "visited_uris.h"
#include "http/http.h"
#include "rtr/rtr.h"
#include "rtr/db/vrps.h"
if (error)
goto just_quit;
+ error = visited_uris_init();
+ if (error)
+ goto vrps_cleanup;
+
error = rtr_listen();
+ visited_uris_destroy();
+vrps_cleanup:
vrps_destroy();
just_quit:
return error;
#include "nid.h"
#include "str.h"
#include "thread_var.h"
+#include "visited_uris.h"
#include "asn1/decode.h"
#include "asn1/oid.h"
#include "asn1/asn1c/IPAddrBlocks.h"
{
access_method_exec *cb_primary;
access_method_exec *cb_secondary;
+ bool primary_rrdp;
int error;
/*
if (sia_uris->caRepository.position > sia_uris->rpkiNotify.position) {
cb_primary = rrdp_cb;
cb_secondary = rsync_cb;
+ primary_rrdp = true;
} else {
cb_primary = rsync_cb;
cb_secondary = rrdp_cb;
+ primary_rrdp = false;
}
/* Try with the preferred; in case of error, try with the next one */
if (!error)
return 0;
+ if (primary_rrdp)
+ pr_warn("Couldn't fetch data from RRDP repository '%s', trying to fetch data now from '%s'.",
+ uri_get_global(sia_uris->rpkiNotify.uri),
+ uri_get_global(sia_uris->caRepository.uri));
+ else
+ pr_warn("Couldn't fetch data from repository '%s', trying to fetch data now from RRDP '%s'.",
+ uri_get_global(sia_uris->caRepository.uri),
+ uri_get_global(sia_uris->rpkiNotify.uri));
+
return cb_secondary(sia_uris);
}
enum rpki_policy policy;
enum cert_type type;
struct rpp *pp;
- bool mft_retry;
+ bool mft_retry, mft_exists;
int error;
state = state_retrieve();
}
/*
- * FIXME (now) Beware: what if RRDP is already being utilized?
- *
* There isn't any restriction about the preferred access method of
* children CAs being the same as the parent CA.
*
* Step (2) must do something different.
* - The manifest should exist at the snapshot file (aka, directory
* structure), so it should be processed from there on.
- * - Verify that the manifest file exists locally (must be the same
- * session_id and serial, load those at start, the serial can
- * be written to a local file, the session_id can be loaded from the
- * directory structure <local-repository>/rrdp/<session_id>).:
+ * - Verify that the manifest file exists locally:
* + Exists: Read and process normally (no need to do rsync)
* + Doesn't exists: check for preferred access method, keep the flow
* from there on.
*/
- error = use_access_method(&sia_uris, exec_rsync_method,
- exec_rrdp_method);
- if (error)
- return error;
+
+ /* Avoid to re-download the repo if the mft was fetched with RRDP */
+ mft_exists = false;
+ switch (type) {
+ case TA:
+ error = use_access_method(&sia_uris, exec_rsync_method,
+ exec_rrdp_method);
+ if (error)
+ return error;
+ break;
+ case CA:
+ if (!visited_uris_exists(uri_get_global(sia_uris.mft.uri))) {
+ mft_exists = true;
+ error = use_access_method(&sia_uris, exec_rsync_method,
+ exec_rrdp_method);
+ if (error)
+ return error;
+ }
+ break;
+ default:
+ break;
+ }
/*
* RFC 6481 section 5: "when the repository publication point contents
error = handle_manifest(sia_uris.mft.uri, &pp);
if (!mft_retry)
uri_refput(sia_uris.mft.uri);
- if (!error || !mft_retry)
+ if (!error || !mft_retry || mft_exists)
break;
pr_info("Retrying repository download to discard 'transient inconsistency' manifest issue (see RFC 6481 section 5) '%s'",
*/
struct validation_handler validation_handler;
- struct rrdp_handler rrdp_handler;
struct validation *state;
struct cert_stack *certstack;
struct deferred_cert deferred;
validation_handler.handle_router_key = handle_router_key;
validation_handler.arg = arg;
- rrdp_handler.uri_cmp = rrdp_uri_cmp;
- rrdp_handler.uri_update = rrdp_uri_update;
- rrdp_handler.uri_get_serial = rrdp_uri_get_serial;
- rrdp_handler.uri_get_last_update = rrdp_uri_get_last_update;
- rrdp_handler.uri_set_last_update = rrdp_uri_set_last_update;
-
- error = validation_prepare(&state, tal, &validation_handler,
- &rrdp_handler);
+ error = validation_prepare(&state, tal, &validation_handler);
if (error)
return ENSURE_NEGATIVE(error);
#include "db_rrdp.h"
#include <stdlib.h>
+#include <stdbool.h>
#include <string.h>
#include <time.h>
#include "data_structure/uthash_nonfatal.h"
char *uri;
struct global_data data;
long last_update;
+ bool visited;
UT_hash_handle hh;
};
tmp->data.serial = serial;
tmp->last_update = 0;
+ tmp->visited = true;
*result = tmp;
return 0;
return get_current_time(&found->last_update);
}
+bool
+db_rrdp_get_visited(struct db_rrdp *db, char const *uri)
+{
+ struct db_rrdp_uri *found;
+
+ HASH_FIND_STR(db->uris, uri, found);
+ if (found == NULL)
+ return false;
+
+ return found->visited;
+}
+
+int
+db_rrdp_set_visited(struct db_rrdp *db, char const *uri, bool value)
+{
+ struct db_rrdp_uri *found;
+
+ HASH_FIND_STR(db->uris, uri, found);
+ if (found == NULL)
+ return -ENOENT;
+
+ found->visited = value;
+ return 0;
+}
+
+int
+db_rrdp_set_all_nonvisited(struct db_rrdp *db)
+{
+ struct db_rrdp_uri *uri_node, *uri_tmp;
+
+ HASH_ITER(hh, db->uris, uri_node, uri_tmp)
+ uri_node->visited = false;
+
+ return 0;
+}
+
int
db_rrdp_create(struct db_rrdp **result)
{
int db_rrdp_set_last_update(struct db_rrdp *, char const *);
+bool db_rrdp_get_visited(struct db_rrdp *, char const *);
+int db_rrdp_set_visited(struct db_rrdp *, char const *, bool);
+int db_rrdp_set_all_nonvisited(struct db_rrdp *);
+
#endif /* SRC_RRDP_DB_RRDP_H_ */
+++ /dev/null
-#include "rrdp_handler.h"
-
-#include "thread_var.h"
-
-#define CALL_HANDLER_FUNC(func_name, func_call) \
- struct rrdp_handler const *handler; \
- int error; \
- \
- error = get_current_threads_handler(&handler); \
- if (error) \
- return error; \
- \
- return (handler->func_name != NULL) \
- ? handler->func_call \
- : 0;
-
-static int
-get_current_threads_handler(struct rrdp_handler const **result)
-{
- struct validation *state;
- struct rrdp_handler const *handler;
-
- state = state_retrieve();
- if (state == NULL)
- return -EINVAL;
-
- handler = validation_get_rrdp_handler(state);
- if (handler == NULL)
- pr_crit("This thread lacks an RRDP handler.");
-
- *result = handler;
- return 0;
-}
-
-rrdp_uri_cmp_result_t
-rhandler_uri_cmp(char const *uri, char const *session_id, unsigned long serial)
-{
- CALL_HANDLER_FUNC(uri_cmp, uri_cmp(uri, session_id, serial))
-}
-
-int
-rhandler_uri_update(char const *uri, char const *session_id,
- unsigned long serial)
-{
- CALL_HANDLER_FUNC(uri_update, uri_update(uri, session_id, serial))
-}
-
-int
-rhandler_uri_get_serial(char const *uri, unsigned long *serial)
-{
- CALL_HANDLER_FUNC(uri_get_serial, uri_get_serial(uri, serial))
-}
-
-int
-rhandler_uri_get_last_update(char const *uri, long *serial)
-{
- CALL_HANDLER_FUNC(uri_get_last_update, uri_get_last_update(uri, serial))
-}
-
-int
-rhandler_uri_set_last_update(char const *uri)
-{
- CALL_HANDLER_FUNC(uri_set_last_update, uri_set_last_update(uri))
-}
+++ /dev/null
-#ifndef SRC_RRDP_RRDP_HANDLER_H_
-#define SRC_RRDP_RRDP_HANDLER_H_
-
-#include "rrdp/rrdp_objects.h"
-
-/*
- * Almost the same idea as 'validation_handler.h', only that the main focus is
- * a multithreaded environment.
- *
- * The RRDP URIs are expected to live at the main thread, the other threads can
- * access such URIs. The handler must assure that the data is safe
- * (handle r/w locks), that's the reason why there isn't any reference to a
- * 'db_rrdp' struct.
- */
-
-struct rrdp_handler {
- /*
- * Search the RRDP URI, returns the corresponding enum to indicate
- * the comparison result.
- */
- rrdp_uri_cmp_result_t (*uri_cmp)(char const *, char const *,
- unsigned long);
- /* Add or update an RRDP URI */
- int (*uri_update)(char const *, char const *, unsigned long);
- /*
- * Get the serial related to an URI, returns -ENOENT if the URI doesn't
- * exists, any other error means that something went wrong.
- */
- int (*uri_get_serial)(char const *, unsigned long *);
- /*
- * Get the last update that an URI was requested, returns -ENOENT if
- * the URI doesn't exists, any other error means that something went
- * wrong.
- */
- int (*uri_get_last_update)(char const *, long *);
- /* Set the last update to now */
- int (*uri_set_last_update)(char const *);
-};
-
-rrdp_uri_cmp_result_t rhandler_uri_cmp(char const *, char const *,
- unsigned long);
-int rhandler_uri_update(char const *, char const *, unsigned long);
-int rhandler_uri_get_serial(char const *, unsigned long *);
-int rhandler_uri_get_last_update(char const *, long *);
-int rhandler_uri_set_last_update(char const *);
-
-#endif /* SRC_RRDP_RRDP_HANDLER_H_ */
#include "rrdp_loader.h"
-#include "rrdp/rrdp_handler.h"
#include "rrdp/rrdp_objects.h"
#include "rrdp/rrdp_parser.h"
+#include "rtr/db/vrps.h"
#include "log.h"
#include "thread_var.h"
unsigned long serial;
int error;
- error = rhandler_uri_get_serial(uri, &serial);
+ error = rrdp_uri_get_serial(uri, &serial);
if (error)
return error;
{
struct update_notification *upd_notification;
rrdp_uri_cmp_result_t res;
- long last_update;
int error;
- last_update = 0;
- error = rhandler_uri_get_last_update(uri_get_global(uri), &last_update);
- if (error && error != -ENOENT)
- return error;
+ /* Avoid multiple requests on the same run */
+ if (rrdp_uri_visited(uri_get_global(uri)))
+ return 0;
- error = rrdp_parse_notification(uri, last_update, &upd_notification);
+ error = rrdp_parse_notification(uri, &upd_notification);
if (error)
return error;
- /* No updates at the file (yet) */
+ /* No updates at the file (yet), didn't pushed to fnstack */
if (upd_notification == NULL)
return 0;
- res = rhandler_uri_cmp(uri_get_global(uri),
+ res = rrdp_uri_cmp(uri_get_global(uri),
upd_notification->global_data.session_id,
upd_notification->global_data.serial);
switch (res) {
/* Any change, and no error during the process, update db */
if (!error) {
- error = rhandler_uri_update(uri_get_global(uri),
+ error = rrdp_uri_update(uri_get_global(uri),
upd_notification->global_data.session_id,
upd_notification->global_data.serial);
if (error)
set_update:
/* Set the last update to now */
- error = rhandler_uri_set_last_update(uri_get_global(uri));
+ error = rrdp_uri_set_last_update(uri_get_global(uri));
end:
- update_notification_destroy(upd_notification);
- fnstack_pop(); /* Pop from rrdp_parse_notification */
+ if (upd_notification != NULL) {
+ update_notification_destroy(upd_notification);
+ fnstack_pop(); /* Pop from rrdp_parse_notification */
+ }
return error;
}
#include "crypto/base64.h"
#include "crypto/hash.h"
#include "http/http.h"
-#include "rrdp/rrdp_handler.h"
+#include "rtr/db/vrps.h"
#include "xml/relax_ng.h"
#include "common.h"
#include "file.h"
#include "log.h"
#include "thread_var.h"
+#include "visited_uris.h"
/* XML Common Namespace of files */
#define RRDP_NAMESPACE "http://www.ripe.net/rpki/rrdp"
unsigned long expected_serial;
};
+static int
+add_mft_to_list(char const *uri)
+{
+ if (strcmp(".mft", strrchr(uri, '.')) != 0)
+ return 0;
+
+ return visited_uris_add(uri);
+}
+
+static int
+rem_mft_from_list(char const *uri)
+{
+ if (strcmp(".mft", strrchr(uri, '.')) != 0)
+ return 0;
+
+ return visited_uris_remove(uri);
+}
+
static size_t
write_local(unsigned char *content, size_t size, size_t nmemb, void *arg)
{
uri_get_local(uri));
}
+ error = add_mft_to_list(uri_get_global(uri));
+ if (error) {
+ uri_refput(uri);
+ file_close(out);
+ return error;
+ }
+
uri_refput(uri);
file_close(out);
return 0;
goto release_str;
} while (true);
+ error = rem_mft_from_list(uri_get_global(uri));
+ if (error)
+ goto release_str;
+
uri_refput(uri);
free(local_uri);
return 0;
ctx->create_snapshot = false;
- res = rhandler_uri_cmp(ctx->uri,
+ res = rrdp_uri_cmp(ctx->uri,
ctx->notification->global_data.session_id,
ctx->notification->global_data.serial);
switch (res) {
* 'If-Modified-Since' header, return 0 and set @result to NULL.
*/
int
-rrdp_parse_notification(struct rpki_uri *uri, long last_update,
+rrdp_parse_notification(struct rpki_uri *uri,
struct update_notification **result)
{
- int error;
+ long last_update;
+ int error, vis_err;
if (uri == NULL || uri_is_rsync(uri))
pr_crit("Wrong call, trying to parse a non HTTPS URI");
+ last_update = 0;
+ error = rrdp_uri_get_last_update(uri_get_global(uri), &last_update);
+ if (error && error != -ENOENT)
+ return error;
+
error = http_download_file_with_ims(uri, write_local, last_update);
if (error < 0)
return error;
+ /*
+ * Mark as visited, if it doesn't exists yet, there's no problem since
+ * this is probably the first time is visited (first run), so it will
+ * be marked as visited when the URI is stored at DB.
+ */
+ vis_err = rrdp_uri_mark_visited(uri_get_global(uri));
+ if (vis_err && vis_err !=-ENOENT)
+ return pr_err("Coudln't mark '%s' as visited",
+ uri_get_global(uri));
+
/* No updates yet */
if (error > 0) {
*result = NULL;
#include "rrdp/rrdp_objects.h"
#include "uri.h"
-int rrdp_parse_notification(struct rpki_uri *, long,
- struct update_notification **);
+int rrdp_parse_notification(struct rpki_uri *, struct update_notification **);
int rrdp_parse_snapshot(struct update_notification *);
int rrdp_process_deltas(struct update_notification *, unsigned long serial);
db_rrdp_set_last_update(state.rrdp_uris, uri))
}
+bool
+rrdp_uri_visited(char const *uri)
+{
+ bool result;
+
+ rwlock_read_lock(&state_lock);
+ result = db_rrdp_get_visited(state.rrdp_uris, uri);
+ rwlock_unlock(&state_lock);
+
+ return result;
+}
+
+int
+rrdp_uri_mark_visited(char const *uri)
+{
+ WLOCK_HANDLER(&state_lock,
+ db_rrdp_set_visited(state.rrdp_uris, uri, true))
+}
+
+static int
+rrdp_uri_reset_visited(void)
+{
+ WLOCK_HANDLER(&state_lock,
+ db_rrdp_set_all_nonvisited(state.rrdp_uris))
+}
+
static int
__perform_standalone_validation(struct db_table **result)
{
if (db == NULL)
return pr_enomem();
+ /*
+ * Reset all RDDP URIs to 'non visited' every cycle, this way we can
+ * assure that the update notification file will be requested when
+ * needed and one time per cycle.
+ */
+ error = rrdp_uri_reset_visited();
+ if (error) {
+ db_table_destroy(db);
+ return error;
+ }
+
error = perform_standalone_validation(db);
if (error) {
terminate_standalone_validation();
int rrdp_uri_get_serial(char const *, unsigned long *);
int rrdp_uri_get_last_update(char const *, long *);
int rrdp_uri_set_last_update(char const *);
+bool rrdp_uri_visited(char const *);
+int rrdp_uri_mark_visited(char const *);
uint16_t get_current_session_id(uint8_t);
char addr_buffer2[INET6_ADDRSTRLEN];
struct validation_handler validation_handler;
-
- /*
- * Handler of RRDP functions, this handler can access data from the
- * main thread, so that multiple threads can use it.
- */
- struct rrdp_handler rrdp_handler;
};
/*
*/
int
validation_prepare(struct validation **out, struct tal *tal,
- struct validation_handler *validation_handler,
- struct rrdp_handler *rrdp_handler)
+ struct validation_handler *validation_handler)
{
struct validation *result;
X509_VERIFY_PARAM *params;
result->pubkey_state = PKS_UNTESTED;
result->validation_handler = *validation_handler;
- result->rrdp_handler = *rrdp_handler;
result->x509_data.params = params; /* Ownership transfered */
*out = result;
{
return &state->validation_handler;
}
-
-struct rrdp_handler const *
-validation_get_rrdp_handler(struct validation *state)
-{
- return &state->rrdp_handler;
-}
#include "cert_stack.h"
#include "validation_handler.h"
#include "object/tal.h"
-#include "rrdp/rrdp_handler.h"
#include "rsync/rsync.h"
struct validation;
int validation_prepare(struct validation **, struct tal *,
- struct validation_handler *, struct rrdp_handler *);
+ struct validation_handler *);
void validation_destroy(struct validation *);
struct tal *validation_tal(struct validation *);
struct validation_handler const *
validation_get_validation_handler(struct validation *);
-struct rrdp_handler const *
-validation_get_rrdp_handler(struct validation *);
-
#endif /* SRC_STATE_H_ */
--- /dev/null
+#include "visited_uris.h"
+
+#include <sys/queue.h>
+#include <pthread.h>
+#include <stddef.h>
+#include <string.h>
+#include "common.h"
+#include "log.h"
+
+/*
+ * FIXME (now) This should be replaced with something better (rtrie?)
+ */
+struct visited_elem {
+ char *uri;
+ SLIST_ENTRY(visited_elem) next;
+};
+
+SLIST_HEAD(visited_uris, visited_elem) uris_db;
+
+/** Read/write lock, which protects @uris_db. */
+static pthread_rwlock_t lock;
+
+static int
+visited_elem_create(struct visited_elem **elem, char const *uri)
+{
+ struct visited_elem *tmp;
+
+ tmp = malloc(sizeof(struct visited_elem));
+ if (tmp == NULL)
+ return pr_enomem();
+
+ tmp->uri = strdup(uri);
+ if (tmp->uri == NULL) {
+ free(tmp);
+ return pr_enomem();
+ }
+
+ *elem = tmp;
+ return 0;
+}
+
+static void
+visited_elem_destroy(struct visited_elem *elem)
+{
+ free(elem->uri);
+ free(elem);
+}
+
+int
+visited_uris_init(void)
+{
+ int error;
+
+ error = pthread_rwlock_init(&lock, NULL);
+ if (error)
+ return pr_errno(error, "Visited uris pthread_rwlock_init() errored");
+
+ SLIST_INIT(&uris_db);
+ return 0;
+}
+
+void
+visited_uris_destroy(void)
+{
+ struct visited_elem *elem;
+
+ while (!SLIST_EMPTY(&uris_db)) {
+ elem = uris_db.slh_first;
+ SLIST_REMOVE_HEAD(&uris_db, next);
+ visited_elem_destroy(elem);
+ }
+ pthread_rwlock_destroy(&lock);
+}
+
+static struct visited_elem *
+elem_find(struct visited_uris *list, char const *uri)
+{
+ struct visited_elem *found;
+
+ rwlock_read_lock(&lock);
+ SLIST_FOREACH(found, list, next) {
+ if (strcmp(uri, found->uri) == 0) {
+ rwlock_unlock(&lock);
+ return found;
+ }
+ }
+ rwlock_unlock(&lock);
+ return NULL;
+}
+
+int
+visited_uris_add(char const *uri)
+{
+ struct visited_elem *elem;
+ int error;
+
+ if (elem_find(&uris_db, uri) != NULL)
+ return 0; /* Exists, don't add again */
+
+ error = visited_elem_create(&elem, uri);
+ if (error)
+ return error;
+
+ rwlock_write_lock(&lock);
+ SLIST_INSERT_HEAD(&uris_db, elem, next);
+ rwlock_unlock(&lock);
+
+ return 0;
+}
+
+int
+visited_uris_remove(char const *uri)
+{
+ struct visited_elem *elem;
+
+ elem = elem_find(&uris_db, uri);
+ if (elem == NULL)
+ return pr_err("Trying to remove a nonexistent URI '%s'", uri);
+
+ rwlock_write_lock(&lock);
+ SLIST_REMOVE(&uris_db, elem, visited_elem, next);
+ visited_elem_destroy(elem);
+ rwlock_unlock(&lock);
+
+ return 0;
+}
+
+bool
+visited_uris_exists(char const *uri)
+{
+ return elem_find(&uris_db, uri) != NULL;
+}
--- /dev/null
+#ifndef SRC_VISITED_URIS_H_
+#define SRC_VISITED_URIS_H_
+
+#include <stdbool.h>
+
+int visited_uris_init(void);
+void visited_uris_destroy(void);
+
+int visited_uris_add(char const *);
+int visited_uris_remove(char const *);
+bool visited_uris_exists(char const *);
+
+#endif /* SRC_VISITED_URIS_H_ */
int
validation_prepare(struct validation **out, struct tal *tal,
- struct validation_handler *validation_handler,
- struct rrdp_handler *rrdp_handler)
+ struct validation_handler *validation_handler)
{
return 0;
}
projects / thirdparty / FORT-validator.git / commitdiff
