From: pcarana <pc.moreno2099@gmail.com>
Date: Mon, 16 Dec 2019 20:49:32 +0000 (-0600)
Subject: Delete temporary XML files and unused TALs RRDP data.
X-Git-Tag: v1.2.0~36
X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=cabf23ff1b350cc6b48c179410c9e91042fd759f;p=thirdparty%2FFORT-validator.git

Delete temporary XML files and unused TALs RRDP data.

+Remove XML files and its directory structure once they have been utilized.
+Mark as visited TAL information related to RRDP once per cycle; if no RRDP information was visited (means, the TAL wasn't validated during the cycle) the data is forgotten, including its local data.
+Move the directory tree removal functions to 'common.h', so that it can be called from multiple parts.
+Remove created file in case of error during an HTTP download.
---

diff --git a/src/common.c b/src/common.c
index 741305e6..6268d085 100644
--- a/src/common.c
+++ b/src/common.c
@@ -4,10 +4,12 @@
 #include <dirent.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 #include <sys/types.h> /* AF_INET, AF_INET6 (needed in OpenBSD) */
 #include <sys/socket.h> /* AF_INET, AF_INET6 (needed in OpenBSD) */
 #include <sys/stat.h>
 
+#include "config.h"
 #include "log.h"
 
 int
@@ -279,3 +281,80 @@ create_dir_recursive(char const *path)
 	free(localuri);
 	return 0;
 }
+
+static int
+remove_file(char const *path)
+{
+	int error;
+
+	errno = 0;
+	error = remove(path);
+	if (error)
+		return pr_errno(errno, "Couldn't delete %s", path);
+
+	return 0;
+}
+
+/*
+ * Delete parent dirs of @path only if dirs are empty, @path must be a file
+ * location and will be deleted first.
+ *
+ * The algorithm is a bit aggressive, but rmdir() won't delete
+ * something unless is empty, so in case the dir still has something in
+ * it the cycle is finished.
+ */
+int
+delete_dir_recursive_bottom_up(char const *path)
+{
+	char *config_repo;
+	char *work_loc, *tmp;
+	size_t config_len;
+	int error;
+
+	error = remove_file(path);
+	if (error)
+		return error;
+
+	config_repo = strdup(config_get_local_repository());
+	if (config_repo == NULL)
+		return pr_enomem();
+
+	/* Stop dir removal when the work_dir has this length */
+	config_len = strlen(config_repo);
+	if (config_repo[config_len - 1] == '/')
+		config_len--;
+	free(config_repo);
+
+	work_loc = strdup(path);
+	if (work_loc == NULL)
+		return pr_enomem();
+
+	do {
+		tmp = strrchr(work_loc, '/');
+		if (tmp == NULL)
+			break;
+		*tmp = '\0';
+
+		/* Stop if the root dir is reached */
+		if (strlen(work_loc) == config_len)
+			break;
+
+		errno = 0;
+		error = rmdir(work_loc);
+		if (!error)
+			continue; /* Keep deleting up */
+
+		/* Stop if there's content in the dir */
+		if (errno == ENOTEMPTY || errno == EEXIST)
+			break;
+
+		error = pr_errno(errno, "Couldn't delete dir %s", work_loc);
+		goto release_str;
+	} while (true);
+
+	free(work_loc);
+	return 0;
+release_str:
+	free(work_loc);
+	return error;
+}
diff --git a/src/common.h b/src/common.h
index 7bb954af..ce64fcdd 100644
--- a/src/common.h
+++ b/src/common.h
@@ -50,5 +50,6 @@ char const *addr2str4(struct in_addr const *, char *);
 char const *addr2str6(struct in6_addr const *, char *);
 
 int create_dir_recursive(char const *);
+int delete_dir_recursive_bottom_up(char const *);
 
 #endif /* SRC_RTR_COMMON_H_ */
diff --git a/src/http/http.c b/src/http/http.c
index cd10ce1a..929675d9 100644
--- a/src/http/http.c
+++ b/src/http/http.c
@@ -163,6 +163,7 @@ __http_download_file(struct rpki_uri *uri, http_write_cb cb,
 	return error;
 close_file:
 	file_close(out);
+	delete_dir_recursive_bottom_up(uri_get_local(uri));
 	return error;
 }
 
diff --git a/src/object/tal.c b/src/object/tal.c
index 80312160..7a56a834 100644
--- a/src/object/tal.c
+++ b/src/object/tal.c
@@ -659,6 +659,9 @@ perform_standalone_validation(struct db_table *table)
 	struct thread *thread;
 	int error;
 
+	/* Set existent tal RRDP info to non visited */
+	db_rrdp_reset_visited_tals();
+
 	SLIST_INIT(&threads);
 	error = process_file_or_dir(config_get_tal(), TAL_FILE_EXTENSION,
 	    __do_file_validation, table);
@@ -676,7 +679,8 @@ perform_standalone_validation(struct db_table *table)
 		thread_destroy(thread);
 	}
 
-	/* FIXME (now) Remove non-visited rrdps URIS by tal */
+	/* Remove non-visited rrdps URIS by tal */
+	db_rrdp_rem_nonvisited_tals();
 
 	return error;
 }
diff --git a/src/rrdp/db/db_rrdp.c b/src/rrdp/db/db_rrdp.c
index c1a688c2..0935a540 100644
--- a/src/rrdp/db/db_rrdp.c
+++ b/src/rrdp/db/db_rrdp.c
@@ -56,8 +56,10 @@ tal_elem_create(struct tal_elem **elem, char const *name)
 }
 
 static void
-tal_elem_destroy(struct tal_elem *elem)
+tal_elem_destroy(struct tal_elem *elem, bool remove_local)
 {
+	if (remove_local)
+		db_rrdp_uris_remove_all_local(elem->uris);
 	db_rrdp_uris_destroy(elem->uris);
 	free(elem->file_name);
 	free(elem);
@@ -84,7 +86,7 @@ db_rrdp_cleanup(void)
 	while (!SLIST_EMPTY(&db.tals)) {
 		elem = db.tals.slh_first;
 		SLIST_REMOVE_HEAD(&db.tals, next);
-		tal_elem_destroy(elem);
+		tal_elem_destroy(elem, false);
 	}
 	pthread_rwlock_destroy(&lock);
 }
@@ -109,12 +111,15 @@ db_rrdp_find_tal(char const *tal_name)
 int
 db_rrdp_add_tal(char const *tal_name)
 {
-	struct tal_elem *elem;
+	struct tal_elem *elem, *found;
 	int error;
 
 	/* Element exists, no need to create it again */
-	if (db_rrdp_find_tal(tal_name) != NULL)
+	found = db_rrdp_find_tal(tal_name);
+	if (found != NULL) {
+		found->visited = true;
 		return 0;
+	}
 
 	error = tal_elem_create(&elem, tal_name);
 	if (error)
@@ -140,7 +145,7 @@ db_rrdp_rem_tal(char const *tal_name)
 	SLIST_REMOVE(&db.tals, found, tal_elem, next);
 	rwlock_unlock(&lock);
 
-	tal_elem_destroy(found);
+	tal_elem_destroy(found, true);
 }
 
 /* Returns the reference to RRDP URIs of a TAL */
@@ -152,3 +157,32 @@ db_rrdp_get_uris(char const *tal_name)
 	found = db_rrdp_find_tal(tal_name);
 	return found != NULL ? found->uris : NULL;
 }
+
+/* Set all tals to non-visited */
+void
+db_rrdp_reset_visited_tals(void)
+{
+	struct tal_elem *found;
+
+	rwlock_read_lock(&lock);
+	SLIST_FOREACH(found, &db.tals, next)
+		found->visited = false;
+
+	rwlock_unlock(&lock);
+}
+
+/* Remove non-visited tals */
+void
+db_rrdp_rem_nonvisited_tals(void)
+{
+	struct tal_elem *found;
+
+	rwlock_read_lock(&lock);
+	SLIST_FOREACH(found, &db.tals, next) {
+		if (!found->visited) {
+			SLIST_REMOVE(&db.tals, found, tal_elem, next);
+			tal_elem_destroy(found, true);
+		}
+	}
+	rwlock_unlock(&lock);
+}
diff --git a/src/rrdp/db/db_rrdp.h b/src/rrdp/db/db_rrdp.h
index dc7e368f..b0440e30 100644
--- a/src/rrdp/db/db_rrdp.h
+++ b/src/rrdp/db/db_rrdp.h
@@ -10,4 +10,7 @@ int db_rrdp_add_tal(char const *);
 void db_rrdp_rem_tal(char const *);
 struct db_rrdp_uri *db_rrdp_get_uris(char const *);
 
+void db_rrdp_reset_visited_tals(void);
+void db_rrdp_rem_nonvisited_tals(void);
+
 #endif /* SRC_RRDP_DB_DB_RRDP_H_ */
diff --git a/src/rrdp/db/db_rrdp_uris.c b/src/rrdp/db/db_rrdp_uris.c
index 752ac9fb..e99c6b3b 100644
--- a/src/rrdp/db/db_rrdp_uris.c
+++ b/src/rrdp/db/db_rrdp_uris.c
@@ -346,3 +346,19 @@ db_rrdp_uris_visited_exists(struct db_rrdp_uri *uris, char const *uri)
 
 	return false;
 }
+
+int
+db_rrdp_uris_remove_all_local(struct db_rrdp_uri *uris)
+{
+	struct uris_table *uri_node, *uri_tmp;
+	int error;
+
+	/* Remove each 'visited_uris' from all the table */
+	HASH_ITER(hh, uris->table, uri_node, uri_tmp) {
+		error = visited_uris_remove_local(uri_node->visited_uris);
+		if (error)
+			return error;
+	}
+
+	return 0;
+}
diff --git a/src/rrdp/db/db_rrdp_uris.h b/src/rrdp/db/db_rrdp_uris.h
index ec316803..f221987a 100644
--- a/src/rrdp/db/db_rrdp_uris.h
+++ b/src/rrdp/db/db_rrdp_uris.h
@@ -31,5 +31,6 @@ int db_rrdp_uris_set_all_nonrequested(void);
 int db_rrdp_uris_get_visited_uris(char const *, struct visited_uris **);
 
 bool db_rrdp_uris_visited_exists(struct db_rrdp_uri *, char const *);
+int db_rrdp_uris_remove_all_local(struct db_rrdp_uri *);
 
 #endif /* SRC_RRDP_DB_DB_RRDP_URIS_H_ */
diff --git a/src/rrdp/rrdp_loader.c b/src/rrdp/rrdp_loader.c
index 8a139252..c4566a4c 100644
--- a/src/rrdp/rrdp_loader.c
+++ b/src/rrdp/rrdp_loader.c
@@ -3,7 +3,6 @@
 #include "rrdp/db/db_rrdp_uris.h"
 #include "rrdp/rrdp_objects.h"
 #include "rrdp/rrdp_parser.h"
-#include "delete_dir_daemon.h"
 #include "log.h"
 #include "thread_var.h"
 #include "visited_uris.h"
@@ -43,7 +42,7 @@ process_snapshot(struct update_notification *notification,
 
 	error = rrdp_parse_snapshot(notification, tmp);
 	if (error) {
-		free(tmp);
+		visited_uris_refput(tmp);
 		return error;
 	}
 
@@ -52,29 +51,17 @@ process_snapshot(struct update_notification *notification,
 }
 
 static int
-remove_rrdp_uri_files(struct update_notification *notification)
+remove_rrdp_uri_files(char const *notification_uri)
 {
 	struct visited_uris *tmp;
-	char *root_path;
 	int error;
 
 	/* Work with the existent visited uris */
-	error = db_rrdp_uris_get_visited_uris(notification->uri, &tmp);
+	error = db_rrdp_uris_get_visited_uris(notification_uri, &tmp);
 	if (error)
 		return error;
 
-	error = visited_uris_get_root(tmp, &root_path);
-	if (error)
-		return error;
-
-	error = delete_dir_daemon_start(root_path);
-	if (error) {
-		free(root_path);
-		return error;
-	}
-
-	free(root_path);
-	return 0;
+	return visited_uris_remove_local(tmp);
 }
 
 int
@@ -123,7 +110,7 @@ rrdp_load(struct rpki_uri *uri)
 		pr_warn("There was an error processing RRDP deltas, using the snapshot instead.");
 	case RRDP_URI_DIFF_SESSION:
 		/* Delete the old session files */
-		error = remove_rrdp_uri_files(upd_notification);
+		error = remove_rrdp_uri_files(upd_notification->uri);
 		if (error)
 			break;
 	case RRDP_URI_NOTFOUND:
diff --git a/src/rrdp/rrdp_parser.c b/src/rrdp/rrdp_parser.c
index 12b2107f..3001ba2e 100644
--- a/src/rrdp/rrdp_parser.c
+++ b/src/rrdp/rrdp_parser.c
@@ -413,9 +413,8 @@ parse_global_data(xmlTextReaderPtr reader, struct global_data *gdata,
 		goto return_val; /* Error O is OK */
 
 	/*
-	 * FIXME (now) Prepare the callers to receive positive error values,
-	 * which means the file was successfully parsed but is has a logic error
-	 * (in this case, session ID doesn't match parent's).
+	 * Positive error value means the file was successfully parsed but it
+	 * has a logic error (in this case, session ID doesn't match parent's).
 	 */
 	if (strcmp(expected_session, session_id) != 0) {
 		pr_info("File session id [%s] doesn't match parent's session id [%s]",
@@ -614,68 +613,33 @@ write_from_uri(char const *location, unsigned char *content, size_t content_len,
 }
 
 static int
-delete_from_uri(char const *location, struct visited_uris *visited_uris)
+delete_from_uri(struct rpki_uri *uri, struct visited_uris *visited_uris)
 {
-	struct rpki_uri *uri;
-	char *local_uri, *work_loc, *tmp;
 	int error;
 
-	error = uri_create_mixed_str(&uri, location, strlen(location));
-	if (error)
-		return error;
-
-	local_uri = strdup(uri_get_local(uri));
-	if (local_uri == NULL) {
-		error = pr_enomem();
-		goto release_uri;
-	}
-
-	errno = 0;
-	error = remove(local_uri);
-	if (error) {
-		error = pr_errno(errno, "Couldn't delete %s", local_uri);
-		goto release_str;
+	if (visited_uris) {
+		error = rem_mft_from_list(visited_uris, uri_get_global(uri));
+		if (error)
+			return error;
 	}
 
-	error = rem_mft_from_list(visited_uris, uri_get_global(uri));
-	if (error)
-		goto release_str;
-
-	/*
-	 * Delete parent dirs only if empty.
-	 *
-	 * The algorithm is a bit aggressive, but rmdir() won't delete
-	 * something unless is empty, so in case the dir still has something in
-	 * it the cycle is finished.
-	 */
-	work_loc = local_uri;
-	do {
-		tmp = strrchr(work_loc, '/');
-		if (tmp == NULL)
-			break;
-		*tmp = '\0';
-
-		/* FIXME (now) use a lock, what if the root dir is reached? */
+	/* Delete parent dirs only if empty. */
+	return delete_dir_recursive_bottom_up(uri_get_local(uri));
+}
 
-		errno = 0;
-		error = rmdir(work_loc);
-		if (!error)
-			continue; /* Keep deleting up */
+int
+__delete_from_uri(char const *location, struct visited_uris *visited_uris)
+{
+	struct rpki_uri *uri;
+	int error;
 
-		/* Stop if there's content in the dir */
-		if (errno == ENOTEMPTY || errno == EEXIST)
-			break;
+	error = uri_create_mixed_str(&uri, location, strlen(location));
+	if (error)
+		return error;
 
-		error = pr_errno(errno, "Couldn't delete dir %s", work_loc);
-		goto release_str;
-	} while (true);
+	error = delete_from_uri(uri, visited_uris);
 
-	uri_refput(uri);
-	free(local_uri);
-	return 0;
-release_str:
-	free(local_uri);
-release_uri:
+	/* Error 0 is ok */
 	uri_refput(uri);
 	return error;
 }
@@ -719,7 +683,7 @@ parse_withdraw_elem(xmlTextReaderPtr reader, struct visited_uris *visited_uris)
 	if (error)
 		return error;
 
-	error = delete_from_uri(tmp->doc_data.uri, visited_uris);
+	error = __delete_from_uri(tmp->doc_data.uri, visited_uris);
 	withdraw_destroy(tmp);
 	if (error)
 		return error;
@@ -1084,6 +1048,7 @@ process_delta(struct delta_head *delta_head, void *arg)
 
 	error = parse_delta(uri, delta_head, arg);
 
+	delete_from_uri(uri, NULL);
 	/* Error 0 its ok */
 release_uri:
 	uri_refput(uri);
@@ -1130,12 +1095,14 @@ rrdp_parse_notification(struct rpki_uri *uri,
 
 	/* No updates yet */
 	if (error > 0) {
+		delete_from_uri(uri, NULL);
 		*result = NULL;
 		return 0;
 	}
 
 	fnstack_push_uri(uri);
 	error = parse_notification(uri, result);
+	delete_from_uri(uri, NULL);
 	if (error) {
 		fnstack_pop();
 		return error;
@@ -1167,6 +1134,7 @@ rrdp_parse_snapshot(struct update_notification *parent,
 
 	error = parse_snapshot(uri, &args);
 
+	delete_from_uri(uri, NULL);
 	/* Error 0 is ok */
 release_uri:
 	uri_refput(uri);
diff --git a/src/visited_uris.c b/src/visited_uris.c
index d4ad0c6d..4ec85d0b 100644
--- a/src/visited_uris.c
+++ b/src/visited_uris.c
@@ -4,6 +4,7 @@
 #include <stddef.h>
 #include <string.h>
 #include "log.h"
+#include "delete_dir_daemon.h"
 
 /*
  * FIXME (now) This should be replaced with something better (rtrie?)
@@ -154,6 +155,12 @@ visited_uris_get_root(struct visited_uris *uris, char **result)
 
 	elem = SLIST_FIRST(uris->list);
 
+	/* No elements yet */
+	if (elem == NULL) {
+		*result = NULL;
+		return 0;
+	}
+
 	i = 0;
 	ptr = strchr(elem->uri, '/');
 	while(i < 2) {
@@ -171,3 +178,26 @@ visited_uris_get_root(struct visited_uris *uris, char **result)
 	*result = tmp;
 	return 0;
 }
+
+int
+visited_uris_remove_local(struct visited_uris *uris)
+{
+	char *root_path;
+	int error;
+
+	error = visited_uris_get_root(uris, &root_path);
+	if (error)
+		return error;
+
+	if (root_path == NULL)
+		return 0;
+
+	error = delete_dir_daemon_start(root_path);
+	if (error) {
+		free(root_path);
+		return error;
+	}
+
+	free(root_path);
+	return 0;
+}
diff --git a/src/visited_uris.h b/src/visited_uris.h
index eff91b32..334966f4 100644
--- a/src/visited_uris.h
+++ b/src/visited_uris.h
@@ -13,5 +13,6 @@ int visited_uris_add(struct visited_uris *, char const *);
 int visited_uris_remove(struct visited_uris *, char const *);
 bool visited_uris_exists(struct visited_uris *, char const *);
 int visited_uris_get_root(struct visited_uris *, char **);
+int visited_uris_remove_local(struct visited_uris *);
 
 #endif /* SRC_VISITED_URIS_H_ */
diff --git a/test/tal_test.c b/test/tal_test.c
index aedfabc6..b7d60d9a 100644
--- a/test/tal_test.c
+++ b/test/tal_test.c
@@ -86,6 +86,18 @@ state_retrieve(void)
 	return NULL;
 }
 
+void
+db_rrdp_reset_visited_tals(void)
+{
+	/* Empty */
+}
+
+void
+db_rrdp_rem_nonvisited_tals(void)
+{
+	/* Empty */
+}
+
 START_TEST(tal_load_normal)
 {
 	struct tal *tal;