From: Michael Tremer <michael.tremer@ipfire.org>
Date: Thu, 1 Sep 2022 11:29:11 +0000 (+0000)
Subject: archive: Refactor reading archives
X-Git-Tag: 0.9.28~342
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=7109ccf367afca211700f1f0189c87f45e221965;p=pakfire.git

archive: Refactor reading archives

This changes that we will always parse any package metadata (because
without it we won't have a valid package).

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>
---

diff --git a/src/libpakfire/archive.c b/src/libpakfire/archive.c
index 246bace00..f97ce86d7 100644
--- a/src/libpakfire/archive.c
+++ b/src/libpakfire/archive.c
@@ -77,30 +77,6 @@ struct pakfire_archive {
 	int verify;
 };
 
-static const char* pakfire_archive_legacy_filename(
-		struct pakfire_archive* archive, const char* filename) {
-	// Do nothing for new package formats
-	if (archive->format >= 6)
-		return filename;
-
-	const struct files {
-		const char* file;
-		const char* legacy;
-	} filenames[] = {
-		{ "DATA",     "data.img" },
-		{ ".PKGINFO", "info" },
-		{ NULL, NULL },
-	};
-
-	for (const struct files* f = filenames; f->file; f++) {
-		if (strcmp(f->file, filename) == 0)
-			return f->legacy;
-	}
-
-	// Nothing found
-	return filename;
-}
-
 static int pakfire_archive_compute_digests(struct pakfire_archive* archive) {
 	int r;
 
@@ -159,7 +135,10 @@ ERROR:
 
 static int pakfire_archive_walk_entries(struct pakfire_archive* archive, struct archive* a,
 		int (*callback)(struct pakfire_archive* archive, struct archive* a,
-			struct archive_entry* e, void* data), void* data) {
+			struct archive_entry* e, void* data),
+		int (*filter_callback)(struct pakfire_archive* archive, struct archive* a,
+			struct archive_entry* e, void* data),
+		void* data) {
 	struct archive_entry* e = NULL;
 
 	// Walk through the archive
@@ -174,6 +153,23 @@ static int pakfire_archive_walk_entries(struct pakfire_archive* archive, struct
 		else if (r)
 			return r;
 
+		// Call the filter callback before we call the actual callback
+		if (filter_callback) {
+			r = filter_callback(archive, a, e, data);
+
+			// End processing the archive if the filter callback asked us to
+			if (r == ARCHIVE_EOF)
+				break;
+
+			// Skip this file
+			else if (r == ARCHIVE_RETRY)
+				continue;
+
+			// Raise any errors
+			else if (r)
+				return r;
+		}
+
 		// Run callback
 		if (callback) {
 			r = callback(archive, a, e, data);
@@ -187,7 +183,10 @@ static int pakfire_archive_walk_entries(struct pakfire_archive* archive, struct
 
 static int pakfire_archive_walk(struct pakfire_archive* archive,
 		int (*callback)(struct pakfire_archive* archive, struct archive* a,
-			struct archive_entry* e, void* data), void* data) {
+			struct archive_entry* e, void* data),
+		int (*filter_callback)(struct pakfire_archive* archive, struct archive* a,
+			struct archive_entry* e, void* data),
+		void* data) {
 	struct archive* a;
 
 	// Open the archive file
@@ -196,7 +195,7 @@ static int pakfire_archive_walk(struct pakfire_archive* archive,
 		return r;
 
 	// Walk through the archive
-	r = pakfire_archive_walk_entries(archive, a, callback, data);
+	r = pakfire_archive_walk_entries(archive, a, callback, filter_callback, data);
 
 	// Close the archive
 	close_archive(archive, a);
@@ -242,9 +241,6 @@ static int open_archive_and_find(struct pakfire_archive* archive, struct archive
 	if (r)
 		return r;
 
-	// Fix filename
-	filename = pakfire_archive_legacy_filename(archive, filename);
-
 	r = find_archive_entry(archive, entry, *a, filename);
 
 	// Close archive on error
@@ -254,25 +250,6 @@ static int open_archive_and_find(struct pakfire_archive* archive, struct archive
 	return r;
 }
 
-static int open_archive_and_read(struct pakfire_archive* archive, const char* filename,
-		char** data, size_t* size) {
-	struct archive* a = NULL;
-	struct archive_entry* e = NULL;
-
-	// Open the archive and find the right file
-	int r = open_archive_and_find(archive, &a, &e, filename);
-	if (r)
-		return r;
-
-	// Read the file into memory
-	r = pakfire_archive_copy_data_to_buffer(archive->pakfire, a, e, data, size);
-
-	// Close the archive
-	close_archive(archive, a);
-
-	return r;
-}
-
 static la_ssize_t pakfire_archive_read_callback(struct archive* a,
 		void* client_data, const void** buffer) {
 	struct archive* archive = (struct archive*)client_data;
@@ -303,7 +280,7 @@ static struct archive* pakfire_archive_open_payload(struct pakfire_archive* arch
 	struct archive* payload = NULL;
 
 	// Find the payload
-	int r = open_archive_and_find(archive, a, &entry, "DATA");
+	int r = open_archive_and_find(archive, a, &entry, "data.img");
 	if (r)
 		goto ERROR;
 
@@ -319,11 +296,8 @@ static struct archive* pakfire_archive_open_payload(struct pakfire_archive* arch
 	// All of our packages are tar balls
 	archive_read_support_format_tar(payload);
 
-	// They are compressed using XZ or ZSTD
-	if (archive->format >= 6)
-		archive_read_support_filter_zstd(payload);
-	else
-		archive_read_support_filter_xz(payload);
+	// They are compressed using XZ
+	archive_read_support_filter_xz(payload);
 
 	// Try opening the payload archive
 	r = archive_read_open2(payload, *a, NULL, pakfire_archive_read_callback, NULL, NULL);
@@ -405,18 +379,9 @@ static struct pakfire_package* pakfire_archive_get_package(struct pakfire_archiv
 
 // Metadata
 
-static int pakfire_archive_parse_json_metadata(struct pakfire_archive* archive) {
-	char* data = NULL;
-	size_t size = 0;
-
-	// Do nothing if metadata has already been loaded
-	if (archive->metadata)
-		return 0;
-
-	// Read the metadata file
-	int r = open_archive_and_read(archive, ".PKGINFO", &data, &size);
-	if (r)
-		return r;
+static int pakfire_archive_parse_json_metadata(struct pakfire_archive* archive,
+		const char* data, const size_t length) {
+	int r = 1;
 
 	// Create tokener
 	struct json_tokener* tokener = json_tokener_new();
@@ -426,7 +391,7 @@ static int pakfire_archive_parse_json_metadata(struct pakfire_archive* archive)
 	}
 
 	// Parse JSON from buffer
-	archive->metadata = json_tokener_parse_ex(tokener, data, size);
+	archive->metadata = json_tokener_parse_ex(tokener, data, length);
 	if (!archive->metadata) {
 		enum json_tokener_error error = json_tokener_get_error(tokener);
 
@@ -439,6 +404,9 @@ static int pakfire_archive_parse_json_metadata(struct pakfire_archive* archive)
 		json_object_to_json_string_ext(archive->metadata,
 			JSON_C_TO_STRING_PRETTY|JSON_C_TO_STRING_PRETTY_TAB));
 
+	// Success
+	r = 0;
+
 ERROR:
 	if (tokener)
 		json_tokener_free(tokener);
@@ -446,18 +414,9 @@ ERROR:
 	return r;
 }
 
-static int pakfire_archive_parse_legacy_metadata(struct pakfire_archive* archive) {
-	char* data = NULL;
-	size_t size;
-
-	// Do nothing if this has already been loaded
-	if (archive->parser)
-		return 0;
-
-	// Read the metadata file
-	int r = open_archive_and_read(archive, ".PKGINFO", &data, &size);
-	if (r)
-		return r;
+static int pakfire_archive_parse_legacy_metadata(struct pakfire_archive* archive,
+		const char* data, const size_t length) {
+	int r = 1;
 
 	// Allocate a new parser
 	archive->parser = pakfire_parser_create(archive->pakfire, NULL, NULL, 0);
@@ -465,15 +424,14 @@ static int pakfire_archive_parse_legacy_metadata(struct pakfire_archive* archive
 		goto ERROR;
 
 	// Parse the metadata
-	r = pakfire_parser_parse_data(archive->parser, data, size, NULL);
+	r = pakfire_parser_parse_data(archive->parser, data, length, NULL);
 	if (r) {
 		ERROR(archive->pakfire, "Error parsing metadata\n");
 		goto ERROR;
 	}
 
 	// Success
-	r = 0;
-	goto CLEANUP;
+	return 0;
 
 ERROR:
 	if (archive->parser) {
@@ -481,70 +439,84 @@ ERROR:
 		archive->parser = NULL;
 	}
 
-CLEANUP:
-	if (data)
-		free(data);
-
 	return r;
 }
 
-static int pakfire_archive_parse_metadata(struct pakfire_archive* archive) {
-	// Parse JSON for newer formats
-	if (archive->format >= 6)
-		return pakfire_archive_parse_json_metadata(archive);
+static int pakfire_archive_parse_format(struct pakfire_archive* archive,
+		const char* data, const size_t length) {
+	// Check if format has already been set
+	if (archive->format) {
+		ERROR(archive->pakfire, "Archive format has already been parsed\n");
+		errno = EINVAL;
+		return 1;
+	}
 
-	// Use legacy parser for others
-	return pakfire_archive_parse_legacy_metadata(archive);
-}
+	// Parse the format
+	archive->format = strtoul(data, NULL, 10);
 
-/*
-	This function tries (very easily) to find out whether something is a valid
-	archive or not.
-*/
-static int pakfire_archive_read_format(struct pakfire_archive* archive, struct archive* a) {
-	struct archive_entry* entry = NULL;
-	char* data = NULL;
-	size_t size;
+	switch (archive->format) {
+		// Handle all supported formats
+		case 1:
+		case 2:
+		case 3:
+		case 4:
+		case 5:
+		case 6:
+			break;
 
-	// Read the first header
-	int r = archive_read_next_header(a, &entry);
-	if (r) {
-		ERROR(archive->pakfire, "Could not read first tar header: %s\n",
-			archive_error_string(a));
-		goto ERROR;
+		// Break on anything else
+		default:
+			ERROR(archive->pakfire, "This version of Pakfire does not support "
+				"archive format %d\n", archive->format);
+			errno = EINVAL;
+			return 1;
 	}
 
-	const char* entry_name = archive_entry_pathname(entry);
+	DEBUG(archive->pakfire, "Archive format is %d\n", archive->format);
 
-	// Check for filename (must be "pakfire-format")
-	if (strcmp(entry_name, "pakfire-format") != 0) {
-		DEBUG(archive->pakfire, "First file is not named \"pakfire-format\"");
-		r = 1;
-		goto ERROR;
-	}
+	return 0;
+}
 
-	// The file cannot be larger than a couple of bytes
-	size = archive_entry_size(entry);
-	if (size == 0 || size >= 128) {
-		DEBUG(archive->pakfire, "pakfire-format is of an invalid size: %zu bytes\n", size);
-		r = 1;
-		goto ERROR;
-	}
+static int __pakfire_archive_read_metadata(struct pakfire_archive* archive,
+		struct archive* a, struct archive_entry* entry, void* p) {
+	char* data = NULL;
+	size_t length = 0;
+	int r;
 
-	// Read the entire payload
-	r = pakfire_archive_copy_data_to_buffer(archive->pakfire, a, entry, &data, &size);
+	const char* path = archive_entry_pathname(entry);
+
+	// Load the file into memory
+	r = pakfire_archive_copy_data_to_buffer(archive->pakfire, a, entry, &data, &length);
 	if (r) {
-		ERROR(archive->pakfire, "Could not read data of first file: %s\n",
+		ERROR(archive->pakfire, "Could not read data from archive: %s\n",
 			archive_error_string(a));
 		goto ERROR;
 	}
 
-	// Parse the format
-	archive->format = strtoul(data, NULL, 10);
+	// Format >= 6
+	if (archive->format >= 6) {
+		// Parse PKGINFO
+		if (strcmp(path, ".PKGINFO") == 0) {
+			r = pakfire_archive_parse_json_metadata(archive, data, length);
+			if (r)
+				goto ERROR;
+		}
 
-	DEBUG(archive->pakfire, "Archive format is %d\n", archive->format);
+	// Format >= 1
+	} else if (archive->format >= 1) {
+		// Parse info
+		if (strcmp(path, "info") == 0) {
+			r = pakfire_archive_parse_legacy_metadata(archive, data, length);
+			if (r)
+				goto ERROR;
+		}
 
-	// XXX check if this version is supported
+	// pakfire-format
+	} else if (strcmp(path, "pakfire-format") == 0) {
+		r = pakfire_archive_parse_format(archive, data, length);
+		if (r)
+			goto ERROR;
+	}
 
 ERROR:
 	if (data)
@@ -553,12 +525,74 @@ ERROR:
 	return r;
 }
 
+static int __pakfire_archive_filter_metadata(struct pakfire_archive* archive,
+		struct archive* a, struct archive_entry* entry, void* data) {
+	const char* path = archive_entry_pathname(entry);
+
+	// Format >= 6
+	if (archive->format >= 6) {
+		// Anything that starts with "." is a metadata file
+		if (*path == '.')
+			return ARCHIVE_OK;
+
+		// Otherwise, the payload begins
+		return ARCHIVE_EOF;
+
+	// Format >= 1
+	} else if (archive->format >= 1) {
+		// info
+		if (strcmp(path, "info") == 0)
+			return ARCHIVE_OK;
+
+		// scriptlets
+		else if (pakfire_string_startswith(path, "scriptlets/"))
+			return ARCHIVE_OK;
+
+		// Ignore anything else
+		return ARCHIVE_RETRY;
+
+	// The pakfire-format file is part of the metadata
+	} else if (strcmp(path, "pakfire-format") == 0) {
+		return ARCHIVE_OK;
+	}
+
+	// Unknown file
+	return 1;
+}
+
+static int pakfire_archive_read_metadata(struct pakfire_archive* archive) {
+	int r;
+
+	// Walk through the archive
+	r = pakfire_archive_walk(archive, __pakfire_archive_read_metadata,
+		__pakfire_archive_filter_metadata, NULL);
+	if (r)
+		return r;
+
+	// Check if we could successfully read something
+	if (!archive->format) {
+		ERROR(archive->pakfire, "Archive has an unknown format\n");
+		errno = EINVAL;
+		return 1;
+	}
+
+	// Check if we have read some metadata
+	if (!archive->metadata && !archive->parser) {
+		ERROR(archive->pakfire, "Archive has no metadata\n");
+		errno = EINVAL;
+		return 1;
+	}
+
+	return 0;
+}
+
 static int pakfire_archive_try_open(struct pakfire_archive* archive, const char* path) {
-	struct archive* a = NULL;
 	int r;
 
-	if (!path)
-		return EINVAL;
+	if (!path) {
+		errno = EINVAL;
+		return 1;
+	}
 
 	DEBUG(archive->pakfire, "Opening archive %s\n", path);
 
@@ -577,26 +611,14 @@ static int pakfire_archive_try_open(struct pakfire_archive* archive, const char*
 		goto ERROR;
 	}
 
-	// Open the archive file for reading.
-	r = open_archive(archive, &a);
-	if (r)
-		goto ERROR;
-
-	// Read the format of this archive
-	r = pakfire_archive_read_format(archive, a);
-	if (r)
+	// Read all package metadata
+	r = pakfire_archive_read_metadata(archive);
+	if (r) {
+		ERROR(archive->pakfire, "Could not open archive: %m\n");
 		goto ERROR;
-
-	// Success
-	r = 0;
-
-	// Reset errno
-	errno = 0;
+	}
 
 ERROR:
-	if (a)
-		close_archive(archive, a);
-
 	return r;
 }
 
@@ -620,11 +642,8 @@ ERROR:
 
 static struct json_object* pakfire_archive_metadata_get_object(
 		struct pakfire_archive* archive, const char* key1, const char* key2) {
-	int r = pakfire_archive_parse_metadata(archive);
-	if (r)
-		return NULL;
-
 	struct json_object* object = archive->metadata;
+	int r;
 
 	const char* keys[] = {
 		key1,
@@ -667,11 +686,8 @@ static int64_t pakfire_archive_metadata_get_int64(
 	return json_object_get_int64(object);
 }
 
-PAKFIRE_EXPORT char* pakfire_archive_get(struct pakfire_archive* archive, const char* namespace, const char* key) {
-	int r = pakfire_archive_parse_metadata(archive);
-	if (r)
-		return NULL;
-
+PAKFIRE_EXPORT char* pakfire_archive_get(struct pakfire_archive* archive,
+		const char* namespace, const char* key) {
 	return pakfire_parser_get(archive->parser, namespace, key);
 }
 
@@ -1338,7 +1354,7 @@ static int pakfire_archive_load_scriptlets(struct pakfire_archive* archive) {
 
 	int counter = 0;
 
-	return pakfire_archive_walk(archive, pakfire_archive_load_scriptlet, &counter);
+	return pakfire_archive_walk(archive, pakfire_archive_load_scriptlet, NULL, &counter);
 }
 
 struct pakfire_scriptlet* pakfire_archive_get_scriptlet(