From: Michael Tremer Date: Tue, 23 Mar 2021 20:07:12 +0000 (+0000) Subject: snapshots: Refactor writing snapshots X-Git-Tag: 0.9.28~1285^2~483 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=88124868a67f7bf59010be818f4358c3170402ae;p=pakfire.git snapshots: Refactor writing snapshots We no longer make the filelist first. This would have been nice for a progress bar, but it should be quick enough to not be worries about this. Signed-off-by: Michael Tremer --- diff --git a/src/libpakfire/include/pakfire/util.h b/src/libpakfire/include/pakfire/util.h index 531032d14..345b67fe5 100644 --- a/src/libpakfire/include/pakfire/util.h +++ b/src/libpakfire/include/pakfire/util.h @@ -58,6 +58,8 @@ char* pakfire_generate_uuid(); #ifdef PAKFIRE_PRIVATE +#include + int pakfire_string_endswith(const char* s, const char* suffix); char* pakfire_lstrip(const char* s); @@ -70,6 +72,11 @@ FILE* pakfire_mktemp(char* path); char* pakfire_mkdtemp(char* path); int pakfire_rmtree(const char* path, int flags); +// Archive Stuff + +int pakfire_archive_copy_data(struct archive* src, struct archive* dst, + struct archive_entry* entry); + // JSON Stuff struct json_object* pakfire_json_parse_from_file(Pakfire pakfire, const char* path); diff --git a/src/libpakfire/snapshot.c b/src/libpakfire/snapshot.c index 73309a599..0d5304385 100644 --- a/src/libpakfire/snapshot.c +++ b/src/libpakfire/snapshot.c @@ -30,6 +30,7 @@ #include #include #include +#include static const char* pakfire_snapshot_excludes[] = { "/dev", @@ -41,6 +42,16 @@ static const char* pakfire_snapshot_excludes[] = { NULL, }; +static int pakfire_snapshot_path_is_excluded(const char* path) { + for (const char** exclude = pakfire_snapshot_excludes; *exclude; exclude++) { + if (pakfire_string_startswith(path, *exclude)) + return 1; + } + + // No match + return 0; +} + static struct archive* pakfire_snapshot_create_archive(Pakfire pakfire, FILE* f) { struct archive* a = archive_write_new(); if (!a) { @@ -77,10 +88,11 @@ ERROR: } PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) { - char buffer[512 * 1024]; - const char* path = pakfire_get_path(pakfire); + int r = 1; + + const char* root = pakfire_get_path(pakfire); - INFO(pakfire, "Creating snapshot of %s...\n", path); + INFO(pakfire, "Creating snapshot of %s...\n", root); struct archive* a = pakfire_snapshot_create_archive(pakfire, f); if (!a) { @@ -88,72 +100,76 @@ PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) { return 1; } - PakfireFilelist filelist = NULL; - int r = 1; - - r = pakfire_filelist_create(&filelist, pakfire); - if (r) - goto ERROR; - - // Search for files to package - r = pakfire_filelist_scan(filelist, path, NULL, pakfire_snapshot_excludes); - if (r) + // Create disk reader + struct archive* reader = pakfire_make_archive_disk_reader(pakfire, 1); + if (!reader) goto ERROR; - const size_t num_files = pakfire_filelist_size(filelist); - if (!num_files) { - ERROR(pakfire, "No files found for snapshot\n"); + // Open path + r = archive_read_disk_open(reader, root); + if (r) { + ERROR(pakfire, "Could not open %s: %s\n", root, archive_error_string(reader)); goto ERROR; } - DEBUG(pakfire, "Found %zu file(s) to snapshot\n", num_files); + struct archive_entry* entry = NULL; + while (1) { + r = archive_read_next_header(reader, &entry); + if (r == ARCHIVE_EOF) + break; + else if (r == ARCHIVE_RETRY) + continue; + else if (r) { + ERROR(pakfire, "Could not read next entry: %s\n", archive_error_string(reader)); + goto ERROR; + } - for (unsigned int i = 0; i < num_files; i++) { - PakfireFile file = pakfire_filelist_get(filelist, i); + // Compute the relative path + const char* path = pakfire_path_relpath(root, archive_entry_pathname(entry)); + if (!path) + continue; - struct archive_entry* entry = pakfire_file_archive_entry(file); - if (!entry) { - r = 1; - goto ERROR; + // Skip excludes + if (pakfire_snapshot_path_is_excluded(path)) { + DEBUG(pakfire, "Skipping %s...\n", path); + continue; } - // Write entry to archive + DEBUG(pakfire, "Processing %s...\n", path); + + archive_read_disk_descend(reader); + + // Reset path to relative path in archive + archive_entry_set_pathname(entry, path); + + // Write header r = archive_write_header(a, entry); if (r) { - ERROR(pakfire, "Could not write header for %s: %s\n", - pakfire_file_get_path(file), archive_error_string(a)); - pakfire_file_unref(file); + ERROR(pakfire, "Could not write header: %s\n", archive_error_string(a)); goto ERROR; } - // Write payload - if (archive_entry_filetype(entry) == AE_IFREG) { - f = pakfire_file_fopen(file, "r"); - if (!f) - goto ERROR; + if (archive_entry_size(entry) > 0) { + r = pakfire_archive_copy_data(reader, a, entry); + if (r) { + ERROR(pakfire, "Could not copy %s\n", archive_entry_pathname(entry)); - while (!feof(f)) { - size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); - - // Check if any error occured - if (ferror(f)) { - ERROR(pakfire, "Error reading from file: %s\n", strerror(errno)); - fclose(f); - goto ERROR; - } - - ssize_t bytes_written = archive_write_data(a, buffer, bytes_read); - if (bytes_written < 0) { - ERROR(pakfire, "Error writing data: %s\n", archive_error_string(a)); - fclose(f); - goto ERROR; - } - } + const char* error = archive_error_string(reader); + if (error) + ERROR(pakfire, "Read error: %s\n", error); - fclose(f); + error = archive_error_string(a); + if (error) + ERROR(pakfire, "Write error: %s\n", error); + + goto ERROR; + } } - pakfire_file_unref(file); + // Write trailer + r = archive_write_finish_entry(a); + if (r) + goto ERROR; } // Close archive @@ -167,9 +183,8 @@ PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) { r = 0; ERROR: - if (filelist) - pakfire_filelist_unref(filelist); - + if (reader) + archive_read_free(reader); archive_write_free(a); return r; diff --git a/src/libpakfire/util.c b/src/libpakfire/util.c index ec6ea4879..8dcf62098 100644 --- a/src/libpakfire/util.c +++ b/src/libpakfire/util.c @@ -33,6 +33,7 @@ #include #include +#include #include #include @@ -584,6 +585,30 @@ int pakfire_rmtree(const char* path, int flags) { return r; } +// Archive Stuff + +int pakfire_archive_copy_data(struct archive* src, struct archive* dst, + struct archive_entry* entry) { + const void* buffer; + size_t size; + off_t offset; + int r; + + for (;;) { + // Read a block of data + r = archive_read_data_block(src, &buffer, &size, &offset); + if (r == ARCHIVE_EOF) + return ARCHIVE_OK; + else if (r) + return r; + + // Write the read block of data + r = archive_write_data(dst, buffer, size); + if (r < 0) + return r; + } +} + // JSON Stuff static struct json_object* pakfire_json_parse(Pakfire pakfire, FILE* f) {