snapshots: Refactor writing snapshots

We no longer make the filelist first. This would have
been nice for a progress bar, but it should be quick enough
to not be worries about this.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/libpakfire/include/pakfire/util.h b/src/libpakfire/include/pakfire/util.h
index 531032d14ae46380d338db316175487ffc354dd1..345b67fe5b280f0c7efcb3a5126438ff490cbe6f 100644 (file)
--- a/src/libpakfire/include/pakfire/util.h
+++ b/src/libpakfire/include/pakfire/util.h
@@ -58,6 +58,8 @@ char* pakfire_generate_uuid();
 
 #ifdef PAKFIRE_PRIVATE
 
+#include <archive.h>
+
 int pakfire_string_endswith(const char* s, const char* suffix);
 char* pakfire_lstrip(const char* s);
 
@@ -70,6 +72,11 @@ FILE* pakfire_mktemp(char* path);
 char* pakfire_mkdtemp(char* path);
 int pakfire_rmtree(const char* path, int flags);
 
+// Archive Stuff
+
+int pakfire_archive_copy_data(struct archive* src, struct archive* dst,
+       struct archive_entry* entry);
+
 // JSON Stuff
 
 struct json_object* pakfire_json_parse_from_file(Pakfire pakfire, const char* path);

diff --git a/src/libpakfire/snapshot.c b/src/libpakfire/snapshot.c
index 73309a5991c3b48f03f77d20063d142e3dae2a06..0d5304385dc57e5b2963d7095acac641afb97575 100644 (file)
--- a/src/libpakfire/snapshot.c
+++ b/src/libpakfire/snapshot.c
@@ -30,6 +30,7 @@
 #include <pakfire/private.h>
 #include <pakfire/snapshot.h>
 #include <pakfire/types.h>
+#include <pakfire/util.h>
 
 static const char* pakfire_snapshot_excludes[] = {
        "/dev",
@@ -41,6 +42,16 @@ static const char* pakfire_snapshot_excludes[] = {
        NULL,
 };
 
+static int pakfire_snapshot_path_is_excluded(const char* path) {
+       for (const char** exclude = pakfire_snapshot_excludes; *exclude; exclude++) {
+               if (pakfire_string_startswith(path, *exclude))
+                       return 1;
+       }
+
+       // No match
+       return 0;
+}
+
 static struct archive* pakfire_snapshot_create_archive(Pakfire pakfire, FILE* f) {
        struct archive* a = archive_write_new();
        if (!a) {
@@ -77,10 +88,11 @@ ERROR:
 }
 
 PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) {
-       char buffer[512 * 1024];
-       const char* path = pakfire_get_path(pakfire);
+       int r = 1;
+
+       const char* root = pakfire_get_path(pakfire);
 
-       INFO(pakfire, "Creating snapshot of %s...\n", path);
+       INFO(pakfire, "Creating snapshot of %s...\n", root);
 
        struct archive* a = pakfire_snapshot_create_archive(pakfire, f);
        if (!a) {
@@ -88,72 +100,76 @@ PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) {
                return 1;
        }
 
-       PakfireFilelist filelist = NULL;
-       int r = 1;
-
-       r = pakfire_filelist_create(&filelist, pakfire);
-       if (r)
-               goto ERROR;
-
-       // Search for files to package
-       r = pakfire_filelist_scan(filelist, path, NULL, pakfire_snapshot_excludes);
-       if (r)
+       // Create disk reader
+       struct archive* reader = pakfire_make_archive_disk_reader(pakfire, 1);
+       if (!reader)
                goto ERROR;
 
-       const size_t num_files = pakfire_filelist_size(filelist);
-       if (!num_files) {
-               ERROR(pakfire, "No files found for snapshot\n");
+       // Open path
+       r = archive_read_disk_open(reader, root);
+       if (r) {
+               ERROR(pakfire, "Could not open %s: %s\n", root, archive_error_string(reader));
                goto ERROR;
        }
 
-       DEBUG(pakfire, "Found %zu file(s) to snapshot\n", num_files);
+       struct archive_entry* entry = NULL;
+       while (1) {
+               r = archive_read_next_header(reader, &entry);
+               if (r == ARCHIVE_EOF)
+                       break;
+               else if (r == ARCHIVE_RETRY)
+                       continue;
+               else if (r) {
+                       ERROR(pakfire, "Could not read next entry: %s\n", archive_error_string(reader));
+                       goto ERROR;
+               }
 
-       for (unsigned int i = 0; i < num_files; i++) {
-               PakfireFile file = pakfire_filelist_get(filelist, i);
+               // Compute the relative path
+               const char* path = pakfire_path_relpath(root, archive_entry_pathname(entry));
+               if (!path)
+                       continue;
 
-               struct archive_entry* entry = pakfire_file_archive_entry(file);
-               if (!entry) {
-                       r = 1;
-                       goto ERROR;
+               // Skip excludes
+               if (pakfire_snapshot_path_is_excluded(path)) {
+                       DEBUG(pakfire, "Skipping %s...\n", path);
+                       continue;
                }
 
-               // Write entry to archive
+               DEBUG(pakfire, "Processing %s...\n", path);
+
+               archive_read_disk_descend(reader);
+
+               // Reset path to relative path in archive
+               archive_entry_set_pathname(entry, path);
+
+               // Write header
                r = archive_write_header(a, entry);
                if (r) {
-                       ERROR(pakfire, "Could not write header for %s: %s\n",
-                               pakfire_file_get_path(file), archive_error_string(a));
-                       pakfire_file_unref(file);
+                       ERROR(pakfire, "Could not write header: %s\n", archive_error_string(a));
                        goto ERROR;
                }
 
-               // Write payload
-               if (archive_entry_filetype(entry) == AE_IFREG) {
-                       f = pakfire_file_fopen(file, "r");
-                       if (!f)
-                               goto ERROR;
+               if (archive_entry_size(entry) > 0) {
+                       r = pakfire_archive_copy_data(reader, a, entry);
+                       if (r) {
+                               ERROR(pakfire, "Could not copy %s\n", archive_entry_pathname(entry));
 
-                       while (!feof(f)) {
-                               size_t bytes_read = fread(buffer, 1, sizeof(buffer), f);
-
-                               // Check if any error occured
-                               if (ferror(f)) {
-                                       ERROR(pakfire, "Error reading from file: %s\n", strerror(errno));
-                                       fclose(f);
-                                       goto ERROR;
-                               }
-
-                               ssize_t bytes_written = archive_write_data(a, buffer, bytes_read);
-                               if (bytes_written < 0) {
-                                       ERROR(pakfire, "Error writing data: %s\n", archive_error_string(a));
-                                       fclose(f);
-                                       goto ERROR;
-                               }
-                       }
+                               const char* error = archive_error_string(reader);
+                               if (error)
+                                       ERROR(pakfire, "Read error: %s\n", error);
 
-                       fclose(f);
+                               error = archive_error_string(a);
+                               if (error)
+                                       ERROR(pakfire, "Write error: %s\n", error);
+
+                               goto ERROR;
+                       }
                }
 
-               pakfire_file_unref(file);
+               // Write trailer
+               r = archive_write_finish_entry(a);
+               if (r)
+                       goto ERROR;
        }
 
        // Close archive
@@ -167,9 +183,8 @@ PAKFIRE_EXPORT int pakfire_snapshot_create(Pakfire pakfire, FILE* f) {
        r = 0;
 
 ERROR:
-       if (filelist)
-               pakfire_filelist_unref(filelist);
-
+       if (reader)
+               archive_read_free(reader);
        archive_write_free(a);
 
        return r;

diff --git a/src/libpakfire/util.c b/src/libpakfire/util.c
index ec6ea4879d8bb0c77b5d3857e2f7ea29f1cbb6c0..8dcf620985673457feda191bbfa060d1de59cec3 100644 (file)
--- a/src/libpakfire/util.c
+++ b/src/libpakfire/util.c
@@ -33,6 +33,7 @@
 #include <time.h>
 #include <unistd.h>
 
+#include <archive.h>
 #include <json.h>
 #include <uuid/uuid.h>
 
@@ -584,6 +585,30 @@ int pakfire_rmtree(const char* path, int flags) {
        return r;
 }
 
+// Archive Stuff
+
+int pakfire_archive_copy_data(struct archive* src, struct archive* dst,
+               struct archive_entry* entry) {
+       const void* buffer;
+       size_t size;
+       off_t offset;
+       int r;
+
+       for (;;) {
+               // Read a block of data
+               r = archive_read_data_block(src, &buffer, &size, &offset);
+               if (r == ARCHIVE_EOF)
+                       return ARCHIVE_OK;
+               else if (r)
+                       return r;
+
+               // Write the read block of data
+               r = archive_write_data(dst, buffer, size);
+               if (r < 0)
+                       return r;
+       }
+}
+
 // JSON Stuff
 
 static struct json_object* pakfire_json_parse(Pakfire pakfire, FILE* f) {