archive: Refactor reading files into buffer

This avoids using the loop and allocates as much memory as is needed.

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/_pakfire/archive.c b/src/_pakfire/archive.c
index 5da0faee71e67e07a6c5221ae63d838feb6383da..88513ae042c9f4e6c563f47ba16aae44aa313ca9 100644 (file)
--- a/src/_pakfire/archive.c
+++ b/src/_pakfire/archive.c
@@ -82,7 +82,7 @@ static PyObject* Archive_read(ArchiveObject* self, PyObject* args, PyObject* kwd
        if (!filename)
                Py_RETURN_NONE;
 
-       void* data = NULL;
+       char* data = NULL;
        size_t data_size = 0;
 
        int ret = pakfire_archive_read(self->archive, filename, &data, &data_size, flags);

diff --git a/src/libpakfire/archive.c b/src/libpakfire/archive.c
index fdb67964853e1a6849504e8f847cb99062c665d8..df3ceaf1711c63864737fbb510346c480763d0b6 100644 (file)
--- a/src/libpakfire/archive.c
+++ b/src/libpakfire/archive.c
@@ -52,8 +52,6 @@
 #include <pakfire/scriptlet.h>
 #include <pakfire/util.h>
 
-#define BLOCKSIZE      1024 * 1024 // 1MB
-
 struct pakfire_archive_chksum {
        STAILQ_ENTRY(pakfire_archive_chksum) nodes;
 
@@ -96,7 +94,7 @@ struct _PakfireArchiveSignature {
 
 struct payload_archive_data {
        struct archive* archive;
-       char buffer[BLOCKSIZE];
+       char buffer[1024 * 1024];
 };
 
 static void configure_archive(struct archive* a) {
@@ -129,27 +127,29 @@ static int archive_open(PakfireArchive archive, struct archive** a) {
        return 0;
 }
 
-static int archive_read(struct archive* a, void** data, size_t* data_size) {
+static int archive_read_to_buffer(Pakfire pakfire, struct archive* a,
+               struct archive_entry* entry, char** data, size_t* data_size) {
        *data = NULL;
        *data_size = 0;
 
-       for (;;) {
-               *data = realloc(*data, *data_size + BLOCKSIZE);
-
-               ssize_t size = archive_read_data(a, *data + *data_size, BLOCKSIZE);
-               if (size == 0)
-                       break;
-
-               if (size < 0) {
-                       free(*data);
-                       *data = NULL;
+       size_t required_size = archive_entry_size(entry);
+       if (!required_size)
+               return 0;
 
-                       return 1;
-               }
+       // Allocate a block of the required size
+       *data = malloc(required_size);
+       if (!*data)
+               return ENOMEM;
 
-               *data_size += size;
+       ssize_t bytes_read = archive_read_data(a, *data, required_size);
+       if (bytes_read < 0) {
+               ERROR(pakfire, "Could not read from archive: %s\n", archive_error_string(a));
+               free(*data);
+               return 1;
        }
 
+       *data_size = bytes_read;
+
        return 0;
 }
 
@@ -424,9 +424,9 @@ PAKFIRE_EXPORT Pakfire pakfire_archive_get_pakfire(PakfireArchive archive) {
 static int pakfire_archive_parse_entry_format(PakfireArchive archive,
                struct archive* a, struct archive_entry* e) {
        char format[10];
-       format[sizeof(*format)] = '\0';
+       size_t size = sizeof(format);
 
-       archive_read_data(a, &format, sizeof(*format));
+       archive_read_to_buffer(archive->pakfire, a, e, (char**)&format, &size);
        archive->format = atoi(format);
 
        DEBUG(archive->pakfire, "Archive at %p format is %d\n",
@@ -437,15 +437,15 @@ static int pakfire_archive_parse_entry_format(PakfireArchive archive,
 
 static int pakfire_archive_parse_entry_metadata(PakfireArchive archive,
                struct archive* a, struct archive_entry* e) {
-       void* data;
+       char* data;
        size_t data_size;
 
-       int r = archive_read(a, &data, &data_size);
+       int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size);
        if (r)
                return r;
 
        // Parse metadata file
-       r = pakfire_parser_parse_data(archive->parser, (const char*)data, data_size, NULL);
+       r = pakfire_parser_parse_data(archive->parser, data, data_size, NULL);
        free(data);
 
        return r;
@@ -456,7 +456,7 @@ static int pakfire_archive_parse_entry_filelist(PakfireArchive archive,
        char* data;
        size_t data_size;
 
-       int r = archive_read(a, (void**)&data, &data_size);
+       int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size);
        if (r) {
                return 1;
        }
@@ -479,7 +479,7 @@ static int pakfire_archive_parse_entry_checksums(PakfireArchive archive,
        char* data;
        size_t data_size;
 
-       int r = archive_read(a, (void**)&data, &data_size);
+       int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size);
        if (r)
                return 1;
 
@@ -548,7 +548,7 @@ static int pakfire_archive_parse_entry_scriptlet(PakfireArchive archive,
        // Set the type
        scriptlet->type = pakfire_scriptlet_type_from_filename(filename);
 
-       int r = archive_read(a, &scriptlet->data, &scriptlet->size);
+       int r = archive_read_to_buffer(archive->pakfire, a, e, &scriptlet->data, &scriptlet->size);
        if (r)
                return r;
 
@@ -798,7 +798,7 @@ static struct archive* archive_open_payload(struct archive* a) {
 }
 
 PAKFIRE_EXPORT int pakfire_archive_read(PakfireArchive archive, const char* filename,
-               void** data, size_t* data_size, int flags) {
+               char** data, size_t* data_size, int flags) {
        struct archive* a;
        struct archive* pa = NULL;
        struct archive_entry* entry;
@@ -823,7 +823,8 @@ PAKFIRE_EXPORT int pakfire_archive_read(PakfireArchive archive, const char* file
                goto out;
        }
 
-       r = archive_read(use_payload ? pa : a, data, data_size);
+       r = archive_read_to_buffer(archive->pakfire, use_payload ? pa : a, entry,
+               data, data_size);
 
 out:
        if (pa)
@@ -911,7 +912,7 @@ static int pakfire_archive_parse_entry_signature(PakfireArchive archive,
        char* data;
        size_t data_size;
 
-       int r = archive_read(a, (void**)&data, &data_size);
+       int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size);
        if (r)
                return 1;
 

diff --git a/src/libpakfire/include/pakfire/archive.h b/src/libpakfire/include/pakfire/archive.h
index 7fa87c11dc36d495c50e697fbcc56df40db846c9..38ee778535ef5aad334ceaba031642b09fa769e7 100644 (file)
--- a/src/libpakfire/include/pakfire/archive.h
+++ b/src/libpakfire/include/pakfire/archive.h
@@ -49,7 +49,7 @@ Pakfire pakfire_archive_get_pakfire(PakfireArchive archive);
 char* pakfire_archive_get(PakfireArchive archive, const char* namespace, const char* key);
 
 int pakfire_archive_read(PakfireArchive archive, const char* filename,
-       void** data, size_t* data_size, int flags);
+       char** data, size_t* data_size, int flags);
 int pakfire_archive_extract(PakfireArchive archive, const char* prefix, int flags);
 char* pakfire_archive_extraction_path(PakfireArchive archive, const char* target);
 

diff --git a/src/libpakfire/include/pakfire/scriptlet.h b/src/libpakfire/include/pakfire/scriptlet.h
index dcaea631238a5bedcdf3ba0903a313b859d5c60d..e70d5ab3b5abb98cb8122bb6ae1a3c2d1369bb66 100644 (file)
--- a/src/libpakfire/include/pakfire/scriptlet.h
+++ b/src/libpakfire/include/pakfire/scriptlet.h
@@ -55,11 +55,11 @@ const struct pakfire_scriptlet_type PAKFIRE_SCRIPTLET_TYPES[NUM_PAKFIRE_SCRIPTLE
 
 struct pakfire_scriptlet {
        pakfire_scriptlet_type type;
-       void* data;
+       char* data;
        size_t size;
 };
 
-struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const void* data, size_t size);
+struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const char* data, size_t size);
 void pakfire_scriptlet_free(struct pakfire_scriptlet* scriptlet);
 
 pakfire_scriptlet_type pakfire_scriptlet_type_from_filename(const char* filename);

diff --git a/src/libpakfire/scriptlet.c b/src/libpakfire/scriptlet.c
index 542b8cc5494024742868d5536b0bd0b445923929..7ef72679d4968bb7eee44a3e4ce93c6b3a5ece1e 100644 (file)
--- a/src/libpakfire/scriptlet.c
+++ b/src/libpakfire/scriptlet.c
@@ -41,7 +41,7 @@ const struct pakfire_scriptlet_type PAKFIRE_SCRIPTLET_TYPES[NUM_PAKFIRE_SCRIPTLE
        { PAKFIRE_SCRIPTLET_UNDEFINED,   NULL,                     NULL },
 };
 
-static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const void* data, size_t size) {
+static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const char* data, size_t size) {
        if (scriptlet->data)
                free(scriptlet->data);
 
@@ -56,7 +56,7 @@ static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const void
        return 0;
 }
 
-struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const void* data, size_t size) {
+struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const char* data, size_t size) {
        struct pakfire_scriptlet* scriptlet = calloc(1, sizeof(*scriptlet));
        if (!scriptlet)
                return NULL;