From: Michael Tremer Date: Wed, 7 Apr 2021 09:58:29 +0000 (+0000) Subject: archive: Refactor reading files into buffer X-Git-Tag: 0.9.28~1285^2~410 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2597edad994e43340fb48a4d7c5e6bd2add1405f;p=pakfire.git archive: Refactor reading files into buffer This avoids using the loop and allocates as much memory as is needed. Signed-off-by: Michael Tremer --- diff --git a/src/_pakfire/archive.c b/src/_pakfire/archive.c index 5da0faee7..88513ae04 100644 --- a/src/_pakfire/archive.c +++ b/src/_pakfire/archive.c @@ -82,7 +82,7 @@ static PyObject* Archive_read(ArchiveObject* self, PyObject* args, PyObject* kwd if (!filename) Py_RETURN_NONE; - void* data = NULL; + char* data = NULL; size_t data_size = 0; int ret = pakfire_archive_read(self->archive, filename, &data, &data_size, flags); diff --git a/src/libpakfire/archive.c b/src/libpakfire/archive.c index fdb679648..df3ceaf17 100644 --- a/src/libpakfire/archive.c +++ b/src/libpakfire/archive.c @@ -52,8 +52,6 @@ #include #include -#define BLOCKSIZE 1024 * 1024 // 1MB - struct pakfire_archive_chksum { STAILQ_ENTRY(pakfire_archive_chksum) nodes; @@ -96,7 +94,7 @@ struct _PakfireArchiveSignature { struct payload_archive_data { struct archive* archive; - char buffer[BLOCKSIZE]; + char buffer[1024 * 1024]; }; static void configure_archive(struct archive* a) { @@ -129,27 +127,29 @@ static int archive_open(PakfireArchive archive, struct archive** a) { return 0; } -static int archive_read(struct archive* a, void** data, size_t* data_size) { +static int archive_read_to_buffer(Pakfire pakfire, struct archive* a, + struct archive_entry* entry, char** data, size_t* data_size) { *data = NULL; *data_size = 0; - for (;;) { - *data = realloc(*data, *data_size + BLOCKSIZE); - - ssize_t size = archive_read_data(a, *data + *data_size, BLOCKSIZE); - if (size == 0) - break; - - if (size < 0) { - free(*data); - *data = NULL; + size_t required_size = archive_entry_size(entry); + if (!required_size) + return 0; - return 1; - } + // Allocate a block of the required size + *data = malloc(required_size); + if (!*data) + return ENOMEM; - *data_size += size; + ssize_t bytes_read = archive_read_data(a, *data, required_size); + if (bytes_read < 0) { + ERROR(pakfire, "Could not read from archive: %s\n", archive_error_string(a)); + free(*data); + return 1; } + *data_size = bytes_read; + return 0; } @@ -424,9 +424,9 @@ PAKFIRE_EXPORT Pakfire pakfire_archive_get_pakfire(PakfireArchive archive) { static int pakfire_archive_parse_entry_format(PakfireArchive archive, struct archive* a, struct archive_entry* e) { char format[10]; - format[sizeof(*format)] = '\0'; + size_t size = sizeof(format); - archive_read_data(a, &format, sizeof(*format)); + archive_read_to_buffer(archive->pakfire, a, e, (char**)&format, &size); archive->format = atoi(format); DEBUG(archive->pakfire, "Archive at %p format is %d\n", @@ -437,15 +437,15 @@ static int pakfire_archive_parse_entry_format(PakfireArchive archive, static int pakfire_archive_parse_entry_metadata(PakfireArchive archive, struct archive* a, struct archive_entry* e) { - void* data; + char* data; size_t data_size; - int r = archive_read(a, &data, &data_size); + int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size); if (r) return r; // Parse metadata file - r = pakfire_parser_parse_data(archive->parser, (const char*)data, data_size, NULL); + r = pakfire_parser_parse_data(archive->parser, data, data_size, NULL); free(data); return r; @@ -456,7 +456,7 @@ static int pakfire_archive_parse_entry_filelist(PakfireArchive archive, char* data; size_t data_size; - int r = archive_read(a, (void**)&data, &data_size); + int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size); if (r) { return 1; } @@ -479,7 +479,7 @@ static int pakfire_archive_parse_entry_checksums(PakfireArchive archive, char* data; size_t data_size; - int r = archive_read(a, (void**)&data, &data_size); + int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size); if (r) return 1; @@ -548,7 +548,7 @@ static int pakfire_archive_parse_entry_scriptlet(PakfireArchive archive, // Set the type scriptlet->type = pakfire_scriptlet_type_from_filename(filename); - int r = archive_read(a, &scriptlet->data, &scriptlet->size); + int r = archive_read_to_buffer(archive->pakfire, a, e, &scriptlet->data, &scriptlet->size); if (r) return r; @@ -798,7 +798,7 @@ static struct archive* archive_open_payload(struct archive* a) { } PAKFIRE_EXPORT int pakfire_archive_read(PakfireArchive archive, const char* filename, - void** data, size_t* data_size, int flags) { + char** data, size_t* data_size, int flags) { struct archive* a; struct archive* pa = NULL; struct archive_entry* entry; @@ -823,7 +823,8 @@ PAKFIRE_EXPORT int pakfire_archive_read(PakfireArchive archive, const char* file goto out; } - r = archive_read(use_payload ? pa : a, data, data_size); + r = archive_read_to_buffer(archive->pakfire, use_payload ? pa : a, entry, + data, data_size); out: if (pa) @@ -911,7 +912,7 @@ static int pakfire_archive_parse_entry_signature(PakfireArchive archive, char* data; size_t data_size; - int r = archive_read(a, (void**)&data, &data_size); + int r = archive_read_to_buffer(archive->pakfire, a, e, &data, &data_size); if (r) return 1; diff --git a/src/libpakfire/include/pakfire/archive.h b/src/libpakfire/include/pakfire/archive.h index 7fa87c11d..38ee77853 100644 --- a/src/libpakfire/include/pakfire/archive.h +++ b/src/libpakfire/include/pakfire/archive.h @@ -49,7 +49,7 @@ Pakfire pakfire_archive_get_pakfire(PakfireArchive archive); char* pakfire_archive_get(PakfireArchive archive, const char* namespace, const char* key); int pakfire_archive_read(PakfireArchive archive, const char* filename, - void** data, size_t* data_size, int flags); + char** data, size_t* data_size, int flags); int pakfire_archive_extract(PakfireArchive archive, const char* prefix, int flags); char* pakfire_archive_extraction_path(PakfireArchive archive, const char* target); diff --git a/src/libpakfire/include/pakfire/scriptlet.h b/src/libpakfire/include/pakfire/scriptlet.h index dcaea6312..e70d5ab3b 100644 --- a/src/libpakfire/include/pakfire/scriptlet.h +++ b/src/libpakfire/include/pakfire/scriptlet.h @@ -55,11 +55,11 @@ const struct pakfire_scriptlet_type PAKFIRE_SCRIPTLET_TYPES[NUM_PAKFIRE_SCRIPTLE struct pakfire_scriptlet { pakfire_scriptlet_type type; - void* data; + char* data; size_t size; }; -struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const void* data, size_t size); +struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const char* data, size_t size); void pakfire_scriptlet_free(struct pakfire_scriptlet* scriptlet); pakfire_scriptlet_type pakfire_scriptlet_type_from_filename(const char* filename); diff --git a/src/libpakfire/scriptlet.c b/src/libpakfire/scriptlet.c index 542b8cc54..7ef72679d 100644 --- a/src/libpakfire/scriptlet.c +++ b/src/libpakfire/scriptlet.c @@ -41,7 +41,7 @@ const struct pakfire_scriptlet_type PAKFIRE_SCRIPTLET_TYPES[NUM_PAKFIRE_SCRIPTLE { PAKFIRE_SCRIPTLET_UNDEFINED, NULL, NULL }, }; -static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const void* data, size_t size) { +static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const char* data, size_t size) { if (scriptlet->data) free(scriptlet->data); @@ -56,7 +56,7 @@ static int pakfire_scriptlet_set(struct pakfire_scriptlet* scriptlet, const void return 0; } -struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const void* data, size_t size) { +struct pakfire_scriptlet* pakfire_scriptlet_create(Pakfire pakfire, const char* data, size_t size) { struct pakfire_scriptlet* scriptlet = calloc(1, sizeof(*scriptlet)); if (!scriptlet) return NULL;