From: Michael Tremer Date: Tue, 16 Mar 2021 17:53:15 +0000 (+0000) Subject: libpakfire: Move transparent compression to own file X-Git-Tag: 0.9.28~1285^2~522 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c352b7763a3a6c13786b008b546cb5b977a531ac;p=pakfire.git libpakfire: Move transparent compression to own file Signed-off-by: Michael Tremer --- diff --git a/Makefile.am b/Makefile.am index 735989ff1..3b1e43e4f 100644 --- a/Makefile.am +++ b/Makefile.am @@ -257,6 +257,7 @@ lib_LTLIBRARIES += \ libpakfire_la_SOURCES = \ src/libpakfire/arch.c \ src/libpakfire/archive.c \ + src/libpakfire/compress.c \ src/libpakfire/db.c \ src/libpakfire/dist.c \ src/libpakfire/downloader.c \ @@ -288,6 +289,7 @@ libpakfire_la_SOURCES = \ pkginclude_HEADERS += \ src/libpakfire/include/pakfire/arch.h \ src/libpakfire/include/pakfire/archive.h \ + src/libpakfire/include/pakfire/compress.h \ src/libpakfire/include/pakfire/constants.h \ src/libpakfire/include/pakfire/db.h \ src/libpakfire/include/pakfire/dist.h \ diff --git a/src/libpakfire/compress.c b/src/libpakfire/compress.c new file mode 100644 index 000000000..f92acea1e --- /dev/null +++ b/src/libpakfire/compress.c @@ -0,0 +1,181 @@ +/*############################################################################# +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2021 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +#############################################################################*/ + +#include +#include + +#include + +#include + +// Read up to N bytes for analyze the magic +#define MAX_MAGIC_LENGTH 6 + +const struct compressor { + char magic[MAX_MAGIC_LENGTH]; + size_t magic_length; + FILE* (*open)(FILE* f, const char* mode); +} compressors[] = { + // XZ + { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, }, + { "", 0, NULL, }, +}; + +struct xz_cookie { + FILE* f; + lzma_stream stream; + int done; + + uint8_t buffer[64 * 1024]; +}; + +// Try to guess the compression +FILE* pakfire_xfopen(FILE* f, const char* mode) { + char buffer[MAX_MAGIC_LENGTH]; + + if (!f) + return NULL; + + fpos_t pos; + + // Store the position + int r = fgetpos(f, &pos); + if (r < 0) + return NULL; + + // Read a couple of bytes + size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); + + // Reset position + r = fsetpos(f, &pos); + if (r < 0) + return NULL; + + // Check if we could read anything + if (!bytes_read || bytes_read < sizeof(buffer)) + return f; + + // Analyze magic + for (const struct compressor* c = compressors; c->open; c++) { + // Check if we have read enough data + if (bytes_read < c->magic_length) + continue; + + // Compare the magic value + r = memcmp(c->magic, buffer, c->magic_length); + if (r) + continue; + + // We found a match! + return c->open(f, mode); + } + + // Nothing seems to match + return f; +} + +static ssize_t xz_read(void* data, char* buffer, size_t size) { + struct xz_cookie* cookie = (struct xz_cookie*)data; + if (!cookie) + return -1; + + // Return nothing after we are done + if (cookie->done) + return 0; + + lzma_action action = LZMA_RUN; + + // Set output to allocated buffer + cookie->stream.next_out = (uint8_t *)buffer; + cookie->stream.avail_out = size; + + while (1) { + // Read something when the input buffer is empty + if (cookie->stream.avail_in == 0) { + cookie->stream.next_in = cookie->buffer; + cookie->stream.avail_in = fread(cookie->buffer, + 1, sizeof(*cookie->buffer), cookie->f); + + // Break if the input file could not be read + if (ferror(cookie->f)) + return -1; + + // Finish after we have reached the end of the input file + if (feof(cookie->f)) { + action = LZMA_FINISH; + cookie->done = 1; + } + } + + lzma_ret ret = lzma_code(&cookie->stream, action); + + // If the stream has ended, we just send the + // remaining output and mark that we are done. + if (ret == LZMA_STREAM_END) { + cookie->done = 1; + return size - cookie->stream.avail_out; + } + + // Break on all other unexpected errors + if (ret != LZMA_OK) + return -1; + + // When we have read enough to fill the entire output buffer, we return + if (cookie->stream.avail_out == 0) + return size; + + if (cookie->done) + return -1; + } +} + +static int xz_close(void* data) { + struct xz_cookie* cookie = (struct xz_cookie*)data; + + // Free the deocder + lzma_end(&cookie->stream); + + // Close input file + fclose(cookie->f); + + return 0; +} + +FILE* pakfire_xzfopen(FILE* f, const char* mode) { + struct xz_cookie cookie = { + .f = f, + .stream = LZMA_STREAM_INIT, + .done = 0, + }; + + // Initialise the decoder + lzma_ret ret = lzma_stream_decoder(&cookie.stream, UINT64_MAX, 0); + if (ret != LZMA_OK) + return NULL; + + cookie_io_functions_t functions = { + .read = xz_read, + .write = NULL, + .seek = NULL, + .close = xz_close, + }; + + return fopencookie(&cookie, "rb", functions); +} diff --git a/src/libpakfire/include/pakfire/compress.h b/src/libpakfire/include/pakfire/compress.h new file mode 100644 index 000000000..5e6e28cc6 --- /dev/null +++ b/src/libpakfire/include/pakfire/compress.h @@ -0,0 +1,34 @@ +/*############################################################################# +# # +# Pakfire - The IPFire package management system # +# Copyright (C) 2021 Pakfire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +#############################################################################*/ + +#ifndef PAKFIRE_COMPRESS_H +#define PAKFIRE_COMPRESS_H + +#ifdef PAKFIRE_PRIVATE + +// Automatically detect +FILE* pakfire_xfopen(FILE* f, const char* mode); + +// XZ +FILE* pakfire_xzfopen(FILE* f, const char* mode); + +#endif + +#endif /* PAKFIRE_COMPRESS_H */ diff --git a/src/libpakfire/repo.c b/src/libpakfire/repo.c index df41f5e67..af235c4b0 100644 --- a/src/libpakfire/repo.c +++ b/src/libpakfire/repo.c @@ -32,9 +32,9 @@ #include #include -#include #include +#include #include #include #include @@ -53,9 +53,6 @@ // Refresh repository metadata every 10 minutes #define REFRESH_AGE_METADATA 600 -const uint8_t XZ_HEADER_MAGIC[] = { 0xFD, '7', 'z', 'X', 'Z', 0x00 }; -const size_t XZ_HEADER_LENGTH = sizeof(XZ_HEADER_MAGIC); - struct pakfire_repo_appdata { Repodata* repodata; @@ -673,123 +670,8 @@ PAKFIRE_EXPORT int pakfire_repo_read_solv(PakfireRepo repo, const char* filename return ret; } -struct xz_cookie { - FILE* f; - lzma_stream stream; - int done; - - uint8_t buffer[1048576]; -}; - -static ssize_t _xz_read(void* data, char* buffer, size_t size) { - struct xz_cookie* cookie = (struct xz_cookie*)data; - if (!cookie) - return -1; - - // Return nothing after we are done - if (cookie->done) - return 0; - - lzma_action action = LZMA_RUN; - - // Set output to allocated buffer - cookie->stream.next_out = (uint8_t *)buffer; - cookie->stream.avail_out = size; - - while (1) { - // Read something when the input buffer is empty - if (cookie->stream.avail_in == 0) { - cookie->stream.next_in = cookie->buffer; - cookie->stream.avail_in = fread(cookie->buffer, - 1, sizeof(*cookie->buffer), cookie->f); - - // Break if the input file could not be read - if (ferror(cookie->f)) - return -1; - - // Finish after we have reached the end of the input file - if (feof(cookie->f)) { - action = LZMA_FINISH; - cookie->done = 1; - } - } - - lzma_ret ret = lzma_code(&cookie->stream, action); - - // If the stream has ended, we just send the - // remaining output and mark that we are done. - if (ret == LZMA_STREAM_END) { - cookie->done = 1; - return size - cookie->stream.avail_out; - } - - // Break on all other unexpected errors - if (ret != LZMA_OK) - return -1; - - // When we have read enough to fill the entire output buffer, we return - if (cookie->stream.avail_out == 0) - return size; - - if (cookie->done) - return -1; - } -} - -static int _xz_close(void* data) { - struct xz_cookie* cookie = (struct xz_cookie*)data; - - // Free the deocder - lzma_end(&cookie->stream); - - // Close input file - fclose(cookie->f); - - return 0; -} - -static FILE* decompression_proxy(FILE* f) { - uint8_t buffer; - - // Search for XZ header - for (unsigned int i = 0; i < XZ_HEADER_LENGTH; i++) { - fread(&buffer, 1, 1, f); - - if (buffer != XZ_HEADER_MAGIC[i]) - goto UNCOMPRESSED; - } - - // Reset to beginning - fseek(f, 0, SEEK_SET); - - // If we get here, an XZ header was found - struct xz_cookie cookie = { - .f = f, - .stream = LZMA_STREAM_INIT, - .done = 0, - }; - - // Initialise the decoder - lzma_ret ret = lzma_stream_decoder(&cookie.stream, UINT64_MAX, 0); - if (ret != LZMA_OK) - return NULL; - - cookie_io_functions_t functions = { - .read = _xz_read, - .write = NULL, - .seek = NULL, - .close = _xz_close, - }; - - return fopencookie(&cookie, "rb", functions); - -UNCOMPRESSED: - fseek(f, 0, SEEK_SET); - return f; -} - PAKFIRE_EXPORT int pakfire_repo_read_solv_fp(PakfireRepo repo, FILE *f, int flags) { - f = decompression_proxy(f); + f = pakfire_xfopen(f, "r"); int ret = repo_add_solv(repo->repo, f, flags);