1 /*#############################################################################
3 # Pakfire - The IPFire package management system #
4 # Copyright (C) 2021 Pakfire development team #
6 # This program is free software: you can redistribute it and/or modify #
7 # it under the terms of the GNU General Public License as published by #
8 # the Free Software Foundation, either version 3 of the License, or #
9 # (at your option) any later version. #
11 # This program is distributed in the hope that it will be useful, #
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
14 # GNU General Public License for more details. #
16 # You should have received a copy of the GNU General Public License #
17 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
19 #############################################################################*/
30 #include <pakfire/compress.h>
31 #include <pakfire/file.h>
32 #include <pakfire/filelist.h>
33 #include <pakfire/logging.h>
34 #include <pakfire/progressbar.h>
35 #include <pakfire/string.h>
36 #include <pakfire/util.h>
38 // Read up to N bytes for analyze the magic
39 #define MAX_MAGIC_LENGTH 6
41 // Compression/Decompression buffer size
42 #define BUFFER_SIZE 64 * 1024
45 #define XZ_COMPRESSION_LEVEL 6
48 #define ZSTD_COMPRESSION_LEVEL 7
50 const struct compressor
{
51 char magic
[MAX_MAGIC_LENGTH
];
53 FILE* (*open
)(FILE* f
, const char* mode
);
56 { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen
, },
58 { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen
, },
63 // Try to guess the compression
64 FILE* pakfire_xfopen(FILE* f
, const char* mode
) {
65 char buffer
[MAX_MAGIC_LENGTH
];
77 // This only works for reading files
86 int r
= fgetpos(f
, &pos
);
90 // Read a couple of bytes
91 size_t bytes_read
= fread(buffer
, 1, sizeof(buffer
), f
);
98 // Check if we could read anything
99 if (!bytes_read
|| bytes_read
< sizeof(buffer
))
103 for (const struct compressor
* c
= compressors
; c
->open
; c
++) {
104 // Check if we have read enough data
105 if (bytes_read
< c
->magic_length
)
108 // Compare the magic value
109 r
= memcmp(c
->magic
, buffer
, c
->magic_length
);
114 return c
->open(f
, mode
);
117 // Nothing seems to match
128 uint8_t buffer
[BUFFER_SIZE
];
131 static ssize_t
xz_read(void* data
, char* buffer
, size_t size
) {
132 struct xz_cookie
* cookie
= (struct xz_cookie
*)data
;
136 // Do not read when mode is "w"
137 if (cookie
->mode
== 'w')
140 lzma_action action
= LZMA_RUN
;
142 // Set output to allocated buffer
143 cookie
->stream
.next_out
= (uint8_t *)buffer
;
144 cookie
->stream
.avail_out
= size
;
147 // Read something when the input buffer is empty
148 if (cookie
->stream
.avail_in
== 0) {
149 cookie
->stream
.next_in
= cookie
->buffer
;
150 cookie
->stream
.avail_in
= fread(cookie
->buffer
,
151 1, sizeof(cookie
->buffer
), cookie
->f
);
153 // Break if the input file could not be read
154 if (ferror(cookie
->f
))
157 // Finish after we have reached the end of the input file
162 lzma_ret ret
= lzma_code(&cookie
->stream
, action
);
164 // If the stream has ended, we just send the
165 // remaining output and mark that we are done.
166 if (ret
== LZMA_STREAM_END
) {
168 return size
- cookie
->stream
.avail_out
;
171 // Break on all other unexpected errors
175 // When we have read enough to fill the entire output buffer, we return
176 if (cookie
->stream
.avail_out
== 0)
184 static ssize_t
xz_write(void* data
, const char* buffer
, size_t size
) {
185 struct xz_cookie
* cookie
= (struct xz_cookie
*)data
;
189 // Do not write when mode is "r"
190 if (cookie
->mode
== 'r')
193 // Return nothing when there is no input
197 // Set input to allocated buffer
198 cookie
->stream
.next_in
= (uint8_t *)buffer
;
199 cookie
->stream
.avail_in
= size
;
202 cookie
->stream
.next_out
= cookie
->buffer
;
203 cookie
->stream
.avail_out
= sizeof(cookie
->buffer
);
205 lzma_ret ret
= lzma_code(&cookie
->stream
, LZMA_RUN
);
209 size_t bytes_to_write
= sizeof(cookie
->buffer
) - cookie
->stream
.avail_out
;
210 if (bytes_to_write
) {
211 size_t bytes_written
= fwrite(cookie
->buffer
, 1, bytes_to_write
, cookie
->f
);
213 if (bytes_written
!= bytes_to_write
)
217 // Report that all data has been written
218 if (cookie
->stream
.avail_in
== 0)
223 static int xz_close(void* data
) {
224 struct xz_cookie
* cookie
= (struct xz_cookie
*)data
;
228 if (cookie
->mode
== 'w') {
230 cookie
->stream
.next_out
= cookie
->buffer
;
231 cookie
->stream
.avail_out
= sizeof(cookie
->buffer
);
233 lzma_ret ret
= lzma_code(&cookie
->stream
, LZMA_FINISH
);
234 if (ret
!= LZMA_OK
&& ret
!= LZMA_STREAM_END
)
237 size_t bytes_to_write
= sizeof(cookie
->buffer
) - cookie
->stream
.avail_out
;
238 if (bytes_to_write
) {
239 size_t bytes_written
= fwrite(cookie
->buffer
, 1, bytes_to_write
, cookie
->f
);
241 if (bytes_written
!= bytes_to_write
)
245 if (ret
== LZMA_STREAM_END
)
250 lzma_end(&cookie
->stream
);
253 int r
= fclose(cookie
->f
);
259 static cookie_io_functions_t xz_functions
= {
266 FILE* pakfire_xzfopen(FILE* f
, const char* mode
) {
279 struct xz_cookie
* cookie
= calloc(1, sizeof(*cookie
));
284 cookie
->mode
= *mode
;
286 switch (cookie
->mode
) {
288 ret
= lzma_stream_decoder(&cookie
->stream
, UINT64_MAX
, 0);
292 ret
= lzma_easy_encoder(&cookie
->stream
, XZ_COMPRESSION_LEVEL
, LZMA_CHECK_SHA256
);
303 return fopencookie(cookie
, mode
, xz_functions
);
318 ZSTD_CStream
* cstream
;
322 ZSTD_DStream
* dstream
;
325 uint8_t buffer
[BUFFER_SIZE
];
328 static ssize_t
zstd_read(void* data
, char* buffer
, size_t size
) {
329 struct zstd_cookie
* cookie
= (struct zstd_cookie
*)data
;
333 // Do not read when mode is "w"
334 if (cookie
->mode
== 'w')
342 // Configure output buffer
343 cookie
->out
.dst
= buffer
;
345 cookie
->out
.size
= size
;
348 if (!feof(cookie
->f
) && (cookie
->in
.pos
== cookie
->in
.size
)) {
350 cookie
->in
.size
= fread(cookie
->buffer
, 1, sizeof(cookie
->buffer
), cookie
->f
);
353 if (r
|| cookie
->in
.size
)
354 r
= ZSTD_decompressStream(cookie
->dstream
, &cookie
->out
, &cookie
->in
);
356 if (r
== 0 && feof(cookie
->f
)) {
358 return cookie
->out
.pos
;
365 if (cookie
->out
.pos
== size
)
370 static ssize_t
zstd_write(void* data
, const char* buffer
, size_t size
) {
371 struct zstd_cookie
* cookie
= (struct zstd_cookie
*)data
;
375 // Do not write when mode is "r"
376 if (cookie
->mode
== 'r')
379 // Return nothing when there is no input
383 // Configure input buffer
384 cookie
->in
.src
= buffer
;
386 cookie
->in
.size
= size
;
391 size_t r
= ZSTD_compressStream(cookie
->cstream
, &cookie
->out
, &cookie
->in
);
395 if (cookie
->out
.pos
> 0) {
396 size_t bytes_written
= fwrite(cookie
->buffer
, 1, cookie
->out
.pos
, cookie
->f
);
398 if (bytes_written
!= cookie
->out
.pos
)
402 // Return when all input has been written
403 if (cookie
->in
.pos
== size
)
408 static int zstd_close(void* data
) {
409 struct zstd_cookie
* cookie
= (struct zstd_cookie
*)data
;
413 if (cookie
->mode
== 'w') {
415 // Reset output buffer
418 size_t r
= ZSTD_endStream(cookie
->cstream
, &cookie
->out
);
422 if (cookie
->out
.pos
> 0) {
423 size_t bytes_written
= fwrite(cookie
->buffer
, 1, cookie
->out
.pos
, cookie
->f
);
425 if (bytes_written
!= cookie
->out
.pos
)
434 int r
= fclose(cookie
->f
);
438 ZSTD_freeCStream(cookie
->cstream
);
440 ZSTD_freeDStream(cookie
->dstream
);
446 static cookie_io_functions_t zstd_functions
= {
453 FILE* pakfire_zstdfopen(FILE* f
, const char* mode
) {
464 struct zstd_cookie
* cookie
= calloc(1, sizeof(*cookie
));
469 cookie
->mode
= *mode
;
472 switch (cookie
->mode
) {
475 cookie
->dstream
= ZSTD_createDStream();
476 if (!cookie
->dstream
)
480 r
= ZSTD_initDStream(cookie
->dstream
);
484 cookie
->in
.src
= cookie
->buffer
;
491 cookie
->cstream
= ZSTD_createCStream();
492 if (!cookie
->cstream
)
496 r
= ZSTD_initCStream(cookie
->cstream
, ZSTD_COMPRESSION_LEVEL
);
500 cookie
->out
.dst
= cookie
->buffer
;
502 cookie
->out
.size
= sizeof(cookie
->buffer
);
510 return fopencookie(cookie
, mode
, zstd_functions
);
514 ZSTD_freeCStream(cookie
->cstream
);
516 ZSTD_freeDStream(cookie
->dstream
);
523 Helper function to conditionally walk through an archive
524 and perform actions based on the callback.
526 int pakfire_walk(struct pakfire
* pakfire
, struct archive
* archive
,
527 pakfire_walk_callback callback
, pakfire_walk_filter_callback filter_callback
,
529 struct archive_entry
* entry
= NULL
;
530 const char* path
= NULL
;
533 // Walk through the archive
535 r
= archive_read_next_header(archive
, &entry
);
537 // Handle the return code
539 // Fall through if everything is okay
543 // Return OK when we reached the end of the archive
547 // Raise any other errors
552 path
= archive_entry_pathname(entry
);
554 DEBUG(pakfire
, "Walking through %s...\n", path
);
556 // Call the filter callback before we call the actual callback
557 if (filter_callback
) {
558 r
= filter_callback(pakfire
, archive
, entry
, p
);
560 // Handle the return code
562 case PAKFIRE_WALK_OK
:
565 case PAKFIRE_WALK_END
:
566 DEBUG(pakfire
, "Filter callback sent END\n");
569 case PAKFIRE_WALK_SKIP
:
570 DEBUG(pakfire
, "Filter callback sent SKIP\n");
573 case PAKFIRE_WALK_DONE
:
574 DEBUG(pakfire
, "Filter callback sent DONE\n");
576 // Clear the callback function
577 filter_callback
= NULL
;
580 // Raise any other errors
582 DEBUG(pakfire
, "Filter callback returned an error: %d\n", r
);
589 r
= callback(pakfire
, archive
, entry
, p
);
591 // Handle the return code
593 case PAKFIRE_WALK_OK
:
596 case PAKFIRE_WALK_DONE
:
597 DEBUG(pakfire
, "Callback sent DONE\n");
600 // Raise any other errors
612 struct pakfire_extract
{
613 // Reference to Pakfire
614 struct pakfire
* pakfire
;
619 // The archive to extract
620 struct archive
* archive
;
622 // The filelist of all extracted files
623 struct pakfire_filelist
* filelist
;
625 // Prepend this prefix
626 char prefix
[PATH_MAX
];
629 struct archive
* writer
;
632 struct pakfire_progressbar
* progressbar
;
635 static int pakfire_extract_progressbar_create(struct pakfire_progressbar
** progressbar
,
636 const char* message
, int flags
) {
639 // Create the progressbar
640 r
= pakfire_progressbar_create(progressbar
, NULL
);
646 r
= pakfire_progressbar_add_string(*progressbar
, "%s", message
);
652 r
= pakfire_progressbar_add_bar(*progressbar
);
657 if (flags
& PAKFIRE_EXTRACT_SHOW_THROUGHPUT
) {
658 r
= pakfire_progressbar_add_transfer_speed(*progressbar
);
664 r
= pakfire_progressbar_add_percentage(*progressbar
);
672 static void pakfire_extract_progress(void* p
) {
673 struct pakfire_extract
* data
= (struct pakfire_extract
*)p
;
675 // Fetch how many bytes have been read
676 const size_t position
= archive_filter_bytes(data
->archive
, -1);
678 // Update progressbar
679 pakfire_progressbar_update(data
->progressbar
, position
);
682 static int __pakfire_extract(struct pakfire
* pakfire
, struct archive
* a
,
683 struct archive_entry
* entry
, void* p
) {
684 struct pakfire_file
* file
= NULL
;
685 char buffer
[PATH_MAX
];
688 struct pakfire_extract
* data
= (struct pakfire_extract
*)p
;
691 const char* path
= archive_entry_pathname(entry
);
693 // Generate a file object
694 r
= pakfire_file_create_from_archive_entry(&file
, pakfire
, entry
);
698 // Add entry to filelist (if requested)
699 if (data
->filelist
) {
700 // Append the file to the list
701 r
= pakfire_filelist_add(data
->filelist
, file
);
706 const int configfile
= pakfire_file_has_flag(file
, PAKFIRE_FILE_CONFIG
);
708 // Prepend the prefix
711 r
= pakfire_path_join(buffer
, data
->prefix
, path
);
713 ERROR(pakfire
, "Could not compose file path: %m\n");
718 archive_entry_set_pathname(entry
, buffer
);
720 // Update hardlink destination
721 const char* link
= archive_entry_hardlink(entry
);
723 r
= pakfire_path_join(buffer
, data
->prefix
, link
);
725 ERROR(pakfire
, "Could not compose hardlink path: %m\n");
730 archive_entry_set_hardlink(entry
, buffer
);
735 // Fetch path again since we changed it
736 path
= archive_entry_pathname(entry
);
738 if (pakfire_path_exists(path
)) {
739 DEBUG(pakfire
, "The configuration file %s exists\n",
740 pakfire_file_get_path(file
));
742 r
= pakfire_string_format(buffer
, "%s.paknew", path
);
744 ERROR(pakfire
, "Could not compose path for configuration file: %m\n");
748 // Set the path again
749 archive_entry_set_pathname(entry
, buffer
);
753 // Create file & extract payload
755 // Fetch path again since we changed it
756 path
= archive_entry_pathname(entry
);
758 DEBUG(pakfire
, "Extracting %s\n", path
);
760 // Remove any extended attributes which we never write to disk
761 archive_entry_xattr_clear(entry
);
764 r
= archive_read_extract2(data
->archive
, entry
, data
->writer
);
771 ERROR(pakfire
, "%s\n", archive_error_string(data
->writer
));
773 // Pretend everything has been okay
778 ERROR(pakfire
, "%s\n", archive_error_string(data
->writer
));
786 pakfire_file_unref(file
);
791 int pakfire_extract(struct pakfire
* pakfire
, struct archive
* archive
,
792 size_t size
, struct pakfire_filelist
* filelist
,
793 const char* prefix
, const char* message
,
794 pakfire_walk_filter_callback filter_callback
, int flags
) {
797 // Use an empty string if no prefix set
801 struct pakfire_extract data
= {
804 .filelist
= filelist
,
809 // Is this a dry run?
810 const int dry_run
= flags
& PAKFIRE_EXTRACT_DRY_RUN
;
812 // Should we show a progress bar?
813 const int no_progress
= flags
& PAKFIRE_EXTRACT_NO_PROGRESS
;
815 // Set prefix (including pakfire path)
816 r
= pakfire_path(pakfire
, data
.prefix
, "%s", prefix
);
822 data
.writer
= pakfire_make_archive_disk_writer(pakfire
, 1);
824 ERROR(pakfire
, "Could not create disk writer: %m\n");
830 // Create the progressbar
832 r
= pakfire_extract_progressbar_create(&data
.progressbar
, message
, flags
);
836 // Register progress callback
837 archive_read_extract_set_progress_callback(data
.archive
,
838 pakfire_extract_progress
, &data
);
841 pakfire_progressbar_start(data
.progressbar
, size
);
844 // Walk through the entire archive
845 r
= pakfire_walk(pakfire
, archive
, __pakfire_extract
, filter_callback
, &data
);
849 // Finish the progressbar
850 if (data
.progressbar
)
851 pakfire_progressbar_finish(data
.progressbar
);
854 if (data
.progressbar
)
855 pakfire_progressbar_unref(data
.progressbar
);
857 archive_write_free(data
.writer
);
862 // Common compression
864 struct pakfire_compress
{
865 // Reference to Pakfire
866 struct pakfire
* pakfire
;
871 // The archive to write to
872 struct archive
* archive
;
874 // Resolver for hardlinks
875 struct archive_entry_linkresolver
* linkresolver
;
877 // The filelist of all files to write
878 struct pakfire_filelist
* filelist
;
881 struct pakfire_progressbar
* progressbar
;
883 // Digests to write to the archive
887 static int pakfire_compress_progressbar_create(struct pakfire_progressbar
** progressbar
,
888 const char* message
, int flags
) {
891 // Create the progressbar
892 r
= pakfire_progressbar_create(progressbar
, NULL
);
898 r
= pakfire_progressbar_add_string(*progressbar
, "%s", message
);
904 r
= pakfire_progressbar_add_bar(*progressbar
);
909 if (flags
& PAKFIRE_COMPRESS_SHOW_THROUGHPUT
) {
910 r
= pakfire_progressbar_add_transfer_speed(*progressbar
);
915 // Show how many bytes have been written
916 r
= pakfire_progressbar_add_bytes_transferred(*progressbar
);
921 r
= pakfire_progressbar_add_percentage(*progressbar
);
929 static int __pakfire_compress_entry(struct pakfire
* pakfire
, struct pakfire_file
* file
,
930 struct pakfire_compress
* data
, struct archive_entry
* entry
) {
935 r
= archive_write_header(data
->archive
, entry
);
937 ERROR(pakfire
, "Error writing file header: %s\n",
938 archive_error_string(data
->archive
));
942 // Copy the data if there is any
943 if (archive_entry_size(entry
)) {
945 f
= pakfire_file_open(file
);
951 // Copy the payload into the archive
952 r
= pakfire_archive_copy_data_from_file(pakfire
, data
->archive
, f
);
958 r
= archive_write_finish_entry(data
->archive
);
969 static int __pakfire_compress(struct pakfire
* pakfire
, struct pakfire_file
* file
, void* p
) {
970 struct archive_entry
* entry
= NULL
;
971 struct archive_entry
* sparse_entry
= NULL
;
974 struct pakfire_compress
* data
= (struct pakfire_compress
*)p
;
976 // Fetch the file size
977 const size_t size
= pakfire_file_get_size(file
);
979 // Generate file metadata into an archive entry
980 entry
= pakfire_file_archive_entry(file
, data
->digests
);
986 // Perform search for hardlinks
987 archive_entry_linkify(data
->linkresolver
, &entry
, &sparse_entry
);
989 // Write the main entry
991 r
= __pakfire_compress_entry(pakfire
, file
, data
, entry
);
996 // Write the sparse entry
998 r
= __pakfire_compress_entry(pakfire
, file
, data
, sparse_entry
);
1003 // Query the link resolver for any more entries
1007 archive_entry_free(entry
);
1011 // Free the sparse entry
1013 archive_entry_free(sparse_entry
);
1014 sparse_entry
= NULL
;
1017 // Fetch the next entry
1018 archive_entry_linkify(data
->linkresolver
, &entry
, &sparse_entry
);
1022 // Write the entry to the archive
1023 r
= __pakfire_compress_entry(pakfire
, file
, data
, entry
);
1028 // Update the progressbar
1029 if (data
->progressbar
)
1030 pakfire_progressbar_increment(data
->progressbar
, size
);
1034 archive_entry_free(entry
);
1036 archive_entry_free(sparse_entry
);
1041 int pakfire_compress(struct pakfire
* pakfire
, struct archive
* archive
,
1042 struct pakfire_filelist
* filelist
, const char* message
, int flags
, int digests
) {
1045 struct pakfire_compress data
= {
1048 .filelist
= filelist
,
1053 // Should we show a progress bar?
1054 const int no_progress
= flags
& PAKFIRE_COMPRESS_NO_PROGRESS
;
1056 // Fetch the length of the filelist
1057 const size_t size
= pakfire_filelist_total_size(filelist
);
1059 // Create the progressbar
1061 r
= pakfire_compress_progressbar_create(&data
.progressbar
, message
, flags
);
1065 // Start progressbar
1066 pakfire_progressbar_start(data
.progressbar
, size
);
1069 // Setup the link resolver
1070 data
.linkresolver
= archive_entry_linkresolver_new();
1071 if (!data
.linkresolver
) {
1072 ERROR(pakfire
, "Could not setup link resolver: m\n");
1076 // Set the link resolver strategy
1077 archive_entry_linkresolver_set_strategy(data
.linkresolver
, archive_format(archive
));
1079 // Walk through the entire filelist
1080 r
= pakfire_filelist_walk(filelist
, __pakfire_compress
, &data
);
1084 // Finish the progressbar
1085 if (data
.progressbar
)
1086 pakfire_progressbar_finish(data
.progressbar
);
1089 if (data
.progressbar
)
1090 pakfire_progressbar_unref(data
.progressbar
);
1091 if (data
.linkresolver
)
1092 archive_entry_linkresolver_free(data
.linkresolver
);
1097 int pakfire_compress_create_archive(struct pakfire
* pakfire
, struct archive
** archive
,
1098 FILE* f
, const enum pakfire_compressions compression
, const unsigned int level
) {
1099 struct archive
* a
= NULL
;
1103 // Open a new archive
1104 a
= archive_write_new();
1106 ERROR(pakfire
, "archive_write_new() failed\n");
1111 // Use the PAX format
1112 r
= archive_write_set_format_pax(a
);
1114 ERROR(pakfire
, "Could not set format to PAX: %s\n", archive_error_string(a
));
1118 // Store any extended attributes in the SCHILY headers
1119 r
= archive_write_set_format_option(a
, "pax", "xattrheader", "SCHILY");
1121 ERROR(pakfire
, "Could not set xattrheader option: %s\n", archive_error_string(a
));
1125 switch (compression
) {
1126 case PAKFIRE_COMPRESS_ZSTD
:
1128 r
= archive_write_add_filter_zstd(a
);
1130 ERROR(pakfire
, "Could not enable Zstandard compression: %s\n",
1131 archive_error_string(a
));
1135 // Do not pad the last block
1136 archive_write_set_bytes_in_last_block(a
, 1);
1138 // Set compression level
1140 r
= pakfire_string_format(value
, "%u", level
);
1144 r
= archive_write_set_filter_option(a
, NULL
, "compression-level", value
);
1146 ERROR(pakfire
, "Could not set Zstandard compression level: %s\n",
1147 archive_error_string(a
));
1152 #if ARCHIVE_VERSION_NUMBER >= 3006000
1153 // Fetch numbers of processors
1154 long processors
= sysconf(_SC_NPROCESSORS_ONLN
);
1156 if (processors
> 1) {
1157 r
= pakfire_string_format(value
, "%ld", processors
);
1159 ERROR(pakfire
, "Could not format threads: %m\n");
1163 // Try using multiple threads
1164 r
= archive_write_set_filter_option(a
, NULL
, "threads", value
);
1166 ERROR(pakfire
, "Could not enable %ld threads for compression: %s\n",
1167 processors
, archive_error_string(a
));
1174 // Write archive to f
1176 r
= archive_write_open_FILE(a
, f
);
1178 ERROR(pakfire
, "archive_write_open_FILE() failed: %s\n", archive_error_string(a
));
1190 archive_write_free(a
);