]>
Commit | Line | Data |
---|---|---|
c352b776 MT |
1 | /*############################################################################# |
2 | # # | |
3 | # Pakfire - The IPFire package management system # | |
4 | # Copyright (C) 2021 Pakfire development team # | |
5 | # # | |
6 | # This program is free software: you can redistribute it and/or modify # | |
7 | # it under the terms of the GNU General Public License as published by # | |
8 | # the Free Software Foundation, either version 3 of the License, or # | |
9 | # (at your option) any later version. # | |
10 | # # | |
11 | # This program is distributed in the hope that it will be useful, # | |
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # | |
14 | # GNU General Public License for more details. # | |
15 | # # | |
16 | # You should have received a copy of the GNU General Public License # | |
17 | # along with this program. If not, see <http://www.gnu.org/licenses/>. # | |
18 | # # | |
19 | #############################################################################*/ | |
20 | ||
64f977a3 | 21 | #include <errno.h> |
c352b776 | 22 | #include <stdio.h> |
99018421 | 23 | #include <stdlib.h> |
c352b776 MT |
24 | #include <string.h> |
25 | ||
79824416 | 26 | #include <archive.h> |
c352b776 | 27 | #include <lzma.h> |
9476d502 | 28 | #include <zstd.h> |
c352b776 MT |
29 | |
30 | #include <pakfire/compress.h> | |
bb7f09cd MT |
31 | #include <pakfire/file.h> |
32 | #include <pakfire/filelist.h> | |
79824416 MT |
33 | #include <pakfire/logging.h> |
34 | #include <pakfire/progressbar.h> | |
24553159 | 35 | #include <pakfire/string.h> |
79824416 | 36 | #include <pakfire/util.h> |
c352b776 MT |
37 | |
38 | // Read up to N bytes for analyze the magic | |
39 | #define MAX_MAGIC_LENGTH 6 | |
40 | ||
5cd454df MT |
41 | // Compression/Decompression buffer size |
42 | #define BUFFER_SIZE 64 * 1024 | |
43 | ||
0bed1e1d MT |
44 | // Settings for XZ |
45 | #define XZ_COMPRESSION_LEVEL 6 | |
46 | ||
9476d502 MT |
47 | // Settings for ZSTD |
48 | #define ZSTD_COMPRESSION_LEVEL 7 | |
49 | ||
c352b776 MT |
50 | const struct compressor { |
51 | char magic[MAX_MAGIC_LENGTH]; | |
52 | size_t magic_length; | |
53 | FILE* (*open)(FILE* f, const char* mode); | |
54 | } compressors[] = { | |
55 | // XZ | |
56 | { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, }, | |
60732b6e MT |
57 | // ZSTD |
58 | { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, }, | |
59 | // End | |
c352b776 MT |
60 | { "", 0, NULL, }, |
61 | }; | |
62 | ||
c352b776 MT |
63 | // Try to guess the compression |
64 | FILE* pakfire_xfopen(FILE* f, const char* mode) { | |
65 | char buffer[MAX_MAGIC_LENGTH]; | |
66 | ||
64f977a3 MT |
67 | if (!f) { |
68 | errno = EBADFD; | |
c352b776 | 69 | return NULL; |
64f977a3 MT |
70 | } |
71 | ||
72 | if (!mode) { | |
73 | errno = EINVAL; | |
74 | return NULL; | |
75 | } | |
76 | ||
77 | // This only works for reading files | |
78 | if (*mode != 'r') { | |
79 | errno = ENOTSUP; | |
80 | return NULL; | |
81 | } | |
c352b776 MT |
82 | |
83 | fpos_t pos; | |
84 | ||
85 | // Store the position | |
86 | int r = fgetpos(f, &pos); | |
87 | if (r < 0) | |
88 | return NULL; | |
89 | ||
90 | // Read a couple of bytes | |
91 | size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); | |
92 | ||
93 | // Reset position | |
94 | r = fsetpos(f, &pos); | |
95 | if (r < 0) | |
96 | return NULL; | |
97 | ||
98 | // Check if we could read anything | |
99 | if (!bytes_read || bytes_read < sizeof(buffer)) | |
100 | return f; | |
101 | ||
102 | // Analyze magic | |
103 | for (const struct compressor* c = compressors; c->open; c++) { | |
104 | // Check if we have read enough data | |
105 | if (bytes_read < c->magic_length) | |
106 | continue; | |
107 | ||
108 | // Compare the magic value | |
109 | r = memcmp(c->magic, buffer, c->magic_length); | |
110 | if (r) | |
111 | continue; | |
112 | ||
113 | // We found a match! | |
114 | return c->open(f, mode); | |
115 | } | |
116 | ||
117 | // Nothing seems to match | |
49492077 | 118 | errno = ENOTSUP; |
c352b776 MT |
119 | return f; |
120 | } | |
121 | ||
5cd454df MT |
122 | struct xz_cookie { |
123 | FILE* f; | |
124 | char mode; | |
125 | lzma_stream stream; | |
126 | int done; | |
127 | ||
128 | uint8_t buffer[BUFFER_SIZE]; | |
129 | }; | |
130 | ||
c352b776 MT |
131 | static ssize_t xz_read(void* data, char* buffer, size_t size) { |
132 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
133 | if (!cookie) | |
134 | return -1; | |
135 | ||
48187ff3 MT |
136 | // Do not read when mode is "w" |
137 | if (cookie->mode == 'w') | |
6923552a | 138 | return -1; |
48187ff3 | 139 | |
c352b776 MT |
140 | lzma_action action = LZMA_RUN; |
141 | ||
142 | // Set output to allocated buffer | |
143 | cookie->stream.next_out = (uint8_t *)buffer; | |
144 | cookie->stream.avail_out = size; | |
145 | ||
146 | while (1) { | |
147 | // Read something when the input buffer is empty | |
148 | if (cookie->stream.avail_in == 0) { | |
149 | cookie->stream.next_in = cookie->buffer; | |
150 | cookie->stream.avail_in = fread(cookie->buffer, | |
5cd454df | 151 | 1, sizeof(cookie->buffer), cookie->f); |
c352b776 MT |
152 | |
153 | // Break if the input file could not be read | |
154 | if (ferror(cookie->f)) | |
155 | return -1; | |
156 | ||
157 | // Finish after we have reached the end of the input file | |
5e12758c | 158 | if (feof(cookie->f)) |
c352b776 | 159 | cookie->done = 1; |
c352b776 MT |
160 | } |
161 | ||
162 | lzma_ret ret = lzma_code(&cookie->stream, action); | |
163 | ||
164 | // If the stream has ended, we just send the | |
165 | // remaining output and mark that we are done. | |
166 | if (ret == LZMA_STREAM_END) { | |
167 | cookie->done = 1; | |
168 | return size - cookie->stream.avail_out; | |
169 | } | |
170 | ||
171 | // Break on all other unexpected errors | |
172 | if (ret != LZMA_OK) | |
173 | return -1; | |
174 | ||
175 | // When we have read enough to fill the entire output buffer, we return | |
176 | if (cookie->stream.avail_out == 0) | |
177 | return size; | |
178 | ||
179 | if (cookie->done) | |
180 | return -1; | |
181 | } | |
182 | } | |
183 | ||
0bed1e1d MT |
184 | static ssize_t xz_write(void* data, const char* buffer, size_t size) { |
185 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
186 | if (!cookie) | |
187 | return -1; | |
188 | ||
189 | // Do not write when mode is "r" | |
190 | if (cookie->mode == 'r') | |
6923552a | 191 | return -1; |
0bed1e1d MT |
192 | |
193 | // Return nothing when there is no input | |
194 | if (size == 0) | |
195 | return 0; | |
196 | ||
197 | // Set input to allocated buffer | |
9d816df3 MT |
198 | cookie->stream.next_in = (uint8_t *)buffer; |
199 | cookie->stream.avail_in = size; | |
0bed1e1d MT |
200 | |
201 | while (1) { | |
202 | cookie->stream.next_out = cookie->buffer; | |
203 | cookie->stream.avail_out = sizeof(cookie->buffer); | |
204 | ||
205 | lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN); | |
206 | if (ret != LZMA_OK) | |
207 | return -1; | |
208 | ||
209 | size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; | |
210 | if (bytes_to_write) { | |
211 | size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); | |
212 | ||
213 | if (bytes_written != bytes_to_write) | |
214 | return -1; | |
215 | } | |
216 | ||
217 | // Report that all data has been written | |
218 | if (cookie->stream.avail_in == 0) | |
219 | return size; | |
220 | } | |
221 | } | |
222 | ||
c352b776 MT |
223 | static int xz_close(void* data) { |
224 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
99018421 MT |
225 | if (!cookie) |
226 | return -1; | |
227 | ||
228 | if (cookie->mode == 'w') { | |
229 | while (1) { | |
230 | cookie->stream.next_out = cookie->buffer; | |
231 | cookie->stream.avail_out = sizeof(cookie->buffer); | |
232 | ||
233 | lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH); | |
234 | if (ret != LZMA_OK && ret != LZMA_STREAM_END) | |
235 | return -1; | |
236 | ||
237 | size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; | |
238 | if (bytes_to_write) { | |
239 | size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); | |
240 | ||
241 | if (bytes_written != bytes_to_write) | |
242 | return -1; | |
243 | } | |
244 | ||
245 | if (ret == LZMA_STREAM_END) | |
246 | break; | |
247 | } | |
248 | } | |
c352b776 | 249 | |
c352b776 MT |
250 | lzma_end(&cookie->stream); |
251 | ||
252 | // Close input file | |
99018421 MT |
253 | int r = fclose(cookie->f); |
254 | free(cookie); | |
c352b776 | 255 | |
99018421 | 256 | return r; |
c352b776 MT |
257 | } |
258 | ||
99018421 MT |
259 | static cookie_io_functions_t xz_functions = { |
260 | .read = xz_read, | |
261 | .write = xz_write, | |
262 | .seek = NULL, | |
263 | .close = xz_close, | |
264 | }; | |
265 | ||
c352b776 | 266 | FILE* pakfire_xzfopen(FILE* f, const char* mode) { |
99018421 MT |
267 | lzma_ret ret; |
268 | ||
48187ff3 MT |
269 | if (!f) { |
270 | errno = EBADFD; | |
271 | return NULL; | |
272 | } | |
273 | ||
274 | if (!mode) { | |
275 | errno = EINVAL; | |
276 | return NULL; | |
277 | } | |
278 | ||
99018421 MT |
279 | struct xz_cookie* cookie = calloc(1, sizeof(*cookie)); |
280 | if (!cookie) | |
281 | return NULL; | |
c352b776 | 282 | |
99018421 MT |
283 | cookie->f = f; |
284 | cookie->mode = *mode; | |
0bed1e1d | 285 | |
99018421 | 286 | switch (cookie->mode) { |
0bed1e1d | 287 | case 'r': |
99018421 | 288 | ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0); |
0bed1e1d MT |
289 | break; |
290 | ||
291 | case 'w': | |
99018421 | 292 | ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256); |
0bed1e1d MT |
293 | break; |
294 | ||
295 | default: | |
296 | errno = ENOTSUP; | |
2d4af759 | 297 | goto ERROR; |
0bed1e1d MT |
298 | } |
299 | ||
c352b776 | 300 | if (ret != LZMA_OK) |
2d4af759 | 301 | goto ERROR; |
c352b776 | 302 | |
99018421 | 303 | return fopencookie(cookie, mode, xz_functions); |
2d4af759 MT |
304 | |
305 | ERROR: | |
306 | free(cookie); | |
307 | return NULL; | |
c352b776 | 308 | } |
9476d502 MT |
309 | |
310 | // ZSTD | |
311 | ||
312 | struct zstd_cookie { | |
313 | FILE* f; | |
314 | char mode; | |
315 | int done; | |
316 | ||
317 | // Encoder | |
318 | ZSTD_CStream* cstream; | |
319 | ZSTD_inBuffer in; | |
320 | ||
321 | // Decoder | |
322 | ZSTD_DStream* dstream; | |
323 | ZSTD_outBuffer out; | |
324 | ||
325 | uint8_t buffer[BUFFER_SIZE]; | |
326 | }; | |
327 | ||
328 | static ssize_t zstd_read(void* data, char* buffer, size_t size) { | |
329 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
330 | if (!cookie) | |
331 | return -1; | |
332 | ||
333 | // Do not read when mode is "w" | |
334 | if (cookie->mode == 'w') | |
335 | return -1; | |
336 | ||
337 | if (cookie->done) | |
338 | return 0; | |
339 | ||
340 | size_t r = 0; | |
341 | ||
342 | // Configure output buffer | |
343 | cookie->out.dst = buffer; | |
344 | cookie->out.pos = 0; | |
345 | cookie->out.size = size; | |
346 | ||
347 | while (1) { | |
348 | if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) { | |
349 | cookie->in.pos = 0; | |
350 | cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f); | |
351 | } | |
352 | ||
353 | if (r || cookie->in.size) | |
354 | r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in); | |
355 | ||
356 | if (r == 0 && feof(cookie->f)) { | |
357 | cookie->done = 1; | |
358 | return cookie->out.pos; | |
359 | } | |
360 | ||
361 | if (ZSTD_isError(r)) | |
362 | return -1; | |
363 | ||
364 | // Buffer full | |
365 | if (cookie->out.pos == size) | |
366 | return size; | |
367 | } | |
368 | } | |
369 | ||
99a1527a MT |
370 | static ssize_t zstd_write(void* data, const char* buffer, size_t size) { |
371 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
372 | if (!cookie) | |
373 | return -1; | |
374 | ||
375 | // Do not write when mode is "r" | |
376 | if (cookie->mode == 'r') | |
377 | return -1; | |
378 | ||
379 | // Return nothing when there is no input | |
380 | if (size == 0) | |
381 | return 0; | |
382 | ||
383 | // Configure input buffer | |
384 | cookie->in.src = buffer; | |
385 | cookie->in.pos = 0; | |
386 | cookie->in.size = size; | |
387 | ||
388 | while (1) { | |
389 | cookie->out.pos = 0; | |
390 | ||
391 | size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in); | |
392 | if (ZSTD_isError(r)) | |
393 | return -1; | |
394 | ||
395 | if (cookie->out.pos > 0) { | |
396 | size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); | |
397 | ||
398 | if (bytes_written != cookie->out.pos) | |
399 | return -1; | |
400 | } | |
401 | ||
402 | // Return when all input has been written | |
403 | if (cookie->in.pos == size) | |
404 | return size; | |
405 | } | |
406 | } | |
407 | ||
9476d502 MT |
408 | static int zstd_close(void* data) { |
409 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
410 | if (!cookie) | |
411 | return -1; | |
412 | ||
99a1527a MT |
413 | if (cookie->mode == 'w') { |
414 | while (1) { | |
415 | // Reset output buffer | |
416 | cookie->out.pos = 0; | |
417 | ||
418 | size_t r = ZSTD_endStream(cookie->cstream, &cookie->out); | |
419 | if (ZSTD_isError(r)) | |
420 | return -1; | |
421 | ||
422 | if (cookie->out.pos > 0) { | |
423 | size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); | |
424 | ||
425 | if (bytes_written != cookie->out.pos) | |
426 | return -1; | |
427 | } | |
428 | ||
429 | if (r == 0) | |
430 | break; | |
431 | } | |
432 | } | |
9476d502 MT |
433 | |
434 | int r = fclose(cookie->f); | |
435 | ||
436 | // Free everything | |
437 | if (cookie->cstream) | |
438 | ZSTD_freeCStream(cookie->cstream); | |
439 | if (cookie->dstream) | |
440 | ZSTD_freeDStream(cookie->dstream); | |
441 | free(cookie); | |
442 | ||
443 | return r; | |
444 | } | |
445 | ||
446 | static cookie_io_functions_t zstd_functions = { | |
447 | .read = zstd_read, | |
99a1527a | 448 | .write = zstd_write, |
9476d502 MT |
449 | .seek = NULL, |
450 | .close = zstd_close, | |
451 | }; | |
452 | ||
453 | FILE* pakfire_zstdfopen(FILE* f, const char* mode) { | |
454 | if (!f) { | |
455 | errno = EBADFD; | |
456 | return NULL; | |
457 | } | |
458 | ||
459 | if (!mode) { | |
460 | errno = EINVAL; | |
461 | return NULL; | |
462 | } | |
463 | ||
464 | struct zstd_cookie* cookie = calloc(1, sizeof(*cookie)); | |
465 | if (!cookie) | |
466 | return NULL; | |
467 | ||
468 | cookie->f = f; | |
469 | cookie->mode = *mode; | |
470 | ||
471 | size_t r; | |
472 | switch (cookie->mode) { | |
473 | case 'r': | |
474 | // Allocate stream | |
475 | cookie->dstream = ZSTD_createDStream(); | |
476 | if (!cookie->dstream) | |
477 | goto ERROR; | |
478 | ||
479 | // Initialize stream | |
480 | r = ZSTD_initDStream(cookie->dstream); | |
481 | if (ZSTD_isError(r)) | |
482 | goto ERROR; | |
483 | ||
484 | cookie->in.src = cookie->buffer; | |
485 | cookie->in.pos = 0; | |
486 | cookie->in.size = 0; | |
487 | break; | |
488 | ||
489 | case 'w': | |
490 | // Allocate stream | |
491 | cookie->cstream = ZSTD_createCStream(); | |
492 | if (!cookie->cstream) | |
493 | goto ERROR; | |
494 | ||
495 | // Initialize stream | |
496 | r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL); | |
497 | if (ZSTD_isError(r)) | |
498 | goto ERROR; | |
499 | ||
500 | cookie->out.dst = cookie->buffer; | |
501 | cookie->out.pos = 0; | |
502 | cookie->out.size = sizeof(cookie->buffer); | |
503 | break; | |
504 | ||
505 | default: | |
506 | errno = ENOTSUP; | |
507 | goto ERROR; | |
508 | } | |
509 | ||
510 | return fopencookie(cookie, mode, zstd_functions); | |
511 | ||
512 | ERROR: | |
513 | if (cookie->cstream) | |
514 | ZSTD_freeCStream(cookie->cstream); | |
515 | if (cookie->dstream) | |
516 | ZSTD_freeDStream(cookie->dstream); | |
517 | free(cookie); | |
518 | ||
519 | return NULL; | |
520 | } | |
79824416 | 521 | |
d9302cc3 MT |
522 | /* |
523 | Helper function to conditionally walk through an archive | |
524 | and perform actions based on the callback. | |
525 | */ | |
526 | int pakfire_walk(struct pakfire* pakfire, struct archive* archive, | |
527 | pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback, | |
528 | void* p) { | |
529 | struct archive_entry* entry = NULL; | |
662ff8e8 | 530 | const char* path = NULL; |
d9302cc3 MT |
531 | int r; |
532 | ||
533 | // Walk through the archive | |
534 | for (;;) { | |
535 | r = archive_read_next_header(archive, &entry); | |
536 | ||
537 | // Handle the return code | |
538 | switch (r) { | |
0e25bebb MT |
539 | // Fall through if everything is okay |
540 | case ARCHIVE_OK: | |
541 | break; | |
542 | ||
d9302cc3 MT |
543 | // Return OK when we reached the end of the archive |
544 | case ARCHIVE_EOF: | |
545 | return 0; | |
546 | ||
547 | // Raise any other errors | |
548 | default: | |
549 | return r; | |
550 | } | |
551 | ||
662ff8e8 MT |
552 | path = archive_entry_pathname(entry); |
553 | ||
554 | DEBUG(pakfire, "Walking through %s...\n", path); | |
555 | ||
d9302cc3 MT |
556 | // Call the filter callback before we call the actual callback |
557 | if (filter_callback) { | |
558 | r = filter_callback(pakfire, archive, entry, p); | |
559 | ||
560 | // Handle the return code | |
561 | switch (r) { | |
562 | case PAKFIRE_WALK_OK: | |
563 | break; | |
564 | ||
2480e018 MT |
565 | case PAKFIRE_WALK_END: |
566 | DEBUG(pakfire, "Filter callback sent END\n"); | |
d9302cc3 MT |
567 | return 0; |
568 | ||
569 | case PAKFIRE_WALK_SKIP: | |
570 | DEBUG(pakfire, "Filter callback sent SKIP\n"); | |
571 | continue; | |
572 | ||
2480e018 MT |
573 | case PAKFIRE_WALK_DONE: |
574 | DEBUG(pakfire, "Filter callback sent DONE\n"); | |
f8a46d2f MT |
575 | |
576 | // Clear the callback function | |
577 | filter_callback = NULL; | |
578 | break; | |
579 | ||
d9302cc3 MT |
580 | // Raise any other errors |
581 | default: | |
662ff8e8 | 582 | DEBUG(pakfire, "Filter callback returned an error: %d\n", r); |
d9302cc3 MT |
583 | return r; |
584 | } | |
585 | } | |
586 | ||
587 | // Run callback | |
588 | if (callback) { | |
589 | r = callback(pakfire, archive, entry, p); | |
590 | ||
591 | // Handle the return code | |
592 | switch (r) { | |
593 | case PAKFIRE_WALK_OK: | |
594 | break; | |
595 | ||
596 | case PAKFIRE_WALK_DONE: | |
597 | DEBUG(pakfire, "Callback sent DONE\n"); | |
598 | return 0; | |
599 | ||
600 | // Raise any other errors | |
601 | default: | |
602 | return r; | |
603 | } | |
604 | } | |
605 | } | |
606 | ||
607 | return 0; | |
608 | } | |
609 | ||
79824416 MT |
610 | // Common extraction |
611 | ||
612 | struct pakfire_extract { | |
613 | // Reference to Pakfire | |
614 | struct pakfire* pakfire; | |
615 | ||
a3b276d1 MT |
616 | // Flags |
617 | int flags; | |
618 | ||
79824416 MT |
619 | // The archive to extract |
620 | struct archive* archive; | |
621 | ||
d05af3ba MT |
622 | // The filelist of all extracted files |
623 | struct pakfire_filelist* filelist; | |
624 | ||
79824416 MT |
625 | // Prepend this prefix |
626 | char prefix[PATH_MAX]; | |
627 | ||
628 | // The writer | |
629 | struct archive* writer; | |
630 | ||
631 | // The progressbar | |
632 | struct pakfire_progressbar* progressbar; | |
633 | }; | |
634 | ||
635 | static int pakfire_extract_progressbar_create(struct pakfire_progressbar** progressbar, | |
636 | const char* message, int flags) { | |
637 | int r; | |
638 | ||
639 | // Create the progressbar | |
640 | r = pakfire_progressbar_create(progressbar, NULL); | |
641 | if (r) | |
642 | return r; | |
643 | ||
644 | // Add message | |
645 | if (message) { | |
646 | r = pakfire_progressbar_add_string(*progressbar, "%s", message); | |
647 | if (r) | |
648 | return r; | |
649 | } | |
650 | ||
651 | // Add bar | |
652 | r = pakfire_progressbar_add_bar(*progressbar); | |
653 | if (r) | |
654 | return r; | |
655 | ||
95a232a7 MT |
656 | // Add throughput |
657 | if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) { | |
658 | r = pakfire_progressbar_add_transfer_speed(*progressbar); | |
659 | if (r) | |
660 | return r; | |
661 | } | |
662 | ||
79824416 MT |
663 | // Add percentage |
664 | r = pakfire_progressbar_add_percentage(*progressbar); | |
665 | if (r) | |
666 | return r; | |
667 | ||
668 | // Success | |
669 | return 0; | |
670 | } | |
671 | ||
672 | static void pakfire_extract_progress(void* p) { | |
673 | struct pakfire_extract* data = (struct pakfire_extract*)p; | |
674 | ||
675 | // Fetch how many bytes have been read | |
676 | const size_t position = archive_filter_bytes(data->archive, -1); | |
677 | ||
678 | // Update progressbar | |
679 | pakfire_progressbar_update(data->progressbar, position); | |
680 | } | |
681 | ||
14eedd80 MT |
682 | static int __pakfire_extract(struct pakfire* pakfire, struct archive* a, |
683 | struct archive_entry* entry, void* p) { | |
bb7f09cd | 684 | struct pakfire_file* file = NULL; |
79824416 MT |
685 | char buffer[PATH_MAX]; |
686 | int r; | |
687 | ||
14eedd80 MT |
688 | struct pakfire_extract* data = (struct pakfire_extract*)p; |
689 | ||
79824416 MT |
690 | // Fetch path |
691 | const char* path = archive_entry_pathname(entry); | |
692 | ||
24553159 MT |
693 | // Generate a file object |
694 | r = pakfire_file_create_from_archive_entry(&file, pakfire, entry); | |
695 | if (r) | |
696 | goto ERROR; | |
697 | ||
e5503d53 MT |
698 | // Add entry to filelist (if requested) |
699 | if (data->filelist) { | |
e5503d53 | 700 | // Append the file to the list |
2f88682d | 701 | r = pakfire_filelist_add(data->filelist, file); |
e5503d53 MT |
702 | if (r) |
703 | goto ERROR; | |
704 | } | |
705 | ||
24553159 MT |
706 | const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG); |
707 | ||
79824416 MT |
708 | // Prepend the prefix |
709 | if (*data->prefix) { | |
710 | // Compose file path | |
711 | r = pakfire_path_join(buffer, data->prefix, path); | |
56796f84 | 712 | if (r) { |
79824416 | 713 | ERROR(pakfire, "Could not compose file path: %m\n"); |
24553159 | 714 | goto ERROR; |
79824416 MT |
715 | } |
716 | ||
717 | // Set file path | |
718 | archive_entry_set_pathname(entry, buffer); | |
719 | ||
720 | // Update hardlink destination | |
721 | const char* link = archive_entry_hardlink(entry); | |
722 | if (link) { | |
723 | r = pakfire_path_join(buffer, data->prefix, link); | |
56796f84 | 724 | if (r) { |
79824416 | 725 | ERROR(pakfire, "Could not compose hardlink path: %m\n"); |
24553159 | 726 | goto ERROR; |
79824416 MT |
727 | } |
728 | ||
729 | // Set hardlink path | |
730 | archive_entry_set_hardlink(entry, buffer); | |
731 | } | |
732 | } | |
733 | ||
24553159 MT |
734 | if (configfile) { |
735 | // Fetch path again since we changed it | |
736 | path = archive_entry_pathname(entry); | |
737 | ||
738 | if (pakfire_path_exists(path)) { | |
739 | DEBUG(pakfire, "The configuration file %s exists\n", | |
740 | pakfire_file_get_path(file)); | |
741 | ||
742 | r = pakfire_string_format(buffer, "%s.paknew", path); | |
743 | if (r) { | |
744 | ERROR(pakfire, "Could not compose path for configuration file: %m\n"); | |
745 | goto ERROR; | |
746 | } | |
747 | ||
748 | // Set the path again | |
749 | archive_entry_set_pathname(entry, buffer); | |
750 | } | |
751 | } | |
752 | ||
79824416 | 753 | // Create file & extract payload |
a3b276d1 | 754 | if (data->writer) { |
07c1c6e0 MT |
755 | // Fetch path again since we changed it |
756 | path = archive_entry_pathname(entry); | |
757 | ||
cf71c992 MT |
758 | DEBUG(pakfire, "Extracting %s\n", path); |
759 | ||
89ee0833 MT |
760 | // Remove any extended attributes which we never write to disk |
761 | archive_entry_xattr_clear(entry); | |
762 | ||
763 | // Write payload | |
a3b276d1 MT |
764 | r = archive_read_extract2(data->archive, entry, data->writer); |
765 | switch (r) { | |
766 | case ARCHIVE_OK: | |
767 | r = 0; | |
768 | break; | |
79824416 | 769 | |
a3b276d1 MT |
770 | case ARCHIVE_WARN: |
771 | ERROR(pakfire, "%s\n", archive_error_string(data->writer)); | |
79824416 | 772 | |
a3b276d1 MT |
773 | // Pretend everything has been okay |
774 | r = 0; | |
775 | break; | |
79824416 | 776 | |
a3b276d1 MT |
777 | case ARCHIVE_FATAL: |
778 | ERROR(pakfire, "%s\n", archive_error_string(data->writer)); | |
779 | r = 1; | |
780 | break; | |
781 | } | |
79824416 MT |
782 | } |
783 | ||
bb7f09cd MT |
784 | ERROR: |
785 | if (file) | |
786 | pakfire_file_unref(file); | |
787 | ||
79824416 MT |
788 | return r; |
789 | } | |
790 | ||
791 | int pakfire_extract(struct pakfire* pakfire, struct archive* archive, | |
bb7f09cd | 792 | size_t size, struct pakfire_filelist* filelist, |
f8a46d2f MT |
793 | const char* prefix, const char* message, |
794 | pakfire_walk_filter_callback filter_callback, int flags) { | |
79824416 MT |
795 | int r = 1; |
796 | ||
a623c421 MT |
797 | // Use an empty string if no prefix set |
798 | if (!prefix) | |
799 | prefix = ""; | |
800 | ||
79824416 | 801 | struct pakfire_extract data = { |
d05af3ba MT |
802 | .pakfire = pakfire, |
803 | .archive = archive, | |
804 | .filelist = filelist, | |
805 | .flags = flags, | |
806 | .writer = NULL, | |
79824416 MT |
807 | }; |
808 | ||
a3b276d1 MT |
809 | // Is this a dry run? |
810 | const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN; | |
811 | ||
5880b6ac MT |
812 | // Should we show a progress bar? |
813 | const int no_progress = flags & PAKFIRE_EXTRACT_NO_PROGRESS; | |
814 | ||
79824416 | 815 | // Set prefix (including pakfire path) |
77e26129 MT |
816 | r = pakfire_path(pakfire, data.prefix, "%s", prefix); |
817 | if (r) | |
818 | goto ERROR; | |
79824416 MT |
819 | |
820 | // Allocate writer | |
a3b276d1 MT |
821 | if (!dry_run) { |
822 | data.writer = pakfire_make_archive_disk_writer(pakfire, 1); | |
823 | if (!data.writer) { | |
824 | ERROR(pakfire, "Could not create disk writer: %m\n"); | |
825 | r = 1; | |
826 | goto ERROR; | |
827 | } | |
79824416 MT |
828 | } |
829 | ||
830 | // Create the progressbar | |
5880b6ac MT |
831 | if (!no_progress) { |
832 | r = pakfire_extract_progressbar_create(&data.progressbar, message, flags); | |
833 | if (r) | |
834 | goto ERROR; | |
79824416 | 835 | |
5880b6ac | 836 | // Register progress callback |
79824416 MT |
837 | archive_read_extract_set_progress_callback(data.archive, |
838 | pakfire_extract_progress, &data); | |
839 | ||
5880b6ac | 840 | // Start progressbar |
79824416 | 841 | pakfire_progressbar_start(data.progressbar, size); |
5880b6ac | 842 | } |
79824416 | 843 | |
14eedd80 | 844 | // Walk through the entire archive |
f8a46d2f | 845 | r = pakfire_walk(pakfire, archive, __pakfire_extract, filter_callback, &data); |
14eedd80 MT |
846 | if (r) |
847 | goto ERROR; | |
79824416 MT |
848 | |
849 | // Finish the progressbar | |
850 | if (data.progressbar) | |
851 | pakfire_progressbar_finish(data.progressbar); | |
852 | ||
853 | ERROR: | |
854 | if (data.progressbar) | |
855 | pakfire_progressbar_unref(data.progressbar); | |
856 | if (data.writer) | |
857 | archive_write_free(data.writer); | |
858 | ||
859 | return r; | |
860 | } | |
1e0d6cca MT |
861 | |
862 | // Common compression | |
863 | ||
864 | struct pakfire_compress { | |
865 | // Reference to Pakfire | |
866 | struct pakfire* pakfire; | |
867 | ||
868 | // Flags | |
869 | int flags; | |
870 | ||
871 | // The archive to write to | |
872 | struct archive* archive; | |
873 | ||
377c9b98 MT |
874 | // Resolver for hardlinks |
875 | struct archive_entry_linkresolver* linkresolver; | |
876 | ||
1e0d6cca MT |
877 | // The filelist of all files to write |
878 | struct pakfire_filelist* filelist; | |
879 | ||
880 | // The progressbar | |
881 | struct pakfire_progressbar* progressbar; | |
882 | ||
883 | // Digests to write to the archive | |
884 | int digests; | |
885 | }; | |
886 | ||
887 | static int pakfire_compress_progressbar_create(struct pakfire_progressbar** progressbar, | |
888 | const char* message, int flags) { | |
889 | int r; | |
890 | ||
891 | // Create the progressbar | |
892 | r = pakfire_progressbar_create(progressbar, NULL); | |
893 | if (r) | |
894 | return r; | |
895 | ||
896 | // Add message | |
897 | if (message) { | |
898 | r = pakfire_progressbar_add_string(*progressbar, "%s", message); | |
899 | if (r) | |
900 | return r; | |
901 | } | |
902 | ||
903 | // Add bar | |
904 | r = pakfire_progressbar_add_bar(*progressbar); | |
905 | if (r) | |
906 | return r; | |
907 | ||
908 | // Add throughput | |
909 | if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT) { | |
910 | r = pakfire_progressbar_add_transfer_speed(*progressbar); | |
911 | if (r) | |
912 | return r; | |
913 | } | |
914 | ||
bd94d753 MT |
915 | // Show how many bytes have been written |
916 | r = pakfire_progressbar_add_bytes_transferred(*progressbar); | |
917 | if (r) | |
918 | return r; | |
919 | ||
1e0d6cca MT |
920 | // Add percentage |
921 | r = pakfire_progressbar_add_percentage(*progressbar); | |
922 | if (r) | |
923 | return r; | |
924 | ||
925 | // Success | |
926 | return 0; | |
927 | } | |
928 | ||
377c9b98 MT |
929 | static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file, |
930 | struct pakfire_compress* data, struct archive_entry* entry) { | |
1e0d6cca | 931 | FILE* f = NULL; |
377c9b98 | 932 | int r; |
1e0d6cca MT |
933 | |
934 | // Write the header | |
935 | r = archive_write_header(data->archive, entry); | |
936 | if (r) { | |
937 | ERROR(pakfire, "Error writing file header: %s\n", | |
938 | archive_error_string(data->archive)); | |
939 | goto ERROR; | |
940 | } | |
941 | ||
377c9b98 MT |
942 | // Copy the data if there is any |
943 | if (archive_entry_size(entry)) { | |
1e0d6cca MT |
944 | // Open the file |
945 | f = pakfire_file_open(file); | |
946 | if (!f) { | |
947 | r = 1; | |
948 | goto ERROR; | |
949 | } | |
950 | ||
951 | // Copy the payload into the archive | |
952 | r = pakfire_archive_copy_data_from_file(pakfire, data->archive, f); | |
953 | if (r) | |
954 | goto ERROR; | |
955 | } | |
956 | ||
957 | // Write trailer | |
958 | r = archive_write_finish_entry(data->archive); | |
959 | if (r) | |
960 | goto ERROR; | |
961 | ||
377c9b98 MT |
962 | ERROR: |
963 | if (f) | |
964 | fclose(f); | |
965 | ||
966 | return r; | |
967 | } | |
968 | ||
969 | static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) { | |
970 | struct archive_entry* entry = NULL; | |
971 | struct archive_entry* sparse_entry = NULL; | |
972 | int r = 1; | |
973 | ||
974 | struct pakfire_compress* data = (struct pakfire_compress*)p; | |
975 | ||
976 | // Fetch the file size | |
977 | const size_t size = pakfire_file_get_size(file); | |
978 | ||
979 | // Generate file metadata into an archive entry | |
980 | entry = pakfire_file_archive_entry(file, data->digests); | |
981 | if (!entry) { | |
982 | r = 1; | |
983 | goto ERROR; | |
984 | } | |
985 | ||
986 | // Perform search for hardlinks | |
987 | archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); | |
988 | ||
989 | // Write the main entry | |
990 | if (entry) { | |
991 | r = __pakfire_compress_entry(pakfire, file, data, entry); | |
992 | if (r) | |
993 | goto ERROR; | |
994 | } | |
995 | ||
996 | // Write the sparse entry | |
997 | if (sparse_entry) { | |
998 | r = __pakfire_compress_entry(pakfire, file, data, sparse_entry); | |
999 | if (r) | |
1000 | goto ERROR; | |
1001 | } | |
1002 | ||
1003 | // Query the link resolver for any more entries | |
1004 | for (;;) { | |
1005 | // Free the entry | |
1006 | if (entry) { | |
1007 | archive_entry_free(entry); | |
1008 | entry = NULL; | |
1009 | } | |
1010 | ||
1011 | // Free the sparse entry | |
1012 | if (sparse_entry) { | |
1013 | archive_entry_free(sparse_entry); | |
1014 | sparse_entry = NULL; | |
1015 | } | |
1016 | ||
1017 | // Fetch the next entry | |
1018 | archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); | |
1019 | if (!entry) | |
1020 | break; | |
1021 | ||
1022 | // Write the entry to the archive | |
1023 | r = __pakfire_compress_entry(pakfire, file, data, entry); | |
1024 | if (r) | |
1025 | goto ERROR; | |
1026 | } | |
1027 | ||
1e0d6cca MT |
1028 | // Update the progressbar |
1029 | if (data->progressbar) | |
1030 | pakfire_progressbar_increment(data->progressbar, size); | |
1031 | ||
1032 | ERROR: | |
1e0d6cca MT |
1033 | if (entry) |
1034 | archive_entry_free(entry); | |
377c9b98 MT |
1035 | if (sparse_entry) |
1036 | archive_entry_free(sparse_entry); | |
1e0d6cca MT |
1037 | |
1038 | return r; | |
1039 | } | |
1040 | ||
1041 | int pakfire_compress(struct pakfire* pakfire, struct archive* archive, | |
1042 | struct pakfire_filelist* filelist, const char* message, int flags, int digests) { | |
1043 | int r = 1; | |
1044 | ||
1045 | struct pakfire_compress data = { | |
1046 | .pakfire = pakfire, | |
1047 | .archive = archive, | |
1048 | .filelist = filelist, | |
1049 | .flags = flags, | |
1050 | .digests = digests, | |
1051 | }; | |
1052 | ||
1053 | // Should we show a progress bar? | |
1054 | const int no_progress = flags & PAKFIRE_COMPRESS_NO_PROGRESS; | |
1055 | ||
1056 | // Fetch the length of the filelist | |
1057 | const size_t size = pakfire_filelist_total_size(filelist); | |
1058 | ||
1059 | // Create the progressbar | |
1060 | if (!no_progress) { | |
1061 | r = pakfire_compress_progressbar_create(&data.progressbar, message, flags); | |
1062 | if (r) | |
1063 | goto ERROR; | |
1064 | ||
1065 | // Start progressbar | |
1066 | pakfire_progressbar_start(data.progressbar, size); | |
1067 | } | |
1068 | ||
377c9b98 MT |
1069 | // Setup the link resolver |
1070 | data.linkresolver = archive_entry_linkresolver_new(); | |
1071 | if (!data.linkresolver) { | |
1072 | ERROR(pakfire, "Could not setup link resolver: m\n"); | |
1073 | goto ERROR; | |
1074 | } | |
1075 | ||
1076 | // Set the link resolver strategy | |
1077 | archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive)); | |
1078 | ||
1e0d6cca | 1079 | // Walk through the entire filelist |
a3f3c077 | 1080 | r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0); |
1e0d6cca MT |
1081 | if (r) |
1082 | goto ERROR; | |
1083 | ||
1084 | // Finish the progressbar | |
1085 | if (data.progressbar) | |
1086 | pakfire_progressbar_finish(data.progressbar); | |
1087 | ||
1088 | ERROR: | |
1089 | if (data.progressbar) | |
1090 | pakfire_progressbar_unref(data.progressbar); | |
377c9b98 MT |
1091 | if (data.linkresolver) |
1092 | archive_entry_linkresolver_free(data.linkresolver); | |
1e0d6cca MT |
1093 | |
1094 | return r; | |
1095 | } | |
e66c6bbc MT |
1096 | |
1097 | int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive, | |
1098 | FILE* f, const enum pakfire_compressions compression, const unsigned int level) { | |
1099 | struct archive* a = NULL; | |
1100 | char value[16]; | |
1101 | int r; | |
1102 | ||
1103 | // Open a new archive | |
1104 | a = archive_write_new(); | |
1105 | if (!a) { | |
1106 | ERROR(pakfire, "archive_write_new() failed\n"); | |
1107 | r = 1; | |
1108 | goto ERROR; | |
1109 | } | |
1110 | ||
1111 | // Use the PAX format | |
1112 | r = archive_write_set_format_pax(a); | |
1113 | if (r) { | |
1114 | ERROR(pakfire, "Could not set format to PAX: %s\n", archive_error_string(a)); | |
1115 | goto ERROR; | |
1116 | } | |
1117 | ||
1118 | // Store any extended attributes in the SCHILY headers | |
1119 | r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY"); | |
1120 | if (r) { | |
1121 | ERROR(pakfire, "Could not set xattrheader option: %s\n", archive_error_string(a)); | |
1122 | goto ERROR; | |
1123 | } | |
1124 | ||
1125 | switch (compression) { | |
1126 | case PAKFIRE_COMPRESS_ZSTD: | |
1127 | // Enable Zstd | |
1128 | r = archive_write_add_filter_zstd(a); | |
1129 | if (r) { | |
1130 | ERROR(pakfire, "Could not enable Zstandard compression: %s\n", | |
1131 | archive_error_string(a)); | |
1132 | goto ERROR; | |
1133 | } | |
1134 | ||
1135 | // Do not pad the last block | |
1136 | archive_write_set_bytes_in_last_block(a, 1); | |
1137 | ||
1138 | // Set compression level | |
1139 | if (level) { | |
1140 | r = pakfire_string_format(value, "%u", level); | |
1141 | if (r) | |
1142 | goto ERROR; | |
1143 | ||
1144 | r = archive_write_set_filter_option(a, NULL, "compression-level", value); | |
1145 | if (r) { | |
1146 | ERROR(pakfire, "Could not set Zstandard compression level: %s\n", | |
1147 | archive_error_string(a)); | |
1148 | goto ERROR; | |
1149 | } | |
1150 | } | |
177f5ff3 MT |
1151 | |
1152 | #if ARCHIVE_VERSION_NUMBER >= 3006000 | |
1153 | // Fetch numbers of processors | |
1154 | long processors = sysconf(_SC_NPROCESSORS_ONLN); | |
1155 | ||
1156 | if (processors > 1) { | |
1157 | r = pakfire_string_format(value, "%ld", processors); | |
1158 | if (r) { | |
1159 | ERROR(pakfire, "Could not format threads: %m\n"); | |
1160 | goto ERROR; | |
1161 | } | |
1162 | ||
1163 | // Try using multiple threads | |
1d7a641f | 1164 | r = archive_write_set_filter_option(a, NULL, "threads", value); |
177f5ff3 | 1165 | if (r) { |
1d7a641f MT |
1166 | ERROR(pakfire, "Could not enable %ld threads for compression: %s\n", |
1167 | processors, archive_error_string(a)); | |
177f5ff3 MT |
1168 | goto ERROR; |
1169 | } | |
1170 | } | |
1171 | #endif | |
e66c6bbc MT |
1172 | } |
1173 | ||
1174 | // Write archive to f | |
1175 | if (f) { | |
1176 | r = archive_write_open_FILE(a, f); | |
1177 | if (r) { | |
1178 | ERROR(pakfire, "archive_write_open_FILE() failed: %s\n", archive_error_string(a)); | |
1179 | goto ERROR; | |
1180 | } | |
1181 | } | |
1182 | ||
1183 | // Success | |
1184 | *archive = a; | |
1185 | ||
1186 | return 0; | |
1187 | ||
1188 | ERROR: | |
1189 | if (a) | |
1190 | archive_write_free(a); | |
1191 | ||
1192 | return r; | |
1193 | } |