]>
Commit | Line | Data |
---|---|---|
c352b776 MT |
1 | /*############################################################################# |
2 | # # | |
3 | # Pakfire - The IPFire package management system # | |
4 | # Copyright (C) 2021 Pakfire development team # | |
5 | # # | |
6 | # This program is free software: you can redistribute it and/or modify # | |
7 | # it under the terms of the GNU General Public License as published by # | |
8 | # the Free Software Foundation, either version 3 of the License, or # | |
9 | # (at your option) any later version. # | |
10 | # # | |
11 | # This program is distributed in the hope that it will be useful, # | |
12 | # but WITHOUT ANY WARRANTY; without even the implied warranty of # | |
13 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # | |
14 | # GNU General Public License for more details. # | |
15 | # # | |
16 | # You should have received a copy of the GNU General Public License # | |
17 | # along with this program. If not, see <http://www.gnu.org/licenses/>. # | |
18 | # # | |
19 | #############################################################################*/ | |
20 | ||
64f977a3 | 21 | #include <errno.h> |
c352b776 | 22 | #include <stdio.h> |
99018421 | 23 | #include <stdlib.h> |
c352b776 MT |
24 | #include <string.h> |
25 | ||
79824416 | 26 | #include <archive.h> |
c352b776 | 27 | #include <lzma.h> |
9476d502 | 28 | #include <zstd.h> |
c352b776 | 29 | |
be12460f | 30 | #include <pakfire/ctx.h> |
c352b776 | 31 | #include <pakfire/compress.h> |
bb7f09cd MT |
32 | #include <pakfire/file.h> |
33 | #include <pakfire/filelist.h> | |
79824416 | 34 | #include <pakfire/logging.h> |
729827f7 | 35 | #include <pakfire/path.h> |
fcaa90ac | 36 | #include <pakfire/progress.h> |
24553159 | 37 | #include <pakfire/string.h> |
79824416 | 38 | #include <pakfire/util.h> |
c352b776 MT |
39 | |
40 | // Read up to N bytes for analyze the magic | |
41 | #define MAX_MAGIC_LENGTH 6 | |
42 | ||
5cd454df MT |
43 | // Compression/Decompression buffer size |
44 | #define BUFFER_SIZE 64 * 1024 | |
45 | ||
0bed1e1d MT |
46 | // Settings for XZ |
47 | #define XZ_COMPRESSION_LEVEL 6 | |
48 | ||
9476d502 MT |
49 | // Settings for ZSTD |
50 | #define ZSTD_COMPRESSION_LEVEL 7 | |
51 | ||
c352b776 MT |
52 | const struct compressor { |
53 | char magic[MAX_MAGIC_LENGTH]; | |
54 | size_t magic_length; | |
55 | FILE* (*open)(FILE* f, const char* mode); | |
56 | } compressors[] = { | |
57 | // XZ | |
58 | { { 0xFD, '7', 'z', 'X', 'Z', 0x00 }, 6, pakfire_xzfopen, }, | |
60732b6e MT |
59 | // ZSTD |
60 | { { 0x28, 0xb5, 0x2f, 0xfd }, 4, pakfire_zstdfopen, }, | |
61 | // End | |
c352b776 MT |
62 | { "", 0, NULL, }, |
63 | }; | |
64 | ||
c352b776 MT |
65 | // Try to guess the compression |
66 | FILE* pakfire_xfopen(FILE* f, const char* mode) { | |
67 | char buffer[MAX_MAGIC_LENGTH]; | |
68 | ||
64f977a3 MT |
69 | if (!f) { |
70 | errno = EBADFD; | |
c352b776 | 71 | return NULL; |
64f977a3 MT |
72 | } |
73 | ||
74 | if (!mode) { | |
75 | errno = EINVAL; | |
76 | return NULL; | |
77 | } | |
78 | ||
79 | // This only works for reading files | |
80 | if (*mode != 'r') { | |
81 | errno = ENOTSUP; | |
82 | return NULL; | |
83 | } | |
c352b776 MT |
84 | |
85 | fpos_t pos; | |
86 | ||
87 | // Store the position | |
88 | int r = fgetpos(f, &pos); | |
89 | if (r < 0) | |
90 | return NULL; | |
91 | ||
92 | // Read a couple of bytes | |
93 | size_t bytes_read = fread(buffer, 1, sizeof(buffer), f); | |
94 | ||
95 | // Reset position | |
96 | r = fsetpos(f, &pos); | |
97 | if (r < 0) | |
98 | return NULL; | |
99 | ||
100 | // Check if we could read anything | |
101 | if (!bytes_read || bytes_read < sizeof(buffer)) | |
102 | return f; | |
103 | ||
104 | // Analyze magic | |
105 | for (const struct compressor* c = compressors; c->open; c++) { | |
106 | // Check if we have read enough data | |
107 | if (bytes_read < c->magic_length) | |
108 | continue; | |
109 | ||
110 | // Compare the magic value | |
111 | r = memcmp(c->magic, buffer, c->magic_length); | |
112 | if (r) | |
113 | continue; | |
114 | ||
115 | // We found a match! | |
116 | return c->open(f, mode); | |
117 | } | |
118 | ||
119 | // Nothing seems to match | |
49492077 | 120 | errno = ENOTSUP; |
c352b776 MT |
121 | return f; |
122 | } | |
123 | ||
5cd454df MT |
124 | struct xz_cookie { |
125 | FILE* f; | |
126 | char mode; | |
127 | lzma_stream stream; | |
128 | int done; | |
129 | ||
130 | uint8_t buffer[BUFFER_SIZE]; | |
131 | }; | |
132 | ||
c352b776 MT |
133 | static ssize_t xz_read(void* data, char* buffer, size_t size) { |
134 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
135 | if (!cookie) | |
136 | return -1; | |
137 | ||
48187ff3 MT |
138 | // Do not read when mode is "w" |
139 | if (cookie->mode == 'w') | |
6923552a | 140 | return -1; |
48187ff3 | 141 | |
c352b776 MT |
142 | lzma_action action = LZMA_RUN; |
143 | ||
144 | // Set output to allocated buffer | |
145 | cookie->stream.next_out = (uint8_t *)buffer; | |
146 | cookie->stream.avail_out = size; | |
147 | ||
148 | while (1) { | |
149 | // Read something when the input buffer is empty | |
150 | if (cookie->stream.avail_in == 0) { | |
151 | cookie->stream.next_in = cookie->buffer; | |
152 | cookie->stream.avail_in = fread(cookie->buffer, | |
5cd454df | 153 | 1, sizeof(cookie->buffer), cookie->f); |
c352b776 MT |
154 | |
155 | // Break if the input file could not be read | |
156 | if (ferror(cookie->f)) | |
157 | return -1; | |
158 | ||
159 | // Finish after we have reached the end of the input file | |
5e12758c | 160 | if (feof(cookie->f)) |
c352b776 | 161 | cookie->done = 1; |
c352b776 MT |
162 | } |
163 | ||
164 | lzma_ret ret = lzma_code(&cookie->stream, action); | |
165 | ||
166 | // If the stream has ended, we just send the | |
167 | // remaining output and mark that we are done. | |
168 | if (ret == LZMA_STREAM_END) { | |
169 | cookie->done = 1; | |
170 | return size - cookie->stream.avail_out; | |
171 | } | |
172 | ||
173 | // Break on all other unexpected errors | |
174 | if (ret != LZMA_OK) | |
175 | return -1; | |
176 | ||
177 | // When we have read enough to fill the entire output buffer, we return | |
178 | if (cookie->stream.avail_out == 0) | |
179 | return size; | |
180 | ||
181 | if (cookie->done) | |
182 | return -1; | |
183 | } | |
184 | } | |
185 | ||
0bed1e1d MT |
186 | static ssize_t xz_write(void* data, const char* buffer, size_t size) { |
187 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
188 | if (!cookie) | |
189 | return -1; | |
190 | ||
191 | // Do not write when mode is "r" | |
192 | if (cookie->mode == 'r') | |
6923552a | 193 | return -1; |
0bed1e1d MT |
194 | |
195 | // Return nothing when there is no input | |
196 | if (size == 0) | |
197 | return 0; | |
198 | ||
199 | // Set input to allocated buffer | |
9d816df3 MT |
200 | cookie->stream.next_in = (uint8_t *)buffer; |
201 | cookie->stream.avail_in = size; | |
0bed1e1d MT |
202 | |
203 | while (1) { | |
204 | cookie->stream.next_out = cookie->buffer; | |
205 | cookie->stream.avail_out = sizeof(cookie->buffer); | |
206 | ||
207 | lzma_ret ret = lzma_code(&cookie->stream, LZMA_RUN); | |
208 | if (ret != LZMA_OK) | |
209 | return -1; | |
210 | ||
211 | size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; | |
212 | if (bytes_to_write) { | |
213 | size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); | |
214 | ||
215 | if (bytes_written != bytes_to_write) | |
216 | return -1; | |
217 | } | |
218 | ||
219 | // Report that all data has been written | |
220 | if (cookie->stream.avail_in == 0) | |
221 | return size; | |
222 | } | |
223 | } | |
224 | ||
c352b776 MT |
225 | static int xz_close(void* data) { |
226 | struct xz_cookie* cookie = (struct xz_cookie*)data; | |
99018421 MT |
227 | if (!cookie) |
228 | return -1; | |
229 | ||
230 | if (cookie->mode == 'w') { | |
231 | while (1) { | |
232 | cookie->stream.next_out = cookie->buffer; | |
233 | cookie->stream.avail_out = sizeof(cookie->buffer); | |
234 | ||
235 | lzma_ret ret = lzma_code(&cookie->stream, LZMA_FINISH); | |
236 | if (ret != LZMA_OK && ret != LZMA_STREAM_END) | |
237 | return -1; | |
238 | ||
239 | size_t bytes_to_write = sizeof(cookie->buffer) - cookie->stream.avail_out; | |
240 | if (bytes_to_write) { | |
241 | size_t bytes_written = fwrite(cookie->buffer, 1, bytes_to_write, cookie->f); | |
242 | ||
243 | if (bytes_written != bytes_to_write) | |
244 | return -1; | |
245 | } | |
246 | ||
247 | if (ret == LZMA_STREAM_END) | |
248 | break; | |
249 | } | |
250 | } | |
c352b776 | 251 | |
c352b776 MT |
252 | lzma_end(&cookie->stream); |
253 | ||
254 | // Close input file | |
99018421 MT |
255 | int r = fclose(cookie->f); |
256 | free(cookie); | |
c352b776 | 257 | |
99018421 | 258 | return r; |
c352b776 MT |
259 | } |
260 | ||
99018421 MT |
261 | static cookie_io_functions_t xz_functions = { |
262 | .read = xz_read, | |
263 | .write = xz_write, | |
264 | .seek = NULL, | |
265 | .close = xz_close, | |
266 | }; | |
267 | ||
c352b776 | 268 | FILE* pakfire_xzfopen(FILE* f, const char* mode) { |
99018421 MT |
269 | lzma_ret ret; |
270 | ||
48187ff3 MT |
271 | if (!f) { |
272 | errno = EBADFD; | |
273 | return NULL; | |
274 | } | |
275 | ||
276 | if (!mode) { | |
277 | errno = EINVAL; | |
278 | return NULL; | |
279 | } | |
280 | ||
99018421 MT |
281 | struct xz_cookie* cookie = calloc(1, sizeof(*cookie)); |
282 | if (!cookie) | |
283 | return NULL; | |
c352b776 | 284 | |
99018421 MT |
285 | cookie->f = f; |
286 | cookie->mode = *mode; | |
0bed1e1d | 287 | |
99018421 | 288 | switch (cookie->mode) { |
0bed1e1d | 289 | case 'r': |
99018421 | 290 | ret = lzma_stream_decoder(&cookie->stream, UINT64_MAX, 0); |
0bed1e1d MT |
291 | break; |
292 | ||
293 | case 'w': | |
99018421 | 294 | ret = lzma_easy_encoder(&cookie->stream, XZ_COMPRESSION_LEVEL, LZMA_CHECK_SHA256); |
0bed1e1d MT |
295 | break; |
296 | ||
297 | default: | |
298 | errno = ENOTSUP; | |
2d4af759 | 299 | goto ERROR; |
0bed1e1d MT |
300 | } |
301 | ||
c352b776 | 302 | if (ret != LZMA_OK) |
2d4af759 | 303 | goto ERROR; |
c352b776 | 304 | |
99018421 | 305 | return fopencookie(cookie, mode, xz_functions); |
2d4af759 MT |
306 | |
307 | ERROR: | |
308 | free(cookie); | |
309 | return NULL; | |
c352b776 | 310 | } |
9476d502 MT |
311 | |
312 | // ZSTD | |
313 | ||
314 | struct zstd_cookie { | |
315 | FILE* f; | |
316 | char mode; | |
317 | int done; | |
318 | ||
319 | // Encoder | |
320 | ZSTD_CStream* cstream; | |
321 | ZSTD_inBuffer in; | |
322 | ||
323 | // Decoder | |
324 | ZSTD_DStream* dstream; | |
325 | ZSTD_outBuffer out; | |
326 | ||
327 | uint8_t buffer[BUFFER_SIZE]; | |
328 | }; | |
329 | ||
330 | static ssize_t zstd_read(void* data, char* buffer, size_t size) { | |
331 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
332 | if (!cookie) | |
333 | return -1; | |
334 | ||
335 | // Do not read when mode is "w" | |
336 | if (cookie->mode == 'w') | |
337 | return -1; | |
338 | ||
339 | if (cookie->done) | |
340 | return 0; | |
341 | ||
342 | size_t r = 0; | |
343 | ||
344 | // Configure output buffer | |
345 | cookie->out.dst = buffer; | |
346 | cookie->out.pos = 0; | |
347 | cookie->out.size = size; | |
348 | ||
349 | while (1) { | |
350 | if (!feof(cookie->f) && (cookie->in.pos == cookie->in.size)) { | |
351 | cookie->in.pos = 0; | |
352 | cookie->in.size = fread(cookie->buffer, 1, sizeof(cookie->buffer), cookie->f); | |
353 | } | |
354 | ||
355 | if (r || cookie->in.size) | |
356 | r = ZSTD_decompressStream(cookie->dstream, &cookie->out, &cookie->in); | |
357 | ||
358 | if (r == 0 && feof(cookie->f)) { | |
359 | cookie->done = 1; | |
360 | return cookie->out.pos; | |
361 | } | |
362 | ||
363 | if (ZSTD_isError(r)) | |
364 | return -1; | |
365 | ||
366 | // Buffer full | |
367 | if (cookie->out.pos == size) | |
368 | return size; | |
369 | } | |
370 | } | |
371 | ||
99a1527a MT |
372 | static ssize_t zstd_write(void* data, const char* buffer, size_t size) { |
373 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
374 | if (!cookie) | |
375 | return -1; | |
376 | ||
377 | // Do not write when mode is "r" | |
378 | if (cookie->mode == 'r') | |
379 | return -1; | |
380 | ||
381 | // Return nothing when there is no input | |
382 | if (size == 0) | |
383 | return 0; | |
384 | ||
385 | // Configure input buffer | |
386 | cookie->in.src = buffer; | |
387 | cookie->in.pos = 0; | |
388 | cookie->in.size = size; | |
389 | ||
390 | while (1) { | |
391 | cookie->out.pos = 0; | |
392 | ||
393 | size_t r = ZSTD_compressStream(cookie->cstream, &cookie->out, &cookie->in); | |
394 | if (ZSTD_isError(r)) | |
395 | return -1; | |
396 | ||
397 | if (cookie->out.pos > 0) { | |
398 | size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); | |
399 | ||
400 | if (bytes_written != cookie->out.pos) | |
401 | return -1; | |
402 | } | |
403 | ||
404 | // Return when all input has been written | |
405 | if (cookie->in.pos == size) | |
406 | return size; | |
407 | } | |
408 | } | |
409 | ||
9476d502 MT |
410 | static int zstd_close(void* data) { |
411 | struct zstd_cookie* cookie = (struct zstd_cookie*)data; | |
412 | if (!cookie) | |
413 | return -1; | |
414 | ||
99a1527a MT |
415 | if (cookie->mode == 'w') { |
416 | while (1) { | |
417 | // Reset output buffer | |
418 | cookie->out.pos = 0; | |
419 | ||
420 | size_t r = ZSTD_endStream(cookie->cstream, &cookie->out); | |
421 | if (ZSTD_isError(r)) | |
422 | return -1; | |
423 | ||
424 | if (cookie->out.pos > 0) { | |
425 | size_t bytes_written = fwrite(cookie->buffer, 1, cookie->out.pos, cookie->f); | |
426 | ||
427 | if (bytes_written != cookie->out.pos) | |
428 | return -1; | |
429 | } | |
430 | ||
431 | if (r == 0) | |
432 | break; | |
433 | } | |
434 | } | |
9476d502 MT |
435 | |
436 | int r = fclose(cookie->f); | |
437 | ||
438 | // Free everything | |
439 | if (cookie->cstream) | |
440 | ZSTD_freeCStream(cookie->cstream); | |
441 | if (cookie->dstream) | |
442 | ZSTD_freeDStream(cookie->dstream); | |
443 | free(cookie); | |
444 | ||
445 | return r; | |
446 | } | |
447 | ||
448 | static cookie_io_functions_t zstd_functions = { | |
449 | .read = zstd_read, | |
99a1527a | 450 | .write = zstd_write, |
9476d502 MT |
451 | .seek = NULL, |
452 | .close = zstd_close, | |
453 | }; | |
454 | ||
455 | FILE* pakfire_zstdfopen(FILE* f, const char* mode) { | |
456 | if (!f) { | |
457 | errno = EBADFD; | |
458 | return NULL; | |
459 | } | |
460 | ||
461 | if (!mode) { | |
462 | errno = EINVAL; | |
463 | return NULL; | |
464 | } | |
465 | ||
466 | struct zstd_cookie* cookie = calloc(1, sizeof(*cookie)); | |
467 | if (!cookie) | |
468 | return NULL; | |
469 | ||
470 | cookie->f = f; | |
471 | cookie->mode = *mode; | |
472 | ||
473 | size_t r; | |
474 | switch (cookie->mode) { | |
475 | case 'r': | |
476 | // Allocate stream | |
477 | cookie->dstream = ZSTD_createDStream(); | |
478 | if (!cookie->dstream) | |
479 | goto ERROR; | |
480 | ||
481 | // Initialize stream | |
482 | r = ZSTD_initDStream(cookie->dstream); | |
483 | if (ZSTD_isError(r)) | |
484 | goto ERROR; | |
485 | ||
486 | cookie->in.src = cookie->buffer; | |
487 | cookie->in.pos = 0; | |
488 | cookie->in.size = 0; | |
489 | break; | |
490 | ||
491 | case 'w': | |
492 | // Allocate stream | |
493 | cookie->cstream = ZSTD_createCStream(); | |
494 | if (!cookie->cstream) | |
495 | goto ERROR; | |
496 | ||
497 | // Initialize stream | |
498 | r = ZSTD_initCStream(cookie->cstream, ZSTD_COMPRESSION_LEVEL); | |
499 | if (ZSTD_isError(r)) | |
500 | goto ERROR; | |
501 | ||
502 | cookie->out.dst = cookie->buffer; | |
503 | cookie->out.pos = 0; | |
504 | cookie->out.size = sizeof(cookie->buffer); | |
505 | break; | |
506 | ||
507 | default: | |
508 | errno = ENOTSUP; | |
509 | goto ERROR; | |
510 | } | |
511 | ||
512 | return fopencookie(cookie, mode, zstd_functions); | |
513 | ||
514 | ERROR: | |
515 | if (cookie->cstream) | |
516 | ZSTD_freeCStream(cookie->cstream); | |
517 | if (cookie->dstream) | |
518 | ZSTD_freeDStream(cookie->dstream); | |
519 | free(cookie); | |
520 | ||
521 | return NULL; | |
522 | } | |
79824416 | 523 | |
d9302cc3 MT |
524 | /* |
525 | Helper function to conditionally walk through an archive | |
526 | and perform actions based on the callback. | |
527 | */ | |
9d956439 | 528 | int pakfire_walk(struct pakfire_ctx* ctx, struct archive* archive, |
d9302cc3 MT |
529 | pakfire_walk_callback callback, pakfire_walk_filter_callback filter_callback, |
530 | void* p) { | |
531 | struct archive_entry* entry = NULL; | |
532 | int r; | |
533 | ||
534 | // Walk through the archive | |
535 | for (;;) { | |
536 | r = archive_read_next_header(archive, &entry); | |
537 | ||
538 | // Handle the return code | |
539 | switch (r) { | |
0e25bebb MT |
540 | // Fall through if everything is okay |
541 | case ARCHIVE_OK: | |
542 | break; | |
543 | ||
d9302cc3 MT |
544 | // Return OK when we reached the end of the archive |
545 | case ARCHIVE_EOF: | |
546 | return 0; | |
547 | ||
548 | // Raise any other errors | |
549 | default: | |
550 | return r; | |
551 | } | |
552 | ||
553 | // Call the filter callback before we call the actual callback | |
554 | if (filter_callback) { | |
9d956439 | 555 | r = filter_callback(ctx, archive, entry, p); |
d9302cc3 MT |
556 | |
557 | // Handle the return code | |
558 | switch (r) { | |
559 | case PAKFIRE_WALK_OK: | |
560 | break; | |
561 | ||
2480e018 | 562 | case PAKFIRE_WALK_END: |
9d956439 | 563 | CTX_DEBUG(ctx, "Filter callback sent END\n"); |
d9302cc3 MT |
564 | return 0; |
565 | ||
566 | case PAKFIRE_WALK_SKIP: | |
9d956439 | 567 | CTX_DEBUG(ctx, "Filter callback sent SKIP\n"); |
d9302cc3 MT |
568 | continue; |
569 | ||
2480e018 | 570 | case PAKFIRE_WALK_DONE: |
9d956439 | 571 | CTX_DEBUG(ctx, "Filter callback sent DONE\n"); |
f8a46d2f MT |
572 | |
573 | // Clear the callback function | |
574 | filter_callback = NULL; | |
575 | break; | |
576 | ||
d8a0adeb | 577 | case PAKFIRE_WALK_AGAIN: |
9d956439 | 578 | CTX_DEBUG(ctx, "Filter callback sent AGAIN\n"); |
d8a0adeb MT |
579 | return -EAGAIN; |
580 | ||
d9302cc3 MT |
581 | // Raise any other errors |
582 | default: | |
9d956439 | 583 | CTX_DEBUG(ctx, "Filter callback returned an error: %d\n", r); |
d9302cc3 MT |
584 | return r; |
585 | } | |
586 | } | |
587 | ||
588 | // Run callback | |
589 | if (callback) { | |
9d956439 | 590 | r = callback(ctx, archive, entry, p); |
d9302cc3 MT |
591 | |
592 | // Handle the return code | |
593 | switch (r) { | |
594 | case PAKFIRE_WALK_OK: | |
595 | break; | |
596 | ||
597 | case PAKFIRE_WALK_DONE: | |
9d956439 | 598 | CTX_DEBUG(ctx, "Callback sent DONE\n"); |
d9302cc3 MT |
599 | return 0; |
600 | ||
601 | // Raise any other errors | |
602 | default: | |
603 | return r; | |
604 | } | |
605 | } | |
606 | } | |
607 | ||
608 | return 0; | |
609 | } | |
610 | ||
79824416 MT |
611 | // Common extraction |
612 | ||
613 | struct pakfire_extract { | |
614 | // Reference to Pakfire | |
615 | struct pakfire* pakfire; | |
616 | ||
a3b276d1 MT |
617 | // Flags |
618 | int flags; | |
619 | ||
79824416 MT |
620 | // The archive to extract |
621 | struct archive* archive; | |
622 | ||
d05af3ba MT |
623 | // The filelist of all extracted files |
624 | struct pakfire_filelist* filelist; | |
625 | ||
79824416 | 626 | // Prepend this prefix |
1ce8f42e | 627 | const char* prefix; |
79824416 MT |
628 | |
629 | // The writer | |
630 | struct archive* writer; | |
631 | ||
32bbeefc MT |
632 | // The progress indicator |
633 | struct pakfire_progress* progress; | |
79824416 MT |
634 | }; |
635 | ||
79824416 MT |
636 | static void pakfire_extract_progress(void* p) { |
637 | struct pakfire_extract* data = (struct pakfire_extract*)p; | |
638 | ||
639 | // Fetch how many bytes have been read | |
640 | const size_t position = archive_filter_bytes(data->archive, -1); | |
641 | ||
32bbeefc MT |
642 | // Update progress |
643 | pakfire_progress_update(data->progress, position); | |
79824416 MT |
644 | } |
645 | ||
9d956439 | 646 | static int __pakfire_extract(struct pakfire_ctx* ctx, struct archive* a, |
14eedd80 | 647 | struct archive_entry* entry, void* p) { |
bb7f09cd | 648 | struct pakfire_file* file = NULL; |
729568d0 | 649 | struct vfs_cap_data cap_data = {}; |
79824416 MT |
650 | char buffer[PATH_MAX]; |
651 | int r; | |
652 | ||
14eedd80 MT |
653 | struct pakfire_extract* data = (struct pakfire_extract*)p; |
654 | ||
79824416 MT |
655 | // Fetch path |
656 | const char* path = archive_entry_pathname(entry); | |
657 | ||
e990815d MT |
658 | // Make sure we have a leading slash on the filelist |
659 | if (!pakfire_string_startswith(path, "/")) { | |
660 | r = pakfire_string_format(buffer, "/%s", path); | |
661 | if (r) | |
662 | goto ERROR; | |
663 | ||
664 | // Store the new name | |
665 | archive_entry_set_pathname(entry, buffer); | |
666 | ||
667 | // Update the path pointer | |
668 | path = archive_entry_pathname(entry); | |
669 | } | |
670 | ||
24553159 | 671 | // Generate a file object |
9d956439 | 672 | r = pakfire_file_create_from_archive_entry(&file, data->pakfire, entry); |
24553159 MT |
673 | if (r) |
674 | goto ERROR; | |
675 | ||
e5503d53 MT |
676 | // Add entry to filelist (if requested) |
677 | if (data->filelist) { | |
e5503d53 | 678 | // Append the file to the list |
2f88682d | 679 | r = pakfire_filelist_add(data->filelist, file); |
e5503d53 MT |
680 | if (r) |
681 | goto ERROR; | |
682 | } | |
683 | ||
24553159 MT |
684 | const int configfile = pakfire_file_has_flag(file, PAKFIRE_FILE_CONFIG); |
685 | ||
79824416 MT |
686 | // Prepend the prefix |
687 | if (*data->prefix) { | |
688 | // Compose file path | |
819232d6 | 689 | r = pakfire_path_append(buffer, data->prefix, path); |
56796f84 | 690 | if (r) { |
9d956439 | 691 | CTX_ERROR(ctx, "Could not compose file path: %m\n"); |
24553159 | 692 | goto ERROR; |
79824416 MT |
693 | } |
694 | ||
695 | // Set file path | |
696 | archive_entry_set_pathname(entry, buffer); | |
697 | ||
698 | // Update hardlink destination | |
699 | const char* link = archive_entry_hardlink(entry); | |
700 | if (link) { | |
819232d6 | 701 | r = pakfire_path_append(buffer, data->prefix, link); |
56796f84 | 702 | if (r) { |
9d956439 | 703 | CTX_ERROR(ctx, "Could not compose hardlink path: %m\n"); |
24553159 | 704 | goto ERROR; |
79824416 MT |
705 | } |
706 | ||
707 | // Set hardlink path | |
708 | archive_entry_set_hardlink(entry, buffer); | |
709 | } | |
710 | } | |
711 | ||
24553159 MT |
712 | if (configfile) { |
713 | // Fetch path again since we changed it | |
714 | path = archive_entry_pathname(entry); | |
715 | ||
716 | if (pakfire_path_exists(path)) { | |
9d956439 | 717 | CTX_DEBUG(ctx, "The configuration file %s exists\n", |
24553159 MT |
718 | pakfire_file_get_path(file)); |
719 | ||
720 | r = pakfire_string_format(buffer, "%s.paknew", path); | |
721 | if (r) { | |
9d956439 | 722 | CTX_ERROR(ctx, "Could not compose path for configuration file: %m\n"); |
24553159 MT |
723 | goto ERROR; |
724 | } | |
725 | ||
726 | // Set the path again | |
727 | archive_entry_set_pathname(entry, buffer); | |
728 | } | |
729 | } | |
730 | ||
79824416 | 731 | // Create file & extract payload |
a3b276d1 | 732 | if (data->writer) { |
07c1c6e0 MT |
733 | // Fetch path again since we changed it |
734 | path = archive_entry_pathname(entry); | |
735 | ||
9d956439 | 736 | CTX_DEBUG(ctx, "Extracting %s\n", path); |
cf71c992 | 737 | |
89ee0833 MT |
738 | // Remove any extended attributes which we never write to disk |
739 | archive_entry_xattr_clear(entry); | |
740 | ||
729568d0 MT |
741 | // Set capabilities |
742 | if (pakfire_file_has_caps(file)) { | |
743 | r = pakfire_file_write_fcaps(file, &cap_data); | |
744 | if (r) | |
745 | goto ERROR; | |
746 | ||
747 | // Store capabilities in archive entry | |
748 | archive_entry_xattr_add_entry(entry, "security.capability", | |
749 | &cap_data, sizeof(cap_data)); | |
750 | } | |
751 | ||
89ee0833 | 752 | // Write payload |
a3b276d1 MT |
753 | r = archive_read_extract2(data->archive, entry, data->writer); |
754 | switch (r) { | |
755 | case ARCHIVE_OK: | |
756 | r = 0; | |
757 | break; | |
79824416 | 758 | |
a3b276d1 | 759 | case ARCHIVE_WARN: |
9d956439 | 760 | CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer)); |
79824416 | 761 | |
a3b276d1 MT |
762 | // Pretend everything has been okay |
763 | r = 0; | |
764 | break; | |
79824416 | 765 | |
a3b276d1 | 766 | case ARCHIVE_FATAL: |
9d956439 | 767 | CTX_ERROR(ctx, "%s\n", archive_error_string(data->writer)); |
a3b276d1 MT |
768 | r = 1; |
769 | break; | |
770 | } | |
79824416 MT |
771 | } |
772 | ||
bb7f09cd MT |
773 | ERROR: |
774 | if (file) | |
775 | pakfire_file_unref(file); | |
776 | ||
79824416 MT |
777 | return r; |
778 | } | |
779 | ||
780 | int pakfire_extract(struct pakfire* pakfire, struct archive* archive, | |
bb7f09cd | 781 | size_t size, struct pakfire_filelist* filelist, |
f8a46d2f MT |
782 | const char* prefix, const char* message, |
783 | pakfire_walk_filter_callback filter_callback, int flags) { | |
32bbeefc | 784 | int progress_flags = PAKFIRE_PROGRESS_SHOW_PERCENTAGE; |
79824416 MT |
785 | int r = 1; |
786 | ||
be12460f MT |
787 | struct pakfire_ctx* ctx = pakfire_ctx(pakfire); |
788 | ||
1ce8f42e | 789 | // Use / if no prefix is set |
a623c421 | 790 | if (!prefix) |
1ce8f42e | 791 | prefix = "/"; |
a623c421 | 792 | |
79824416 | 793 | struct pakfire_extract data = { |
d05af3ba MT |
794 | .pakfire = pakfire, |
795 | .archive = archive, | |
796 | .filelist = filelist, | |
1ce8f42e | 797 | .prefix = prefix, |
d05af3ba MT |
798 | .flags = flags, |
799 | .writer = NULL, | |
79824416 MT |
800 | }; |
801 | ||
a3b276d1 MT |
802 | // Is this a dry run? |
803 | const int dry_run = flags & PAKFIRE_EXTRACT_DRY_RUN; | |
804 | ||
79824416 | 805 | // Allocate writer |
a3b276d1 MT |
806 | if (!dry_run) { |
807 | data.writer = pakfire_make_archive_disk_writer(pakfire, 1); | |
808 | if (!data.writer) { | |
809 | ERROR(pakfire, "Could not create disk writer: %m\n"); | |
810 | r = 1; | |
811 | goto ERROR; | |
812 | } | |
79824416 MT |
813 | } |
814 | ||
32bbeefc MT |
815 | // Should we show any progress? |
816 | if (flags & PAKFIRE_EXTRACT_NO_PROGRESS) | |
817 | progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS; | |
79824416 | 818 | |
32bbeefc MT |
819 | // Show throughput? |
820 | if (flags & PAKFIRE_EXTRACT_SHOW_THROUGHPUT) | |
821 | progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED; | |
79824416 | 822 | |
fcaa90ac | 823 | // Create the progress indicator |
be12460f | 824 | r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL); |
d1076560 MT |
825 | if (r) |
826 | goto ERROR; | |
827 | ||
828 | // Set the title | |
829 | r = pakfire_progress_set_title(data.progress, "%s", message); | |
32bbeefc MT |
830 | if (r) |
831 | goto ERROR; | |
832 | ||
833 | // Register progress callback | |
834 | archive_read_extract_set_progress_callback(data.archive, | |
835 | pakfire_extract_progress, &data); | |
836 | ||
837 | // Start progress | |
838 | r = pakfire_progress_start(data.progress, size); | |
839 | if (r) | |
840 | goto ERROR; | |
79824416 | 841 | |
14eedd80 | 842 | // Walk through the entire archive |
9d956439 | 843 | r = pakfire_walk(ctx, archive, __pakfire_extract, filter_callback, &data); |
14eedd80 MT |
844 | if (r) |
845 | goto ERROR; | |
79824416 | 846 | |
fcaa90ac | 847 | // Finish the progress |
32bbeefc MT |
848 | r = pakfire_progress_finish(data.progress); |
849 | if (r) | |
850 | goto ERROR; | |
79824416 MT |
851 | |
852 | ERROR: | |
32bbeefc MT |
853 | if (data.progress) |
854 | pakfire_progress_unref(data.progress); | |
79824416 MT |
855 | if (data.writer) |
856 | archive_write_free(data.writer); | |
be12460f MT |
857 | if (ctx) |
858 | pakfire_ctx_unref(ctx); | |
79824416 MT |
859 | |
860 | return r; | |
861 | } | |
1e0d6cca MT |
862 | |
863 | // Common compression | |
864 | ||
865 | struct pakfire_compress { | |
866 | // Reference to Pakfire | |
867 | struct pakfire* pakfire; | |
868 | ||
869 | // Flags | |
870 | int flags; | |
871 | ||
872 | // The archive to write to | |
873 | struct archive* archive; | |
874 | ||
377c9b98 MT |
875 | // Resolver for hardlinks |
876 | struct archive_entry_linkresolver* linkresolver; | |
877 | ||
1e0d6cca MT |
878 | // The filelist of all files to write |
879 | struct pakfire_filelist* filelist; | |
880 | ||
fcaa90ac MT |
881 | // The progress indicator |
882 | struct pakfire_progress* progress; | |
1e0d6cca MT |
883 | |
884 | // Digests to write to the archive | |
885 | int digests; | |
886 | }; | |
887 | ||
7ce5af3a MT |
888 | static int pakfire_copy_data_from_file(struct pakfire* pakfire, |
889 | struct archive* archive, FILE* f) { | |
890 | char buffer[BUFFER_SIZE]; | |
891 | ||
892 | ssize_t bytes_read = 0; | |
893 | ssize_t bytes_written = 0; | |
894 | ||
895 | // Read file from the very beginning - also allows calling this multiple times | |
896 | rewind(f); | |
897 | ||
898 | // Loop through the entire length of the file | |
899 | while (!feof(f)) { | |
900 | // Read a block from file | |
901 | bytes_read = fread(buffer, 1, sizeof(buffer), f); | |
902 | ||
903 | // Check if any error occured | |
904 | if (ferror(f)) { | |
905 | ERROR(pakfire, "Read error: %m\n"); | |
906 | return -errno; | |
907 | } | |
908 | ||
909 | // Write the block to the archive | |
910 | bytes_written = archive_write_data(archive, buffer, bytes_read); | |
911 | if (bytes_written < bytes_read) { | |
912 | ERROR(pakfire, "Write error: %s\n", archive_error_string(archive)); | |
913 | return -errno; | |
914 | } | |
915 | } | |
916 | ||
917 | return 0; | |
918 | } | |
919 | ||
377c9b98 MT |
920 | static int __pakfire_compress_entry(struct pakfire* pakfire, struct pakfire_file* file, |
921 | struct pakfire_compress* data, struct archive_entry* entry) { | |
1e0d6cca | 922 | FILE* f = NULL; |
377c9b98 | 923 | int r; |
1e0d6cca | 924 | |
e990815d MT |
925 | const char* path = archive_entry_pathname(entry); |
926 | ||
927 | // Remove any leading slahes | |
928 | while (*path == '/') | |
929 | path++; | |
930 | ||
931 | archive_entry_set_pathname(entry, path); | |
932 | ||
1e0d6cca MT |
933 | // Write the header |
934 | r = archive_write_header(data->archive, entry); | |
935 | if (r) { | |
936 | ERROR(pakfire, "Error writing file header: %s\n", | |
937 | archive_error_string(data->archive)); | |
938 | goto ERROR; | |
939 | } | |
940 | ||
377c9b98 MT |
941 | // Copy the data if there is any |
942 | if (archive_entry_size(entry)) { | |
1e0d6cca MT |
943 | // Open the file |
944 | f = pakfire_file_open(file); | |
945 | if (!f) { | |
946 | r = 1; | |
947 | goto ERROR; | |
948 | } | |
949 | ||
950 | // Copy the payload into the archive | |
7ce5af3a | 951 | r = pakfire_copy_data_from_file(pakfire, data->archive, f); |
1e0d6cca MT |
952 | if (r) |
953 | goto ERROR; | |
954 | } | |
955 | ||
956 | // Write trailer | |
957 | r = archive_write_finish_entry(data->archive); | |
958 | if (r) | |
959 | goto ERROR; | |
960 | ||
377c9b98 MT |
961 | ERROR: |
962 | if (f) | |
963 | fclose(f); | |
964 | ||
965 | return r; | |
966 | } | |
967 | ||
968 | static int __pakfire_compress(struct pakfire* pakfire, struct pakfire_file* file, void* p) { | |
969 | struct archive_entry* entry = NULL; | |
970 | struct archive_entry* sparse_entry = NULL; | |
971 | int r = 1; | |
972 | ||
973 | struct pakfire_compress* data = (struct pakfire_compress*)p; | |
974 | ||
975 | // Fetch the file size | |
976 | const size_t size = pakfire_file_get_size(file); | |
977 | ||
978 | // Generate file metadata into an archive entry | |
979 | entry = pakfire_file_archive_entry(file, data->digests); | |
980 | if (!entry) { | |
981 | r = 1; | |
982 | goto ERROR; | |
983 | } | |
984 | ||
985 | // Perform search for hardlinks | |
986 | archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); | |
987 | ||
988 | // Write the main entry | |
989 | if (entry) { | |
990 | r = __pakfire_compress_entry(pakfire, file, data, entry); | |
991 | if (r) | |
992 | goto ERROR; | |
993 | } | |
994 | ||
995 | // Write the sparse entry | |
996 | if (sparse_entry) { | |
997 | r = __pakfire_compress_entry(pakfire, file, data, sparse_entry); | |
998 | if (r) | |
999 | goto ERROR; | |
1000 | } | |
1001 | ||
1002 | // Query the link resolver for any more entries | |
1003 | for (;;) { | |
1004 | // Free the entry | |
1005 | if (entry) { | |
1006 | archive_entry_free(entry); | |
1007 | entry = NULL; | |
1008 | } | |
1009 | ||
1010 | // Free the sparse entry | |
1011 | if (sparse_entry) { | |
1012 | archive_entry_free(sparse_entry); | |
1013 | sparse_entry = NULL; | |
1014 | } | |
1015 | ||
1016 | // Fetch the next entry | |
1017 | archive_entry_linkify(data->linkresolver, &entry, &sparse_entry); | |
1018 | if (!entry) | |
1019 | break; | |
1020 | ||
1021 | // Write the entry to the archive | |
1022 | r = __pakfire_compress_entry(pakfire, file, data, entry); | |
1023 | if (r) | |
1024 | goto ERROR; | |
1025 | } | |
1026 | ||
fcaa90ac MT |
1027 | // Update the progress |
1028 | pakfire_progress_increment(data->progress, size); | |
1e0d6cca MT |
1029 | |
1030 | ERROR: | |
1e0d6cca MT |
1031 | if (entry) |
1032 | archive_entry_free(entry); | |
377c9b98 MT |
1033 | if (sparse_entry) |
1034 | archive_entry_free(sparse_entry); | |
1e0d6cca MT |
1035 | |
1036 | return r; | |
1037 | } | |
1038 | ||
1039 | int pakfire_compress(struct pakfire* pakfire, struct archive* archive, | |
1040 | struct pakfire_filelist* filelist, const char* message, int flags, int digests) { | |
fcaa90ac MT |
1041 | int progress_flags = |
1042 | PAKFIRE_PROGRESS_SHOW_PERCENTAGE | | |
1043 | PAKFIRE_PROGRESS_SHOW_BYTES_TRANSFERRED; | |
1e0d6cca MT |
1044 | int r = 1; |
1045 | ||
be12460f MT |
1046 | struct pakfire_ctx* ctx = pakfire_ctx(pakfire); |
1047 | ||
1e0d6cca MT |
1048 | struct pakfire_compress data = { |
1049 | .pakfire = pakfire, | |
1050 | .archive = archive, | |
1051 | .filelist = filelist, | |
1052 | .flags = flags, | |
1053 | .digests = digests, | |
1054 | }; | |
1055 | ||
1056 | // Should we show a progress bar? | |
fcaa90ac MT |
1057 | if (flags & PAKFIRE_COMPRESS_NO_PROGRESS) |
1058 | progress_flags |= PAKFIRE_PROGRESS_NO_PROGRESS; | |
1059 | ||
1060 | // Should we show throughput? | |
1061 | if (flags & PAKFIRE_COMPRESS_SHOW_THROUGHPUT) | |
1062 | progress_flags |= PAKFIRE_PROGRESS_SHOW_TRANSFER_SPEED; | |
1e0d6cca MT |
1063 | |
1064 | // Fetch the length of the filelist | |
1065 | const size_t size = pakfire_filelist_total_size(filelist); | |
1066 | ||
fcaa90ac | 1067 | // Create the progress indicator |
be12460f | 1068 | r = pakfire_progress_create(&data.progress, ctx, progress_flags, NULL); |
fcaa90ac MT |
1069 | if (r) |
1070 | goto ERROR; | |
1e0d6cca | 1071 | |
fcaa90ac MT |
1072 | // Set progress title |
1073 | r = pakfire_progress_set_title(data.progress, "%s", message); | |
1074 | if (r) | |
1075 | goto ERROR; | |
1076 | ||
1077 | // Start progress | |
1078 | r = pakfire_progress_start(data.progress, size); | |
1079 | if (r) | |
1080 | goto ERROR; | |
1e0d6cca | 1081 | |
377c9b98 MT |
1082 | // Setup the link resolver |
1083 | data.linkresolver = archive_entry_linkresolver_new(); | |
1084 | if (!data.linkresolver) { | |
1085 | ERROR(pakfire, "Could not setup link resolver: m\n"); | |
1086 | goto ERROR; | |
1087 | } | |
1088 | ||
1089 | // Set the link resolver strategy | |
1090 | archive_entry_linkresolver_set_strategy(data.linkresolver, archive_format(archive)); | |
1091 | ||
1e0d6cca | 1092 | // Walk through the entire filelist |
a3f3c077 | 1093 | r = pakfire_filelist_walk(filelist, __pakfire_compress, &data, 0); |
1e0d6cca MT |
1094 | if (r) |
1095 | goto ERROR; | |
1096 | ||
fcaa90ac MT |
1097 | // Finish the progress |
1098 | if (data.progress) | |
1099 | pakfire_progress_finish(data.progress); | |
1e0d6cca MT |
1100 | |
1101 | ERROR: | |
fcaa90ac MT |
1102 | if (data.progress) |
1103 | pakfire_progress_unref(data.progress); | |
377c9b98 MT |
1104 | if (data.linkresolver) |
1105 | archive_entry_linkresolver_free(data.linkresolver); | |
be12460f MT |
1106 | if (ctx) |
1107 | pakfire_ctx_unref(ctx); | |
1e0d6cca MT |
1108 | |
1109 | return r; | |
1110 | } | |
e66c6bbc MT |
1111 | |
1112 | int pakfire_compress_create_archive(struct pakfire* pakfire, struct archive** archive, | |
1113 | FILE* f, const enum pakfire_compressions compression, const unsigned int level) { | |
1114 | struct archive* a = NULL; | |
1115 | char value[16]; | |
1116 | int r; | |
1117 | ||
1118 | // Open a new archive | |
1119 | a = archive_write_new(); | |
1120 | if (!a) { | |
1121 | ERROR(pakfire, "archive_write_new() failed\n"); | |
1122 | r = 1; | |
1123 | goto ERROR; | |
1124 | } | |
1125 | ||
1126 | // Use the PAX format | |
1127 | r = archive_write_set_format_pax(a); | |
1128 | if (r) { | |
1129 | ERROR(pakfire, "Could not set format to PAX: %s\n", archive_error_string(a)); | |
1130 | goto ERROR; | |
1131 | } | |
1132 | ||
1133 | // Store any extended attributes in the SCHILY headers | |
1134 | r = archive_write_set_format_option(a, "pax", "xattrheader", "SCHILY"); | |
1135 | if (r) { | |
1136 | ERROR(pakfire, "Could not set xattrheader option: %s\n", archive_error_string(a)); | |
1137 | goto ERROR; | |
1138 | } | |
1139 | ||
1140 | switch (compression) { | |
1141 | case PAKFIRE_COMPRESS_ZSTD: | |
1142 | // Enable Zstd | |
1143 | r = archive_write_add_filter_zstd(a); | |
1144 | if (r) { | |
1145 | ERROR(pakfire, "Could not enable Zstandard compression: %s\n", | |
1146 | archive_error_string(a)); | |
1147 | goto ERROR; | |
1148 | } | |
1149 | ||
1150 | // Do not pad the last block | |
1151 | archive_write_set_bytes_in_last_block(a, 1); | |
1152 | ||
1153 | // Set compression level | |
1154 | if (level) { | |
1155 | r = pakfire_string_format(value, "%u", level); | |
1156 | if (r) | |
1157 | goto ERROR; | |
1158 | ||
1159 | r = archive_write_set_filter_option(a, NULL, "compression-level", value); | |
1160 | if (r) { | |
1161 | ERROR(pakfire, "Could not set Zstandard compression level: %s\n", | |
1162 | archive_error_string(a)); | |
1163 | goto ERROR; | |
1164 | } | |
1165 | } | |
177f5ff3 MT |
1166 | |
1167 | #if ARCHIVE_VERSION_NUMBER >= 3006000 | |
1168 | // Fetch numbers of processors | |
1169 | long processors = sysconf(_SC_NPROCESSORS_ONLN); | |
1170 | ||
1171 | if (processors > 1) { | |
1172 | r = pakfire_string_format(value, "%ld", processors); | |
1173 | if (r) { | |
1174 | ERROR(pakfire, "Could not format threads: %m\n"); | |
1175 | goto ERROR; | |
1176 | } | |
1177 | ||
1178 | // Try using multiple threads | |
1d7a641f | 1179 | r = archive_write_set_filter_option(a, NULL, "threads", value); |
177f5ff3 | 1180 | if (r) { |
1d7a641f MT |
1181 | ERROR(pakfire, "Could not enable %ld threads for compression: %s\n", |
1182 | processors, archive_error_string(a)); | |
177f5ff3 MT |
1183 | goto ERROR; |
1184 | } | |
1185 | } | |
1186 | #endif | |
e66c6bbc MT |
1187 | } |
1188 | ||
1189 | // Write archive to f | |
1190 | if (f) { | |
1191 | r = archive_write_open_FILE(a, f); | |
1192 | if (r) { | |
1193 | ERROR(pakfire, "archive_write_open_FILE() failed: %s\n", archive_error_string(a)); | |
1194 | goto ERROR; | |
1195 | } | |
1196 | } | |
1197 | ||
1198 | // Success | |
1199 | *archive = a; | |
1200 | ||
1201 | return 0; | |
1202 | ||
1203 | ERROR: | |
1204 | if (a) | |
1205 | archive_write_free(a); | |
1206 | ||
1207 | return r; | |
1208 | } |