]> git.ipfire.org Git - thirdparty/libarchive.git/blob - libarchive/archive_write_add_filter_zstd.c
projects / thirdparty / libarchive.git / blob
? search:


94249accd08b61d8fe2ec4331320a91a02a70272
[thirdparty/libarchive.git] / libarchive / archive_write_add_filter_zstd.c
1 /*-
2 * Copyright (c) 2017 Sean Purcell
3 * Copyright (c) 2023-2024 Klara, Inc.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27 #include "archive_platform.h"
28
29 #ifdef HAVE_ERRNO_H
30 #include <errno.h>
31 #endif
32 #ifdef HAVE_STDINT_H
33 #include <stdint.h>
34 #endif
35 #ifdef HAVE_STDLIB_H
36 #include <stdlib.h>
37 #endif
38 #ifdef HAVE_STRING_H
39 #include <string.h>
40 #endif
41 #ifdef HAVE_ZSTD_H
42 #include <zstd.h>
43 #endif
44
45 #include "archive.h"
46 #include "archive_private.h"
47 #include "archive_string.h"
48 #include "archive_write_private.h"
49
50 /* Don't compile this if we don't have zstd.h */
51
52 struct private_data {
53 int compression_level;
54 int threads;
55 int long_distance;
56 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
57 enum {
58 running,
59 finishing,
60 resetting,
61 } state;
62 int frame_per_file;
63 size_t min_frame_in;
64 size_t max_frame_in;
65 size_t min_frame_out;
66 size_t max_frame_out;
67 size_t cur_frame;
68 size_t cur_frame_in;
69 size_t cur_frame_out;
70 size_t total_in;
71 ZSTD_CStream *cstream;
72 ZSTD_outBuffer out;
73 #else
74 struct archive_write_program_data *pdata;
75 #endif
76 };
77
78 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */
79 #define CLEVEL_MIN -99
80 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
81 #define CLEVEL_DEFAULT 3
82 #define CLEVEL_STD_MAX 19 /* without using --ultra */
83 #define CLEVEL_MAX 22
84
85 #define LONG_STD 27
86
87 #define MINVER_NEGCLEVEL 10304
88 #define MINVER_MINCLEVEL 10306
89 #define MINVER_LONG 10302
90
91 static int archive_compressor_zstd_options(struct archive_write_filter *,
92 const char *, const char *);
93 static int archive_compressor_zstd_open(struct archive_write_filter *);
94 static int archive_compressor_zstd_write(struct archive_write_filter *,
95 const void *, size_t);
96 static int archive_compressor_zstd_flush(struct archive_write_filter *);
97 static int archive_compressor_zstd_close(struct archive_write_filter *);
98 static int archive_compressor_zstd_free(struct archive_write_filter *);
99 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
100 static int drive_compressor(struct archive_write_filter *,
101 struct private_data *, int, const void *, size_t);
102 #endif
103
104
105 /*
106 * Add a zstd compression filter to this write handle.
107 */
108 int
109 archive_write_add_filter_zstd(struct archive *_a)
110 {
111 struct archive_write *a = (struct archive_write *)_a;
112 struct archive_write_filter *f = __archive_write_allocate_filter(_a);
113 struct private_data *data;
114 archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
115 ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
116
117 data = calloc(1, sizeof(*data));
118 if (data == NULL) {
119 archive_set_error(&a->archive, ENOMEM, "Out of memory");
120 return (ARCHIVE_FATAL);
121 }
122 f->data = data;
123 f->open = &archive_compressor_zstd_open;
124 f->options = &archive_compressor_zstd_options;
125 f->flush = &archive_compressor_zstd_flush;
126 f->close = &archive_compressor_zstd_close;
127 f->free = &archive_compressor_zstd_free;
128 f->code = ARCHIVE_FILTER_ZSTD;
129 f->name = "zstd";
130 data->compression_level = CLEVEL_DEFAULT;
131 data->threads = 0;
132 data->long_distance = 0;
133 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
134 data->frame_per_file = 0;
135 data->min_frame_in = 0;
136 data->max_frame_in = SIZE_MAX;
137 data->min_frame_out = 0;
138 data->max_frame_out = SIZE_MAX;
139 data->cur_frame_in = 0;
140 data->cur_frame_out = 0;
141 data->cstream = ZSTD_createCStream();
142 if (data->cstream == NULL) {
143 free(data);
144 archive_set_error(&a->archive, ENOMEM,
145 "Failed to allocate zstd compressor object");
146 return (ARCHIVE_FATAL);
147 }
148
149 return (ARCHIVE_OK);
150 #else
151 data->pdata = __archive_write_program_allocate("zstd");
152 if (data->pdata == NULL) {
153 free(data);
154 archive_set_error(&a->archive, ENOMEM, "Out of memory");
155 return (ARCHIVE_FATAL);
156 }
157 archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
158 "Using external zstd program");
159 return (ARCHIVE_WARN);
160 #endif
161 }
162
163 static int
164 archive_compressor_zstd_free(struct archive_write_filter *f)
165 {
166 struct private_data *data = (struct private_data *)f->data;
167 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
168 ZSTD_freeCStream(data->cstream);
169 free(data->out.dst);
170 #else
171 __archive_write_program_free(data->pdata);
172 #endif
173 free(data);
174 f->data = NULL;
175 return (ARCHIVE_OK);
176 }
177
178 static int
179 string_to_number(const char *string, intmax_t *numberp)
180 {
181 char *end;
182
183 if (string == NULL || *string == '\0')
184 return (ARCHIVE_WARN);
185 *numberp = strtoimax(string, &end, 10);
186 if (end == string || *end != '\0' || errno == EOVERFLOW) {
187 *numberp = 0;
188 return (ARCHIVE_WARN);
189 }
190 return (ARCHIVE_OK);
191 }
192
193 static int
194 string_to_size(const char *string, size_t *numberp)
195 {
196 uintmax_t number;
197 char *end;
198 unsigned int shift = 0;
199
200 if (string == NULL || *string == '\0' || *string == '-')
201 return (ARCHIVE_WARN);
202 number = strtoumax(string, &end, 10);
203 if (end > string) {
204 if (*end == 'K' || *end == 'k') {
205 shift = 10;
206 end++;
207 } else if (*end == 'M' || *end == 'm') {
208 shift = 20;
209 end++;
210 } else if (*end == 'G' || *end == 'g') {
211 shift = 30;
212 end++;
213 }
214 if (*end == 'B' || *end == 'b') {
215 end++;
216 }
217 }
218 if (end == string || *end != '\0' || errno == EOVERFLOW) {
219 return (ARCHIVE_WARN);
220 }
221 if (number > (uintmax_t)SIZE_MAX >> shift) {
222 return (ARCHIVE_WARN);
223 }
224 *numberp = (size_t)(number << shift);
225 return (ARCHIVE_OK);
226 }
227
228 /*
229 * Set write options.
230 */
231 static int
232 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
233 const char *value)
234 {
235 struct private_data *data = (struct private_data *)f->data;
236
237 if (strcmp(key, "compression-level") == 0) {
238 intmax_t level;
239 if (string_to_number(value, &level) != ARCHIVE_OK) {
240 return (ARCHIVE_WARN);
241 }
242 /* If we don't have the library, hard-code the max level */
243 int minimum = CLEVEL_MIN;
244 int maximum = CLEVEL_MAX;
245 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
246 maximum = ZSTD_maxCLevel();
247 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
248 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
249 minimum = ZSTD_minCLevel();
250 }
251 else
252 #endif
253 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
254 minimum = CLEVEL_STD_MIN;
255 }
256 #endif
257 if (level < minimum || level > maximum) {
258 return (ARCHIVE_WARN);
259 }
260 data->compression_level = (int)level;
261 return (ARCHIVE_OK);
262 } else if (strcmp(key, "threads") == 0) {
263 intmax_t threads;
264 if (string_to_number(value, &threads) != ARCHIVE_OK) {
265 return (ARCHIVE_WARN);
266 }
267 if (threads < 0) {
268 return (ARCHIVE_WARN);
269 }
270 data->threads = (int)threads;
271 return (ARCHIVE_OK);
272 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
273 } else if (strcmp(key, "frame-per-file") == 0) {
274 data->frame_per_file = 1;
275 return (ARCHIVE_OK);
276 } else if (strcmp(key, "min-frame-in") == 0) {
277 if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) {
278 return (ARCHIVE_WARN);
279 }
280 return (ARCHIVE_OK);
281 } else if (strcmp(key, "min-frame-out") == 0 ||
282 strcmp(key, "min-frame-size") == 0) {
283 if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) {
284 return (ARCHIVE_WARN);
285 }
286 return (ARCHIVE_OK);
287 } else if (strcmp(key, "max-frame-in") == 0 ||
288 strcmp(key, "max-frame-size") == 0) {
289 if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK ||
290 data->max_frame_in < 1024) {
291 return (ARCHIVE_WARN);
292 }
293 return (ARCHIVE_OK);
294 } else if (strcmp(key, "max-frame-out") == 0) {
295 if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK ||
296 data->max_frame_out < 1024) {
297 return (ARCHIVE_WARN);
298 }
299 return (ARCHIVE_OK);
300 #endif
301 }
302 else if (strcmp(key, "long") == 0) {
303 intmax_t long_distance;
304 if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
305 return (ARCHIVE_WARN);
306 }
307 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
308 ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
309 if (ZSTD_isError(bounds.error)) {
310 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
311 if (((int)long_distance) < 10 || (int)long_distance > max_distance)
312 return (ARCHIVE_WARN);
313 } else {
314 if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound)
315 return (ARCHIVE_WARN);
316 }
317 #else
318 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
319 if (((int)long_distance) < 10 || (int)long_distance > max_distance)
320 return (ARCHIVE_WARN);
321 #endif
322 data->long_distance = (int)long_distance;
323 return (ARCHIVE_OK);
324 }
325
326 /* Note: The "warn" return is just to inform the options
327 * supervisor that we didn't handle it. It will generate
328 * a suitable error if no one used this option. */
329 return (ARCHIVE_WARN);
330 }
331
332 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
333 /*
334 * Setup callback.
335 */
336 static int
337 archive_compressor_zstd_open(struct archive_write_filter *f)
338 {
339 struct private_data *data = (struct private_data *)f->data;
340
341 if (data->out.dst == NULL) {
342 size_t bs = ZSTD_CStreamOutSize(), bpb;
343 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
344 /* Buffer size should be a multiple number of
345 * the of bytes per block for performance. */
346 bpb = archive_write_get_bytes_per_block(f->archive);
347 if (bpb > bs)
348 bs = bpb;
349 else if (bpb != 0)
350 bs -= bs % bpb;
351 }
352 data->out.size = bs;
353 data->out.pos = 0;
354 data->out.dst
355 = (unsigned char *)malloc(data->out.size);
356 if (data->out.dst == NULL) {
357 archive_set_error(f->archive, ENOMEM,
358 "Can't allocate data for compression buffer");
359 return (ARCHIVE_FATAL);
360 }
361 }
362
363 f->write = archive_compressor_zstd_write;
364
365 if (ZSTD_isError(ZSTD_initCStream(data->cstream,
366 data->compression_level))) {
367 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
368 "Internal error initializing zstd compressor object");
369 return (ARCHIVE_FATAL);
370 }
371
372 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);
373
374 #if ZSTD_VERSION_NUMBER >= MINVER_LONG
375 ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance);
376 #endif
377
378 return (ARCHIVE_OK);
379 }
380
381 /*
382 * Write data to the compressed stream.
383 */
384 static int
385 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
386 size_t length)
387 {
388 struct private_data *data = (struct private_data *)f->data;
389
390 return (drive_compressor(f, data, 0, buff, length));
391 }
392
393 /*
394 * Flush the compressed stream.
395 */
396 static int
397 archive_compressor_zstd_flush(struct archive_write_filter *f)
398 {
399 struct private_data *data = (struct private_data *)f->data;
400
401 if (data->frame_per_file && data->state == running) {
402 if (data->cur_frame_in > data->min_frame_in &&
403 data->cur_frame_out > data->min_frame_out) {
404 data->state = finishing;
405 }
406 }
407 return (drive_compressor(f, data, 1, NULL, 0));
408 }
409
410 /*
411 * Finish the compression...
412 */
413 static int
414 archive_compressor_zstd_close(struct archive_write_filter *f)
415 {
416 struct private_data *data = (struct private_data *)f->data;
417
418 if (data->state == running)
419 data->state = finishing;
420 return (drive_compressor(f, data, 1, NULL, 0));
421 }
422
423 /*
424 * Utility function to push input data through compressor,
425 * writing full output blocks as necessary.
426 */
427 static int
428 drive_compressor(struct archive_write_filter *f,
429 struct private_data *data, int flush, const void *src, size_t length)
430 {
431 ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
432 size_t ipos, opos, zstdret = 0;
433 int ret;
434
435 for (;;) {
436 ipos = in.pos;
437 opos = data->out.pos;
438 switch (data->state) {
439 case running:
440 if (in.pos == in.size)
441 return (ARCHIVE_OK);
442 zstdret = ZSTD_compressStream(data->cstream,
443 &data->out, &in);
444 if (ZSTD_isError(zstdret))
445 goto zstd_fatal;
446 break;
447 case finishing:
448 zstdret = ZSTD_endStream(data->cstream, &data->out);
449 if (ZSTD_isError(zstdret))
450 goto zstd_fatal;
451 if (zstdret == 0)
452 data->state = resetting;
453 break;
454 case resetting:
455 ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only);
456 data->cur_frame++;
457 data->cur_frame_in = 0;
458 data->cur_frame_out = 0;
459 data->state = running;
460 break;
461 }
462 data->total_in += in.pos - ipos;
463 data->cur_frame_in += in.pos - ipos;
464 data->cur_frame_out += data->out.pos - opos;
465 if (data->state == running) {
466 if (data->cur_frame_in >= data->max_frame_in ||
467 data->cur_frame_out >= data->max_frame_out) {
468 data->state = finishing;
469 }
470 }
471 if (data->out.pos == data->out.size ||
472 (flush && data->out.pos > 0)) {
473 ret = __archive_write_filter(f->next_filter,
474 data->out.dst, data->out.pos);
475 if (ret != ARCHIVE_OK)
476 goto fatal;
477 data->out.pos = 0;
478 }
479 }
480 zstd_fatal:
481 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
482 "Zstd compression failed: %s",
483 ZSTD_getErrorName(zstdret));
484 fatal:
485 return (ARCHIVE_FATAL);
486 }
487
488 #else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
489
490 static int
491 archive_compressor_zstd_open(struct archive_write_filter *f)
492 {
493 struct private_data *data = (struct private_data *)f->data;
494 struct archive_string as;
495 int r;
496
497 archive_string_init(&as);
498 /* --no-check matches library default */
499 archive_strcpy(&as, "zstd --no-check");
500
501 if (data->compression_level < CLEVEL_STD_MIN) {
502 archive_string_sprintf(&as, " --fast=%d", -data->compression_level);
503 } else {
504 archive_string_sprintf(&as, " -%d", data->compression_level);
505 }
506
507 if (data->compression_level > CLEVEL_STD_MAX) {
508 archive_strcat(&as, " --ultra");
509 }
510
511 if (data->threads != 0) {
512 archive_string_sprintf(&as, " --threads=%d", data->threads);
513 }
514
515 if (data->long_distance != 0) {
516 archive_string_sprintf(&as, " --long=%d", data->long_distance);
517 }
518
519 f->write = archive_compressor_zstd_write;
520 r = __archive_write_program_open(f, data->pdata, as.s);
521 archive_string_free(&as);
522 return (r);
523 }
524
525 static int
526 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
527 size_t length)
528 {
529 struct private_data *data = (struct private_data *)f->data;
530
531 return __archive_write_program_write(f, data->pdata, buff, length);
532 }
533
534 static int
535 archive_compressor_zstd_flush(struct archive_write_filter *f)
536 {
537 (void)f; /* UNUSED */
538
539 return (ARCHIVE_OK);
540 }
541
542 static int
543 archive_compressor_zstd_close(struct archive_write_filter *f)
544 {
545 struct private_data *data = (struct private_data *)f->data;
546
547 return __archive_write_program_close(f, data->pdata);
548 }
549
550 #endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
Mirror of https://github.com/libarchive/libarchive.git