libarchive/archive_write_add_filter_zstd.c

   1 /*-
   2  * Copyright (c) 2017 Sean Purcell
   3  * Copyright (c) 2023-2024 Klara, Inc.
   4  * All rights reserved.
   5  *
   6  * Redistribution and use in source and binary forms, with or without
   7  * modification, are permitted provided that the following conditions
   8  * are met:
   9  * 1. Redistributions of source code must retain the above copyright
  10  *    notice, this list of conditions and the following disclaimer.
  11  * 2. Redistributions in binary form must reproduce the above copyright
  12  *    notice, this list of conditions and the following disclaimer in the
  13  *    documentation and/or other materials provided with the distribution.
  14  *
  15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) ``AS IS'' AND ANY EXPRESS OR
  16  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  17  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  18  * IN NO EVENT SHALL THE AUTHOR(S) BE LIABLE FOR ANY DIRECT, INDIRECT,
  19  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  20  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  21  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  22  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  23  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  24  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  25  */
  26
  27 #include "archive_platform.h"
  28
  29 #ifdef HAVE_ERRNO_H
  30 #include <errno.h>
  31 #endif
  32 #ifdef HAVE_STDINT_H
  33 #include <stdint.h>
  34 #endif
  35 #ifdef HAVE_STDLIB_H
  36 #include <stdlib.h>
  37 #endif
  38 #ifdef HAVE_STRING_H
  39 #include <string.h>
  40 #endif
  41 #ifdef HAVE_ZSTD_H
  42 #include <zstd.h>
  43 #endif
  44
  45 #include "archive.h"
  46 #include "archive_private.h"
  47 #include "archive_string.h"
  48 #include "archive_write_private.h"
  49
  50 /* Don't compile this if we don't have zstd.h */
  51
  52 struct private_data {
  53         int              compression_level;
  54         int              threads;
  55         int              long_distance;
  56 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
  57         enum {
  58                 running,
  59                 finishing,
  60                 resetting,
  61         } state;
  62         int              frame_per_file;
  63         size_t           min_frame_in;
  64         size_t           max_frame_in;
  65         size_t           min_frame_out;
  66         size_t           max_frame_out;
  67         size_t           cur_frame;
  68         size_t           cur_frame_in;
  69         size_t           cur_frame_out;
  70         size_t           total_in;
  71         ZSTD_CStream    *cstream;
  72         ZSTD_outBuffer   out;
  73 #else
  74         struct archive_write_program_data *pdata;
  75 #endif
  76 };
  77
  78 /* If we don't have the library use default range values (zstdcli.c v1.4.0) */
  79 #define CLEVEL_MIN -99
  80 #define CLEVEL_STD_MIN 0 /* prior to 1.3.4 and more recent without using --fast */
  81 #define CLEVEL_DEFAULT 3
  82 #define CLEVEL_STD_MAX 19 /* without using --ultra */
  83 #define CLEVEL_MAX 22
  84
  85 #define LONG_STD 27
  86
  87 #define MINVER_NEGCLEVEL 10304
  88 #define MINVER_MINCLEVEL 10306
  89 #define MINVER_LONG 10302
  90
  91 static int archive_compressor_zstd_options(struct archive_write_filter *,
  92                     const char *, const char *);
  93 static int archive_compressor_zstd_open(struct archive_write_filter *);
  94 static int archive_compressor_zstd_write(struct archive_write_filter *,
  95                     const void *, size_t);
  96 static int archive_compressor_zstd_flush(struct archive_write_filter *);
  97 static int archive_compressor_zstd_close(struct archive_write_filter *);
  98 static int archive_compressor_zstd_free(struct archive_write_filter *);
  99 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 100 static int drive_compressor(struct archive_write_filter *,
 101                     struct private_data *, int, const void *, size_t);
 102 #endif
 103
 104
 105 /*
 106  * Add a zstd compression filter to this write handle.
 107  */
 108 int
 109 archive_write_add_filter_zstd(struct archive *_a)
 110 {
 111         struct archive_write *a = (struct archive_write *)_a;
 112         struct archive_write_filter *f = __archive_write_allocate_filter(_a);
 113         struct private_data *data;
 114         archive_check_magic(&a->archive, ARCHIVE_WRITE_MAGIC,
 115             ARCHIVE_STATE_NEW, "archive_write_add_filter_zstd");
 116
 117         data = calloc(1, sizeof(*data));
 118         if (data == NULL) {
 119                 archive_set_error(&a->archive, ENOMEM, "Out of memory");
 120                 return (ARCHIVE_FATAL);
 121         }
 122         f->data = data;
 123         f->open = &archive_compressor_zstd_open;
 124         f->options = &archive_compressor_zstd_options;
 125         f->flush = &archive_compressor_zstd_flush;
 126         f->close = &archive_compressor_zstd_close;
 127         f->free = &archive_compressor_zstd_free;
 128         f->code = ARCHIVE_FILTER_ZSTD;
 129         f->name = "zstd";
 130         data->compression_level = CLEVEL_DEFAULT;
 131         data->threads = 0;
 132         data->long_distance = 0;
 133 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 134         data->frame_per_file = 0;
 135         data->min_frame_in = 0;
 136         data->max_frame_in = SIZE_MAX;
 137         data->min_frame_out = 0;
 138         data->max_frame_out = SIZE_MAX;
 139         data->cur_frame_in = 0;
 140         data->cur_frame_out = 0;
 141         data->cstream = ZSTD_createCStream();
 142         if (data->cstream == NULL) {
 143                 free(data);
 144                 archive_set_error(&a->archive, ENOMEM,
 145                     "Failed to allocate zstd compressor object");
 146                 return (ARCHIVE_FATAL);
 147         }
 148
 149         return (ARCHIVE_OK);
 150 #else
 151         data->pdata = __archive_write_program_allocate("zstd");
 152         if (data->pdata == NULL) {
 153                 free(data);
 154                 archive_set_error(&a->archive, ENOMEM, "Out of memory");
 155                 return (ARCHIVE_FATAL);
 156         }
 157         archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
 158             "Using external zstd program");
 159         return (ARCHIVE_WARN);
 160 #endif
 161 }
 162
 163 static int
 164 archive_compressor_zstd_free(struct archive_write_filter *f)
 165 {
 166         struct private_data *data = (struct private_data *)f->data;
 167 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 168         ZSTD_freeCStream(data->cstream);
 169         free(data->out.dst);
 170 #else
 171         __archive_write_program_free(data->pdata);
 172 #endif
 173         free(data);
 174         f->data = NULL;
 175         return (ARCHIVE_OK);
 176 }
 177
 178 static int
 179 string_to_number(const char *string, intmax_t *numberp)
 180 {
 181         char *end;
 182
 183         if (string == NULL || *string == '\0')
 184                 return (ARCHIVE_WARN);
 185         *numberp = strtoimax(string, &end, 10);
 186         if (end == string || *end != '\0' || errno == EOVERFLOW) {
 187                 *numberp = 0;
 188                 return (ARCHIVE_WARN);
 189         }
 190         return (ARCHIVE_OK);
 191 }
 192
 193 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 194 static int
 195 string_to_size(const char *string, size_t *numberp)
 196 {
 197         uintmax_t number;
 198         char *end;
 199         unsigned int shift = 0;
 200
 201         if (string == NULL || *string == '\0' || *string == '-')
 202                 return (ARCHIVE_WARN);
 203         number = strtoumax(string, &end, 10);
 204         if (end > string) {
 205                 if (*end == 'K' || *end == 'k') {
 206                         shift = 10;
 207                         end++;
 208                 } else if (*end == 'M' || *end == 'm') {
 209                         shift = 20;
 210                         end++;
 211                 } else if (*end == 'G' || *end == 'g') {
 212                         shift = 30;
 213                         end++;
 214                 }
 215                 if (*end == 'B' || *end == 'b') {
 216                         end++;
 217                 }
 218         }
 219         if (end == string || *end != '\0' || errno == EOVERFLOW) {
 220                 return (ARCHIVE_WARN);
 221         }
 222         if (number > (uintmax_t)SIZE_MAX >> shift) {
 223                 return (ARCHIVE_WARN);
 224         }
 225         *numberp = (size_t)(number << shift);
 226         return (ARCHIVE_OK);
 227 }
 228 #endif
 229
 230 /*
 231  * Set write options.
 232  */
 233 static int
 234 archive_compressor_zstd_options(struct archive_write_filter *f, const char *key,
 235     const char *value)
 236 {
 237         struct private_data *data = (struct private_data *)f->data;
 238
 239         if (strcmp(key, "compression-level") == 0) {
 240                 intmax_t level;
 241                 if (string_to_number(value, &level) != ARCHIVE_OK) {
 242                         return (ARCHIVE_WARN);
 243                 }
 244                 /* If we don't have the library, hard-code the max level */
 245                 int minimum = CLEVEL_MIN;
 246                 int maximum = CLEVEL_MAX;
 247 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 248                 maximum = ZSTD_maxCLevel();
 249 #if ZSTD_VERSION_NUMBER >= MINVER_MINCLEVEL
 250                 if (ZSTD_versionNumber() >= MINVER_MINCLEVEL) {
 251                         minimum = ZSTD_minCLevel();
 252                 }
 253                 else
 254 #endif
 255                 if (ZSTD_versionNumber() < MINVER_NEGCLEVEL) {
 256                         minimum = CLEVEL_STD_MIN;
 257                 }
 258 #endif
 259                 if (level < minimum || level > maximum) {
 260                         return (ARCHIVE_WARN);
 261                 }
 262                 data->compression_level = (int)level;
 263                 return (ARCHIVE_OK);
 264         } else if (strcmp(key, "threads") == 0) {
 265                 intmax_t threads;
 266                 if (string_to_number(value, &threads) != ARCHIVE_OK) {
 267                         return (ARCHIVE_WARN);
 268                 }
 269                 if (threads < 0) {
 270                         return (ARCHIVE_WARN);
 271                 }
 272                 data->threads = (int)threads;
 273                 return (ARCHIVE_OK);
 274 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 275         } else if (strcmp(key, "frame-per-file") == 0) {
 276                 data->frame_per_file = 1;
 277                 return (ARCHIVE_OK);
 278         } else if (strcmp(key, "min-frame-in") == 0) {
 279                 if (string_to_size(value, &data->min_frame_in) != ARCHIVE_OK) {
 280                         return (ARCHIVE_WARN);
 281                 }
 282                 return (ARCHIVE_OK);
 283         } else if (strcmp(key, "min-frame-out") == 0 ||
 284             strcmp(key, "min-frame-size") == 0) {
 285                 if (string_to_size(value, &data->min_frame_out) != ARCHIVE_OK) {
 286                         return (ARCHIVE_WARN);
 287                 }
 288                 return (ARCHIVE_OK);
 289         } else if (strcmp(key, "max-frame-in") == 0 ||
 290             strcmp(key, "max-frame-size") == 0) {
 291                 if (string_to_size(value, &data->max_frame_in) != ARCHIVE_OK ||
 292                     data->max_frame_in < 1024) {
 293                         return (ARCHIVE_WARN);
 294                 }
 295                 return (ARCHIVE_OK);
 296         } else if (strcmp(key, "max-frame-out") == 0) {
 297                 if (string_to_size(value, &data->max_frame_out) != ARCHIVE_OK ||
 298                     data->max_frame_out < 1024) {
 299                         return (ARCHIVE_WARN);
 300                 }
 301                 return (ARCHIVE_OK);
 302 #endif
 303         }
 304         else if (strcmp(key, "long") == 0) {
 305                 intmax_t long_distance;
 306                 if (string_to_number(value, &long_distance) != ARCHIVE_OK) {
 307                         return (ARCHIVE_WARN);
 308                 }
 309 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream && ZSTD_VERSION_NUMBER >= MINVER_LONG
 310                 ZSTD_bounds bounds = ZSTD_cParam_getBounds(ZSTD_c_windowLog);
 311                 if (ZSTD_isError(bounds.error)) {
 312                         int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
 313                         if (((int)long_distance) < 10 || (int)long_distance > max_distance)
 314                                 return (ARCHIVE_WARN);
 315                 } else {
 316                         if ((int)long_distance < bounds.lowerBound || (int)long_distance > bounds.upperBound)
 317                                 return (ARCHIVE_WARN);
 318                 }
 319 #else
 320                 int max_distance = ((int)(sizeof(size_t) == 4 ? 30 : 31));
 321                 if (((int)long_distance) < 10 || (int)long_distance > max_distance)
 322                     return (ARCHIVE_WARN);
 323 #endif
 324                 data->long_distance = (int)long_distance;
 325                 return (ARCHIVE_OK);
 326         }
 327
 328         /* Note: The "warn" return is just to inform the options
 329          * supervisor that we didn't handle it.  It will generate
 330          * a suitable error if no one used this option. */
 331         return (ARCHIVE_WARN);
 332 }
 333
 334 #if HAVE_ZSTD_H && HAVE_ZSTD_compressStream
 335 /*
 336  * Setup callback.
 337  */
 338 static int
 339 archive_compressor_zstd_open(struct archive_write_filter *f)
 340 {
 341         struct private_data *data = (struct private_data *)f->data;
 342
 343         if (data->out.dst == NULL) {
 344                 size_t bs = ZSTD_CStreamOutSize(), bpb;
 345                 if (f->archive->magic == ARCHIVE_WRITE_MAGIC) {
 346                         /* Buffer size should be a multiple number of
 347                          * the of bytes per block for performance. */
 348                         bpb = archive_write_get_bytes_per_block(f->archive);
 349                         if (bpb > bs)
 350                                 bs = bpb;
 351                         else if (bpb != 0)
 352                                 bs -= bs % bpb;
 353                 }
 354                 data->out.size = bs;
 355                 data->out.pos = 0;
 356                 data->out.dst
 357                     = (unsigned char *)malloc(data->out.size);
 358                 if (data->out.dst == NULL) {
 359                         archive_set_error(f->archive, ENOMEM,
 360                             "Can't allocate data for compression buffer");
 361                         return (ARCHIVE_FATAL);
 362                 }
 363         }
 364
 365         f->write = archive_compressor_zstd_write;
 366
 367         if (ZSTD_isError(ZSTD_initCStream(data->cstream,
 368             data->compression_level))) {
 369                 archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 370                     "Internal error initializing zstd compressor object");
 371                 return (ARCHIVE_FATAL);
 372         }
 373
 374         ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_nbWorkers, data->threads);
 375
 376 #if ZSTD_VERSION_NUMBER >= MINVER_LONG
 377         ZSTD_CCtx_setParameter(data->cstream, ZSTD_c_windowLog, data->long_distance);
 378 #endif
 379
 380         return (ARCHIVE_OK);
 381 }
 382
 383 /*
 384  * Write data to the compressed stream.
 385  */
 386 static int
 387 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
 388     size_t length)
 389 {
 390         struct private_data *data = (struct private_data *)f->data;
 391
 392         return (drive_compressor(f, data, 0, buff, length));
 393 }
 394
 395 /*
 396  * Flush the compressed stream.
 397  */
 398 static int
 399 archive_compressor_zstd_flush(struct archive_write_filter *f)
 400 {
 401         struct private_data *data = (struct private_data *)f->data;
 402
 403         if (data->frame_per_file && data->state == running) {
 404                 if (data->cur_frame_in > data->min_frame_in &&
 405                     data->cur_frame_out > data->min_frame_out) {
 406                         data->state = finishing;
 407                 }
 408         }
 409         return (drive_compressor(f, data, 1, NULL, 0));
 410 }
 411
 412 /*
 413  * Finish the compression...
 414  */
 415 static int
 416 archive_compressor_zstd_close(struct archive_write_filter *f)
 417 {
 418         struct private_data *data = (struct private_data *)f->data;
 419
 420         if (data->state == running)
 421                 data->state = finishing;
 422         return (drive_compressor(f, data, 1, NULL, 0));
 423 }
 424
 425 /*
 426  * Utility function to push input data through compressor,
 427  * writing full output blocks as necessary.
 428  */
 429 static int
 430 drive_compressor(struct archive_write_filter *f,
 431     struct private_data *data, int flush, const void *src, size_t length)
 432 {
 433         ZSTD_inBuffer in = { .src = src, .size = length, .pos = 0 };
 434         size_t ipos, opos, zstdret = 0;
 435         int ret;
 436
 437         for (;;) {
 438                 ipos = in.pos;
 439                 opos = data->out.pos;
 440                 switch (data->state) {
 441                 case running:
 442                         if (in.pos == in.size)
 443                                 return (ARCHIVE_OK);
 444                         zstdret = ZSTD_compressStream(data->cstream,
 445                             &data->out, &in);
 446                         if (ZSTD_isError(zstdret))
 447                                 goto zstd_fatal;
 448                         break;
 449                 case finishing:
 450                         zstdret = ZSTD_endStream(data->cstream, &data->out);
 451                         if (ZSTD_isError(zstdret))
 452                                 goto zstd_fatal;
 453                         if (zstdret == 0)
 454                                 data->state = resetting;
 455                         break;
 456                 case resetting:
 457                         ZSTD_CCtx_reset(data->cstream, ZSTD_reset_session_only);
 458                         data->cur_frame++;
 459                         data->cur_frame_in = 0;
 460                         data->cur_frame_out = 0;
 461                         data->state = running;
 462                         break;
 463                 }
 464                 data->total_in += in.pos - ipos;
 465                 data->cur_frame_in += in.pos - ipos;
 466                 data->cur_frame_out += data->out.pos - opos;
 467                 if (data->state == running) {
 468                         if (data->cur_frame_in >= data->max_frame_in ||
 469                             data->cur_frame_out >= data->max_frame_out) {
 470                                 data->state = finishing;
 471                         }
 472                 }
 473                 if (data->out.pos == data->out.size ||
 474                     (flush && data->out.pos > 0)) {
 475                         ret = __archive_write_filter(f->next_filter,
 476                             data->out.dst, data->out.pos);
 477                         if (ret != ARCHIVE_OK)
 478                                 goto fatal;
 479                         data->out.pos = 0;
 480                 }
 481         }
 482 zstd_fatal:
 483         archive_set_error(f->archive, ARCHIVE_ERRNO_MISC,
 484             "Zstd compression failed: %s",
 485             ZSTD_getErrorName(zstdret));
 486 fatal:
 487         return (ARCHIVE_FATAL);
 488 }
 489
 490 #else /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */
 491
 492 static int
 493 archive_compressor_zstd_open(struct archive_write_filter *f)
 494 {
 495         struct private_data *data = (struct private_data *)f->data;
 496         struct archive_string as;
 497         int r;
 498
 499         archive_string_init(&as);
 500         /* --no-check matches library default */
 501         archive_strcpy(&as, "zstd --no-check");
 502
 503         if (data->compression_level < CLEVEL_STD_MIN) {
 504                 archive_string_sprintf(&as, " --fast=%d", -data->compression_level);
 505         } else {
 506                 archive_string_sprintf(&as, " -%d", data->compression_level);
 507         }
 508
 509         if (data->compression_level > CLEVEL_STD_MAX) {
 510                 archive_strcat(&as, " --ultra");
 511         }
 512
 513         if (data->threads != 0) {
 514                 archive_string_sprintf(&as, " --threads=%d", data->threads);
 515         }
 516
 517         if (data->long_distance != 0) {
 518                 archive_string_sprintf(&as, " --long=%d", data->long_distance);
 519         }
 520
 521         f->write = archive_compressor_zstd_write;
 522         r = __archive_write_program_open(f, data->pdata, as.s);
 523         archive_string_free(&as);
 524         return (r);
 525 }
 526
 527 static int
 528 archive_compressor_zstd_write(struct archive_write_filter *f, const void *buff,
 529     size_t length)
 530 {
 531         struct private_data *data = (struct private_data *)f->data;
 532
 533         return __archive_write_program_write(f, data->pdata, buff, length);
 534 }
 535
 536 static int
 537 archive_compressor_zstd_flush(struct archive_write_filter *f)
 538 {
 539         (void)f; /* UNUSED */
 540
 541         return (ARCHIVE_OK);
 542 }
 543
 544 static int
 545 archive_compressor_zstd_close(struct archive_write_filter *f)
 546 {
 547         struct private_data *data = (struct private_data *)f->data;
 548
 549         return __archive_write_program_close(f, data->pdata);
 550 }
 551
 552 #endif /* HAVE_ZSTD_H && HAVE_ZSTD_compressStream */