src/test/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <sys/stat.h>
   4
   5 #if HAVE_LZ4
   6 #include <lz4.h>
   7 #endif
   8
   9 #include "alloc-util.h"
  10 #include "compress.h"
  11 #include "fd-util.h"
  12 #include "fs-util.h"
  13 #include "macro.h"
  14 #include "memory-util.h"
  15 #include "path-util.h"
  16 #include "random-util.h"
  17 #include "tests.h"
  18 #include "tmpfile-util.h"
  19
  20 #if HAVE_XZ
  21 # define XZ_OK 0
  22 #else
  23 # define XZ_OK -EPROTONOSUPPORT
  24 #endif
  25
  26 #if HAVE_LZ4
  27 # define LZ4_OK 0
  28 #else
  29 # define LZ4_OK -EPROTONOSUPPORT
  30 #endif
  31
  32 #define HUGE_SIZE (4096*1024)
  33
  34 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  35                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  36 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  37                                 void **dst,
  38                                 size_t* dst_size, size_t dst_max);
  39 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  40                               void **buffer,
  41                               const void *prefix, size_t prefix_len,
  42                               uint8_t extra);
  43
  44 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes, uint64_t *uncompressed_size);
  45 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  46
  47 #if HAVE_COMPRESSION
  48 _unused_ static void test_compress_decompress(
  49                 const char *compression,
  50                 compress_blob_t compress,
  51                 decompress_blob_t decompress,
  52                 const char *data,
  53                 size_t data_len,
  54                 bool may_fail) {
  55
  56         char compressed[512];
  57         size_t csize;
  58         _cleanup_free_ char *decompressed = NULL;
  59         int r;
  60
  61         log_info("/* testing %s %s blob compression/decompression */",
  62                  compression, data);
  63
  64         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  65         if (r == -ENOBUFS) {
  66                 log_info_errno(r, "compression failed: %m");
  67                 assert_se(may_fail);
  68         } else {
  69                 assert_se(r >= 0);
  70                 r = decompress(compressed, csize,
  71                                (void **) &decompressed, &csize, 0);
  72                 assert_se(r == 0);
  73                 assert_se(decompressed);
  74                 assert_se(memcmp(decompressed, data, data_len) == 0);
  75         }
  76
  77         r = decompress("garbage", 7,
  78                        (void **) &decompressed, &csize, 0);
  79         assert_se(r < 0);
  80
  81         /* make sure to have the minimal lz4 compressed size */
  82         r = decompress("00000000\1g", 9,
  83                        (void **) &decompressed, &csize, 0);
  84         assert_se(r < 0);
  85
  86         r = decompress("\100000000g", 9,
  87                        (void **) &decompressed, &csize, 0);
  88         assert_se(r < 0);
  89
  90         explicit_bzero_safe(decompressed, MALLOC_SIZEOF_SAFE(decompressed));
  91 }
  92
  93 _unused_ static void test_decompress_startswith(const char *compression,
  94                                                 compress_blob_t compress,
  95                                                 decompress_sw_t decompress_sw,
  96                                                 const char *data,
  97                                                 size_t data_len,
  98                                                 bool may_fail) {
  99
 100         char *compressed;
 101         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 102         size_t csize, len;
 103         int r;
 104
 105         log_info("/* testing decompress_startswith with %s on %.20s text */",
 106                  compression, data);
 107
 108 #define BUFSIZE_1 512
 109 #define BUFSIZE_2 20000
 110
 111         compressed = compressed1 = malloc(BUFSIZE_1);
 112         assert_se(compressed1);
 113         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 114         if (r == -ENOBUFS) {
 115                 log_info_errno(r, "compression failed: %m");
 116                 assert_se(may_fail);
 117
 118                 compressed = compressed2 = malloc(BUFSIZE_2);
 119                 assert_se(compressed2);
 120                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 121         }
 122         assert_se(r >= 0);
 123
 124         len = strlen(data);
 125
 126         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
 127         assert_se(r > 0);
 128         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, 'w');
 129         assert_se(r == 0);
 130         r = decompress_sw(compressed, csize, (void **) &decompressed, "barbarbar", 9, ' ');
 131         assert_se(r == 0);
 132         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, data[len-1]);
 133         assert_se(r > 0);
 134         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, 'w');
 135         assert_se(r == 0);
 136         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
 137         assert_se(r > 0);
 138 }
 139
 140 _unused_ static void test_decompress_startswith_short(const char *compression,
 141                                                       compress_blob_t compress,
 142                                                       decompress_sw_t decompress_sw) {
 143
 144 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 145
 146         char buf[1024];
 147         size_t csize;
 148         int r;
 149
 150         log_info("/* %s with %s */", __func__, compression);
 151
 152         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 153         assert_se(r >= 0);
 154
 155         for (size_t i = 1; i < strlen(TEXT); i++) {
 156                 _cleanup_free_ void *buf2 = NULL;
 157
 158                 assert_se(buf2 = malloc(i));
 159
 160                 assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, TEXT[i]) == 1);
 161                 assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, 'y') == 0);
 162         }
 163 }
 164
 165 _unused_ static void test_compress_stream(const char *compression,
 166                                           const char *cat,
 167                                           compress_stream_t compress,
 168                                           decompress_stream_t decompress,
 169                                           const char *srcfile) {
 170
 171         _cleanup_close_ int src = -EBADF, dst = -EBADF, dst2 = -EBADF;
 172         _cleanup_(unlink_tempfilep) char
 173                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 174                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 175         int r;
 176         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 177         struct stat st = {};
 178         uint64_t uncompressed_size;
 179
 180         r = find_executable(cat, NULL);
 181         if (r < 0) {
 182                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 183                 return;
 184         }
 185
 186         log_debug("/* testing %s compression */", compression);
 187
 188         log_debug("/* create source from %s */", srcfile);
 189
 190         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 191
 192         log_debug("/* test compression */");
 193
 194         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 195
 196         assert_se(compress(src, dst, -1, &uncompressed_size) >= 0);
 197
 198         if (cat) {
 199                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 200                 assert_se(system(cmd) == 0);
 201         }
 202
 203         log_debug("/* test decompression */");
 204
 205         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 206
 207         assert_se(stat(srcfile, &st) == 0);
 208         assert_se((uint64_t)st.st_size == uncompressed_size);
 209
 210         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 211         r = decompress(dst, dst2, st.st_size);
 212         assert_se(r == 0);
 213
 214         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 215         assert_se(system(cmd2) == 0);
 216
 217         log_debug("/* test faulty decompression */");
 218
 219         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 220         r = decompress(dst, dst2, st.st_size);
 221         assert_se(IN_SET(r, 0, -EBADMSG));
 222
 223         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 224         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 225         r = decompress(dst, dst2, st.st_size - 1);
 226         assert_se(r == -EFBIG);
 227 }
 228 #endif
 229
 230 #if HAVE_LZ4
 231 static void test_lz4_decompress_partial(void) {
 232         char buf[20000], buf2[100];
 233         size_t buf_size = sizeof(buf), compressed;
 234         int r;
 235         _cleanup_free_ char *huge = NULL;
 236
 237         log_debug("/* %s */", __func__);
 238
 239         assert_se(huge = malloc(HUGE_SIZE));
 240         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 241         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 242         huge[HUGE_SIZE - 1] = '\0';
 243
 244         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 245         assert_se(r >= 0);
 246         compressed = r;
 247         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 248
 249         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 250         assert_se(r >= 0);
 251         log_info("Decompressed → %i", r);
 252
 253         r = LZ4_decompress_safe_partial(buf, huge,
 254                                         compressed,
 255                                         12, HUGE_SIZE);
 256         assert_se(r >= 0);
 257         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 258
 259         for (size_t size = 1; size < sizeof(buf2); size++) {
 260                 /* This failed in older lz4s but works in newer ones. */
 261                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 262                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 263                                                                    r < 0 ? "bad" : "good");
 264                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 265                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 266                         assert_se(memcmp(buf2, huge, r) == 0);
 267         }
 268 }
 269 #endif
 270
 271 int main(int argc, char *argv[]) {
 272 #if HAVE_COMPRESSION
 273         _unused_ const char text[] =
 274                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 275                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 276
 277         /* The file to test compression on can be specified as the first argument */
 278         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 279
 280         char data[512] = "random\0";
 281
 282         _cleanup_free_ char *huge = NULL;
 283
 284         assert_se(huge = malloc(HUGE_SIZE));
 285         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 286         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 287         huge[HUGE_SIZE - 1] = '\0';
 288
 289         test_setup_logging(LOG_DEBUG);
 290
 291         random_bytes(data + 7, sizeof(data) - 7);
 292
 293 #if HAVE_XZ
 294         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 295                                  text, sizeof(text), false);
 296         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 297                                  data, sizeof(data), true);
 298
 299         test_decompress_startswith("XZ",
 300                                    compress_blob_xz, decompress_startswith_xz,
 301                                    text, sizeof(text), false);
 302         test_decompress_startswith("XZ",
 303                                    compress_blob_xz, decompress_startswith_xz,
 304                                    data, sizeof(data), true);
 305         test_decompress_startswith("XZ",
 306                                    compress_blob_xz, decompress_startswith_xz,
 307                                    huge, HUGE_SIZE, true);
 308
 309         test_compress_stream("XZ", "xzcat",
 310                              compress_stream_xz, decompress_stream_xz, srcfile);
 311
 312         test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
 313
 314 #else
 315         log_info("/* XZ test skipped */");
 316 #endif
 317
 318 #if HAVE_LZ4
 319         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 320                                  text, sizeof(text), false);
 321         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 322                                  data, sizeof(data), true);
 323
 324         test_decompress_startswith("LZ4",
 325                                    compress_blob_lz4, decompress_startswith_lz4,
 326                                    text, sizeof(text), false);
 327         test_decompress_startswith("LZ4",
 328                                    compress_blob_lz4, decompress_startswith_lz4,
 329                                    data, sizeof(data), true);
 330         test_decompress_startswith("LZ4",
 331                                    compress_blob_lz4, decompress_startswith_lz4,
 332                                    huge, HUGE_SIZE, true);
 333
 334         test_compress_stream("LZ4", "lz4cat",
 335                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 336
 337         test_lz4_decompress_partial();
 338
 339         test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
 340
 341 #else
 342         log_info("/* LZ4 test skipped */");
 343 #endif
 344
 345 #if HAVE_ZSTD
 346         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 347                                  text, sizeof(text), false);
 348         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 349                                  data, sizeof(data), true);
 350
 351         test_decompress_startswith("ZSTD",
 352                                    compress_blob_zstd, decompress_startswith_zstd,
 353                                    text, sizeof(text), false);
 354         test_decompress_startswith("ZSTD",
 355                                    compress_blob_zstd, decompress_startswith_zstd,
 356                                    data, sizeof(data), true);
 357         test_decompress_startswith("ZSTD",
 358                                    compress_blob_zstd, decompress_startswith_zstd,
 359                                    huge, HUGE_SIZE, true);
 360
 361         test_compress_stream("ZSTD", "zstdcat",
 362                              compress_stream_zstd, decompress_stream_zstd, srcfile);
 363
 364         test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
 365 #else
 366         log_info("/* ZSTD test skipped */");
 367 #endif
 368
 369         return 0;
 370 #else
 371         log_info("/* XZ, LZ4 and ZSTD tests skipped */");
 372         return EXIT_TEST_SKIP;
 373 #endif
 374 }