src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <sys/stat.h>
   4
   5 #if HAVE_LZ4
   6 #include <lz4.h>
   7 #endif
   8
   9 #include "alloc-util.h"
  10 #include "compress.h"
  11 #include "fd-util.h"
  12 #include "fs-util.h"
  13 #include "macro.h"
  14 #include "memory-util.h"
  15 #include "path-util.h"
  16 #include "random-util.h"
  17 #include "tests.h"
  18 #include "tmpfile-util.h"
  19
  20 #if HAVE_XZ
  21 # define XZ_OK 0
  22 #else
  23 # define XZ_OK -EPROTONOSUPPORT
  24 #endif
  25
  26 #if HAVE_LZ4
  27 # define LZ4_OK 0
  28 #else
  29 # define LZ4_OK -EPROTONOSUPPORT
  30 #endif
  31
  32 #define HUGE_SIZE (4096*1024)
  33
  34 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  35                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  36 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  37                                 void **dst, size_t *dst_alloc_size,
  38                                 size_t* dst_size, size_t dst_max);
  39 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  40                               void **buffer, size_t *buffer_size,
  41                               const void *prefix, size_t prefix_len,
  42                               uint8_t extra);
  43
  44 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  45 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  46
  47 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD
  48 _unused_ static void test_compress_decompress(const char *compression,
  49                                               compress_blob_t compress,
  50                                               decompress_blob_t decompress,
  51                                               const char *data,
  52                                               size_t data_len,
  53                                               bool may_fail) {
  54         char compressed[512];
  55         size_t csize, usize = 0;
  56         _cleanup_free_ char *decompressed = NULL;
  57         int r;
  58
  59         log_info("/* testing %s %s blob compression/decompression */",
  60                  compression, data);
  61
  62         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  63         if (r == -ENOBUFS) {
  64                 log_info_errno(r, "compression failed: %m");
  65                 assert_se(may_fail);
  66         } else {
  67                 assert_se(r == 0);
  68                 r = decompress(compressed, csize,
  69                                (void **) &decompressed, &usize, &csize, 0);
  70                 assert_se(r == 0);
  71                 assert_se(decompressed);
  72                 assert_se(memcmp(decompressed, data, data_len) == 0);
  73         }
  74
  75         r = decompress("garbage", 7,
  76                        (void **) &decompressed, &usize, &csize, 0);
  77         assert_se(r < 0);
  78
  79         /* make sure to have the minimal lz4 compressed size */
  80         r = decompress("00000000\1g", 9,
  81                        (void **) &decompressed, &usize, &csize, 0);
  82         assert_se(r < 0);
  83
  84         r = decompress("\100000000g", 9,
  85                        (void **) &decompressed, &usize, &csize, 0);
  86         assert_se(r < 0);
  87
  88         memzero(decompressed, usize);
  89 }
  90
  91 _unused_ static void test_decompress_startswith(const char *compression,
  92                                                 compress_blob_t compress,
  93                                                 decompress_sw_t decompress_sw,
  94                                                 const char *data,
  95                                                 size_t data_len,
  96                                                 bool may_fail) {
  97
  98         char *compressed;
  99         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 100         size_t csize, usize = 0, len;
 101         int r;
 102
 103         log_info("/* testing decompress_startswith with %s on %.20s text */",
 104                  compression, data);
 105
 106 #define BUFSIZE_1 512
 107 #define BUFSIZE_2 20000
 108
 109         compressed = compressed1 = malloc(BUFSIZE_1);
 110         assert_se(compressed1);
 111         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 112         if (r == -ENOBUFS) {
 113                 log_info_errno(r, "compression failed: %m");
 114                 assert_se(may_fail);
 115
 116                 compressed = compressed2 = malloc(BUFSIZE_2);
 117                 assert_se(compressed2);
 118                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 119                 assert(r == 0);
 120         }
 121         assert_se(r == 0);
 122
 123         len = strlen(data);
 124
 125         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 126         assert_se(r > 0);
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 128         assert_se(r == 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 130         assert_se(r == 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 132         assert_se(r > 0);
 133         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 134         assert_se(r == 0);
 135         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 136         assert_se(r > 0);
 137 }
 138
 139 _unused_ static void test_decompress_startswith_short(const char *compression,
 140                                                       compress_blob_t compress,
 141                                                       decompress_sw_t decompress_sw) {
 142
 143 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 144
 145         char buf[1024];
 146         size_t i, csize;
 147         int r;
 148
 149         log_info("/* %s with %s */", __func__, compression);
 150
 151         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 152         assert_se(r == 0);
 153
 154         for (i = 1; i < strlen(TEXT); i++) {
 155                 size_t alloc_size = i;
 156                 _cleanup_free_ void *buf2 = NULL;
 157
 158                 assert_se(buf2 = malloc(i));
 159
 160                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, TEXT[i]) == 1);
 161                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, 'y') == 0);
 162         }
 163 }
 164
 165 _unused_ static void test_compress_stream(const char *compression,
 166                                           const char *cat,
 167                                           compress_stream_t compress,
 168                                           decompress_stream_t decompress,
 169                                           const char *srcfile) {
 170
 171         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 172         _cleanup_(unlink_tempfilep) char
 173                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 174                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 175         int r;
 176         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 177         struct stat st = {};
 178
 179         r = find_binary(cat, NULL);
 180         if (r < 0) {
 181                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 182                 return;
 183         }
 184
 185         log_debug("/* testing %s compression */", compression);
 186
 187         log_debug("/* create source from %s */", srcfile);
 188
 189         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 190
 191         log_debug("/* test compression */");
 192
 193         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 194
 195         assert_se(compress(src, dst, -1) == 0);
 196
 197         if (cat) {
 198                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 199                 assert_se(system(cmd) == 0);
 200         }
 201
 202         log_debug("/* test decompression */");
 203
 204         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 205
 206         assert_se(stat(srcfile, &st) == 0);
 207
 208         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 209         r = decompress(dst, dst2, st.st_size);
 210         assert_se(r == 0);
 211
 212         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 213         assert_se(system(cmd2) == 0);
 214
 215         log_debug("/* test faulty decompression */");
 216
 217         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 218         r = decompress(dst, dst2, st.st_size);
 219         assert_se(IN_SET(r, 0, -EBADMSG));
 220
 221         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 222         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 223         r = decompress(dst, dst2, st.st_size - 1);
 224         assert_se(r == -EFBIG);
 225 }
 226 #endif
 227
 228 #if HAVE_LZ4
 229 static void test_lz4_decompress_partial(void) {
 230         char buf[20000], buf2[100];
 231         size_t buf_size = sizeof(buf), compressed;
 232         int r;
 233         _cleanup_free_ char *huge = NULL;
 234
 235         assert_se(huge = malloc(HUGE_SIZE));
 236         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 237         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 238         huge[HUGE_SIZE - 1] = '\0';
 239
 240         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 241         assert_se(r >= 0);
 242         compressed = r;
 243         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 244
 245         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 246         assert_se(r >= 0);
 247         log_info("Decompressed → %i", r);
 248
 249         r = LZ4_decompress_safe_partial(buf, huge,
 250                                         compressed,
 251                                         12, HUGE_SIZE);
 252         assert_se(r >= 0);
 253         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 254
 255         for (size_t size = 1; size < sizeof(buf2); size++) {
 256                 /* This failed in older lz4s but works in newer ones. */
 257                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 258                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 259                                                                    r < 0 ? "bad" : "good");
 260                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 261                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 262                         assert_se(memcmp(buf2, huge, r) == 0);
 263         }
 264 }
 265 #endif
 266
 267 int main(int argc, char *argv[]) {
 268 #if HAVE_XZ || HAVE_LZ4 || HAVE_ZSTD
 269         _unused_ const char text[] =
 270                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 271                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 272
 273         /* The file to test compression on can be specified as the first argument */
 274         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 275
 276         char data[512] = "random\0";
 277
 278         _cleanup_free_ char *huge = NULL;
 279
 280         assert_se(huge = malloc(HUGE_SIZE));
 281         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 282         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 283         huge[HUGE_SIZE - 1] = '\0';
 284
 285         test_setup_logging(LOG_DEBUG);
 286
 287         random_bytes(data + 7, sizeof(data) - 7);
 288
 289 #if HAVE_XZ
 290         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 291                                  text, sizeof(text), false);
 292         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 293                                  data, sizeof(data), true);
 294
 295         test_decompress_startswith("XZ",
 296                                    compress_blob_xz, decompress_startswith_xz,
 297                                    text, sizeof(text), false);
 298         test_decompress_startswith("XZ",
 299                                    compress_blob_xz, decompress_startswith_xz,
 300                                    data, sizeof(data), true);
 301         test_decompress_startswith("XZ",
 302                                    compress_blob_xz, decompress_startswith_xz,
 303                                    huge, HUGE_SIZE, true);
 304
 305         test_compress_stream("XZ", "xzcat",
 306                              compress_stream_xz, decompress_stream_xz, srcfile);
 307
 308         test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
 309
 310 #else
 311         log_info("/* XZ test skipped */");
 312 #endif
 313
 314 #if HAVE_LZ4
 315         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 316                                  text, sizeof(text), false);
 317         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 318                                  data, sizeof(data), true);
 319
 320         test_decompress_startswith("LZ4",
 321                                    compress_blob_lz4, decompress_startswith_lz4,
 322                                    text, sizeof(text), false);
 323         test_decompress_startswith("LZ4",
 324                                    compress_blob_lz4, decompress_startswith_lz4,
 325                                    data, sizeof(data), true);
 326         test_decompress_startswith("LZ4",
 327                                    compress_blob_lz4, decompress_startswith_lz4,
 328                                    huge, HUGE_SIZE, true);
 329
 330         test_compress_stream("LZ4", "lz4cat",
 331                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 332
 333         test_lz4_decompress_partial();
 334
 335         test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
 336
 337 #else
 338         log_info("/* LZ4 test skipped */");
 339 #endif
 340
 341 #if HAVE_ZSTD
 342         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 343                                  text, sizeof(text), false);
 344         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 345                                  data, sizeof(data), true);
 346
 347         test_decompress_startswith("ZSTD",
 348                                    compress_blob_zstd, decompress_startswith_zstd,
 349                                    text, sizeof(text), false);
 350         test_decompress_startswith("ZSTD",
 351                                    compress_blob_zstd, decompress_startswith_zstd,
 352                                    data, sizeof(data), true);
 353         test_decompress_startswith("ZSTD",
 354                                    compress_blob_zstd, decompress_startswith_zstd,
 355                                    huge, HUGE_SIZE, true);
 356
 357         test_compress_stream("ZSTD", "zstdcat",
 358                              compress_stream_zstd, decompress_stream_zstd, srcfile);
 359
 360         test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
 361 #else
 362         log_info("/* ZSTD test skipped */");
 363 #endif
 364
 365         return 0;
 366 #else
 367         log_info("/* XZ, LZ4 and ZSTD tests skipped */");
 368         return EXIT_TEST_SKIP;
 369 #endif
 370 }