src/libsystemd/sd-journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1-or-later */
   2
   3 #include <sys/stat.h>
   4
   5 #if HAVE_LZ4
   6 #include <lz4.h>
   7 #endif
   8
   9 #include "alloc-util.h"
  10 #include "compress.h"
  11 #include "fd-util.h"
  12 #include "fs-util.h"
  13 #include "macro.h"
  14 #include "memory-util.h"
  15 #include "path-util.h"
  16 #include "random-util.h"
  17 #include "tests.h"
  18 #include "tmpfile-util.h"
  19
  20 #if HAVE_XZ
  21 # define XZ_OK 0
  22 #else
  23 # define XZ_OK -EPROTONOSUPPORT
  24 #endif
  25
  26 #if HAVE_LZ4
  27 # define LZ4_OK 0
  28 #else
  29 # define LZ4_OK -EPROTONOSUPPORT
  30 #endif
  31
  32 #define HUGE_SIZE (4096*1024)
  33
  34 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  35                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  36 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  37                                 void **dst,
  38                                 size_t* dst_size, size_t dst_max);
  39 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  40                               void **buffer,
  41                               const void *prefix, size_t prefix_len,
  42                               uint8_t extra);
  43
  44 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  45 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  46
  47 #if HAVE_COMPRESSION
  48 _unused_ static void test_compress_decompress(
  49                 const char *compression,
  50                 compress_blob_t compress,
  51                 decompress_blob_t decompress,
  52                 const char *data,
  53                 size_t data_len,
  54                 bool may_fail) {
  55
  56         char compressed[512];
  57         size_t csize;
  58         _cleanup_free_ char *decompressed = NULL;
  59         int r;
  60
  61         log_info("/* testing %s %s blob compression/decompression */",
  62                  compression, data);
  63
  64         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  65         if (r == -ENOBUFS) {
  66                 log_info_errno(r, "compression failed: %m");
  67                 assert_se(may_fail);
  68         } else {
  69                 assert_se(r == 0);
  70                 r = decompress(compressed, csize,
  71                                (void **) &decompressed, &csize, 0);
  72                 assert_se(r == 0);
  73                 assert_se(decompressed);
  74                 assert_se(memcmp(decompressed, data, data_len) == 0);
  75         }
  76
  77         r = decompress("garbage", 7,
  78                        (void **) &decompressed, &csize, 0);
  79         assert_se(r < 0);
  80
  81         /* make sure to have the minimal lz4 compressed size */
  82         r = decompress("00000000\1g", 9,
  83                        (void **) &decompressed, &csize, 0);
  84         assert_se(r < 0);
  85
  86         r = decompress("\100000000g", 9,
  87                        (void **) &decompressed, &csize, 0);
  88         assert_se(r < 0);
  89
  90         explicit_bzero_safe(decompressed, MALLOC_SIZEOF_SAFE(decompressed));
  91 }
  92
  93 _unused_ static void test_decompress_startswith(const char *compression,
  94                                                 compress_blob_t compress,
  95                                                 decompress_sw_t decompress_sw,
  96                                                 const char *data,
  97                                                 size_t data_len,
  98                                                 bool may_fail) {
  99
 100         char *compressed;
 101         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 102         size_t csize, len;
 103         int r;
 104
 105         log_info("/* testing decompress_startswith with %s on %.20s text */",
 106                  compression, data);
 107
 108 #define BUFSIZE_1 512
 109 #define BUFSIZE_2 20000
 110
 111         compressed = compressed1 = malloc(BUFSIZE_1);
 112         assert_se(compressed1);
 113         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 114         if (r == -ENOBUFS) {
 115                 log_info_errno(r, "compression failed: %m");
 116                 assert_se(may_fail);
 117
 118                 compressed = compressed2 = malloc(BUFSIZE_2);
 119                 assert_se(compressed2);
 120                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 121                 assert(r == 0);
 122         }
 123         assert_se(r == 0);
 124
 125         len = strlen(data);
 126
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
 128         assert_se(r > 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, 'w');
 130         assert_se(r == 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, "barbarbar", 9, ' ');
 132         assert_se(r == 0);
 133         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, data[len-1]);
 134         assert_se(r > 0);
 135         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len - 1, 'w');
 136         assert_se(r == 0);
 137         r = decompress_sw(compressed, csize, (void **) &decompressed, data, len, '\0');
 138         assert_se(r > 0);
 139 }
 140
 141 _unused_ static void test_decompress_startswith_short(const char *compression,
 142                                                       compress_blob_t compress,
 143                                                       decompress_sw_t decompress_sw) {
 144
 145 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 146
 147         char buf[1024];
 148         size_t i, csize;
 149         int r;
 150
 151         log_info("/* %s with %s */", __func__, compression);
 152
 153         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 154         assert_se(r == 0);
 155
 156         for (i = 1; i < strlen(TEXT); i++) {
 157                 _cleanup_free_ void *buf2 = NULL;
 158
 159                 assert_se(buf2 = malloc(i));
 160
 161                 assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, TEXT[i]) == 1);
 162                 assert_se(decompress_sw(buf, csize, &buf2, TEXT, i, 'y') == 0);
 163         }
 164 }
 165
 166 _unused_ static void test_compress_stream(const char *compression,
 167                                           const char *cat,
 168                                           compress_stream_t compress,
 169                                           decompress_stream_t decompress,
 170                                           const char *srcfile) {
 171
 172         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 173         _cleanup_(unlink_tempfilep) char
 174                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 175                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 176         int r;
 177         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 178         struct stat st = {};
 179
 180         r = find_executable(cat, NULL);
 181         if (r < 0) {
 182                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 183                 return;
 184         }
 185
 186         log_debug("/* testing %s compression */", compression);
 187
 188         log_debug("/* create source from %s */", srcfile);
 189
 190         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 191
 192         log_debug("/* test compression */");
 193
 194         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 195
 196         assert_se(compress(src, dst, -1) == 0);
 197
 198         if (cat) {
 199                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 200                 assert_se(system(cmd) == 0);
 201         }
 202
 203         log_debug("/* test decompression */");
 204
 205         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 206
 207         assert_se(stat(srcfile, &st) == 0);
 208
 209         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 210         r = decompress(dst, dst2, st.st_size);
 211         assert_se(r == 0);
 212
 213         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 214         assert_se(system(cmd2) == 0);
 215
 216         log_debug("/* test faulty decompression */");
 217
 218         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 219         r = decompress(dst, dst2, st.st_size);
 220         assert_se(IN_SET(r, 0, -EBADMSG));
 221
 222         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 223         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 224         r = decompress(dst, dst2, st.st_size - 1);
 225         assert_se(r == -EFBIG);
 226 }
 227 #endif
 228
 229 #if HAVE_LZ4
 230 static void test_lz4_decompress_partial(void) {
 231         char buf[20000], buf2[100];
 232         size_t buf_size = sizeof(buf), compressed;
 233         int r;
 234         _cleanup_free_ char *huge = NULL;
 235
 236         log_debug("/* %s */", __func__);
 237
 238         assert_se(huge = malloc(HUGE_SIZE));
 239         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 240         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 241         huge[HUGE_SIZE - 1] = '\0';
 242
 243         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 244         assert_se(r >= 0);
 245         compressed = r;
 246         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 247
 248         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 249         assert_se(r >= 0);
 250         log_info("Decompressed → %i", r);
 251
 252         r = LZ4_decompress_safe_partial(buf, huge,
 253                                         compressed,
 254                                         12, HUGE_SIZE);
 255         assert_se(r >= 0);
 256         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 257
 258         for (size_t size = 1; size < sizeof(buf2); size++) {
 259                 /* This failed in older lz4s but works in newer ones. */
 260                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 261                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 262                                                                    r < 0 ? "bad" : "good");
 263                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 264                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 265                         assert_se(memcmp(buf2, huge, r) == 0);
 266         }
 267 }
 268 #endif
 269
 270 int main(int argc, char *argv[]) {
 271 #if HAVE_COMPRESSION
 272         _unused_ const char text[] =
 273                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 274                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 275
 276         /* The file to test compression on can be specified as the first argument */
 277         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 278
 279         char data[512] = "random\0";
 280
 281         _cleanup_free_ char *huge = NULL;
 282
 283         assert_se(huge = malloc(HUGE_SIZE));
 284         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 285         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 286         huge[HUGE_SIZE - 1] = '\0';
 287
 288         test_setup_logging(LOG_DEBUG);
 289
 290         random_bytes(data + 7, sizeof(data) - 7);
 291
 292 #if HAVE_XZ
 293         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 294                                  text, sizeof(text), false);
 295         test_compress_decompress("XZ", compress_blob_xz, decompress_blob_xz,
 296                                  data, sizeof(data), true);
 297
 298         test_decompress_startswith("XZ",
 299                                    compress_blob_xz, decompress_startswith_xz,
 300                                    text, sizeof(text), false);
 301         test_decompress_startswith("XZ",
 302                                    compress_blob_xz, decompress_startswith_xz,
 303                                    data, sizeof(data), true);
 304         test_decompress_startswith("XZ",
 305                                    compress_blob_xz, decompress_startswith_xz,
 306                                    huge, HUGE_SIZE, true);
 307
 308         test_compress_stream("XZ", "xzcat",
 309                              compress_stream_xz, decompress_stream_xz, srcfile);
 310
 311         test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
 312
 313 #else
 314         log_info("/* XZ test skipped */");
 315 #endif
 316
 317 #if HAVE_LZ4
 318         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 319                                  text, sizeof(text), false);
 320         test_compress_decompress("LZ4", compress_blob_lz4, decompress_blob_lz4,
 321                                  data, sizeof(data), true);
 322
 323         test_decompress_startswith("LZ4",
 324                                    compress_blob_lz4, decompress_startswith_lz4,
 325                                    text, sizeof(text), false);
 326         test_decompress_startswith("LZ4",
 327                                    compress_blob_lz4, decompress_startswith_lz4,
 328                                    data, sizeof(data), true);
 329         test_decompress_startswith("LZ4",
 330                                    compress_blob_lz4, decompress_startswith_lz4,
 331                                    huge, HUGE_SIZE, true);
 332
 333         test_compress_stream("LZ4", "lz4cat",
 334                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 335
 336         test_lz4_decompress_partial();
 337
 338         test_decompress_startswith_short("LZ4", compress_blob_lz4, decompress_startswith_lz4);
 339
 340 #else
 341         log_info("/* LZ4 test skipped */");
 342 #endif
 343
 344 #if HAVE_ZSTD
 345         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 346                                  text, sizeof(text), false);
 347         test_compress_decompress("ZSTD", compress_blob_zstd, decompress_blob_zstd,
 348                                  data, sizeof(data), true);
 349
 350         test_decompress_startswith("ZSTD",
 351                                    compress_blob_zstd, decompress_startswith_zstd,
 352                                    text, sizeof(text), false);
 353         test_decompress_startswith("ZSTD",
 354                                    compress_blob_zstd, decompress_startswith_zstd,
 355                                    data, sizeof(data), true);
 356         test_decompress_startswith("ZSTD",
 357                                    compress_blob_zstd, decompress_startswith_zstd,
 358                                    huge, HUGE_SIZE, true);
 359
 360         test_compress_stream("ZSTD", "zstdcat",
 361                              compress_stream_zstd, decompress_stream_zstd, srcfile);
 362
 363         test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
 364 #else
 365         log_info("/* ZSTD test skipped */");
 366 #endif
 367
 368         return 0;
 369 #else
 370         log_info("/* XZ, LZ4 and ZSTD tests skipped */");
 371         return EXIT_TEST_SKIP;
 372 #endif
 373 }