src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <sys/stat.h>
   4
   5 #if HAVE_LZ4
   6 #include <lz4.h>
   7 #endif
   8
   9 #include "alloc-util.h"
  10 #include "compress.h"
  11 #include "fd-util.h"
  12 #include "fs-util.h"
  13 #include "macro.h"
  14 #include "memory-util.h"
  15 #include "path-util.h"
  16 #include "random-util.h"
  17 #include "tests.h"
  18 #include "tmpfile-util.h"
  19
  20 #if HAVE_XZ
  21 # define XZ_OK 0
  22 #else
  23 # define XZ_OK -EPROTONOSUPPORT
  24 #endif
  25
  26 #if HAVE_LZ4
  27 # define LZ4_OK 0
  28 #else
  29 # define LZ4_OK -EPROTONOSUPPORT
  30 #endif
  31
  32 #define HUGE_SIZE (4096*1024)
  33
  34 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  35                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  36 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  37                                 void **dst, size_t *dst_alloc_size,
  38                                 size_t* dst_size, size_t dst_max);
  39 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  40                               void **buffer, size_t *buffer_size,
  41                               const void *prefix, size_t prefix_len,
  42                               uint8_t extra);
  43
  44 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  45 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  46
  47 #if HAVE_XZ || HAVE_LZ4
  48 static void test_compress_decompress(int compression,
  49                                      compress_blob_t compress,
  50                                      decompress_blob_t decompress,
  51                                      const char *data,
  52                                      size_t data_len,
  53                                      bool may_fail) {
  54         char compressed[512];
  55         size_t csize, usize = 0;
  56         _cleanup_free_ char *decompressed = NULL;
  57         int r;
  58
  59         log_info("/* testing %s %s blob compression/decompression */",
  60                  object_compressed_to_string(compression), data);
  61
  62         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  63         if (r == -ENOBUFS) {
  64                 log_info_errno(r, "compression failed: %m");
  65                 assert_se(may_fail);
  66         } else {
  67                 assert_se(r == 0);
  68                 r = decompress(compressed, csize,
  69                                (void **) &decompressed, &usize, &csize, 0);
  70                 assert_se(r == 0);
  71                 assert_se(decompressed);
  72                 assert_se(memcmp(decompressed, data, data_len) == 0);
  73         }
  74
  75         r = decompress("garbage", 7,
  76                        (void **) &decompressed, &usize, &csize, 0);
  77         assert_se(r < 0);
  78
  79         /* make sure to have the minimal lz4 compressed size */
  80         r = decompress("00000000\1g", 9,
  81                        (void **) &decompressed, &usize, &csize, 0);
  82         assert_se(r < 0);
  83
  84         r = decompress("\100000000g", 9,
  85                        (void **) &decompressed, &usize, &csize, 0);
  86         assert_se(r < 0);
  87
  88         memzero(decompressed, usize);
  89 }
  90
  91 static void test_decompress_startswith(int compression,
  92                                        compress_blob_t compress,
  93                                        decompress_sw_t decompress_sw,
  94                                        const char *data,
  95                                        size_t data_len,
  96                                        bool may_fail) {
  97
  98         char *compressed;
  99         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 100         size_t csize, usize = 0, len;
 101         int r;
 102
 103         log_info("/* testing decompress_startswith with %s on %.20s text */",
 104                  object_compressed_to_string(compression), data);
 105
 106 #define BUFSIZE_1 512
 107 #define BUFSIZE_2 20000
 108
 109         compressed = compressed1 = malloc(BUFSIZE_1);
 110         assert_se(compressed1);
 111         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 112         if (r == -ENOBUFS) {
 113                 log_info_errno(r, "compression failed: %m");
 114                 assert_se(may_fail);
 115
 116                 compressed = compressed2 = malloc(BUFSIZE_2);
 117                 assert_se(compressed2);
 118                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 119                 assert(r == 0);
 120         }
 121         assert_se(r == 0);
 122
 123         len = strlen(data);
 124
 125         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 126         assert_se(r > 0);
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 128         assert_se(r == 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 130         assert_se(r == 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 132         assert_se(r > 0);
 133         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 134         assert_se(r == 0);
 135         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 136         assert_se(r > 0);
 137 }
 138
 139 static void test_decompress_startswith_short(int compression,
 140                                              compress_blob_t compress,
 141                                              decompress_sw_t decompress_sw) {
 142
 143 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 144
 145         char buf[1024];
 146         size_t i, csize;
 147         int r;
 148
 149         log_info("/* %s with %s */", __func__, object_compressed_to_string(compression));
 150
 151         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 152         assert_se(r == 0);
 153
 154         for (i = 1; i < strlen(TEXT); i++) {
 155                 size_t alloc_size = i;
 156                 _cleanup_free_ void *buf2 = NULL;
 157
 158                 assert_se(buf2 = malloc(i));
 159
 160                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, TEXT[i]) == 1);
 161                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, 'y') == 0);
 162         }
 163 }
 164
 165 static void test_compress_stream(int compression,
 166                                  const char* cat,
 167                                  compress_stream_t compress,
 168                                  decompress_stream_t decompress,
 169                                  const char *srcfile) {
 170
 171         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 172         _cleanup_(unlink_tempfilep) char
 173                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 174                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 175         int r;
 176         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 177         struct stat st = {};
 178
 179         r = find_binary(cat, NULL);
 180         if (r < 0) {
 181                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 182                 return;
 183         }
 184
 185         log_debug("/* testing %s compression */",
 186                   object_compressed_to_string(compression));
 187
 188         log_debug("/* create source from %s */", srcfile);
 189
 190         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 191
 192         log_debug("/* test compression */");
 193
 194         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 195
 196         assert_se(compress(src, dst, -1) == 0);
 197
 198         if (cat) {
 199                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 200                 assert_se(system(cmd) == 0);
 201         }
 202
 203         log_debug("/* test decompression */");
 204
 205         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 206
 207         assert_se(stat(srcfile, &st) == 0);
 208
 209         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 210         r = decompress(dst, dst2, st.st_size);
 211         assert_se(r == 0);
 212
 213         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 214         assert_se(system(cmd2) == 0);
 215
 216         log_debug("/* test faulty decompression */");
 217
 218         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 219         r = decompress(dst, dst2, st.st_size);
 220         assert_se(IN_SET(r, 0, -EBADMSG));
 221
 222         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 223         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 224         r = decompress(dst, dst2, st.st_size - 1);
 225         assert_se(r == -EFBIG);
 226 }
 227 #endif
 228
 229 #if HAVE_LZ4
 230 static void test_lz4_decompress_partial(void) {
 231         char buf[20000], buf2[100];
 232         size_t buf_size = sizeof(buf), compressed;
 233         int r;
 234         _cleanup_free_ char *huge = NULL;
 235
 236         assert_se(huge = malloc(HUGE_SIZE));
 237         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 238         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 239         huge[HUGE_SIZE - 1] = '\0';
 240
 241         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 242         assert_se(r >= 0);
 243         compressed = r;
 244         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 245
 246         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 247         assert_se(r >= 0);
 248         log_info("Decompressed → %i", r);
 249
 250         r = LZ4_decompress_safe_partial(buf, huge,
 251                                         compressed,
 252                                         12, HUGE_SIZE);
 253         assert_se(r >= 0);
 254         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 255
 256         for (size_t size = 1; size < sizeof(buf2); size++) {
 257                 /* This failed in older lz4s but works in newer ones. */
 258                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 259                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 260                                                                    r < 0 ? "bad" : "good");
 261                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 262                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 263                         assert_se(memcmp(buf2, huge, r) == 0);
 264         }
 265 }
 266 #endif
 267
 268 int main(int argc, char *argv[]) {
 269 #if HAVE_XZ || HAVE_LZ4
 270         const char text[] =
 271                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 272                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 273
 274         /* The file to test compression on can be specified as the first argument */
 275         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 276
 277         char data[512] = "random\0";
 278
 279         _cleanup_free_ char *huge = NULL;
 280
 281         assert_se(huge = malloc(HUGE_SIZE));
 282         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 283         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 284         huge[HUGE_SIZE - 1] = '\0';
 285
 286         test_setup_logging(LOG_DEBUG);
 287
 288         random_bytes(data + 7, sizeof(data) - 7);
 289
 290 #if HAVE_XZ
 291         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 292                                  text, sizeof(text), false);
 293         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 294                                  data, sizeof(data), true);
 295
 296         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 297                                    compress_blob_xz, decompress_startswith_xz,
 298                                    text, sizeof(text), false);
 299         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 300                                    compress_blob_xz, decompress_startswith_xz,
 301                                    data, sizeof(data), true);
 302         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 303                                    compress_blob_xz, decompress_startswith_xz,
 304                                    huge, HUGE_SIZE, true);
 305
 306         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 307                              compress_stream_xz, decompress_stream_xz, srcfile);
 308
 309         test_decompress_startswith_short(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz);
 310
 311 #else
 312         log_info("/* XZ test skipped */");
 313 #endif
 314
 315 #if HAVE_LZ4
 316         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 317                                  text, sizeof(text), false);
 318         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 319                                  data, sizeof(data), true);
 320
 321         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 322                                    compress_blob_lz4, decompress_startswith_lz4,
 323                                    text, sizeof(text), false);
 324         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 325                                    compress_blob_lz4, decompress_startswith_lz4,
 326                                    data, sizeof(data), true);
 327         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 328                                    compress_blob_lz4, decompress_startswith_lz4,
 329                                    huge, HUGE_SIZE, true);
 330
 331         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 332                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 333
 334         test_lz4_decompress_partial();
 335
 336         test_decompress_startswith_short(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4);
 337
 338 #else
 339         log_info("/* LZ4 test skipped */");
 340 #endif
 341
 342         return 0;
 343 #else
 344         log_info("/* XZ and LZ4 tests skipped */");
 345         return EXIT_TEST_SKIP;
 346 #endif
 347 }