src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #if HAVE_LZ4
   4 #include <lz4.h>
   5 #endif
   6
   7 #include "alloc-util.h"
   8 #include "compress.h"
   9 #include "fd-util.h"
  10 #include "fs-util.h"
  11 #include "macro.h"
  12 #include "path-util.h"
  13 #include "random-util.h"
  14 #include "tests.h"
  15 #include "tmpfile-util.h"
  16 #include "util.h"
  17
  18 #if HAVE_XZ
  19 # define XZ_OK 0
  20 #else
  21 # define XZ_OK -EPROTONOSUPPORT
  22 #endif
  23
  24 #if HAVE_LZ4
  25 # define LZ4_OK 0
  26 #else
  27 # define LZ4_OK -EPROTONOSUPPORT
  28 #endif
  29
  30 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  31                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  32 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  33                                 void **dst, size_t *dst_alloc_size,
  34                                 size_t* dst_size, size_t dst_max);
  35 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  36                               void **buffer, size_t *buffer_size,
  37                               const void *prefix, size_t prefix_len,
  38                               uint8_t extra);
  39
  40 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  41 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  42
  43 #if HAVE_XZ || HAVE_LZ4
  44 static void test_compress_decompress(int compression,
  45                                      compress_blob_t compress,
  46                                      decompress_blob_t decompress,
  47                                      const char *data,
  48                                      size_t data_len,
  49                                      bool may_fail) {
  50         char compressed[512];
  51         size_t csize, usize = 0;
  52         _cleanup_free_ char *decompressed = NULL;
  53         int r;
  54
  55         log_info("/* testing %s %s blob compression/decompression */",
  56                  object_compressed_to_string(compression), data);
  57
  58         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  59         if (r == -ENOBUFS) {
  60                 log_info_errno(r, "compression failed: %m");
  61                 assert_se(may_fail);
  62         } else {
  63                 assert_se(r == 0);
  64                 r = decompress(compressed, csize,
  65                                (void **) &decompressed, &usize, &csize, 0);
  66                 assert_se(r == 0);
  67                 assert_se(decompressed);
  68                 assert_se(memcmp(decompressed, data, data_len) == 0);
  69         }
  70
  71         r = decompress("garbage", 7,
  72                        (void **) &decompressed, &usize, &csize, 0);
  73         assert_se(r < 0);
  74
  75         /* make sure to have the minimal lz4 compressed size */
  76         r = decompress("00000000\1g", 9,
  77                        (void **) &decompressed, &usize, &csize, 0);
  78         assert_se(r < 0);
  79
  80         r = decompress("\100000000g", 9,
  81                        (void **) &decompressed, &usize, &csize, 0);
  82         assert_se(r < 0);
  83
  84         memzero(decompressed, usize);
  85 }
  86
  87 static void test_decompress_startswith(int compression,
  88                                        compress_blob_t compress,
  89                                        decompress_sw_t decompress_sw,
  90                                        const char *data,
  91                                        size_t data_len,
  92                                        bool may_fail) {
  93
  94         char *compressed;
  95         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
  96         size_t csize, usize = 0, len;
  97         int r;
  98
  99         log_info("/* testing decompress_startswith with %s on %.20s text */",
 100                  object_compressed_to_string(compression), data);
 101
 102 #define BUFSIZE_1 512
 103 #define BUFSIZE_2 20000
 104
 105         compressed = compressed1 = malloc(BUFSIZE_1);
 106         assert_se(compressed1);
 107         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 108         if (r == -ENOBUFS) {
 109                 log_info_errno(r, "compression failed: %m");
 110                 assert_se(may_fail);
 111
 112                 compressed = compressed2 = malloc(BUFSIZE_2);
 113                 assert_se(compressed2);
 114                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 115                 assert(r == 0);
 116         }
 117         assert_se(r == 0);
 118
 119         len = strlen(data);
 120
 121         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 122         assert_se(r > 0);
 123         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 124         assert_se(r == 0);
 125         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 126         assert_se(r == 0);
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 128         assert_se(r > 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 130         assert_se(r == 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 132         assert_se(r > 0);
 133 }
 134
 135 static void test_decompress_startswith_short(int compression,
 136                                              compress_blob_t compress,
 137                                              decompress_sw_t decompress_sw) {
 138
 139 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 140
 141         char buf[1024];
 142         size_t i, csize;
 143         int r;
 144
 145         log_info("/* %s with %s */", __func__, object_compressed_to_string(compression));
 146
 147         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 148         assert_se(r == 0);
 149
 150         for (i = 1; i < strlen(TEXT); i++) {
 151                 size_t alloc_size = i;
 152                 _cleanup_free_ void *buf2 = NULL;
 153
 154                 assert_se(buf2 = malloc(i));
 155
 156                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, TEXT[i]) == 1);
 157                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, 'y') == 0);
 158         }
 159 }
 160
 161 static void test_compress_stream(int compression,
 162                                  const char* cat,
 163                                  compress_stream_t compress,
 164                                  decompress_stream_t decompress,
 165                                  const char *srcfile) {
 166
 167         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 168         _cleanup_(unlink_tempfilep) char
 169                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 170                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 171         int r;
 172         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 173         struct stat st = {};
 174
 175         r = find_binary(cat, NULL);
 176         if (r < 0) {
 177                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 178                 return;
 179         }
 180
 181         log_debug("/* testing %s compression */",
 182                   object_compressed_to_string(compression));
 183
 184         log_debug("/* create source from %s */", srcfile);
 185
 186         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 187
 188         log_debug("/* test compression */");
 189
 190         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 191
 192         assert_se(compress(src, dst, -1) == 0);
 193
 194         if (cat) {
 195                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 196                 assert_se(system(cmd) == 0);
 197         }
 198
 199         log_debug("/* test decompression */");
 200
 201         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 202
 203         assert_se(stat(srcfile, &st) == 0);
 204
 205         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 206         r = decompress(dst, dst2, st.st_size);
 207         assert_se(r == 0);
 208
 209         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 210         assert_se(system(cmd2) == 0);
 211
 212         log_debug("/* test faulty decompression */");
 213
 214         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 215         r = decompress(dst, dst2, st.st_size);
 216         assert_se(IN_SET(r, 0, -EBADMSG));
 217
 218         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 219         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 220         r = decompress(dst, dst2, st.st_size - 1);
 221         assert_se(r == -EFBIG);
 222 }
 223 #endif
 224
 225 #if HAVE_LZ4
 226 static void test_lz4_decompress_partial(void) {
 227         char buf[20000], buf2[100];
 228         size_t buf_size = sizeof(buf), compressed;
 229         int r;
 230         _cleanup_free_ char *huge = NULL;
 231
 232 #define HUGE_SIZE (4096*1024)
 233         assert_se(huge = malloc(HUGE_SIZE));
 234         memset(huge, 'x', HUGE_SIZE);
 235         memcpy(huge, "HUGE=", 5);
 236
 237         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 238         assert_se(r >= 0);
 239         compressed = r;
 240         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 241
 242         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 243         assert_se(r >= 0);
 244         log_info("Decompressed → %i", r);
 245
 246         r = LZ4_decompress_safe_partial(buf, huge,
 247                                         compressed,
 248                                         12, HUGE_SIZE);
 249         assert_se(r >= 0);
 250         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 251
 252         for (size_t size = 1; size < sizeof(buf2); size++) {
 253                 /* This failed in older lz4s but works in newer ones. */
 254                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 255                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 256                                                                    r < 0 ? "bad" : "good");
 257                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 258                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 259                         assert_se(memcmp(buf2, huge, r) == 0);
 260         }
 261 }
 262 #endif
 263
 264 int main(int argc, char *argv[]) {
 265 #if HAVE_XZ || HAVE_LZ4
 266         const char text[] =
 267                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 268                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 269
 270         /* The file to test compression on can be specified as the first argument */
 271         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 272
 273         char data[512] = "random\0";
 274
 275         char huge[4096*1024];
 276         memset(huge, 'x', sizeof(huge));
 277         memcpy(huge, "HUGE=", 5);
 278         char_array_0(huge);
 279
 280         test_setup_logging(LOG_DEBUG);
 281
 282         random_bytes(data + 7, sizeof(data) - 7);
 283
 284 #if HAVE_XZ
 285         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 286                                  text, sizeof(text), false);
 287         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 288                                  data, sizeof(data), true);
 289
 290         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 291                                    compress_blob_xz, decompress_startswith_xz,
 292                                    text, sizeof(text), false);
 293         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 294                                    compress_blob_xz, decompress_startswith_xz,
 295                                    data, sizeof(data), true);
 296         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 297                                    compress_blob_xz, decompress_startswith_xz,
 298                                    huge, sizeof(huge), true);
 299
 300         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 301                              compress_stream_xz, decompress_stream_xz, srcfile);
 302
 303         test_decompress_startswith_short(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz);
 304
 305 #else
 306         log_info("/* XZ test skipped */");
 307 #endif
 308
 309 #if HAVE_LZ4
 310         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 311                                  text, sizeof(text), false);
 312         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 313                                  data, sizeof(data), true);
 314
 315         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 316                                    compress_blob_lz4, decompress_startswith_lz4,
 317                                    text, sizeof(text), false);
 318         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 319                                    compress_blob_lz4, decompress_startswith_lz4,
 320                                    data, sizeof(data), true);
 321         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 322                                    compress_blob_lz4, decompress_startswith_lz4,
 323                                    huge, sizeof(huge), true);
 324
 325         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 326                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 327
 328         test_lz4_decompress_partial();
 329
 330         test_decompress_startswith_short(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4);
 331
 332 #else
 333         log_info("/* LZ4 test skipped */");
 334 #endif
 335
 336         return 0;
 337 #else
 338         log_info("/* XZ and LZ4 tests skipped */");
 339         return EXIT_TEST_SKIP;
 340 #endif
 341 }