src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #include <sys/stat.h>
   4
   5 #if HAVE_LZ4
   6 #include <lz4.h>
   7 #endif
   8
   9 #include "alloc-util.h"
  10 #include "compress.h"
  11 #include "fd-util.h"
  12 #include "fs-util.h"
  13 #include "macro.h"
  14 #include "memory-util.h"
  15 #include "path-util.h"
  16 #include "random-util.h"
  17 #include "tests.h"
  18 #include "tmpfile-util.h"
  19
  20 #if HAVE_XZ
  21 # define XZ_OK 0
  22 #else
  23 # define XZ_OK -EPROTONOSUPPORT
  24 #endif
  25
  26 #if HAVE_LZ4
  27 # define LZ4_OK 0
  28 #else
  29 # define LZ4_OK -EPROTONOSUPPORT
  30 #endif
  31
  32 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  33                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  34 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  35                                 void **dst, size_t *dst_alloc_size,
  36                                 size_t* dst_size, size_t dst_max);
  37 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  38                               void **buffer, size_t *buffer_size,
  39                               const void *prefix, size_t prefix_len,
  40                               uint8_t extra);
  41
  42 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  43 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  44
  45 #if HAVE_XZ || HAVE_LZ4
  46 static void test_compress_decompress(int compression,
  47                                      compress_blob_t compress,
  48                                      decompress_blob_t decompress,
  49                                      const char *data,
  50                                      size_t data_len,
  51                                      bool may_fail) {
  52         char compressed[512];
  53         size_t csize, usize = 0;
  54         _cleanup_free_ char *decompressed = NULL;
  55         int r;
  56
  57         log_info("/* testing %s %s blob compression/decompression */",
  58                  object_compressed_to_string(compression), data);
  59
  60         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  61         if (r == -ENOBUFS) {
  62                 log_info_errno(r, "compression failed: %m");
  63                 assert_se(may_fail);
  64         } else {
  65                 assert_se(r == 0);
  66                 r = decompress(compressed, csize,
  67                                (void **) &decompressed, &usize, &csize, 0);
  68                 assert_se(r == 0);
  69                 assert_se(decompressed);
  70                 assert_se(memcmp(decompressed, data, data_len) == 0);
  71         }
  72
  73         r = decompress("garbage", 7,
  74                        (void **) &decompressed, &usize, &csize, 0);
  75         assert_se(r < 0);
  76
  77         /* make sure to have the minimal lz4 compressed size */
  78         r = decompress("00000000\1g", 9,
  79                        (void **) &decompressed, &usize, &csize, 0);
  80         assert_se(r < 0);
  81
  82         r = decompress("\100000000g", 9,
  83                        (void **) &decompressed, &usize, &csize, 0);
  84         assert_se(r < 0);
  85
  86         memzero(decompressed, usize);
  87 }
  88
  89 static void test_decompress_startswith(int compression,
  90                                        compress_blob_t compress,
  91                                        decompress_sw_t decompress_sw,
  92                                        const char *data,
  93                                        size_t data_len,
  94                                        bool may_fail) {
  95
  96         char *compressed;
  97         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
  98         size_t csize, usize = 0, len;
  99         int r;
 100
 101         log_info("/* testing decompress_startswith with %s on %.20s text */",
 102                  object_compressed_to_string(compression), data);
 103
 104 #define BUFSIZE_1 512
 105 #define BUFSIZE_2 20000
 106
 107         compressed = compressed1 = malloc(BUFSIZE_1);
 108         assert_se(compressed1);
 109         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 110         if (r == -ENOBUFS) {
 111                 log_info_errno(r, "compression failed: %m");
 112                 assert_se(may_fail);
 113
 114                 compressed = compressed2 = malloc(BUFSIZE_2);
 115                 assert_se(compressed2);
 116                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 117                 assert(r == 0);
 118         }
 119         assert_se(r == 0);
 120
 121         len = strlen(data);
 122
 123         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 124         assert_se(r > 0);
 125         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 126         assert_se(r == 0);
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 128         assert_se(r == 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 130         assert_se(r > 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 132         assert_se(r == 0);
 133         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 134         assert_se(r > 0);
 135 }
 136
 137 static void test_decompress_startswith_short(int compression,
 138                                              compress_blob_t compress,
 139                                              decompress_sw_t decompress_sw) {
 140
 141 #define TEXT "HUGE=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"
 142
 143         char buf[1024];
 144         size_t i, csize;
 145         int r;
 146
 147         log_info("/* %s with %s */", __func__, object_compressed_to_string(compression));
 148
 149         r = compress(TEXT, sizeof TEXT, buf, sizeof buf, &csize);
 150         assert_se(r == 0);
 151
 152         for (i = 1; i < strlen(TEXT); i++) {
 153                 size_t alloc_size = i;
 154                 _cleanup_free_ void *buf2 = NULL;
 155
 156                 assert_se(buf2 = malloc(i));
 157
 158                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, TEXT[i]) == 1);
 159                 assert_se(decompress_sw(buf, csize, &buf2, &alloc_size, TEXT, i, 'y') == 0);
 160         }
 161 }
 162
 163 static void test_compress_stream(int compression,
 164                                  const char* cat,
 165                                  compress_stream_t compress,
 166                                  decompress_stream_t decompress,
 167                                  const char *srcfile) {
 168
 169         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 170         _cleanup_(unlink_tempfilep) char
 171                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 172                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 173         int r;
 174         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 175         struct stat st = {};
 176
 177         r = find_binary(cat, NULL);
 178         if (r < 0) {
 179                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 180                 return;
 181         }
 182
 183         log_debug("/* testing %s compression */",
 184                   object_compressed_to_string(compression));
 185
 186         log_debug("/* create source from %s */", srcfile);
 187
 188         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 189
 190         log_debug("/* test compression */");
 191
 192         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 193
 194         assert_se(compress(src, dst, -1) == 0);
 195
 196         if (cat) {
 197                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 198                 assert_se(system(cmd) == 0);
 199         }
 200
 201         log_debug("/* test decompression */");
 202
 203         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 204
 205         assert_se(stat(srcfile, &st) == 0);
 206
 207         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 208         r = decompress(dst, dst2, st.st_size);
 209         assert_se(r == 0);
 210
 211         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 212         assert_se(system(cmd2) == 0);
 213
 214         log_debug("/* test faulty decompression */");
 215
 216         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 217         r = decompress(dst, dst2, st.st_size);
 218         assert_se(IN_SET(r, 0, -EBADMSG));
 219
 220         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 221         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 222         r = decompress(dst, dst2, st.st_size - 1);
 223         assert_se(r == -EFBIG);
 224 }
 225 #endif
 226
 227 #if HAVE_LZ4
 228 static void test_lz4_decompress_partial(void) {
 229         char buf[20000], buf2[100];
 230         size_t buf_size = sizeof(buf), compressed;
 231         int r;
 232         _cleanup_free_ char *huge = NULL;
 233
 234 #define HUGE_SIZE (4096*1024)
 235         assert_se(huge = malloc(HUGE_SIZE));
 236         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 237         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 238         huge[HUGE_SIZE - 1] = '\0';
 239
 240         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 241         assert_se(r >= 0);
 242         compressed = r;
 243         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 244
 245         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 246         assert_se(r >= 0);
 247         log_info("Decompressed → %i", r);
 248
 249         r = LZ4_decompress_safe_partial(buf, huge,
 250                                         compressed,
 251                                         12, HUGE_SIZE);
 252         assert_se(r >= 0);
 253         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 254
 255         for (size_t size = 1; size < sizeof(buf2); size++) {
 256                 /* This failed in older lz4s but works in newer ones. */
 257                 r = LZ4_decompress_safe_partial(buf, buf2, compressed, size, size);
 258                 log_info("Decompressed partial %zu/%zu → %i (%s)", size, size, r,
 259                                                                    r < 0 ? "bad" : "good");
 260                 if (r >= 0 && LZ4_versionNumber() >= 10803)
 261                         /* lz4 <= 1.8.2 should fail that test, let's only check for newer ones */
 262                         assert_se(memcmp(buf2, huge, r) == 0);
 263         }
 264 }
 265 #endif
 266
 267 int main(int argc, char *argv[]) {
 268 #if HAVE_XZ || HAVE_LZ4
 269         const char text[] =
 270                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 271                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 272
 273         /* The file to test compression on can be specified as the first argument */
 274         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 275
 276         char data[512] = "random\0";
 277
 278         _cleanup_free_ char *huge = NULL;
 279
 280         assert_se(huge = malloc(HUGE_SIZE));
 281         memcpy(huge, "HUGE=", STRLEN("HUGE="));
 282         memset(&huge[STRLEN("HUGE=")], 'x', HUGE_SIZE - STRLEN("HUGE=") - 1);
 283         huge[HUGE_SIZE - 1] = '\0';
 284
 285         test_setup_logging(LOG_DEBUG);
 286
 287         random_bytes(data + 7, sizeof(data) - 7);
 288
 289 #if HAVE_XZ
 290         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 291                                  text, sizeof(text), false);
 292         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 293                                  data, sizeof(data), true);
 294
 295         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 296                                    compress_blob_xz, decompress_startswith_xz,
 297                                    text, sizeof(text), false);
 298         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 299                                    compress_blob_xz, decompress_startswith_xz,
 300                                    data, sizeof(data), true);
 301         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 302                                    compress_blob_xz, decompress_startswith_xz,
 303                                    huge, HUGE_SIZE, true);
 304
 305         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 306                              compress_stream_xz, decompress_stream_xz, srcfile);
 307
 308         test_decompress_startswith_short(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_startswith_xz);
 309
 310 #else
 311         log_info("/* XZ test skipped */");
 312 #endif
 313
 314 #if HAVE_LZ4
 315         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 316                                  text, sizeof(text), false);
 317         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 318                                  data, sizeof(data), true);
 319
 320         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 321                                    compress_blob_lz4, decompress_startswith_lz4,
 322                                    text, sizeof(text), false);
 323         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 324                                    compress_blob_lz4, decompress_startswith_lz4,
 325                                    data, sizeof(data), true);
 326         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 327                                    compress_blob_lz4, decompress_startswith_lz4,
 328                                    huge, HUGE_SIZE, true);
 329
 330         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 331                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 332
 333         test_lz4_decompress_partial();
 334
 335         test_decompress_startswith_short(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_startswith_lz4);
 336
 337 #else
 338         log_info("/* LZ4 test skipped */");
 339 #endif
 340
 341         return 0;
 342 #else
 343         log_info("/* XZ and LZ4 tests skipped */");
 344         return EXIT_TEST_SKIP;
 345 #endif
 346 }