src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2
   3 #if HAVE_LZ4
   4 #include <lz4.h>
   5 #endif
   6
   7 #include "alloc-util.h"
   8 #include "compress.h"
   9 #include "fd-util.h"
  10 #include "fileio.h"
  11 #include "fs-util.h"
  12 #include "macro.h"
  13 #include "path-util.h"
  14 #include "random-util.h"
  15 #include "tests.h"
  16 #include "util.h"
  17
  18 #if HAVE_XZ
  19 # define XZ_OK 0
  20 #else
  21 # define XZ_OK -EPROTONOSUPPORT
  22 #endif
  23
  24 #if HAVE_LZ4
  25 # define LZ4_OK 0
  26 #else
  27 # define LZ4_OK -EPROTONOSUPPORT
  28 #endif
  29
  30 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  31                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  32 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  33                                 void **dst, size_t *dst_alloc_size,
  34                                 size_t* dst_size, size_t dst_max);
  35 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  36                               void **buffer, size_t *buffer_size,
  37                               const void *prefix, size_t prefix_len,
  38                               uint8_t extra);
  39
  40 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  41 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  42
  43 #if HAVE_XZ || HAVE_LZ4
  44 static void test_compress_decompress(int compression,
  45                                      compress_blob_t compress,
  46                                      decompress_blob_t decompress,
  47                                      const char *data,
  48                                      size_t data_len,
  49                                      bool may_fail) {
  50         char compressed[512];
  51         size_t csize, usize = 0;
  52         _cleanup_free_ char *decompressed = NULL;
  53         int r;
  54
  55         log_info("/* testing %s %s blob compression/decompression */",
  56                  object_compressed_to_string(compression), data);
  57
  58         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  59         if (r == -ENOBUFS) {
  60                 log_info_errno(r, "compression failed: %m");
  61                 assert_se(may_fail);
  62         } else {
  63                 assert_se(r == 0);
  64                 r = decompress(compressed, csize,
  65                                (void **) &decompressed, &usize, &csize, 0);
  66                 assert_se(r == 0);
  67                 assert_se(decompressed);
  68                 assert_se(memcmp(decompressed, data, data_len) == 0);
  69         }
  70
  71         r = decompress("garbage", 7,
  72                        (void **) &decompressed, &usize, &csize, 0);
  73         assert_se(r < 0);
  74
  75         /* make sure to have the minimal lz4 compressed size */
  76         r = decompress("00000000\1g", 9,
  77                        (void **) &decompressed, &usize, &csize, 0);
  78         assert_se(r < 0);
  79
  80         r = decompress("\100000000g", 9,
  81                        (void **) &decompressed, &usize, &csize, 0);
  82         assert_se(r < 0);
  83
  84         memzero(decompressed, usize);
  85 }
  86
  87 static void test_decompress_startswith(int compression,
  88                                        compress_blob_t compress,
  89                                        decompress_sw_t decompress_sw,
  90                                        const char *data,
  91                                        size_t data_len,
  92                                        bool may_fail) {
  93
  94         char *compressed;
  95         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
  96         size_t csize, usize = 0, len;
  97         int r;
  98
  99         log_info("/* testing decompress_startswith with %s on %.20s text */",
 100                  object_compressed_to_string(compression), data);
 101
 102 #define BUFSIZE_1 512
 103 #define BUFSIZE_2 20000
 104
 105         compressed = compressed1 = malloc(BUFSIZE_1);
 106         assert_se(compressed1);
 107         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 108         if (r == -ENOBUFS) {
 109                 log_info_errno(r, "compression failed: %m");
 110                 assert_se(may_fail);
 111
 112                 compressed = compressed2 = malloc(BUFSIZE_2);
 113                 assert_se(compressed2);
 114                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 115                 assert(r == 0);
 116         }
 117         assert_se(r == 0);
 118
 119         len = strlen(data);
 120
 121         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 122         assert_se(r > 0);
 123         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 124         assert_se(r == 0);
 125         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 126         assert_se(r == 0);
 127         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 128         assert_se(r > 0);
 129         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 130         assert_se(r == 0);
 131         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 132         assert_se(r > 0);
 133 }
 134
 135 static void test_compress_stream(int compression,
 136                                  const char* cat,
 137                                  compress_stream_t compress,
 138                                  decompress_stream_t decompress,
 139                                  const char *srcfile) {
 140
 141         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 142         _cleanup_(unlink_tempfilep) char
 143                 pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 144                 pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 145         int r;
 146         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 147         struct stat st = {};
 148
 149         r = find_binary(cat, NULL);
 150         if (r < 0) {
 151                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 152                 return;
 153         }
 154
 155         log_debug("/* testing %s compression */",
 156                   object_compressed_to_string(compression));
 157
 158         log_debug("/* create source from %s */", srcfile);
 159
 160         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 161
 162         log_debug("/* test compression */");
 163
 164         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 165
 166         assert_se(compress(src, dst, -1) == 0);
 167
 168         if (cat) {
 169                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 170                 assert_se(system(cmd) == 0);
 171         }
 172
 173         log_debug("/* test decompression */");
 174
 175         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 176
 177         assert_se(stat(srcfile, &st) == 0);
 178
 179         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 180         r = decompress(dst, dst2, st.st_size);
 181         assert_se(r == 0);
 182
 183         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 184         assert_se(system(cmd2) == 0);
 185
 186         log_debug("/* test faulty decompression */");
 187
 188         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 189         r = decompress(dst, dst2, st.st_size);
 190         assert_se(IN_SET(r, 0, -EBADMSG));
 191
 192         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 193         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 194         r = decompress(dst, dst2, st.st_size - 1);
 195         assert_se(r == -EFBIG);
 196 }
 197 #endif
 198
 199 #if HAVE_LZ4
 200 static void test_lz4_decompress_partial(void) {
 201         char buf[20000];
 202         size_t buf_size = sizeof(buf), compressed;
 203         int r;
 204         _cleanup_free_ char *huge = NULL;
 205
 206 #define HUGE_SIZE (4096*1024)
 207         huge = malloc(HUGE_SIZE);
 208         memset(huge, 'x', HUGE_SIZE);
 209         memcpy(huge, "HUGE=", 5);
 210
 211 #if LZ4_VERSION_NUMBER >= 10700
 212         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 213 #else
 214         r = LZ4_compress_limitedOutput(huge, buf, HUGE_SIZE, buf_size);
 215 #endif
 216         assert_se(r >= 0);
 217         compressed = r;
 218         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 219
 220         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 221         assert_se(r >= 0);
 222         log_info("Decompressed → %i", r);
 223
 224         r = LZ4_decompress_safe_partial(buf, huge,
 225                                         compressed,
 226                                         12, HUGE_SIZE);
 227         assert_se(r >= 0);
 228         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 229
 230         /* We expect this to fail, because that's how current lz4 works. If this
 231          * call succeeds, then lz4 has been fixed, and we need to change our code.
 232          */
 233         r = LZ4_decompress_safe_partial(buf, huge,
 234                                         compressed,
 235                                         12, HUGE_SIZE-1);
 236         assert_se(r < 0);
 237         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE-1, r);
 238 }
 239 #endif
 240
 241 int main(int argc, char *argv[]) {
 242 #if HAVE_XZ || HAVE_LZ4
 243         const char text[] =
 244                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 245                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 246
 247         /* The file to test compression on can be specified as the first argument */
 248         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 249
 250         char data[512] = "random\0";
 251
 252         char huge[4096*1024];
 253         memset(huge, 'x', sizeof(huge));
 254         memcpy(huge, "HUGE=", 5);
 255         char_array_0(huge);
 256
 257         test_setup_logging(LOG_DEBUG);
 258
 259         random_bytes(data + 7, sizeof(data) - 7);
 260
 261 #if HAVE_XZ
 262         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 263                                  text, sizeof(text), false);
 264         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 265                                  data, sizeof(data), true);
 266
 267         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 268                                    compress_blob_xz, decompress_startswith_xz,
 269                                    text, sizeof(text), false);
 270         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 271                                    compress_blob_xz, decompress_startswith_xz,
 272                                    data, sizeof(data), true);
 273         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 274                                    compress_blob_xz, decompress_startswith_xz,
 275                                    huge, sizeof(huge), true);
 276
 277         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 278                              compress_stream_xz, decompress_stream_xz, srcfile);
 279 #else
 280         log_info("/* XZ test skipped */");
 281 #endif
 282
 283 #if HAVE_LZ4
 284         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 285                                  text, sizeof(text), false);
 286         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 287                                  data, sizeof(data), true);
 288
 289         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 290                                    compress_blob_lz4, decompress_startswith_lz4,
 291                                    text, sizeof(text), false);
 292         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 293                                    compress_blob_lz4, decompress_startswith_lz4,
 294                                    data, sizeof(data), true);
 295         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 296                                    compress_blob_lz4, decompress_startswith_lz4,
 297                                    huge, sizeof(huge), true);
 298
 299         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 300                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 301
 302         test_lz4_decompress_partial();
 303 #else
 304         log_info("/* LZ4 test skipped */");
 305 #endif
 306
 307         return 0;
 308 #else
 309         log_info("/* XZ and LZ4 tests skipped */");
 310         return EXIT_TEST_SKIP;
 311 #endif
 312 }