src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2 /***
   3   This file is part of systemd
   4
   5   Copyright 2014 Ronny Chevalier
   6
   7   systemd is free software; you can redistribute it and/or modify it
   8   under the terms of the GNU Lesser General Public License as published by
   9   the Free Software Foundation; either version 2.1 of the License, or
  10   (at your option) any later version.
  11
  12   systemd is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   Lesser General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  19 ***/
  20
  21 #if HAVE_LZ4
  22 #include <lz4.h>
  23 #endif
  24
  25 #include "alloc-util.h"
  26 #include "compress.h"
  27 #include "fd-util.h"
  28 #include "fileio.h"
  29 #include "macro.h"
  30 #include "path-util.h"
  31 #include "random-util.h"
  32 #include "util.h"
  33
  34 #if HAVE_XZ
  35 # define XZ_OK 0
  36 #else
  37 # define XZ_OK -EPROTONOSUPPORT
  38 #endif
  39
  40 #if HAVE_LZ4
  41 # define LZ4_OK 0
  42 #else
  43 # define LZ4_OK -EPROTONOSUPPORT
  44 #endif
  45
  46 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  47                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  48 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  49                                 void **dst, size_t *dst_alloc_size,
  50                                 size_t* dst_size, size_t dst_max);
  51 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  52                               void **buffer, size_t *buffer_size,
  53                               const void *prefix, size_t prefix_len,
  54                               uint8_t extra);
  55
  56 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  57 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  58
  59 #if HAVE_XZ || HAVE_LZ4
  60 static void test_compress_decompress(int compression,
  61                                      compress_blob_t compress,
  62                                      decompress_blob_t decompress,
  63                                      const char *data,
  64                                      size_t data_len,
  65                                      bool may_fail) {
  66         char compressed[512];
  67         size_t csize, usize = 0;
  68         _cleanup_free_ char *decompressed = NULL;
  69         int r;
  70
  71         log_info("/* testing %s %s blob compression/decompression */",
  72                  object_compressed_to_string(compression), data);
  73
  74         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  75         if (r == -ENOBUFS) {
  76                 log_info_errno(r, "compression failed: %m");
  77                 assert_se(may_fail);
  78         } else {
  79                 assert_se(r == 0);
  80                 r = decompress(compressed, csize,
  81                                (void **) &decompressed, &usize, &csize, 0);
  82                 assert_se(r == 0);
  83                 assert_se(decompressed);
  84                 assert_se(memcmp(decompressed, data, data_len) == 0);
  85         }
  86
  87         r = decompress("garbage", 7,
  88                        (void **) &decompressed, &usize, &csize, 0);
  89         assert_se(r < 0);
  90
  91         /* make sure to have the minimal lz4 compressed size */
  92         r = decompress("00000000\1g", 9,
  93                        (void **) &decompressed, &usize, &csize, 0);
  94         assert_se(r < 0);
  95
  96         r = decompress("\100000000g", 9,
  97                        (void **) &decompressed, &usize, &csize, 0);
  98         assert_se(r < 0);
  99
 100         memzero(decompressed, usize);
 101 }
 102
 103 static void test_decompress_startswith(int compression,
 104                                        compress_blob_t compress,
 105                                        decompress_sw_t decompress_sw,
 106                                        const char *data,
 107                                        size_t data_len,
 108                                        bool may_fail) {
 109
 110         char *compressed;
 111         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 112         size_t csize, usize = 0, len;
 113         int r;
 114
 115         log_info("/* testing decompress_startswith with %s on %.20s text */",
 116                  object_compressed_to_string(compression), data);
 117
 118 #define BUFSIZE_1 512
 119 #define BUFSIZE_2 20000
 120
 121         compressed = compressed1 = malloc(BUFSIZE_1);
 122         assert_se(compressed1);
 123         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 124         if (r == -ENOBUFS) {
 125                 log_info_errno(r, "compression failed: %m");
 126                 assert_se(may_fail);
 127
 128                 compressed = compressed2 = malloc(BUFSIZE_2);
 129                 assert_se(compressed2);
 130                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 131                 assert(r == 0);
 132         }
 133         assert_se(r == 0);
 134
 135         len = strlen(data);
 136
 137         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 138         assert_se(r > 0);
 139         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 140         assert_se(r == 0);
 141         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 142         assert_se(r == 0);
 143         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 144         assert_se(r > 0);
 145         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 146         assert_se(r == 0);
 147         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 148         assert_se(r > 0);
 149 }
 150
 151 static void test_compress_stream(int compression,
 152                                  const char* cat,
 153                                  compress_stream_t compress,
 154                                  decompress_stream_t decompress,
 155                                  const char *srcfile) {
 156
 157         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 158         char pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 159              pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 160         int r;
 161         _cleanup_free_ char *cmd = NULL, *cmd2 = NULL;
 162         struct stat st = {};
 163
 164         r = find_binary(cat, NULL);
 165         if (r < 0) {
 166                 log_error_errno(r, "Skipping %s, could not find %s binary: %m", __func__, cat);
 167                 return;
 168         }
 169
 170         log_debug("/* testing %s compression */",
 171                   object_compressed_to_string(compression));
 172
 173         log_debug("/* create source from %s */", srcfile);
 174
 175         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 176
 177         log_debug("/* test compression */");
 178
 179         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 180
 181         assert_se(compress(src, dst, -1) == 0);
 182
 183         if (cat) {
 184                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 185                 assert_se(system(cmd) == 0);
 186         }
 187
 188         log_debug("/* test decompression */");
 189
 190         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 191
 192         assert_se(stat(srcfile, &st) == 0);
 193
 194         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 195         r = decompress(dst, dst2, st.st_size);
 196         assert_se(r == 0);
 197
 198         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 199         assert_se(system(cmd2) == 0);
 200
 201         log_debug("/* test faulty decompression */");
 202
 203         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 204         r = decompress(dst, dst2, st.st_size);
 205         assert_se(IN_SET(r, 0, -EBADMSG));
 206
 207         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 208         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 209         r = decompress(dst, dst2, st.st_size - 1);
 210         assert_se(r == -EFBIG);
 211
 212         assert_se(unlink(pattern) == 0);
 213         assert_se(unlink(pattern2) == 0);
 214 }
 215 #endif
 216
 217 #if HAVE_LZ4
 218 static void test_lz4_decompress_partial(void) {
 219         char buf[20000];
 220         size_t buf_size = sizeof(buf), compressed;
 221         int r;
 222         _cleanup_free_ char *huge = NULL;
 223
 224 #define HUGE_SIZE (4096*1024)
 225         huge = malloc(HUGE_SIZE);
 226         memset(huge, 'x', HUGE_SIZE);
 227         memcpy(huge, "HUGE=", 5);
 228
 229 #if LZ4_VERSION_NUMBER >= 10700
 230         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 231 #else
 232         r = LZ4_compress_limitedOutput(huge, buf, HUGE_SIZE, buf_size);
 233 #endif
 234         assert_se(r >= 0);
 235         compressed = r;
 236         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 237
 238         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 239         assert_se(r >= 0);
 240         log_info("Decompressed → %i", r);
 241
 242         r = LZ4_decompress_safe_partial(buf, huge,
 243                                         compressed,
 244                                         12, HUGE_SIZE);
 245         assert_se(r >= 0);
 246         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 247
 248         /* We expect this to fail, because that's how current lz4 works. If this
 249          * call succeeds, then lz4 has been fixed, and we need to change our code.
 250          */
 251         r = LZ4_decompress_safe_partial(buf, huge,
 252                                         compressed,
 253                                         12, HUGE_SIZE-1);
 254         assert_se(r < 0);
 255         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE-1, r);
 256 }
 257 #endif
 258
 259 int main(int argc, char *argv[]) {
 260 #if HAVE_XZ || HAVE_LZ4
 261         const char text[] =
 262                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 263                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 264
 265         /* The file to test compression on can be specified as the first argument */
 266         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 267
 268         char data[512] = "random\0";
 269
 270         char huge[4096*1024];
 271         memset(huge, 'x', sizeof(huge));
 272         memcpy(huge, "HUGE=", 5);
 273         char_array_0(huge);
 274
 275         log_set_max_level(LOG_DEBUG);
 276
 277         random_bytes(data + 7, sizeof(data) - 7);
 278
 279 #if HAVE_XZ
 280         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 281                                  text, sizeof(text), false);
 282         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 283                                  data, sizeof(data), true);
 284
 285         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 286                                    compress_blob_xz, decompress_startswith_xz,
 287                                    text, sizeof(text), false);
 288         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 289                                    compress_blob_xz, decompress_startswith_xz,
 290                                    data, sizeof(data), true);
 291         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 292                                    compress_blob_xz, decompress_startswith_xz,
 293                                    huge, sizeof(huge), true);
 294
 295         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 296                              compress_stream_xz, decompress_stream_xz, srcfile);
 297 #else
 298         log_info("/* XZ test skipped */");
 299 #endif
 300
 301 #if HAVE_LZ4
 302         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 303                                  text, sizeof(text), false);
 304         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 305                                  data, sizeof(data), true);
 306
 307         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 308                                    compress_blob_lz4, decompress_startswith_lz4,
 309                                    text, sizeof(text), false);
 310         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 311                                    compress_blob_lz4, decompress_startswith_lz4,
 312                                    data, sizeof(data), true);
 313         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 314                                    compress_blob_lz4, decompress_startswith_lz4,
 315                                    huge, sizeof(huge), true);
 316
 317         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 318                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 319
 320         test_lz4_decompress_partial();
 321 #else
 322         log_info("/* LZ4 test skipped */");
 323 #endif
 324
 325         return 0;
 326 #else
 327         return EXIT_TEST_SKIP;
 328 #endif
 329 }