src/journal/test-compress.c

   1 /* SPDX-License-Identifier: LGPL-2.1+ */
   2 /***
   3   This file is part of systemd
   4
   5   Copyright 2014 Ronny Chevalier
   6
   7   systemd is free software; you can redistribute it and/or modify it
   8   under the terms of the GNU Lesser General Public License as published by
   9   the Free Software Foundation; either version 2.1 of the License, or
  10   (at your option) any later version.
  11
  12   systemd is distributed in the hope that it will be useful, but
  13   WITHOUT ANY WARRANTY; without even the implied warranty of
  14   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15   Lesser General Public License for more details.
  16
  17   You should have received a copy of the GNU Lesser General Public License
  18   along with systemd; If not, see <http://www.gnu.org/licenses/>.
  19 ***/
  20
  21 #if HAVE_LZ4
  22 #include <lz4.h>
  23 #endif
  24
  25 #include "alloc-util.h"
  26 #include "compress.h"
  27 #include "fd-util.h"
  28 #include "fileio.h"
  29 #include "macro.h"
  30 #include "random-util.h"
  31 #include "util.h"
  32
  33 #if HAVE_XZ
  34 # define XZ_OK 0
  35 #else
  36 # define XZ_OK -EPROTONOSUPPORT
  37 #endif
  38
  39 #if HAVE_LZ4
  40 # define LZ4_OK 0
  41 #else
  42 # define LZ4_OK -EPROTONOSUPPORT
  43 #endif
  44
  45 typedef int (compress_blob_t)(const void *src, uint64_t src_size,
  46                               void *dst, size_t dst_alloc_size, size_t *dst_size);
  47 typedef int (decompress_blob_t)(const void *src, uint64_t src_size,
  48                                 void **dst, size_t *dst_alloc_size,
  49                                 size_t* dst_size, size_t dst_max);
  50 typedef int (decompress_sw_t)(const void *src, uint64_t src_size,
  51                               void **buffer, size_t *buffer_size,
  52                               const void *prefix, size_t prefix_len,
  53                               uint8_t extra);
  54
  55 typedef int (compress_stream_t)(int fdf, int fdt, uint64_t max_bytes);
  56 typedef int (decompress_stream_t)(int fdf, int fdt, uint64_t max_size);
  57
  58 #if HAVE_XZ || HAVE_LZ4
  59 static void test_compress_decompress(int compression,
  60                                      compress_blob_t compress,
  61                                      decompress_blob_t decompress,
  62                                      const char *data,
  63                                      size_t data_len,
  64                                      bool may_fail) {
  65         char compressed[512];
  66         size_t csize, usize = 0;
  67         _cleanup_free_ char *decompressed = NULL;
  68         int r;
  69
  70         log_info("/* testing %s %s blob compression/decompression */",
  71                  object_compressed_to_string(compression), data);
  72
  73         r = compress(data, data_len, compressed, sizeof(compressed), &csize);
  74         if (r == -ENOBUFS) {
  75                 log_info_errno(r, "compression failed: %m");
  76                 assert_se(may_fail);
  77         } else {
  78                 assert_se(r == 0);
  79                 r = decompress(compressed, csize,
  80                                (void **) &decompressed, &usize, &csize, 0);
  81                 assert_se(r == 0);
  82                 assert_se(decompressed);
  83                 assert_se(memcmp(decompressed, data, data_len) == 0);
  84         }
  85
  86         r = decompress("garbage", 7,
  87                        (void **) &decompressed, &usize, &csize, 0);
  88         assert_se(r < 0);
  89
  90         /* make sure to have the minimal lz4 compressed size */
  91         r = decompress("00000000\1g", 9,
  92                        (void **) &decompressed, &usize, &csize, 0);
  93         assert_se(r < 0);
  94
  95         r = decompress("\100000000g", 9,
  96                        (void **) &decompressed, &usize, &csize, 0);
  97         assert_se(r < 0);
  98
  99         memzero(decompressed, usize);
 100 }
 101
 102 static void test_decompress_startswith(int compression,
 103                                        compress_blob_t compress,
 104                                        decompress_sw_t decompress_sw,
 105                                        const char *data,
 106                                        size_t data_len,
 107                                        bool may_fail) {
 108
 109         char *compressed;
 110         _cleanup_free_ char *compressed1 = NULL, *compressed2 = NULL, *decompressed = NULL;
 111         size_t csize, usize = 0, len;
 112         int r;
 113
 114         log_info("/* testing decompress_startswith with %s on %.20s text */",
 115                  object_compressed_to_string(compression), data);
 116
 117 #define BUFSIZE_1 512
 118 #define BUFSIZE_2 20000
 119
 120         compressed = compressed1 = malloc(BUFSIZE_1);
 121         assert_se(compressed1);
 122         r = compress(data, data_len, compressed, BUFSIZE_1, &csize);
 123         if (r == -ENOBUFS) {
 124                 log_info_errno(r, "compression failed: %m");
 125                 assert_se(may_fail);
 126
 127                 compressed = compressed2 = malloc(BUFSIZE_2);
 128                 assert_se(compressed2);
 129                 r = compress(data, data_len, compressed, BUFSIZE_2, &csize);
 130                 assert(r == 0);
 131         }
 132         assert_se(r == 0);
 133
 134         len = strlen(data);
 135
 136         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 137         assert_se(r > 0);
 138         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, 'w');
 139         assert_se(r == 0);
 140         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, "barbarbar", 9, ' ');
 141         assert_se(r == 0);
 142         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, data[len-1]);
 143         assert_se(r > 0);
 144         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len - 1, 'w');
 145         assert_se(r == 0);
 146         r = decompress_sw(compressed, csize, (void **) &decompressed, &usize, data, len, '\0');
 147         assert_se(r > 0);
 148 }
 149
 150 static void test_compress_stream(int compression,
 151                                  const char* cat,
 152                                  compress_stream_t compress,
 153                                  decompress_stream_t decompress,
 154                                  const char *srcfile) {
 155
 156         _cleanup_close_ int src = -1, dst = -1, dst2 = -1;
 157         char pattern[] = "/tmp/systemd-test.compressed.XXXXXX",
 158              pattern2[] = "/tmp/systemd-test.compressed.XXXXXX";
 159         int r;
 160         _cleanup_free_ char *cmd = NULL, *cmd2;
 161         struct stat st = {};
 162
 163         log_debug("/* testing %s compression */",
 164                   object_compressed_to_string(compression));
 165
 166         log_debug("/* create source from %s */", srcfile);
 167
 168         assert_se((src = open(srcfile, O_RDONLY|O_CLOEXEC)) >= 0);
 169
 170         log_debug("/* test compression */");
 171
 172         assert_se((dst = mkostemp_safe(pattern)) >= 0);
 173
 174         assert_se(compress(src, dst, -1) == 0);
 175
 176         if (cat) {
 177                 assert_se(asprintf(&cmd, "%s %s | diff %s -", cat, pattern, srcfile) > 0);
 178                 assert_se(system(cmd) == 0);
 179         }
 180
 181         log_debug("/* test decompression */");
 182
 183         assert_se((dst2 = mkostemp_safe(pattern2)) >= 0);
 184
 185         assert_se(stat(srcfile, &st) == 0);
 186
 187         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 188         r = decompress(dst, dst2, st.st_size);
 189         assert_se(r == 0);
 190
 191         assert_se(asprintf(&cmd2, "diff %s %s", srcfile, pattern2) > 0);
 192         assert_se(system(cmd2) == 0);
 193
 194         log_debug("/* test faulty decompression */");
 195
 196         assert_se(lseek(dst, 1, SEEK_SET) == 1);
 197         r = decompress(dst, dst2, st.st_size);
 198         assert_se(IN_SET(r, 0, -EBADMSG));
 199
 200         assert_se(lseek(dst, 0, SEEK_SET) == 0);
 201         assert_se(lseek(dst2, 0, SEEK_SET) == 0);
 202         r = decompress(dst, dst2, st.st_size - 1);
 203         assert_se(r == -EFBIG);
 204
 205         assert_se(unlink(pattern) == 0);
 206         assert_se(unlink(pattern2) == 0);
 207 }
 208 #endif
 209
 210 #if HAVE_LZ4
 211 static void test_lz4_decompress_partial(void) {
 212         char buf[20000];
 213         size_t buf_size = sizeof(buf), compressed;
 214         int r;
 215         _cleanup_free_ char *huge = NULL;
 216
 217 #define HUGE_SIZE (4096*1024)
 218         huge = malloc(HUGE_SIZE);
 219         memset(huge, 'x', HUGE_SIZE);
 220         memcpy(huge, "HUGE=", 5);
 221
 222 #if LZ4_VERSION_NUMBER >= 10700
 223         r = LZ4_compress_default(huge, buf, HUGE_SIZE, buf_size);
 224 #else
 225         r = LZ4_compress_limitedOutput(huge, buf, HUGE_SIZE, buf_size);
 226 #endif
 227         assert_se(r >= 0);
 228         compressed = r;
 229         log_info("Compressed %i → %zu", HUGE_SIZE, compressed);
 230
 231         r = LZ4_decompress_safe(buf, huge, r, HUGE_SIZE);
 232         assert_se(r >= 0);
 233         log_info("Decompressed → %i", r);
 234
 235         r = LZ4_decompress_safe_partial(buf, huge,
 236                                         compressed,
 237                                         12, HUGE_SIZE);
 238         assert_se(r >= 0);
 239         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE, r);
 240
 241         /* We expect this to fail, because that's how current lz4 works. If this
 242          * call succeeds, then lz4 has been fixed, and we need to change our code.
 243          */
 244         r = LZ4_decompress_safe_partial(buf, huge,
 245                                         compressed,
 246                                         12, HUGE_SIZE-1);
 247         assert_se(r < 0);
 248         log_info("Decompressed partial %i/%i → %i", 12, HUGE_SIZE-1, r);
 249 }
 250 #endif
 251
 252 int main(int argc, char *argv[]) {
 253 #if HAVE_XZ || HAVE_LZ4
 254         const char text[] =
 255                 "text\0foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF"
 256                 "foofoofoofoo AAAA aaaaaaaaa ghost busters barbarbar FFF";
 257
 258         /* The file to test compression on can be specified as the first argument */
 259         const char *srcfile = argc > 1 ? argv[1] : argv[0];
 260
 261         char data[512] = "random\0";
 262
 263         char huge[4096*1024];
 264         memset(huge, 'x', sizeof(huge));
 265         memcpy(huge, "HUGE=", 5);
 266         char_array_0(huge);
 267
 268         log_set_max_level(LOG_DEBUG);
 269
 270         random_bytes(data + 7, sizeof(data) - 7);
 271
 272 #if HAVE_XZ
 273         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 274                                  text, sizeof(text), false);
 275         test_compress_decompress(OBJECT_COMPRESSED_XZ, compress_blob_xz, decompress_blob_xz,
 276                                  data, sizeof(data), true);
 277
 278         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 279                                    compress_blob_xz, decompress_startswith_xz,
 280                                    text, sizeof(text), false);
 281         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 282                                    compress_blob_xz, decompress_startswith_xz,
 283                                    data, sizeof(data), true);
 284         test_decompress_startswith(OBJECT_COMPRESSED_XZ,
 285                                    compress_blob_xz, decompress_startswith_xz,
 286                                    huge, sizeof(huge), true);
 287
 288         test_compress_stream(OBJECT_COMPRESSED_XZ, "xzcat",
 289                              compress_stream_xz, decompress_stream_xz, srcfile);
 290 #else
 291         log_info("/* XZ test skipped */");
 292 #endif
 293
 294 #if HAVE_LZ4
 295         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 296                                  text, sizeof(text), false);
 297         test_compress_decompress(OBJECT_COMPRESSED_LZ4, compress_blob_lz4, decompress_blob_lz4,
 298                                  data, sizeof(data), true);
 299
 300         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 301                                    compress_blob_lz4, decompress_startswith_lz4,
 302                                    text, sizeof(text), false);
 303         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 304                                    compress_blob_lz4, decompress_startswith_lz4,
 305                                    data, sizeof(data), true);
 306         test_decompress_startswith(OBJECT_COMPRESSED_LZ4,
 307                                    compress_blob_lz4, decompress_startswith_lz4,
 308                                    huge, sizeof(huge), true);
 309
 310         test_compress_stream(OBJECT_COMPRESSED_LZ4, "lz4cat",
 311                              compress_stream_lz4, decompress_stream_lz4, srcfile);
 312
 313         test_lz4_decompress_partial();
 314 #else
 315         log_info("/* LZ4 test skipped */");
 316 #endif
 317
 318         return 0;
 319 #else
 320         return EXIT_TEST_SKIP;
 321 #endif
 322 }