endif
conf.set('DEFAULT_COMPRESSION',
compression == 'none' ? 0 :
- 'OBJECT_COMPRESSED_@0@'.format(compression.to_upper()))
+ 'COMPRESSION_@0@'.format(compression.to_upper()))
want_xkbcommon = get_option('xkbcommon')
if want_xkbcommon != 'false' and not skip_deps
link_args : ['-shared',
'-Wl,--version-script=' + libsystemd_sym_path],
link_with : [libbasic,
- libbasic_gcrypt],
+ libbasic_gcrypt,
+ libbasic_compress],
link_whole : [libsystemd_static],
dependencies : [threads,
- librt,
- libxz,
- libzstd,
- liblz4],
+ librt],
link_depends : libsystemd_sym,
install : true,
install_dir : rootlibdir)
libsystemd_sources,
basic_sources,
basic_gcrypt_sources,
+ basic_compress_sources,
fundamental_sources,
disable_mempool_c,
include_directories : libsystemd_includes,
'systemd-coredump',
systemd_coredump_sources,
include_directories : includes,
- link_with : [libshared],
+ link_with : [libshared,
+ libbasic_compress],
dependencies : [threads,
libacl,
libxz,
'coredumpctl',
coredumpctl_sources,
include_directories : includes,
- link_with : [libshared],
+ link_with : [libshared,
+ libbasic_compress],
dependencies : [threads,
libxz,
liblz4,
#include "fd-util.h"
#include "fileio.h"
#include "io-util.h"
-#include "journal-def.h"
#include "macro.h"
#include "sparse-endian.h"
#include "string-table.h"
#define ALIGN_8(l) ALIGN_TO(l, sizeof(size_t))
-static const char* const object_compressed_table[_OBJECT_COMPRESSED_MAX] = {
- [OBJECT_COMPRESSED_XZ] = "XZ",
- [OBJECT_COMPRESSED_LZ4] = "LZ4",
- [OBJECT_COMPRESSED_ZSTD] = "ZSTD",
+static const char* const compression_table[_COMPRESSION_MAX] = {
+ [COMPRESSION_XZ] = "XZ",
+ [COMPRESSION_LZ4] = "LZ4",
+ [COMPRESSION_ZSTD] = "ZSTD",
/* If we add too many more entries here, it's going to grow quite large (and be mostly sparse), since
* the array key is actually a bitmask, not a plain enum */
};
-DEFINE_STRING_TABLE_LOOKUP(object_compressed, int);
+DEFINE_STRING_TABLE_LOOKUP(compression, Compression);
int compress_blob_xz(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size) {
return -ENOBUFS;
*dst_size = out_pos;
- return OBJECT_COMPRESSED_XZ;
+ return COMPRESSION_XZ;
#else
return -EPROTONOSUPPORT;
#endif
unaligned_write_le64(dst, src_size);
*dst_size = r + 8;
- return OBJECT_COMPRESSED_LZ4;
+ return COMPRESSION_LZ4;
#else
return -EPROTONOSUPPORT;
#endif
return zstd_ret_to_errno(k);
*dst_size = k;
- return OBJECT_COMPRESSED_ZSTD;
+ return COMPRESSION_ZSTD;
#else
return -EPROTONOSUPPORT;
#endif
}
int decompress_blob(
- int compression,
+ Compression compression,
const void *src,
uint64_t src_size,
void **dst,
size_t* dst_size,
size_t dst_max) {
- if (compression == OBJECT_COMPRESSED_XZ)
+ if (compression == COMPRESSION_XZ)
return decompress_blob_xz(
src, src_size,
dst, dst_size, dst_max);
- else if (compression == OBJECT_COMPRESSED_LZ4)
+ else if (compression == COMPRESSION_LZ4)
return decompress_blob_lz4(
src, src_size,
dst, dst_size, dst_max);
- else if (compression == OBJECT_COMPRESSED_ZSTD)
+ else if (compression == COMPRESSION_ZSTD)
return decompress_blob_zstd(
src, src_size,
dst, dst_size, dst_max);
}
int decompress_startswith(
- int compression,
+ Compression compression,
const void *src,
uint64_t src_size,
void **buffer,
size_t prefix_len,
uint8_t extra) {
- if (compression == OBJECT_COMPRESSED_XZ)
+ if (compression == COMPRESSION_XZ)
return decompress_startswith_xz(
src, src_size,
buffer,
prefix, prefix_len,
extra);
- else if (compression == OBJECT_COMPRESSED_LZ4)
+ else if (compression == COMPRESSION_LZ4)
return decompress_startswith_lz4(
src, src_size,
buffer,
prefix, prefix_len,
extra);
- else if (compression == OBJECT_COMPRESSED_ZSTD)
+ else if (compression == COMPRESSION_ZSTD)
return decompress_startswith_zstd(
src, src_size,
buffer,
s.total_in, s.total_out,
(double) s.total_out / s.total_in * 100);
- return OBJECT_COMPRESSED_XZ;
+ return COMPRESSION_XZ;
}
}
}
total_in, total_out,
(double) total_out / total_in * 100);
- return OBJECT_COMPRESSED_LZ4;
+ return COMPRESSION_LZ4;
#else
return -EPROTONOSUPPORT;
#endif
log_debug("ZSTD compression finished (%" PRIu64 " -> %" PRIu64 " bytes)",
in_bytes, max_bytes - left);
- return OBJECT_COMPRESSED_ZSTD;
+ return COMPRESSION_ZSTD;
#else
return -EPROTONOSUPPORT;
#endif
#include <unistd.h>
-#include "journal-def.h"
+typedef enum Compression {
+ /* These are defined the same way as the relevant object types in journal-def.h,
+ * i.e. OBJECT_COMPRESSED_XZ, … */
+ COMPRESSION_NONE = 0,
+ COMPRESSION_XZ = 1,
+ COMPRESSION_LZ4 = 2,
+ COMPRESSION_ZSTD = 4,
+ _COMPRESSION_MAX,
+ _COMPRESSION_INVALID = -EINVAL,
+} Compression;
-const char* object_compressed_to_string(int compression);
-int object_compressed_from_string(const char *compression);
+const char* compression_to_string(Compression compression);
+Compression compression_from_string(const char *compression);
int compress_blob_xz(const void *src, uint64_t src_size,
void *dst, size_t dst_alloc_size, size_t *dst_size);
void **dst, size_t* dst_size, size_t dst_max);
int decompress_blob_zstd(const void *src, uint64_t src_size,
void **dst, size_t* dst_size, size_t dst_max);
-int decompress_blob(int compression,
+int decompress_blob(Compression compression,
const void *src, uint64_t src_size,
void **dst, size_t* dst_size, size_t dst_max);
void **buffer,
const void *prefix, size_t prefix_len,
uint8_t extra);
-int decompress_startswith(int compression,
+int decompress_startswith(Compression compression,
const void *src, uint64_t src_size,
void **buffer,
const void *prefix, size_t prefix_len,
int decompress_stream_lz4(int fdf, int fdt, uint64_t max_size);
int decompress_stream_zstd(int fdf, int fdt, uint64_t max_size);
-static inline int compress_blob(const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size) {
- switch (DEFAULT_COMPRESSION) {
- case OBJECT_COMPRESSED_ZSTD:
+static inline int compress_blob_explicit(
+ Compression compression,
+ const void *src, uint64_t src_size,
+ void *dst, size_t dst_alloc_size, size_t *dst_size) {
+
+ switch (compression) {
+ case COMPRESSION_ZSTD:
return compress_blob_zstd(src, src_size, dst, dst_alloc_size, dst_size);
- case OBJECT_COMPRESSED_LZ4:
+ case COMPRESSION_LZ4:
return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size);
- case OBJECT_COMPRESSED_XZ:
+ case COMPRESSION_XZ:
return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size);
default:
return -EOPNOTSUPP;
}
}
+#define compress_blob(src, src_size, dst, dst_alloc_size, dst_size) \
+ compress_blob_explicit( \
+ DEFAULT_COMPRESSION, \
+ src, src_size, \
+ dst, dst_alloc_size, dst_size)
+
static inline int compress_stream(int fdf, int fdt, uint64_t max_bytes, uint64_t *ret_uncompressed_size) {
switch (DEFAULT_COMPRESSION) {
- case OBJECT_COMPRESSED_ZSTD:
+ case COMPRESSION_ZSTD:
return compress_stream_zstd(fdf, fdt, max_bytes, ret_uncompressed_size);
- case OBJECT_COMPRESSED_LZ4:
+ case COMPRESSION_LZ4:
return compress_stream_lz4(fdf, fdt, max_bytes, ret_uncompressed_size);
- case OBJECT_COMPRESSED_XZ:
+ case COMPRESSION_XZ:
return compress_stream_xz(fdf, fdt, max_bytes, ret_uncompressed_size);
default:
return -EOPNOTSUPP;
static inline const char* default_compression_extension(void) {
switch (DEFAULT_COMPRESSION) {
- case OBJECT_COMPRESSED_ZSTD:
+ case COMPRESSION_ZSTD:
return ".zst";
- case OBJECT_COMPRESSED_LZ4:
+ case COMPRESSION_LZ4:
return ".lz4";
- case OBJECT_COMPRESSED_XZ:
+ case COMPRESSION_XZ:
return ".xz";
default:
return "";
dependencies : [libgcrypt],
c_args : ['-fvisibility=default'],
build_by_default : false)
+
+############################################################
+
+basic_compress_sources = files(
+ 'compress.c',
+ 'compress.h')
+
+# A convenience library that is separate from libbasic to avoid unnecessary
+# linking to the compression libraries.
+libbasic_compress = static_library(
+ 'basic-compress',
+ basic_compress_sources,
+ include_directories : basic_includes,
+ dependencies : [libxz,
+ libzstd,
+ liblz4],
+ c_args : ['-fvisibility=default'],
+ build_by_default : false)
#include "compress.h"
#include "fuzz.h"
-static int compress(int alg,
- const void *src, uint64_t src_size,
- void *dst, size_t dst_alloc_size, size_t *dst_size) {
-
- if (alg == OBJECT_COMPRESSED_LZ4)
- return compress_blob_lz4(src, src_size, dst, dst_alloc_size, dst_size);
- if (alg == OBJECT_COMPRESSED_XZ)
- return compress_blob_xz(src, src_size, dst, dst_alloc_size, dst_size);
- return -EOPNOTSUPP;
-}
-
typedef struct header {
- uint32_t alg:2; /* We have only two compression algorithms so far, but we might add
- * more in the future. Let's make this a bit wider so our fuzzer
- * cases remain stable in the future. */
+ uint32_t alg:2; /* We have only three compression algorithms so far, but we might add more in the
+ * future. Let's make this a bit wider so our fuzzer cases remain stable in the
+ * future. */
uint32_t sw_len;
uint32_t sw_alloc;
uint32_t reserved[3]; /* Extra space to keep fuzz cases stable in case we need to
log_set_max_level(LOG_CRIT);
log_info("Using compression %s, data size=%zu",
- object_compressed_to_string(alg) ?: "(none)",
+ compression_to_string(alg) ?: "(none)",
data_len);
buf = malloc(MAX(size, 128u)); /* Make the buffer a bit larger for very small data */
}
size_t csize;
- r = compress(alg, h->data, data_len, buf, size, &csize);
+ r = compress_blob_explicit(alg, h->data, data_len, buf, size, &csize);
if (r < 0) {
log_error_errno(r, "Compression failed: %m");
return 0;
p = p + ALIGN64(le64toh(o->object.size));
}
- is_compressed = (o->object.flags & OBJECT_COMPRESSION_MASK) != 0;
+ is_compressed = COMPRESSION_FROM_OBJECT(o) != COMPRESSION_NONE;
(void) managed_journal_file_close(f);
'sd-journal/audit-type.h',
'sd-journal/catalog.c',
'sd-journal/catalog.h',
- 'sd-journal/compress.c',
- 'sd-journal/compress.h',
'sd-journal/journal-def.h',
'sd-journal/journal-file.c',
'sd-journal/journal-file.h',
libsystemd_sources,
include_directories : libsystemd_includes,
c_args : libsystemd_c_args,
- link_with : libbasic,
+ link_with : [libbasic,
+ libbasic_compress],
dependencies : [threads,
librt],
build_by_default : false)
[files('sd-journal/test-catalog.c')],
- [files('sd-journal/test-compress.c'),
- [],
- [liblz4,
- libzstd,
- libxz]],
-
- [files('sd-journal/test-compress-benchmark.c'),
- [],
- [liblz4,
- libzstd,
- libxz],
- [], '', 'timeout=90'],
-
[files('sd-journal/test-audit-type.c')],
]
_OBJECT_TYPE_MAX
} ObjectType;
-/* Object flags
- * The per-compression enums are also redefined in meson.build so that config.h is self-contained */
+/* Object flags (note that src/basic/compress.h uses the same values for the compression types) */
enum {
OBJECT_COMPRESSED_XZ = 1 << 0,
OBJECT_COMPRESSED_LZ4 = 1 << 1,
OBJECT_COMPRESSED_ZSTD = 1 << 2,
- OBJECT_COMPRESSION_MASK = OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4 | OBJECT_COMPRESSED_ZSTD,
- _OBJECT_COMPRESSED_MAX = OBJECT_COMPRESSION_MASK,
+ _OBJECT_COMPRESSED_MASK = OBJECT_COMPRESSED_XZ | OBJECT_COMPRESSED_LZ4 | OBJECT_COMPRESSED_ZSTD,
+ _OBJECT_COMPRESSED_MAX = _OBJECT_COMPRESSED_MASK,
};
struct ObjectHeader {
p = le64toh(f->data_hash_table[h].head_hash_offset);
while (p > 0) {
+ Compression c;
Object *o;
r = journal_file_move_to_object(f, OBJECT_DATA, p, &o);
if (le64toh(o->data.hash) != hash)
goto next;
- if (o->object.flags & OBJECT_COMPRESSION_MASK) {
+ c = COMPRESSION_FROM_OBJECT(o);
+ if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION
uint64_t l;
size_t rsize = 0;
l -= offsetof(Object, data.payload);
- r = decompress_blob(o->object.flags & OBJECT_COMPRESSION_MASK,
- o->data.payload, l, &f->compress_buffer, &rsize, 0);
+ r = decompress_blob(c, o->data.payload, l, &f->compress_buffer, &rsize, 0);
if (r < 0)
return r;
o->object.flags |= compression;
log_debug("Compressed data object %"PRIu64" -> %zu using %s",
- size, rsize, object_compressed_to_string(compression));
+ size, rsize, compression_to_string(compression));
} else
/* Compression didn't work, we don't really care why, let's continue without compression */
compression = 0;
p = le64toh(READ_NOW(f->header->header_size));
while (p != 0) {
const char *s;
+ Compression c;
r = journal_file_move_to_object(f, OBJECT_UNUSED, p, &o);
if (r < 0)
break;
}
- if (o->object.flags & OBJECT_COMPRESSION_MASK)
+ c = COMPRESSION_FROM_OBJECT(o);
+ if (c > COMPRESSION_NONE)
printf("Flags: %s\n",
- object_compressed_to_string(o->object.flags & OBJECT_COMPRESSION_MASK));
+ compression_to_string(c));
if (p == le64toh(f->header->tail_object_offset))
p = 0;
#endif
};
- if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_ZSTD)
+ if (DEFAULT_COMPRESSION == COMPRESSION_ZSTD)
f->compress_zstd = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
- else if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_LZ4)
+ else if (DEFAULT_COMPRESSION == COMPRESSION_LZ4)
f->compress_lz4 = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
- else if (DEFAULT_COMPRESSION == OBJECT_COMPRESSED_XZ)
+ else if (DEFAULT_COMPRESSION == COMPRESSION_XZ)
f->compress_xz = FLAGS_SET(file_flags, JOURNAL_COMPRESS);
/* We turn on keyed hashes by default, but provide an environment variable to turn them off, if
items = newa(EntryItem, n);
for (uint64_t i = 0; i < n; i++) {
+ Compression c;
uint64_t l, h;
size_t t;
void *data;
if ((uint64_t) t != l)
return -E2BIG;
- if (o->object.flags & OBJECT_COMPRESSION_MASK) {
+ c = COMPRESSION_FROM_OBJECT(o);
+ if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION
size_t rsize = 0;
r = decompress_blob(
- o->object.flags & OBJECT_COMPRESSION_MASK,
+ c,
o->data.payload, l,
&from->compress_buffer, &rsize,
0);
#include "sd-event.h"
#include "sd-id128.h"
+#include "compress.h"
#include "hashmap.h"
#include "journal-def.h"
#include "mmap-cache.h"
bool journal_field_valid(const char *p, size_t l, bool allow_protected);
const char* journal_object_type_to_string(ObjectType type) _const_;
+
+static inline Compression COMPRESSION_FROM_OBJECT(const Object *o) {
+ assert(o);
+
+ /* These flags and enums are defined the same way. Make sure this is really the case. */
+ assert_cc((Compression) 0 == COMPRESSION_NONE);
+ assert_cc((Compression) OBJECT_COMPRESSED_XZ == COMPRESSION_XZ);
+ assert_cc((Compression) OBJECT_COMPRESSED_LZ4 == COMPRESSION_LZ4);
+ assert_cc((Compression) OBJECT_COMPRESSED_ZSTD == COMPRESSION_ZSTD);
+
+ return o->object.flags & _OBJECT_COMPRESSED_MASK;
+}
} while (0)
static int hash_payload(JournalFile *f, Object *o, uint64_t offset, const uint8_t *src, uint64_t size, uint64_t *res_hash) {
- int compression, r;
+ Compression c;
+ int r;
assert(o);
assert(src);
assert(res_hash);
- compression = o->object.flags & OBJECT_COMPRESSION_MASK;
- if (compression) {
+ c = COMPRESSION_FROM_OBJECT(o);
+ if (c != COMPRESSION_NONE) {
_cleanup_free_ void *b = NULL;
size_t b_size;
- r = decompress_blob(compression, src, size, &b, &b_size, 0);
+ r = decompress_blob(c, src, size, &b, &b_size, 0);
if (r < 0) {
error_errno(offset, r, "%s decompression failed: %m",
- object_compressed_to_string(compression));
+ compression_to_string(c));
return r;
}
Object *d;
uint64_t p, l;
size_t t;
- int compression;
+ Compression c;
p = le64toh(o->entry.items[i].object_offset);
r = journal_file_move_to_object(f, OBJECT_DATA, p, &d);
l = le64toh(d->object.size) - offsetof(Object, data.payload);
- compression = d->object.flags & OBJECT_COMPRESSION_MASK;
- if (compression) {
+ c = COMPRESSION_FROM_OBJECT(d);
+ if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION
- r = decompress_startswith(compression,
- d->data.payload, l,
- &f->compress_buffer,
- field, field_length, '=');
+ r = decompress_startswith(
+ c,
+ d->data.payload, l,
+ &f->compress_buffer,
+ field, field_length, '=');
if (r < 0)
log_debug_errno(r, "Cannot decompress %s object of length %"PRIu64" at offset "OFSfmt": %m",
- object_compressed_to_string(compression), l, p);
+ compression_to_string(c), l, p);
else if (r > 0) {
size_t rsize;
- r = decompress_blob(compression,
- d->data.payload, l,
- &f->compress_buffer, &rsize,
- j->data_threshold);
+ r = decompress_blob(
+ c,
+ d->data.payload, l,
+ &f->compress_buffer, &rsize,
+ j->data_threshold);
if (r < 0)
return r;
const void **ret_data,
size_t *ret_size) {
- size_t t;
+ Compression c;
uint64_t l;
- int compression;
+ size_t t;
assert(j);
assert(f);
if ((uint64_t) t != l)
return -E2BIG;
- compression = o->object.flags & OBJECT_COMPRESSION_MASK;
- if (compression) {
+ c = COMPRESSION_FROM_OBJECT(o);
+ if (c != COMPRESSION_NONE) {
#if HAVE_COMPRESSION
size_t rsize;
int r;
r = decompress_blob(
- compression,
+ c,
o->data.payload, l,
&f->compress_buffer, &rsize,
j->data_threshold);
[files('test-recurse-dir.c')],
+ [files('test-compress.c'),
+ [libshared,
+ libbasic_compress]],
+
+ [files('test-compress-benchmark.c'),
+ [libshared,
+ libbasic_compress],
+ [],
+ [], '', 'timeout=90'],
+
[files('test-data-fd-util.c')],
[files('test-static-destruct.c')],
random_bytes(data + 7, sizeof(data) - 7);
#if HAVE_XZ
- test_compress_decompress(OBJECT_COMPRESSED_XZ, "XZ",
+ test_compress_decompress(COMPRESSION_XZ, "XZ",
compress_blob_xz, decompress_blob_xz,
text, sizeof(text), false);
- test_compress_decompress(OBJECT_COMPRESSED_XZ, "XZ",
+ test_compress_decompress(COMPRESSION_XZ, "XZ",
compress_blob_xz, decompress_blob_xz,
data, sizeof(data), true);
compress_blob_xz, decompress_startswith_xz,
huge, HUGE_SIZE, true);
- test_compress_stream(OBJECT_COMPRESSED_XZ, "XZ", "xzcat",
+ test_compress_stream(COMPRESSION_XZ, "XZ", "xzcat",
compress_stream_xz, decompress_stream_xz, srcfile);
test_decompress_startswith_short("XZ", compress_blob_xz, decompress_startswith_xz);
#endif
#if HAVE_LZ4
- test_compress_decompress(OBJECT_COMPRESSED_LZ4, "LZ4",
+ test_compress_decompress(COMPRESSION_LZ4, "LZ4",
compress_blob_lz4, decompress_blob_lz4,
text, sizeof(text), false);
- test_compress_decompress(OBJECT_COMPRESSED_LZ4, "LZ4",
+ test_compress_decompress(COMPRESSION_LZ4, "LZ4",
compress_blob_lz4, decompress_blob_lz4,
data, sizeof(data), true);
compress_blob_lz4, decompress_startswith_lz4,
huge, HUGE_SIZE, true);
- test_compress_stream(OBJECT_COMPRESSED_LZ4, "LZ4", "lz4cat",
+ test_compress_stream(COMPRESSION_LZ4, "LZ4", "lz4cat",
compress_stream_lz4, decompress_stream_lz4, srcfile);
test_lz4_decompress_partial();
#endif
#if HAVE_ZSTD
- test_compress_decompress(OBJECT_COMPRESSED_ZSTD, "ZSTD",
+ test_compress_decompress(COMPRESSION_ZSTD, "ZSTD",
compress_blob_zstd, decompress_blob_zstd,
text, sizeof(text), false);
- test_compress_decompress(OBJECT_COMPRESSED_ZSTD, "ZSTD",
+ test_compress_decompress(COMPRESSION_ZSTD, "ZSTD",
compress_blob_zstd, decompress_blob_zstd,
data, sizeof(data), true);
compress_blob_zstd, decompress_startswith_zstd,
huge, HUGE_SIZE, true);
- test_compress_stream(OBJECT_COMPRESSED_ZSTD, "ZSTD", "zstdcat",
+ test_compress_stream(COMPRESSION_ZSTD, "ZSTD", "zstdcat",
compress_stream_zstd, decompress_stream_zstd, srcfile);
test_decompress_startswith_short("ZSTD", compress_blob_zstd, decompress_startswith_zstd);
test_table(unit_type, UNIT_TYPE);
test_table(virtualization, VIRTUALIZATION);
- test_table_sparse(object_compressed, OBJECT_COMPRESSED);
+ test_table_sparse(compression, COMPRESSION);
assert_cc(sizeof(sd_device_action_t) == sizeof(int64_t));
projects / thirdparty / systemd.git / commitdiff
