From d4a5c2040932e85731ffc1c762e7df824cf6a4ed Mon Sep 17 00:00:00 2001 From: =?utf8?q?Anders=20Bj=C3=B6rklund?= Date: Wed, 26 Jun 2019 21:09:49 +0200 Subject: [PATCH] Allow using zstd compresssion instead of zlib (#437) --- Makefile.in | 2 + configure.ac | 9 +++ src/compr_zstd.c | 118 ++++++++++++++++++++++++++++++++++ src/compression.c | 32 +++++++++- src/compression.h | 8 ++- src/conf.c | 4 ++ src/decompr_zstd.c | 123 ++++++++++++++++++++++++++++++++++++ src/result.c | 3 +- unittest/test_compr_zstd.c | 126 +++++++++++++++++++++++++++++++++++++ unittest/test_conf.c | 4 ++ 10 files changed, 425 insertions(+), 4 deletions(-) create mode 100644 src/compr_zstd.c create mode 100644 src/decompr_zstd.c create mode 100644 unittest/test_compr_zstd.c diff --git a/Makefile.in b/Makefile.in index 0f199c4bd..a01d675cc 100644 --- a/Makefile.in +++ b/Makefile.in @@ -37,11 +37,13 @@ non_3pp_sources = \ src/compr_none.c \ src/compr_zlib.c \ src/compression.c \ + src/compr_zstd.c \ src/conf.c \ src/confitems.c \ src/counters.c \ src/decompr_none.c \ src/decompr_zlib.c \ + src/decompr_zstd.c \ src/execute.c \ src/exitfn.c \ src/hash.c \ diff --git a/configure.ac b/configure.ac index bb0f2be9e..42df7a9aa 100644 --- a/configure.ac +++ b/configure.ac @@ -163,6 +163,15 @@ else LIBS="$LIBS -lz" fi +AC_ARG_ENABLE(zstd, + [AS_HELP_STRING([--enable-zstd], + [use zstd compression instead of zlib])]) +if test x${enable_zstd} = xyes; then + CPPFLAGS="$CPPFLAGS -DUSE_ZSTD" + AC_CHECK_HEADERS(zstd.h) + AC_CHECK_LIB(zstd, ZSTD_versionString) +fi + AC_ARG_ENABLE(man, [AS_HELP_STRING([--disable-man], [disable installing man pages])]) diff --git a/src/compr_zstd.c b/src/compr_zstd.c new file mode 100644 index 000000000..49840101f --- /dev/null +++ b/src/compr_zstd.c @@ -0,0 +1,118 @@ +// Copyright (C) 2019 Joel Rosdahl +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ccache.h" +#include "compression.h" + +#ifdef HAVE_ZSTD_H +#include +#endif + +#ifdef HAVE_LIBZSTD +struct state +{ + FILE *output; + ZSTD_CStream *stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + bool failed; +}; + +static struct compr_state * +compr_zstd_init(FILE *output, int level) +{ + struct state *state = malloc(sizeof(struct state)); + state->output = output; + state->stream = ZSTD_createCStream(); + state->failed = false; + + size_t ret = ZSTD_initCStream(state->stream, level); + if (ZSTD_isError(ret)) { + ZSTD_freeCStream(state->stream); + free(state); + return NULL; + } + return (struct compr_state *)state; +} + +static bool +compr_zstd_write(struct compr_state *handle, const void *data, size_t size) +{ + if (!handle) { + return false; + } + struct state *state = (struct state *)handle; + + state->in.src = data; + state->in.size = size; + state->in.pos = 0; + + int flush = data ? 0 : 1; + + size_t ret; + while (state->in.pos < state->in.size) { + unsigned char buffer[READ_BUFFER_SIZE]; + state->out.dst = buffer; + state->out.size = sizeof(buffer); + state->out.pos = 0; + ret = ZSTD_compressStream(state->stream, &state->out, &state->in); + assert(!(ZSTD_isError(ret))); + size_t compressed_bytes = state->out.pos; + if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes + || ferror(state->output)) { + state->failed = true; + return false; + } + } + ret = flush; + while (ret) { + unsigned char buffer[READ_BUFFER_SIZE]; + state->out.dst = buffer; + state->out.size = sizeof(buffer); + state->out.pos = 0; + ret = ZSTD_endStream(state->stream, &state->out); + size_t compressed_bytes = state->out.pos; + if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes + || ferror(state->output)) { + state->failed = true; + return false; + } + } + + return true; +} + +static bool +compr_zstd_free(struct compr_state *handle) +{ + if (!handle) { + return false; + } + struct state *state = (struct state *)handle; + + compr_zstd_write(handle, NULL, 0); + ZSTD_freeCStream(state->stream); + bool success = !state->failed; + free(state); + return success; +} + +struct compressor compressor_zstd_impl = { + compr_zstd_init, + compr_zstd_write, + compr_zstd_free +}; +#endif //HAVE_LIBZSTD diff --git a/src/compression.c b/src/compression.c index 355a0a751..ec0687376 100644 --- a/src/compression.c +++ b/src/compression.c @@ -21,12 +21,24 @@ extern struct conf *conf; int8_t compression_level_from_config(void) { - return conf->compression ? conf->compression_level : 0; + unsigned conf_compression_level; +#ifdef USE_ZSTD + conf_compression_level = conf->compression_level; +#else + conf_compression_level = conf->compression_level; +#endif + return conf->compression ? conf_compression_level : 0; } enum compression_type compression_type_from_config(void) { - return conf->compression ? COMPR_TYPE_ZLIB : COMPR_TYPE_NONE; + enum compression_type conf_compression_type; +#ifdef USE_ZSTD + conf_compression_type = COMPR_TYPE_ZSTD; +#else + conf_compression_type = COMPR_TYPE_ZLIB; +#endif + return conf->compression ? conf_compression_type : COMPR_TYPE_NONE; } const char *compression_type_to_string(uint8_t type) @@ -37,6 +49,9 @@ const char *compression_type_to_string(uint8_t type) case COMPR_TYPE_ZLIB: return "zlib"; + + case COMPR_TYPE_ZSTD: + return "zstd"; } return "unknown"; @@ -50,6 +65,13 @@ struct compressor *compressor_from_type(uint8_t type) case COMPR_TYPE_ZLIB: return &compressor_zlib_impl; + + case COMPR_TYPE_ZSTD: +#ifdef HAVE_LIBZSTD + return &compressor_zstd_impl; +#else + return NULL; +#endif } return NULL; @@ -64,6 +86,12 @@ struct decompressor *decompressor_from_type(uint8_t type) case COMPR_TYPE_ZLIB: return &decompressor_zlib_impl; + case COMPR_TYPE_ZSTD: +#ifdef HAVE_LIBZSTD + return &decompressor_zstd_impl; +#else + return NULL; +#endif } return NULL; diff --git a/src/compression.h b/src/compression.h index 505c47a92..b087ea81c 100644 --- a/src/compression.h +++ b/src/compression.h @@ -21,7 +21,8 @@ struct decompressor { enum compression_type { COMPR_TYPE_NONE = 0, - COMPR_TYPE_ZLIB = 1 + COMPR_TYPE_ZLIB = 1, + COMPR_TYPE_ZSTD = 2 }; extern struct compressor compressor_none_impl; @@ -30,6 +31,11 @@ extern struct decompressor decompressor_none_impl; extern struct compressor compressor_zlib_impl; extern struct decompressor decompressor_zlib_impl; +#ifdef HAVE_LIBZSTD +extern struct compressor compressor_zstd_impl; +extern struct decompressor decompressor_zstd_impl; +#endif //HAVE_LIBZSTD + int8_t compression_level_from_config(void); enum compression_type compression_type_from_config(void); const char *compression_type_to_string(uint8_t type); diff --git a/src/conf.c b/src/conf.c index 64bccf5cb..bcdfd3c28 100644 --- a/src/conf.c +++ b/src/conf.c @@ -133,7 +133,11 @@ conf_create(void) conf->compiler = x_strdup(""); conf->compiler_check = x_strdup("mtime"); conf->compression = false; +#ifdef USE_ZSTD + conf->compression_level = 3; +#else conf->compression_level = 6; +#endif conf->cpp_extension = x_strdup(""); conf->debug = false; conf->depend_mode = false; diff --git a/src/decompr_zstd.c b/src/decompr_zstd.c new file mode 100644 index 000000000..8fc3ace4f --- /dev/null +++ b/src/decompr_zstd.c @@ -0,0 +1,123 @@ +// Copyright (C) 2019 Joel Rosdahl +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "ccache.h" +#include "compression.h" + +#ifdef HAVE_ZSTD_H +#include +#endif + +#ifdef HAVE_LIBZSTD +enum stream_state { + STREAM_STATE_READING, + STREAM_STATE_FAILED, + STREAM_STATE_END +}; + +struct state +{ + FILE *input; + char input_buffer[READ_BUFFER_SIZE]; + size_t input_size; + size_t input_consumed; + ZSTD_DStream *stream; + ZSTD_inBuffer in; + ZSTD_outBuffer out; + enum stream_state stream_state; +}; + +static struct decompr_state * +decompr_zstd_init(FILE *input) +{ + struct state *state = malloc(sizeof(struct state)); + + state->input = input; + state->input_size = 0; + state->input_consumed = 0; + state->stream = ZSTD_createDStream(); + state->stream_state = STREAM_STATE_READING; + + size_t ret = ZSTD_initDStream(state->stream); + if (ZSTD_isError(ret)) { + ZSTD_freeDStream(state->stream); + free(state); + return NULL; + } + return (struct decompr_state *)state; +} + +static bool +decompr_zstd_read(struct decompr_state *handle, void *data, size_t size) +{ + if (!handle) { + return false; + } + struct state *state = (struct state *)handle; + + size_t bytes_read = 0; + while (bytes_read < size) { + assert(state->input_size >= state->input_consumed); + if (state->input_size == state->input_consumed) { + state->input_size = fread( + state->input_buffer, 1, sizeof(state->input_buffer), state->input); + if (state->input_size == 0) { + state->stream_state = STREAM_STATE_FAILED; + return false; + } + state->input_consumed = 0; + } + + state->in.src = (state->input_buffer + state->input_consumed); + state->in.size = state->input_size - state->input_consumed; + state->in.pos = 0; + + state->out.dst = ((char *)data + bytes_read); + state->out.size = size - bytes_read; + state->out.pos = 0; + size_t ret = ZSTD_decompressStream(state->stream, &state->out, &state->in); + if (ZSTD_isError(ret)) { + state->stream_state = STREAM_STATE_FAILED; + return false; + } else if (ret == 0) { + state->stream_state = STREAM_STATE_END; + break; + } + bytes_read += state->out.pos; + state->input_consumed += state->in.pos; + } + + return true; +} + +static bool decompr_zstd_free(struct decompr_state *handle) +{ + if (!handle) { + return false; + } + struct state *state = (struct state *)handle; + ZSTD_freeDStream(state->stream); + bool success = state->stream_state == STREAM_STATE_END; + free(handle); + return success; +} + +struct decompressor decompressor_zstd_impl = { + decompr_zstd_init, + decompr_zstd_read, + decompr_zstd_free +}; +#endif //HAVE_LIBZSTD diff --git a/src/result.c b/src/result.c index 607794e82..c544c5fdd 100644 --- a/src/result.c +++ b/src/result.c @@ -25,9 +25,10 @@ // ::= * ; body is potentially compressed // ::= 4 bytes ("cCrS") // ::= uint8_t -// ::= | +// ::= | | // ::= 0 (uint8_t) // ::= 1 (uint8_t) +// ::= 2 (uint8_t) // ::= int8_t // ::= uint64_t ; size of file if stored uncompressed // ::= uint8_t diff --git a/unittest/test_compr_zstd.c b/unittest/test_compr_zstd.c new file mode 100644 index 000000000..16e4ff927 --- /dev/null +++ b/unittest/test_compr_zstd.c @@ -0,0 +1,126 @@ +// Copyright (C) 2019 Joel Rosdahl +// +// This program is free software; you can redistribute it and/or modify it +// under the terms of the GNU General Public License as published by the Free +// Software Foundation; either version 3 of the License, or (at your option) +// any later version. +// +// This program is distributed in the hope that it will be useful, but WITHOUT +// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +// more details. +// +// You should have received a copy of the GNU General Public License along with +// this program; if not, write to the Free Software Foundation, Inc., 51 +// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +#include "../src/compression.h" +#include "framework.h" +#include "util.h" + +TEST_SUITE(compr_zstd) + +#ifdef HAVE_LIBZSTD +TEST(zstd_small_roundtrip) +{ + FILE *f = fopen("data.zstd", "w"); + struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); + struct compr_state *c_state = compr_zstd->init(f, -1); + CHECK(c_state); + + CHECK(compr_zstd->write(c_state, "foobar", 6)); + + CHECK(compr_zstd->free(c_state)); + fclose(f); + + f = fopen("data.zstd", "r"); + struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); + struct decompr_state *d_state = decompr_zstd->init(f); + CHECK(d_state); + + char buffer[4]; + CHECK(decompr_zstd->read(d_state, buffer, 4)); + CHECK(memcmp(buffer, "foob", 4) == 0); + CHECK(decompr_zstd->read(d_state, buffer, 2)); + CHECK(memcmp(buffer, "ar", 2) == 0); + + // Nothing left to read. + CHECK(!decompr_zstd->read(d_state, buffer, 1)); + + // Error state is remembered. + CHECK(!decompr_zstd->free(d_state)); + fclose(f); +} + +TEST(zstd_large_compressible_roundtrip) +{ + char data[] = "The quick brown fox jumps over the lazy dog"; + + FILE *f = fopen("data.zstd", "w"); + struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); + struct compr_state *c_state = compr_zstd->init(f, 1); + CHECK(c_state); + + for (size_t i = 0; i < 1000; i++) { + CHECK(compr_zstd->write(c_state, data, sizeof(data))); + } + + CHECK(compr_zstd->free(c_state)); + fclose(f); + + f = fopen("data.zstd", "r"); + struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); + struct decompr_state *d_state = decompr_zstd->init(f); + CHECK(d_state); + + char buffer[sizeof(data)]; + for (size_t i = 0; i < 1000; i++) { + CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer))); + CHECK(memcmp(buffer, data, sizeof(data)) == 0); + } + + // Nothing left to read. + CHECK(!decompr_zstd->read(d_state, buffer, 1)); + + // Error state is remembered. + CHECK(!decompr_zstd->free(d_state)); + fclose(f); +} + +TEST(zstd_large_uncompressible_roundtrip) +{ + char data[100000]; + for (size_t i = 0; i < sizeof(data); i++) { + data[i] = rand() % 256; + } + + FILE *f = fopen("data.zstd", "w"); + struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD); + struct compr_state *c_state = compr_zstd->init(f, 1); + CHECK(c_state); + + CHECK(compr_zstd->write(c_state, data, sizeof(data))); + + CHECK(compr_zstd->free(c_state)); + fclose(f); + + f = fopen("data.zstd", "r"); + struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD); + struct decompr_state *d_state = decompr_zstd->init(f); + CHECK(d_state); + + char buffer[sizeof(data)]; + CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer))); + CHECK(memcmp(buffer, data, sizeof(data)) == 0); + + CHECK(decompr_zstd->free(d_state)); + fclose(f); +} +#else +TEST(zstd_skip) +{ + // disabled +} +#endif // HAVE_LIBZSTD + +TEST_SUITE_END diff --git a/unittest/test_conf.c b/unittest/test_conf.c index e7b0f753c..278507342 100644 --- a/unittest/test_conf.c +++ b/unittest/test_conf.c @@ -56,7 +56,11 @@ TEST(conf_create) CHECK_STR_EQ("", conf->compiler); CHECK_STR_EQ("mtime", conf->compiler_check); CHECK(!conf->compression); +#ifdef USE_ZSTD + CHECK_INT_EQ(3, conf->compression_level); +#else CHECK_INT_EQ(6, conf->compression_level); +#endif CHECK_STR_EQ("", conf->cpp_extension); CHECK(!conf->debug); CHECK(!conf->depend_mode); -- 2.47.2