src/compr_none.c \
src/compr_zlib.c \
src/compression.c \
+ src/compr_zstd.c \
src/conf.c \
src/confitems.c \
src/counters.c \
src/decompr_none.c \
src/decompr_zlib.c \
+ src/decompr_zstd.c \
src/execute.c \
src/exitfn.c \
src/hash.c \
LIBS="$LIBS -lz"
fi
+AC_ARG_ENABLE(zstd,
+ [AS_HELP_STRING([--enable-zstd],
+ [use zstd compression instead of zlib])])
+if test x${enable_zstd} = xyes; then
+ CPPFLAGS="$CPPFLAGS -DUSE_ZSTD"
+ AC_CHECK_HEADERS(zstd.h)
+ AC_CHECK_LIB(zstd, ZSTD_versionString)
+fi
+
AC_ARG_ENABLE(man,
[AS_HELP_STRING([--disable-man],
[disable installing man pages])])
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ccache.h"
+#include "compression.h"
+
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#ifdef HAVE_LIBZSTD
+struct state
+{
+ FILE *output;
+ ZSTD_CStream *stream;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+ bool failed;
+};
+
+static struct compr_state *
+compr_zstd_init(FILE *output, int level)
+{
+ struct state *state = malloc(sizeof(struct state));
+ state->output = output;
+ state->stream = ZSTD_createCStream();
+ state->failed = false;
+
+ size_t ret = ZSTD_initCStream(state->stream, level);
+ if (ZSTD_isError(ret)) {
+ ZSTD_freeCStream(state->stream);
+ free(state);
+ return NULL;
+ }
+ return (struct compr_state *)state;
+}
+
+static bool
+compr_zstd_write(struct compr_state *handle, const void *data, size_t size)
+{
+ if (!handle) {
+ return false;
+ }
+ struct state *state = (struct state *)handle;
+
+ state->in.src = data;
+ state->in.size = size;
+ state->in.pos = 0;
+
+ int flush = data ? 0 : 1;
+
+ size_t ret;
+ while (state->in.pos < state->in.size) {
+ unsigned char buffer[READ_BUFFER_SIZE];
+ state->out.dst = buffer;
+ state->out.size = sizeof(buffer);
+ state->out.pos = 0;
+ ret = ZSTD_compressStream(state->stream, &state->out, &state->in);
+ assert(!(ZSTD_isError(ret)));
+ size_t compressed_bytes = state->out.pos;
+ if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes
+ || ferror(state->output)) {
+ state->failed = true;
+ return false;
+ }
+ }
+ ret = flush;
+ while (ret) {
+ unsigned char buffer[READ_BUFFER_SIZE];
+ state->out.dst = buffer;
+ state->out.size = sizeof(buffer);
+ state->out.pos = 0;
+ ret = ZSTD_endStream(state->stream, &state->out);
+ size_t compressed_bytes = state->out.pos;
+ if (fwrite(buffer, 1, compressed_bytes, state->output) != compressed_bytes
+ || ferror(state->output)) {
+ state->failed = true;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static bool
+compr_zstd_free(struct compr_state *handle)
+{
+ if (!handle) {
+ return false;
+ }
+ struct state *state = (struct state *)handle;
+
+ compr_zstd_write(handle, NULL, 0);
+ ZSTD_freeCStream(state->stream);
+ bool success = !state->failed;
+ free(state);
+ return success;
+}
+
+struct compressor compressor_zstd_impl = {
+ compr_zstd_init,
+ compr_zstd_write,
+ compr_zstd_free
+};
+#endif //HAVE_LIBZSTD
int8_t compression_level_from_config(void)
{
- return conf->compression ? conf->compression_level : 0;
+ unsigned conf_compression_level;
+#ifdef USE_ZSTD
+ conf_compression_level = conf->compression_level;
+#else
+ conf_compression_level = conf->compression_level;
+#endif
+ return conf->compression ? conf_compression_level : 0;
}
enum compression_type compression_type_from_config(void)
{
- return conf->compression ? COMPR_TYPE_ZLIB : COMPR_TYPE_NONE;
+ enum compression_type conf_compression_type;
+#ifdef USE_ZSTD
+ conf_compression_type = COMPR_TYPE_ZSTD;
+#else
+ conf_compression_type = COMPR_TYPE_ZLIB;
+#endif
+ return conf->compression ? conf_compression_type : COMPR_TYPE_NONE;
}
const char *compression_type_to_string(uint8_t type)
case COMPR_TYPE_ZLIB:
return "zlib";
+
+ case COMPR_TYPE_ZSTD:
+ return "zstd";
}
return "unknown";
case COMPR_TYPE_ZLIB:
return &compressor_zlib_impl;
+
+ case COMPR_TYPE_ZSTD:
+#ifdef HAVE_LIBZSTD
+ return &compressor_zstd_impl;
+#else
+ return NULL;
+#endif
}
return NULL;
case COMPR_TYPE_ZLIB:
return &decompressor_zlib_impl;
+ case COMPR_TYPE_ZSTD:
+#ifdef HAVE_LIBZSTD
+ return &decompressor_zstd_impl;
+#else
+ return NULL;
+#endif
}
return NULL;
enum compression_type {
COMPR_TYPE_NONE = 0,
- COMPR_TYPE_ZLIB = 1
+ COMPR_TYPE_ZLIB = 1,
+ COMPR_TYPE_ZSTD = 2
};
extern struct compressor compressor_none_impl;
extern struct compressor compressor_zlib_impl;
extern struct decompressor decompressor_zlib_impl;
+#ifdef HAVE_LIBZSTD
+extern struct compressor compressor_zstd_impl;
+extern struct decompressor decompressor_zstd_impl;
+#endif //HAVE_LIBZSTD
+
int8_t compression_level_from_config(void);
enum compression_type compression_type_from_config(void);
const char *compression_type_to_string(uint8_t type);
conf->compiler = x_strdup("");
conf->compiler_check = x_strdup("mtime");
conf->compression = false;
+#ifdef USE_ZSTD
+ conf->compression_level = 3;
+#else
conf->compression_level = 6;
+#endif
conf->cpp_extension = x_strdup("");
conf->debug = false;
conf->depend_mode = false;
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "ccache.h"
+#include "compression.h"
+
+#ifdef HAVE_ZSTD_H
+#include <zstd.h>
+#endif
+
+#ifdef HAVE_LIBZSTD
+enum stream_state {
+ STREAM_STATE_READING,
+ STREAM_STATE_FAILED,
+ STREAM_STATE_END
+};
+
+struct state
+{
+ FILE *input;
+ char input_buffer[READ_BUFFER_SIZE];
+ size_t input_size;
+ size_t input_consumed;
+ ZSTD_DStream *stream;
+ ZSTD_inBuffer in;
+ ZSTD_outBuffer out;
+ enum stream_state stream_state;
+};
+
+static struct decompr_state *
+decompr_zstd_init(FILE *input)
+{
+ struct state *state = malloc(sizeof(struct state));
+
+ state->input = input;
+ state->input_size = 0;
+ state->input_consumed = 0;
+ state->stream = ZSTD_createDStream();
+ state->stream_state = STREAM_STATE_READING;
+
+ size_t ret = ZSTD_initDStream(state->stream);
+ if (ZSTD_isError(ret)) {
+ ZSTD_freeDStream(state->stream);
+ free(state);
+ return NULL;
+ }
+ return (struct decompr_state *)state;
+}
+
+static bool
+decompr_zstd_read(struct decompr_state *handle, void *data, size_t size)
+{
+ if (!handle) {
+ return false;
+ }
+ struct state *state = (struct state *)handle;
+
+ size_t bytes_read = 0;
+ while (bytes_read < size) {
+ assert(state->input_size >= state->input_consumed);
+ if (state->input_size == state->input_consumed) {
+ state->input_size = fread(
+ state->input_buffer, 1, sizeof(state->input_buffer), state->input);
+ if (state->input_size == 0) {
+ state->stream_state = STREAM_STATE_FAILED;
+ return false;
+ }
+ state->input_consumed = 0;
+ }
+
+ state->in.src = (state->input_buffer + state->input_consumed);
+ state->in.size = state->input_size - state->input_consumed;
+ state->in.pos = 0;
+
+ state->out.dst = ((char *)data + bytes_read);
+ state->out.size = size - bytes_read;
+ state->out.pos = 0;
+ size_t ret = ZSTD_decompressStream(state->stream, &state->out, &state->in);
+ if (ZSTD_isError(ret)) {
+ state->stream_state = STREAM_STATE_FAILED;
+ return false;
+ } else if (ret == 0) {
+ state->stream_state = STREAM_STATE_END;
+ break;
+ }
+ bytes_read += state->out.pos;
+ state->input_consumed += state->in.pos;
+ }
+
+ return true;
+}
+
+static bool decompr_zstd_free(struct decompr_state *handle)
+{
+ if (!handle) {
+ return false;
+ }
+ struct state *state = (struct state *)handle;
+ ZSTD_freeDStream(state->stream);
+ bool success = state->stream_state == STREAM_STATE_END;
+ free(handle);
+ return success;
+}
+
+struct decompressor decompressor_zstd_impl = {
+ decompr_zstd_init,
+ decompr_zstd_read,
+ decompr_zstd_free
+};
+#endif //HAVE_LIBZSTD
// <body> ::= <n_entries> <entry>* ; body is potentially compressed
// <magic> ::= 4 bytes ("cCrS")
// <version> ::= uint8_t
-// <compr_type> ::= <compr_none> | <compr_zlib>
+// <compr_type> ::= <compr_none> | <compr_zlib> | <compr_zstd>
// <compr_none> ::= 0 (uint8_t)
// <compr_zlib> ::= 1 (uint8_t)
+// <compr_zstd> ::= 2 (uint8_t)
// <compr_level> ::= int8_t
// <content_len> ::= uint64_t ; size of file if stored uncompressed
// <n_entries> ::= uint8_t
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/compression.h"
+#include "framework.h"
+#include "util.h"
+
+TEST_SUITE(compr_zstd)
+
+#ifdef HAVE_LIBZSTD
+TEST(zstd_small_roundtrip)
+{
+ FILE *f = fopen("data.zstd", "w");
+ struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
+ struct compr_state *c_state = compr_zstd->init(f, -1);
+ CHECK(c_state);
+
+ CHECK(compr_zstd->write(c_state, "foobar", 6));
+
+ CHECK(compr_zstd->free(c_state));
+ fclose(f);
+
+ f = fopen("data.zstd", "r");
+ struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
+ struct decompr_state *d_state = decompr_zstd->init(f);
+ CHECK(d_state);
+
+ char buffer[4];
+ CHECK(decompr_zstd->read(d_state, buffer, 4));
+ CHECK(memcmp(buffer, "foob", 4) == 0);
+ CHECK(decompr_zstd->read(d_state, buffer, 2));
+ CHECK(memcmp(buffer, "ar", 2) == 0);
+
+ // Nothing left to read.
+ CHECK(!decompr_zstd->read(d_state, buffer, 1));
+
+ // Error state is remembered.
+ CHECK(!decompr_zstd->free(d_state));
+ fclose(f);
+}
+
+TEST(zstd_large_compressible_roundtrip)
+{
+ char data[] = "The quick brown fox jumps over the lazy dog";
+
+ FILE *f = fopen("data.zstd", "w");
+ struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
+ struct compr_state *c_state = compr_zstd->init(f, 1);
+ CHECK(c_state);
+
+ for (size_t i = 0; i < 1000; i++) {
+ CHECK(compr_zstd->write(c_state, data, sizeof(data)));
+ }
+
+ CHECK(compr_zstd->free(c_state));
+ fclose(f);
+
+ f = fopen("data.zstd", "r");
+ struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
+ struct decompr_state *d_state = decompr_zstd->init(f);
+ CHECK(d_state);
+
+ char buffer[sizeof(data)];
+ for (size_t i = 0; i < 1000; i++) {
+ CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer)));
+ CHECK(memcmp(buffer, data, sizeof(data)) == 0);
+ }
+
+ // Nothing left to read.
+ CHECK(!decompr_zstd->read(d_state, buffer, 1));
+
+ // Error state is remembered.
+ CHECK(!decompr_zstd->free(d_state));
+ fclose(f);
+}
+
+TEST(zstd_large_uncompressible_roundtrip)
+{
+ char data[100000];
+ for (size_t i = 0; i < sizeof(data); i++) {
+ data[i] = rand() % 256;
+ }
+
+ FILE *f = fopen("data.zstd", "w");
+ struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
+ struct compr_state *c_state = compr_zstd->init(f, 1);
+ CHECK(c_state);
+
+ CHECK(compr_zstd->write(c_state, data, sizeof(data)));
+
+ CHECK(compr_zstd->free(c_state));
+ fclose(f);
+
+ f = fopen("data.zstd", "r");
+ struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
+ struct decompr_state *d_state = decompr_zstd->init(f);
+ CHECK(d_state);
+
+ char buffer[sizeof(data)];
+ CHECK(decompr_zstd->read(d_state, buffer, sizeof(buffer)));
+ CHECK(memcmp(buffer, data, sizeof(data)) == 0);
+
+ CHECK(decompr_zstd->free(d_state));
+ fclose(f);
+}
+#else
+TEST(zstd_skip)
+{
+ // disabled
+}
+#endif // HAVE_LIBZSTD
+
+TEST_SUITE_END
CHECK_STR_EQ("", conf->compiler);
CHECK_STR_EQ("mtime", conf->compiler_check);
CHECK(!conf->compression);
+#ifdef USE_ZSTD
+ CHECK_INT_EQ(3, conf->compression_level);
+#else
CHECK_INT_EQ(6, conf->compression_level);
+#endif
CHECK_STR_EQ("", conf->cpp_extension);
CHECK(!conf->debug);
CHECK(!conf->depend_mode);
projects / thirdparty / ccache.git / commitdiff
