Both compressed and uncompressed content are checksummed and verified.
The chosen checksum algorithm is XXH64, which is the same that the zstd
frame format uses (but ccache stores all 64 bits instead of only 32,
because why not?).
* Easy installation.
* Low overhead.
* Compresses data in the cache to save disk space.
+* Checksums data in the cache to detect corruption.
Limitations
#include "compression.h"
+struct state {
+ FILE *output;
+ XXH64_state_t *checksum;
+};
+
static struct compr_state *
-compr_none_init(FILE *output, int8_t level)
+compr_none_init(FILE *output, int8_t level, XXH64_state_t *checksum)
{
+ struct state *state = malloc(sizeof(struct state));
+ state->output = output;
+ state->checksum = checksum;
(void)level;
- return (struct compr_state *)output;
+ return (struct compr_state *)state;
}
static int8_t
static bool
compr_none_write(struct compr_state *handle, const void *data, size_t size)
{
- FILE *output = (FILE *)handle;
- return fwrite(data, 1, size, output) == size;
+ struct state *state = (struct state *)handle;
+ size_t ret = fwrite(data, size, 1, state->output);
+ if (state->checksum) {
+ XXH64_update(state->checksum, data, size);
+ }
+ return ret == 1;
}
static bool
compr_none_free(struct compr_state *handle)
{
- FILE *output = (FILE *)handle;
- return ferror(output) == 0;
+ struct state *state = (struct state *)handle;
+ bool result = ferror(state->output) == 0;
+ free(state);
+ return result;
}
struct compressor compressor_none_impl = {
struct state {
FILE *output;
+ XXH64_state_t *checksum;
ZSTD_CStream *stream;
ZSTD_inBuffer in;
ZSTD_outBuffer out;
};
static struct compr_state *
-compr_zstd_init(FILE *output, int8_t level)
+compr_zstd_init(FILE *output, int8_t level, XXH64_state_t *checksum)
{
struct state *state = malloc(sizeof(struct state));
state->output = output;
+ state->checksum = checksum;
state->stream = ZSTD_createCStream();
state->failed = false;
}
struct state *state = (struct state *)handle;
+ if (state->checksum) {
+ XXH64_update(state->checksum, data, size);
+ }
+
state->in.src = data;
state->in.size = size;
state->in.pos = 0;
if (!handle) {
return false;
}
+
struct state *state = (struct state *)handle;
compr_zstd_write(handle, NULL, 0);
#define COMPRESSION_H
#include "system.h"
+#include "xxhash.h"
struct compr_state;
//
// output: The file to write compressed data to.
// compression_level: Desired compression level.
- struct compr_state *(*init)(FILE *output, int8_t compression_level);
+ // checksum: Checksum state to update (NULL for no checksum).
+ struct compr_state *(*init)(
+ FILE *output,
+ int8_t compression_level,
+ XXH64_state_t *checksum);
// Get the actual compression level that will be used.
int8_t (*get_actual_compression_level)(struct compr_state *state);
// Create and initialize a decompressor.
//
// input: The file to read compressed data from.
- struct decompr_state *(*init)(FILE *input);
+ // checksum: Checksum state to update (NULL for no checksum).
+ struct decompr_state *(*init)(FILE *input, XXH64_state_t *checksum);
// Decompress data.
//
#include "compression.h"
+struct state {
+ FILE *input;
+ XXH64_state_t *checksum;
+ bool failed;
+};
+
static struct decompr_state *
-decompr_none_init(FILE *input)
+decompr_none_init(FILE *input, XXH64_state_t *checksum)
{
- return (struct decompr_state *)input;
+ struct state *state = malloc(sizeof(struct state));
+ state->input = input;
+ state->checksum = checksum;
+ state->failed = false;
+ return (struct decompr_state *)state;
}
static bool
decompr_none_read(struct decompr_state *handle, void *data, size_t size)
{
- FILE *input = (FILE *)handle;
- return fread(data, 1, size, input) == size;
+ struct state *state = (struct state *)handle;
+
+ bool result = fread(data, 1, size, state->input) == size;
+ if (result && state->checksum) {
+ XXH64_update(state->checksum, data, size);
+ }
+ if (!result) {
+ state->failed = true;
+ }
+ return result;
}
static bool
decompr_none_free(struct decompr_state *handle)
{
- FILE *input = (FILE *)handle;
- return ferror(input) == 0;
+ struct state *state = (struct state *)handle;
+ bool result = !state->failed;
+ free(state);
+ return result;
}
struct decompressor decompressor_none_impl = {
struct state {
FILE *input;
+ XXH64_state_t *checksum;
char input_buffer[READ_BUFFER_SIZE];
size_t input_size;
size_t input_consumed;
};
static struct decompr_state *
-decompr_zstd_init(FILE *input)
+decompr_zstd_init(FILE *input, XXH64_state_t *checksum)
{
struct state *state = malloc(sizeof(struct state));
state->input = input;
+ state->checksum = checksum;
state->input_size = 0;
state->input_consumed = 0;
state->stream = ZSTD_createDStream();
if (ZSTD_isError(ret)) {
state->stream_state = STREAM_STATE_FAILED;
return false;
- } else if (ret == 0) {
+ }
+ if (state->checksum) {
+ XXH64_update(state->checksum, state->out.dst, state->out.pos);
+ }
+ if (ret == 0) {
state->stream_state = STREAM_STATE_END;
break;
}
#include "manifest.h"
#include "xxhash.h"
-// Manifest data format (big-endian integers):
+// Manifest data format
+// ====================
//
-// <manifest> ::= <header> <body>
+// Integers are big-endian.
+//
+// <manifest> ::= <header> <body> <epilogue
// <header> ::= <magic> <version> <compr_type> <compr_level>
// <content_len>
// <magic> ::= 4 bytes ("cCrS")
// <n_indexes> ::= uint32_t
// <include_index> ::= uint32_t
// <name> ::= DIGEST_SIZE bytes
+// <epilogue> ::= <checksum>
+// <checksum> ::= uint64_t ; XXH64 of content bytes
//
// Sketch of concrete layout:
// ...
// <name> DIGEST_SIZE bytes
// ...
+// checksum 8 bytes
+//
//
-// Version history:
+// Version history
+// ===============
//
// 1: Introduced in ccache 3.0. (Files are always compressed with gzip.)
// 2: Introduced in ccache 3.8.
struct decompressor *decompressor = NULL;
struct decompr_state *decompr_state = NULL;
*errmsg = NULL;
+ XXH64_state_t *checksum = NULL;
FILE *f = fopen(path, "rb");
if (!f) {
goto out;
}
- decompr_state = decompressor->init(f);
+ checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+ XXH64_update(checksum, header_bytes, sizeof(header_bytes));
+
+ decompr_state = decompressor->init(f, checksum);
if (!decompr_state) {
*errmsg = x_strdup("Failed to initialize decompressor");
goto out;
READ_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
}
- success = true;
+ uint64_t actual_checksum = XXH64_digest(checksum);
+ uint64_t expected_checksum;
+ READ_UINT64(expected_checksum);
+ if (actual_checksum == expected_checksum) {
+ success = true;
+ } else {
+ *errmsg = format(
+ "Incorrect checksum (actual %016llx, expected %016llx)",
+ (unsigned long long)actual_checksum,
+ (unsigned long long)expected_checksum);
+ }
out:
if (decompressor && !decompressor->free(decompr_state)) {
if (f) {
fclose(f);
}
+ if (checksum) {
+ XXH64_freeState(checksum);
+ }
if (!success) {
if (!*errmsg) {
*errmsg = x_strdup("Corrupt manifest file");
#define WRITE_BYTES(buf, length) \
do { \
if (!compressor->write(compr_state, buf, length)) { \
- goto error; \
+ goto out; \
} \
} while (false)
static bool
write_manifest(FILE *f, const struct manifest *mf)
{
+ int ret = false;
+ XXH64_state_t *checksum = NULL;
+
uint64_t content_size = COMMON_HEADER_SIZE;
content_size += 4; // n_files
for (size_t i = 0; i < mf->n_files; i++) {
content_size += mf->results[i].n_file_info_indexes * 4;
content_size += DIGEST_SIZE;
}
+ content_size += 8; // checksum
struct common_header header;
common_header_from_config(&header, MAGIC, MANIFEST_VERSION, content_size);
+ checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+
struct compressor *compressor =
compressor_from_type(header.compression_type);
assert(compressor);
struct compr_state *compr_state =
- compressor->init(f, header.compression_level);
+ compressor->init(f, header.compression_level, checksum);
if (!compr_state) {
cc_log("Failed to initialize compressor");
- goto error;
+ goto out;
}
header.compression_level =
compressor->get_actual_compression_level(compr_state);
uint8_t header_bytes[COMMON_HEADER_SIZE];
common_header_to_bytes(&header, header_bytes);
if (fwrite(header_bytes, sizeof(header_bytes), 1, f) != 1) {
- goto error;
+ goto out;
}
+ XXH64_update(checksum, header_bytes, sizeof(header_bytes));
WRITE_UINT32(mf->n_files);
for (uint32_t i = 0; i < mf->n_files; i++) {
WRITE_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
}
- return compressor->free(compr_state);
+ WRITE_UINT64(XXH64_digest(checksum));
-error:
- cc_log("Error writing to manifest file");
- return false;
+ ret = compressor->free(compr_state);
+
+out:
+ XXH64_freeState(checksum);
+ if (!ret) {
+ cc_log("Error writing to manifest file");
+ }
+ return ret;
}
static bool
#include "common_header.h"
#include "int_bytes_conversion.h"
#include "compression.h"
+#include "xxhash.h"
#include "result.h"
-// Result data format (big-endian integers):
+// Result data format
+// ==================
//
-// <result> ::= <header> <body>
+// Integers are big-endian.
+//
+// <result> ::= <header> <body> <epilogue>
// <header> ::= <magic> <version> <compr_type> <compr_level> <content_len>
// <magic> ::= 4 bytes ("cCrS")
// <version> ::= uint8_t
// <ref_marker> ::= 1 (uint8_t)
// <key_len> ::= uint8_t
// <key> ::= key_len bytes
+// <epilogue> ::= <checksum>
+// <checksum> ::= uint64_t ; XXH64 of content bytes
//
// Sketch of concrete layout:
//
// <key_len> 1 byte
// <key> key_len bytes
// ...
+// checksum 8 bytes
+//
//
-// Version history:
+// Version history
+// ===============
//
// 1: Introduced in ccache 3.8.
struct decompressor *decompressor = NULL;
struct decompr_state *decompr_state = NULL;
FILE *subfile = NULL;
+ XXH64_state_t *checksum = NULL;
FILE *f = fopen(path, "rb");
if (!f) {
goto out;
}
+ checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+ XXH64_update(checksum, header_bytes, sizeof(header_bytes));
+
struct common_header header;
common_header_from_bytes(&header, header_bytes);
goto out;
}
- decompr_state = decompressor->init(f);
+ decompr_state = decompressor->init(f, checksum);
if (!decompr_state) {
*errmsg = x_strdup("Failed to initialize decompressor");
goto out;
}
}
- if (i == n_entries) {
+ if (i != n_entries) {
+ *errmsg = format("Too few entries (read %u, expected %u)", i, n_entries);
+ goto out;
+ }
+
+ uint64_t actual_checksum = XXH64_digest(checksum);
+ uint8_t expected_checksum_bytes[8];
+ READ_BYTES(expected_checksum_bytes, sizeof(expected_checksum_bytes));
+ uint64_t expected_checksum = UINT64_FROM_BYTES(expected_checksum_bytes);
+ if (actual_checksum == expected_checksum) {
success = true;
} else {
- *errmsg = format("Too few entries (read %u, expected %u)", i, n_entries);
+ *errmsg = format(
+ "Incorrect checksum (actual %016llx, expected %016llx)",
+ (unsigned long long)actual_checksum,
+ (unsigned long long)expected_checksum);
}
out:
if (f) {
fclose(f);
}
+ if (checksum) {
+ XXH64_freeState(checksum);
+ }
if (!success && !*errmsg) {
*errmsg = x_strdup("Corrupt result file");
}
write_result(
const struct result_files *list,
struct compressor *compressor,
- struct compr_state *compr_state)
+ struct compr_state *compr_state,
+ XXH64_state_t *checksum)
{
WRITE_BYTE(list->n_files);
fclose(f);
}
+ WRITE_INT(8, XXH64_digest(checksum));
+
return true;
error:
bool result_put(const char *path, struct result_files *list)
{
bool ret = false;
+ XXH64_state_t *checksum = NULL;
+
char *tmp_file = format("%s.tmp", path);
int fd = create_tmp_fd(&tmp_file);
FILE *f = fdopen(fd, "wb");
content_size += 8; // data_len
content_size += list->files[i].size; // data
}
+ content_size += 8; // checksum
struct common_header header;
common_header_from_config(&header, MAGIC, RESULT_VERSION, content_size);
+ checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+
struct compressor *compressor =
compressor_from_type(header.compression_type);
assert(compressor);
struct compr_state *compr_state =
- compressor->init(f, header.compression_level);
+ compressor->init(f, header.compression_level, checksum);
if (!compr_state) {
cc_log("Failed to initialize compressor");
goto out;
cc_log("Failed to write result file header to %s", tmp_file);
goto out;
}
+ XXH64_update(checksum, header_bytes, sizeof(header_bytes));
- bool ok = write_result(list, compressor, compr_state)
+ bool ok = write_result(list, compressor, compr_state, checksum)
&& compressor->free(compr_state);
if (!ok) {
cc_log("Failed to write result file");
if (f) {
fclose(f);
}
+ if (checksum) {
+ XXH64_freeState(checksum);
+ }
return ret;
}
--- /dev/null
+// Copyright (C) 2019 Joel Rosdahl
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/compression.h"
+#include "framework.h"
+#include "util.h"
+
+TEST_SUITE(compr_type_none)
+
+TEST(small_roundtrip)
+{
+ const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
+
+ XXH64_state_t *checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+
+ FILE *f = fopen("data.uncompressed", "w");
+ struct compressor *compr_none = compressor_from_type(COMPR_TYPE_NONE);
+ struct compr_state *c_state = compr_none->init(f, -1, checksum);
+ CHECK(c_state);
+
+ CHECK(compr_none->write(c_state, "foobar", 6));
+
+ CHECK(compr_none->free(c_state));
+ fclose(f);
+
+ CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+
+ XXH64_reset(checksum, 0);
+ f = fopen("data.uncompressed", "r");
+ struct decompressor *decompr_none = decompressor_from_type(COMPR_TYPE_NONE);
+ struct decompr_state *d_state = decompr_none->init(f, checksum);
+ CHECK(d_state);
+
+ char buffer[4];
+ CHECK(decompr_none->read(d_state, buffer, 4));
+ CHECK(memcmp(buffer, "foob", 4) == 0);
+ CHECK(decompr_none->read(d_state, buffer, 2));
+ CHECK(memcmp(buffer, "ar", 2) == 0);
+
+ // Nothing left to read.
+ CHECK(!decompr_none->read(d_state, buffer, 1));
+
+ // Error state is remembered.
+ CHECK(!decompr_none->free(d_state));
+ fclose(f);
+
+ CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+
+ XXH64_freeState(checksum);
+}
+
+TEST_SUITE_END
#include "framework.h"
#include "util.h"
-TEST_SUITE(compr_zstd)
+TEST_SUITE(compr_type_zstd)
-TEST(zstd_small_roundtrip)
+TEST(small_roundtrip)
{
+ const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
+
+ XXH64_state_t *checksum = XXH64_createState();
+ XXH64_reset(checksum, 0);
+
FILE *f = fopen("data.zstd", "w");
struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state *c_state = compr_zstd->init(f, -1);
+ struct compr_state *c_state = compr_zstd->init(f, -1, checksum);
CHECK(c_state);
CHECK(compr_zstd->write(c_state, "foobar", 6));
CHECK(compr_zstd->free(c_state));
fclose(f);
+ CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+
+ XXH64_reset(checksum, 0);
f = fopen("data.zstd", "r");
struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state *d_state = decompr_zstd->init(f);
+ struct decompr_state *d_state = decompr_zstd->init(f, checksum);
CHECK(d_state);
char buffer[4];
// Error state is remembered.
CHECK(!decompr_zstd->free(d_state));
fclose(f);
+
+ CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+
+ XXH64_freeState(checksum);
}
-TEST(zstd_large_compressible_roundtrip)
+TEST(large_compressible_roundtrip)
{
char data[] = "The quick brown fox jumps over the lazy dog";
FILE *f = fopen("data.zstd", "w");
struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state *c_state = compr_zstd->init(f, 1);
+ struct compr_state *c_state = compr_zstd->init(f, 1, NULL);
CHECK(c_state);
for (size_t i = 0; i < 1000; i++) {
f = fopen("data.zstd", "r");
struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state *d_state = decompr_zstd->init(f);
+ struct decompr_state *d_state = decompr_zstd->init(f, NULL);
CHECK(d_state);
char buffer[sizeof(data)];
fclose(f);
}
-TEST(zstd_large_uncompressible_roundtrip)
+TEST(large_uncompressible_roundtrip)
{
char data[100000];
for (size_t i = 0; i < sizeof(data); i++) {
FILE *f = fopen("data.zstd", "w");
struct compressor *compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
- struct compr_state *c_state = compr_zstd->init(f, 1);
+ struct compr_state *c_state = compr_zstd->init(f, 1, NULL);
CHECK(c_state);
CHECK(compr_zstd->write(c_state, data, sizeof(data)));
f = fopen("data.zstd", "r");
struct decompressor *decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
- struct decompr_state *d_state = decompr_zstd->init(f);
+ struct decompr_state *d_state = decompr_zstd->init(f, NULL);
CHECK(d_state);
char buffer[sizeof(data)];
projects / thirdparty / ccache.git / commitdiff
