From: Joel Rosdahl Date: Mon, 1 Jul 2019 14:35:43 +0000 (+0200) Subject: Extract common header setup code to functions X-Git-Tag: v4.0~921 X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=2644e4ea36d08694b5bbaf1757149b7c03f84946;p=thirdparty%2Fccache.git Extract common header setup code to functions --- diff --git a/src/common_header.c b/src/common_header.c index 50e998978..d272e4c5d 100644 --- a/src/common_header.c +++ b/src/common_header.c @@ -16,64 +16,126 @@ #include "ccache.h" #include "int_bytes_conversion.h" -#include "compression.h" #include "common_header.h" -void common_header_from_config( +bool +common_header_initialize_for_writing( struct common_header *header, const char magic[4], - uint8_t RESULT_VERSION, - uint64_t content_size) + uint8_t version, + uint64_t content_size, + XXH64_state_t *checksum, + struct compressor **compressor, + struct compr_state **compr_state, + FILE *output) { enum compression_type compr_type = compression_type_from_config(); int8_t compr_level = compression_level_from_config(); memcpy(header->magic, magic, 4); - header->version = RESULT_VERSION; + header->version = version; header->compression_type = compr_type; header->compression_level = compr_level; header->content_size = content_size; -} -void -common_header_from_bytes(struct common_header *header, uint8_t *buffer) -{ - memcpy(header->magic, buffer, 4); - header->version = buffer[4]; - header->compression_type = buffer[5]; - header->compression_level = buffer[6]; - header->content_size = UINT64_FROM_BYTES(buffer + 7); -} + XXH64_reset(checksum, 0); -void -common_header_to_bytes(const struct common_header *header, uint8_t *buffer) -{ - memcpy(buffer, header->magic, 4); - buffer[4] = header->version; - buffer[5] = header->compression_type; - buffer[6] = header->compression_level; - BYTES_FROM_UINT64(buffer + 7, header->content_size); + *compressor = compressor_from_type(header->compression_type); + assert(*compressor); + *compr_state = + (*compressor)->init(output, header->compression_level, checksum); + if (!*compr_state) { + cc_log("Failed to initialize compressor"); + return false; + } + header->compression_level = + (*compressor)->get_actual_compression_level(*compr_state); + + uint8_t header_bytes[COMMON_HEADER_SIZE]; + memcpy(header_bytes, header->magic, 4); + header_bytes[4] = header->version; + header_bytes[5] = header->compression_type; + header_bytes[6] = header->compression_level; + BYTES_FROM_UINT64(header_bytes + 7, header->content_size); + if (fwrite(header_bytes, sizeof(header_bytes), 1, output) != 1) { + cc_log("Failed to write common file header"); + return false; + } + XXH64_update(checksum, header_bytes, sizeof(header_bytes)); + return true; } -bool common_header_verify( - const struct common_header *header, int fd, const char *name, char **errmsg) +bool common_header_initialize_for_reading( + struct common_header *header, + FILE *input, + const char magic[4], + uint8_t accepted_version, + struct decompressor **decompressor, + struct decompr_state **decompr_state, + XXH64_state_t *checksum, + char **errmsg) { + uint8_t header_bytes[COMMON_HEADER_SIZE]; + if (fread(header_bytes, sizeof(header_bytes), 1, input) != 1) { + *errmsg = format("Failed to read common header"); + return false; + } + + memcpy(header->magic, header_bytes, 4); + header->version = header_bytes[4]; + header->compression_type = header_bytes[5]; + header->compression_level = header_bytes[6]; + header->content_size = UINT64_FROM_BYTES(header_bytes + 7); + + if (memcmp(header->magic, magic, sizeof(header->magic)) != 0) { + *errmsg = format( + "Bad magic value 0x%x%x%x%x", + header->magic[0], + header->magic[1], + header->magic[2], + header->magic[3]); + return false; + } + + if (header->version != accepted_version) { + *errmsg = format( + "Unknown version (actual %u, expected %u)", + header->version, + accepted_version); + return false; + } + if (header->compression_type == COMPR_TYPE_NONE) { // Since we have the size available, let's use it as a super primitive // consistency check for the non-compressed case. (A real checksum is used // for compressed data.) struct stat st; - if (x_fstat(fd, &st) != 0 + if (x_fstat(fileno(input), &st) != 0 || (uint64_t)st.st_size != header->content_size) { *errmsg = format( - "Corrupt %s file (actual %lu bytes, expected %lu bytes)", - name, + "Bad uncompressed file size (actual %lu bytes, expected %lu bytes)", (unsigned long)st.st_size, (unsigned long)header->content_size); return false; } } + *decompressor = decompressor_from_type(header->compression_type); + if (!*decompressor) { + *errmsg = format( + "Unknown compression type: %u", header->compression_type); + return false; + } + + XXH64_reset(checksum, 0); + XXH64_update(checksum, header_bytes, sizeof(header_bytes)); + + *decompr_state = (*decompressor)->init(input, checksum); + if (!*decompr_state) { + *errmsg = x_strdup("Failed to initialize decompressor"); + return false; + } + return true; } diff --git a/src/common_header.h b/src/common_header.h index f25e3a1b5..598d8ec16 100644 --- a/src/common_header.h +++ b/src/common_header.h @@ -1,6 +1,9 @@ #ifndef COMMON_HEADER_H #define COMMON_HEADER_H +#include "compression.h" +#include "xxhash.h" + #define COMMON_HEADER_SIZE 15 struct common_header { @@ -11,16 +14,26 @@ struct common_header { uint64_t content_size; }; -void common_header_from_config( +bool common_header_initialize_for_writing( struct common_header *header, const char magic[4], - uint8_t RESULT_VERSION, - uint64_t content_size); -void common_header_from_bytes(struct common_header *header, uint8_t *bytes); -void common_header_to_bytes( - const struct common_header *header, uint8_t *bytes); -bool common_header_verify( - const struct common_header *header, int fd, const char *name, char **errmsg); + uint8_t version, + uint64_t content_size, + XXH64_state_t *checksum, + struct compressor **compressor, + struct compr_state **compr_state, + FILE *output); + +bool common_header_initialize_for_reading( + struct common_header *header, + FILE *input, + const char magic[4], + uint8_t accepted_version, + struct decompressor **decompressor, + struct decompr_state **decompr_state, + XXH64_state_t *checksum, + char **errmsg); + void common_header_dump(const struct common_header *header, FILE *f); #endif diff --git a/src/manifest.c b/src/manifest.c index 1011b0ddf..302a43711 100644 --- a/src/manifest.c +++ b/src/manifest.c @@ -98,7 +98,7 @@ // 1: Introduced in ccache 3.0. (Files are always compressed with gzip.) // 2: Introduced in ccache 3.8. -static const char MAGIC[4] = "cCmF"; +static const char MANIFEST_MAGIC[4] = "cCmF"; static const uint32_t MAX_MANIFEST_ENTRIES = 100; static const uint32_t MAX_MANIFEST_FILE_INFO_ENTRIES = 10000; @@ -253,7 +253,7 @@ read_manifest(const char *path, char **errmsg) struct decompressor *decompressor = NULL; struct decompr_state *decompr_state = NULL; *errmsg = NULL; - XXH64_state_t *checksum = NULL; + XXH64_state_t *checksum = XXH64_createState(); FILE *f = fopen(path, "rb"); if (!f) { @@ -261,50 +261,15 @@ read_manifest(const char *path, char **errmsg) goto out; } - uint8_t header_bytes[COMMON_HEADER_SIZE]; - if (fread(header_bytes, sizeof(header_bytes), 1, f) != 1) { - *errmsg = format("Failed to read header from %s", path); - goto out; - } - - common_header_from_bytes(&mf->header, header_bytes); - - if (memcmp(mf->header.magic, MAGIC, sizeof(MAGIC)) != 0) { - *errmsg = format( - "Result file has bad magic value 0x%x%x%x%x", - mf->header.magic[0], - mf->header.magic[1], - mf->header.magic[2], - mf->header.magic[3]); - goto out; - } - - if (mf->header.version != MANIFEST_VERSION) { - *errmsg = format( - "Unknown manifest version (actual %u, expected %u)", - mf->header.version, - MANIFEST_VERSION); - goto out; - } - - if (!common_header_verify(&mf->header, fileno(f), "manifest", errmsg)) { - goto out; - } - - decompressor = decompressor_from_type(mf->header.compression_type); - if (!decompressor) { - *errmsg = format( - "Unknown compression type: %u", mf->header.compression_type); - goto out; - } - - checksum = XXH64_createState(); - XXH64_reset(checksum, 0); - XXH64_update(checksum, header_bytes, sizeof(header_bytes)); - - decompr_state = decompressor->init(f, checksum); - if (!decompr_state) { - *errmsg = x_strdup("Failed to initialize decompressor"); + if (!common_header_initialize_for_reading( + &mf->header, + f, + MANIFEST_MAGIC, + MANIFEST_VERSION, + &decompressor, + &decompr_state, + checksum, + errmsg)) { goto out; } @@ -408,7 +373,7 @@ static bool write_manifest(FILE *f, const struct manifest *mf) { int ret = false; - XXH64_state_t *checksum = NULL; + XXH64_state_t *checksum = XXH64_createState(); uint64_t content_size = COMMON_HEADER_SIZE; content_size += 4; // n_files @@ -426,29 +391,19 @@ write_manifest(FILE *f, const struct manifest *mf) content_size += 8; // checksum struct common_header header; - common_header_from_config(&header, MAGIC, MANIFEST_VERSION, content_size); - - checksum = XXH64_createState(); - XXH64_reset(checksum, 0); - - struct compressor *compressor = - compressor_from_type(header.compression_type); - assert(compressor); - struct compr_state *compr_state = - compressor->init(f, header.compression_level, checksum); - if (!compr_state) { - cc_log("Failed to initialize compressor"); - goto out; - } - header.compression_level = - compressor->get_actual_compression_level(compr_state); - - uint8_t header_bytes[COMMON_HEADER_SIZE]; - common_header_to_bytes(&header, header_bytes); - if (fwrite(header_bytes, sizeof(header_bytes), 1, f) != 1) { + struct compressor *compressor; + struct compr_state *compr_state; + if (!common_header_initialize_for_writing( + &header, + MANIFEST_MAGIC, + MANIFEST_VERSION, + content_size, + checksum, + &compressor, + &compr_state, + f)) { goto out; } - XXH64_update(checksum, header_bytes, sizeof(header_bytes)); WRITE_UINT32(mf->n_files); for (uint32_t i = 0; i < mf->n_files; i++) { diff --git a/src/result.c b/src/result.c index 23537602e..26d3230b8 100644 --- a/src/result.c +++ b/src/result.c @@ -18,7 +18,6 @@ #include "common_header.h" #include "int_bytes_conversion.h" #include "compression.h" -#include "xxhash.h" #include "result.h" // Result data format @@ -78,7 +77,7 @@ // // 1: Introduced in ccache 3.8. -static const char MAGIC[4] = "cCrS"; +static const char RESULT_MAGIC[4] = "cCrS"; enum { FILE_MARKER = 0, @@ -163,7 +162,7 @@ read_result( struct decompressor *decompressor = NULL; struct decompr_state *decompr_state = NULL; FILE *subfile = NULL; - XXH64_state_t *checksum = NULL; + XXH64_state_t *checksum = XXH64_createState(); FILE *f = fopen(path, "rb"); if (!f) { @@ -172,23 +171,16 @@ read_result( goto out; } - uint8_t header_bytes[COMMON_HEADER_SIZE]; - if (fread(header_bytes, sizeof(header_bytes), 1, f) != 1) { - *errmsg = format("Failed to read header from %s", path); - goto out; - } - - checksum = XXH64_createState(); - XXH64_reset(checksum, 0); - XXH64_update(checksum, header_bytes, sizeof(header_bytes)); - struct common_header header; - common_header_from_bytes(&header, header_bytes); - - if (memcmp(header.magic, MAGIC, sizeof(MAGIC)) != 0) { - *errmsg = format( - "Result file has bad magic value 0x%x%x%x%x", - header.magic[0], header.magic[1], header.magic[2], header.magic[3]); + if (!common_header_initialize_for_reading( + &header, + f, + RESULT_MAGIC, + RESULT_VERSION, + &decompressor, + &decompr_state, + checksum, + errmsg)) { goto out; } @@ -196,30 +188,6 @@ read_result( common_header_dump(&header, dump_stream); } - if (header.version != RESULT_VERSION) { - *errmsg = format( - "Unknown result version (actual %u, expected %u)", - header.version, - RESULT_VERSION); - goto out; - } - - if (!common_header_verify(&header, fileno(f), "result", errmsg)) { - goto out; - } - - decompressor = decompressor_from_type(header.compression_type); - if (!decompressor) { - *errmsg = format("Unknown compression type: %u", header.compression_type); - goto out; - } - - decompr_state = decompressor->init(f, checksum); - if (!decompr_state) { - *errmsg = x_strdup("Failed to initialize decompressor"); - goto out; - } - uint8_t n_entries; READ_BYTE(n_entries); @@ -419,7 +387,7 @@ bool result_get(const char *path, struct result_files *list) bool result_put(const char *path, struct result_files *list) { bool ret = false; - XXH64_state_t *checksum = NULL; + XXH64_state_t *checksum = XXH64_createState(); char *tmp_file = format("%s.tmp", path); int fd = create_tmp_fd(&tmp_file); @@ -441,30 +409,19 @@ bool result_put(const char *path, struct result_files *list) content_size += 8; // checksum struct common_header header; - common_header_from_config(&header, MAGIC, RESULT_VERSION, content_size); - - checksum = XXH64_createState(); - XXH64_reset(checksum, 0); - - struct compressor *compressor = - compressor_from_type(header.compression_type); - assert(compressor); - struct compr_state *compr_state = - compressor->init(f, header.compression_level, checksum); - if (!compr_state) { - cc_log("Failed to initialize compressor"); - goto out; - } - header.compression_level = - compressor->get_actual_compression_level(compr_state); - - uint8_t header_bytes[COMMON_HEADER_SIZE]; - common_header_to_bytes(&header, header_bytes); - if (fwrite(header_bytes, sizeof(header_bytes), 1, f) != 1) { - cc_log("Failed to write result file header to %s", tmp_file); + struct compressor *compressor; + struct compr_state *compr_state; + if (!common_header_initialize_for_writing( + &header, + RESULT_MAGIC, + RESULT_VERSION, + content_size, + checksum, + &compressor, + &compr_state, + f)) { goto out; } - XXH64_update(checksum, header_bytes, sizeof(header_bytes)); bool ok = write_result(list, compressor, compr_state, checksum) && compressor->free(compr_state);