From: Joel Rosdahl <joel@rosdahl.net>
Date: Wed, 18 Sep 2019 19:17:47 +0000 (+0200)
Subject: Add and use Checksum class
X-Git-Tag: v4.0~768
X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d5eb883a628119c8436750f312099d4ee046de10;p=thirdparty%2Fccache.git

Add and use Checksum class

The class wraps an XXH64 state.
---

diff --git a/Makefile.in b/Makefile.in
index 36d1eb531..0b58e3e3a 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -73,6 +73,7 @@ non_third_party_objs = $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(non_third_p
 ccache_sources = src/main.cpp $(base_sources)
 ccache_objs = $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(ccache_sources)))
 
+test_suites += unittest/test_Checksum.cpp
 test_suites += unittest/test_Config.cpp
 test_suites += unittest/test_args.cpp
 test_suites += unittest/test_argument_processing.cpp
diff --git a/dev.mk.in b/dev.mk.in
index c8527c96d..4399297fe 100644
--- a/dev.mk.in
+++ b/dev.mk.in
@@ -37,6 +37,7 @@ built_dist_files = $(generated_sources) $(generated_docs)
 non_third_party_headers = \
     src/AtomicFile.hpp \
     src/CacheFile.hpp \
+    src/Checksum.hpp \
     src/Config.hpp \
     src/Error.hpp \
     src/ProgressBar.hpp \
diff --git a/src/Checksum.hpp b/src/Checksum.hpp
new file mode 100644
index 000000000..7033b87cc
--- /dev/null
+++ b/src/Checksum.hpp
@@ -0,0 +1,64 @@
+// Copyright (C) 2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#pragma once
+
+#include <cstdint>
+#include <third_party/xxhash.h>
+
+class Checksum
+{
+public:
+  Checksum();
+  ~Checksum();
+
+  void reset();
+  void update(const void* data, size_t length);
+  uint64_t digest() const;
+
+private:
+  XXH64_state_t* m_state;
+};
+
+inline Checksum::Checksum() : m_state(XXH64_createState())
+{
+  reset();
+}
+
+inline Checksum::~Checksum()
+{
+  XXH64_freeState(m_state);
+}
+
+inline void
+Checksum::reset()
+{
+  XXH64_reset(m_state, 0);
+}
+
+inline void
+Checksum::update(const void* data, size_t length)
+{
+  XXH64_update(m_state, data, length);
+}
+
+inline uint64_t
+Checksum::digest() const
+{
+  return XXH64_digest(m_state);
+}
diff --git a/src/common_header.cpp b/src/common_header.cpp
index 83f3f09cb..c8baba6f0 100644
--- a/src/common_header.cpp
+++ b/src/common_header.cpp
@@ -29,7 +29,7 @@ common_header_initialize_for_writing(struct common_header* header,
                                      uint8_t compression_type,
                                      int8_t compression_level,
                                      uint64_t content_size,
-                                     XXH64_state_t* checksum,
+                                     Checksum& checksum,
                                      struct compressor** compressor,
                                      struct compr_state** compr_state)
 {
@@ -39,12 +39,10 @@ common_header_initialize_for_writing(struct common_header* header,
   header->compression_level = compression_level;
   header->content_size = content_size;
 
-  XXH64_reset(checksum, 0);
-
   *compressor = compressor_from_type(header->compression_type);
   assert(*compressor);
   *compr_state =
-    (*compressor)->init(output, header->compression_level, checksum);
+    (*compressor)->init(output, header->compression_level, &checksum);
   if (!*compr_state) {
     cc_log("Failed to initialize compressor");
     return false;
@@ -62,7 +60,7 @@ common_header_initialize_for_writing(struct common_header* header,
     cc_log("Failed to write common file header");
     return false;
   }
-  XXH64_update(checksum, header_bytes, sizeof(header_bytes));
+  checksum.update(header_bytes, sizeof(header_bytes));
   return true;
 }
 
@@ -73,7 +71,7 @@ common_header_initialize_for_reading(struct common_header* header,
                                      uint8_t expected_version,
                                      struct decompressor** decompressor,
                                      struct decompr_state** decompr_state,
-                                     XXH64_state_t* checksum,
+                                     Checksum* checksum,
                                      char** errmsg)
 {
   uint8_t header_bytes[COMMON_HEADER_SIZE];
@@ -132,8 +130,7 @@ common_header_initialize_for_reading(struct common_header* header,
   }
 
   if (checksum) {
-    XXH64_reset(checksum, 0);
-    XXH64_update(checksum, header_bytes, sizeof(header_bytes));
+    checksum->update(header_bytes, sizeof(header_bytes));
   }
 
   *decompr_state = (*decompressor)->init(input, checksum);
diff --git a/src/common_header.hpp b/src/common_header.hpp
index f522eaf1d..5c32fb623 100644
--- a/src/common_header.hpp
+++ b/src/common_header.hpp
@@ -20,10 +20,9 @@
 
 #include "system.hpp"
 
+#include "Checksum.hpp"
 #include "compression.hpp"
 
-#include "third_party/xxhash.h"
-
 #define COMMON_HEADER_SIZE 15
 
 struct common_header
@@ -55,7 +54,7 @@ bool common_header_initialize_for_writing(struct common_header* header,
                                           uint8_t compression_type,
                                           int8_t compression_level,
                                           uint64_t content_size,
-                                          XXH64_state_t* checksum,
+                                          Checksum& checksum,
                                           struct compressor** compressor,
                                           struct compr_state** compr_state);
 
@@ -77,7 +76,7 @@ bool common_header_initialize_for_reading(struct common_header* header,
                                           uint8_t expected_version,
                                           struct decompressor** decompressor,
                                           struct decompr_state** decompr_state,
-                                          XXH64_state_t* checksum,
+                                          Checksum* checksum,
                                           char** errmsg);
 
 void common_header_dump(const struct common_header* header, FILE* f);
diff --git a/src/compr_none.cpp b/src/compr_none.cpp
index 7093d3aa2..e9dc628de 100644
--- a/src/compr_none.cpp
+++ b/src/compr_none.cpp
@@ -21,11 +21,11 @@
 struct state
 {
   FILE* output;
-  XXH64_state_t* checksum;
+  Checksum* checksum;
 };
 
 static struct compr_state*
-compr_none_init(FILE* output, int8_t level, XXH64_state_t* checksum)
+compr_none_init(FILE* output, int8_t level, Checksum* checksum)
 {
   auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
   state->output = output;
@@ -47,7 +47,7 @@ compr_none_write(struct compr_state* handle, const void* data, size_t size)
   struct state* state = (struct state*)handle;
   size_t ret = fwrite(data, size, 1, state->output);
   if (state->checksum) {
-    XXH64_update(state->checksum, data, size);
+    state->checksum->update(data, size);
   }
   return ret == 1;
 }
diff --git a/src/compr_zstd.cpp b/src/compr_zstd.cpp
index 3f726c7f7..dd9b41ad4 100644
--- a/src/compr_zstd.cpp
+++ b/src/compr_zstd.cpp
@@ -27,7 +27,7 @@
 struct state
 {
   FILE* output;
-  XXH64_state_t* checksum;
+  Checksum* checksum;
   ZSTD_CStream* stream;
   ZSTD_inBuffer in;
   ZSTD_outBuffer out;
@@ -36,7 +36,7 @@ struct state
 };
 
 static struct compr_state*
-compr_zstd_init(FILE* output, int8_t level, XXH64_state_t* checksum)
+compr_zstd_init(FILE* output, int8_t level, Checksum* checksum)
 {
   auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
   state->output = output;
@@ -92,7 +92,7 @@ compr_zstd_write(struct compr_state* handle, const void* data, size_t size)
   struct state* state = (struct state*)handle;
 
   if (state->checksum) {
-    XXH64_update(state->checksum, data, size);
+    state->checksum->update(data, size);
   }
 
   state->in.src = data;
diff --git a/src/compression.hpp b/src/compression.hpp
index 93514a711..4bd60d030 100644
--- a/src/compression.hpp
+++ b/src/compression.hpp
@@ -20,7 +20,7 @@
 
 #include "system.hpp"
 
-#include "third_party/xxhash.h"
+#include "Checksum.hpp"
 
 struct compr_state;
 
@@ -33,7 +33,7 @@ struct compressor
   // checksum: Checksum state to update (NULL for no checksum).
   struct compr_state* (*init)(FILE* output,
                               int8_t compression_level,
-                              XXH64_state_t* checksum);
+                              Checksum* checksum);
 
   // Get the actual compression level that will be used.
   int8_t (*get_actual_compression_level)(struct compr_state* state);
@@ -61,7 +61,7 @@ struct decompressor
   //
   // input: The file to read compressed data from.
   // checksum: Checksum state to update (NULL for no checksum).
-  struct decompr_state* (*init)(FILE* input, XXH64_state_t* checksum);
+  struct decompr_state* (*init)(FILE* input, Checksum* checksum);
 
   // Decompress data.
   //
diff --git a/src/decompr_none.cpp b/src/decompr_none.cpp
index 4c40cd49b..29018d22f 100644
--- a/src/decompr_none.cpp
+++ b/src/decompr_none.cpp
@@ -21,12 +21,12 @@
 struct state
 {
   FILE* input;
-  XXH64_state_t* checksum;
+  Checksum* checksum;
   bool failed;
 };
 
 static struct decompr_state*
-decompr_none_init(FILE* input, XXH64_state_t* checksum)
+decompr_none_init(FILE* input, Checksum* checksum)
 {
   auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
   state->input = input;
@@ -42,7 +42,7 @@ decompr_none_read(struct decompr_state* handle, void* data, size_t size)
 
   bool result = fread(data, 1, size, state->input) == size;
   if (result && state->checksum) {
-    XXH64_update(state->checksum, data, size);
+    state->checksum->update(data, size);
   }
   if (!result) {
     state->failed = true;
diff --git a/src/decompr_zstd.cpp b/src/decompr_zstd.cpp
index 9fb895c31..e187abc33 100644
--- a/src/decompr_zstd.cpp
+++ b/src/decompr_zstd.cpp
@@ -30,7 +30,7 @@ enum stream_state {
 struct state
 {
   FILE* input;
-  XXH64_state_t* checksum;
+  Checksum* checksum;
   char input_buffer[READ_BUFFER_SIZE];
   size_t input_size;
   size_t input_consumed;
@@ -41,7 +41,7 @@ struct state
 };
 
 static struct decompr_state*
-decompr_zstd_init(FILE* input, XXH64_state_t* checksum)
+decompr_zstd_init(FILE* input, Checksum* checksum)
 {
   auto state = static_cast<struct state*>(malloc(sizeof(struct state)));
 
@@ -95,7 +95,7 @@ decompr_zstd_read(struct decompr_state* handle, void* data, size_t size)
       return false;
     }
     if (state->checksum) {
-      XXH64_update(state->checksum, state->out.dst, state->out.pos);
+      state->checksum->update(state->out.dst, state->out.pos);
     }
     if (ret == 0) {
       state->stream_state = STREAM_STATE_END;
diff --git a/src/manifest.cpp b/src/manifest.cpp
index f54da6df5..527f999e6 100644
--- a/src/manifest.cpp
+++ b/src/manifest.cpp
@@ -18,14 +18,13 @@
 
 #include "manifest.hpp"
 
+#include "Checksum.hpp"
 #include "ccache.hpp"
 #include "common_header.hpp"
 #include "compression.hpp"
 #include "hashutil.hpp"
 #include "int_bytes_conversion.hpp"
 
-#include "third_party/xxhash.h"
-
 // Manifest data format
 // ====================
 //
@@ -145,7 +144,9 @@ template<> struct hash<FileInfo>
   operator()(const FileInfo& file_info) const
   {
     static_assert(sizeof(FileInfo) == 48, "unexpected size"); // No padding.
-    return XXH64(&file_info, sizeof(file_info), 0);
+    Checksum checksum;
+    checksum.update(&file_info, sizeof(file_info));
+    return checksum.digest();
   }
 };
 
@@ -265,7 +266,7 @@ read_manifest(const char* path, char** errmsg)
   struct decompressor* decompressor = NULL;
   struct decompr_state* decompr_state = NULL;
   *errmsg = NULL;
-  XXH64_state_t* checksum = XXH64_createState();
+  Checksum checksum;
   uint64_t actual_checksum;
   uint64_t expected_checksum;
 
@@ -281,7 +282,7 @@ read_manifest(const char* path, char** errmsg)
                                             MANIFEST_VERSION,
                                             &decompressor,
                                             &decompr_state,
-                                            checksum,
+                                            &checksum,
                                             errmsg)) {
     goto out;
   }
@@ -315,7 +316,7 @@ read_manifest(const char* path, char** errmsg)
     READ_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
   }
 
-  actual_checksum = XXH64_digest(checksum);
+  actual_checksum = checksum.digest();
   READ_UINT64(expected_checksum);
   if (actual_checksum == expected_checksum) {
     success = true;
@@ -332,9 +333,6 @@ out:
   if (f) {
     fclose(f);
   }
-  if (checksum) {
-    XXH64_freeState(checksum);
-  }
   if (!success) {
     if (!*errmsg) {
       *errmsg = x_strdup("Corrupt manifest file");
@@ -384,7 +382,6 @@ static bool
 write_manifest(FILE* f, const struct manifest* mf)
 {
   int ret = false;
-  XXH64_state_t* checksum = XXH64_createState();
 
   uint64_t content_size = COMMON_HEADER_SIZE;
   content_size += 4; // n_files
@@ -401,6 +398,7 @@ write_manifest(FILE* f, const struct manifest* mf)
   }
   content_size += 8; // checksum
 
+  Checksum checksum;
   struct common_header header;
   struct compressor* compressor;
   struct compr_state* compr_state;
@@ -441,12 +439,11 @@ write_manifest(FILE* f, const struct manifest* mf)
     WRITE_BYTES(mf->results[i].name.bytes, DIGEST_SIZE);
   }
 
-  WRITE_UINT64(XXH64_digest(checksum));
+  WRITE_UINT64(checksum.digest());
 
   ret = compressor->free(compr_state);
 
 out:
-  XXH64_freeState(checksum);
   if (!ret) {
     cc_log("Error writing to manifest file");
   }
diff --git a/src/result.cpp b/src/result.cpp
index c7e152e0a..b3f3d2e94 100644
--- a/src/result.cpp
+++ b/src/result.cpp
@@ -380,7 +380,7 @@ read_result(const char* path,
   bool success = false;
   struct decompressor* decompressor = NULL;
   struct decompr_state* decompr_state = NULL;
-  XXH64_state_t* checksum = XXH64_createState();
+  Checksum checksum;
 
   FILE* f = fopen(path, "rb");
   if (!f) {
@@ -395,7 +395,7 @@ read_result(const char* path,
                                             RESULT_VERSION,
                                             &decompressor,
                                             &decompr_state,
-                                            checksum,
+                                            &checksum,
                                             errmsg)) {
     goto out;
   }
@@ -439,7 +439,7 @@ read_result(const char* path,
   }
 
   {
-    uint64_t actual_checksum = XXH64_digest(checksum);
+    uint64_t actual_checksum = checksum.digest();
     uint64_t expected_checksum;
     READ_UINT64(expected_checksum);
 
@@ -459,9 +459,6 @@ out:
   if (f) {
     fclose(f);
   }
-  if (checksum) {
-    XXH64_freeState(checksum);
-  }
   if (!success && !cache_miss && !*errmsg) {
     *errmsg = x_strdup("Corrupt result");
   }
@@ -609,7 +606,7 @@ static bool
 write_result(const struct result_files* list,
              struct compressor* compressor,
              struct compr_state* compr_state,
-             XXH64_state_t* checksum,
+             Checksum& checksum,
              const char* result_path_in_cache)
 {
   WRITE_BYTE(list->n_files);
@@ -624,7 +621,7 @@ write_result(const struct result_files* list,
     }
   }
 
-  WRITE_UINT64(XXH64_digest(checksum));
+  WRITE_UINT64(checksum.digest());
 
   return true;
 
@@ -656,7 +653,7 @@ bool
 result_put(const char* path, struct result_files* list)
 {
   bool ret = false;
-  XXH64_state_t* checksum = XXH64_createState();
+  Checksum checksum;
   bool ok;
   uint64_t content_size;
 
@@ -715,9 +712,6 @@ out:
   if (f) {
     fclose(f);
   }
-  if (checksum) {
-    XXH64_freeState(checksum);
-  }
   return ret;
 }
 
diff --git a/unittest/test_Checksum.cpp b/unittest/test_Checksum.cpp
new file mode 100644
index 000000000..0c5359367
--- /dev/null
+++ b/unittest/test_Checksum.cpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2011-2019 Joel Rosdahl and other contributors
+//
+// See doc/AUTHORS.adoc for a complete list of contributors.
+//
+// This program is free software; you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation; either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along with
+// this program; if not, write to the Free Software Foundation, Inc., 51
+// Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+
+#include "../src/Checksum.hpp"
+
+#include <catch.hpp>
+
+TEST_CASE("Checksums")
+{
+  Checksum checksum;
+  CHECK(checksum.digest() == 0xef46db3751d8e999);
+
+  checksum.update("foo", 3);
+  CHECK(checksum.digest() == 0x33bf00a859c4ba3f);
+
+  checksum.update("t", 1);
+  CHECK(checksum.digest() == 0xef78250064b0eea7);
+
+  checksum.reset();
+  CHECK(checksum.digest() == 0xef46db3751d8e999);
+}
diff --git a/unittest/test_compr_none.cpp b/unittest/test_compr_none.cpp
index f19704c0c..69563c95f 100644
--- a/unittest/test_compr_none.cpp
+++ b/unittest/test_compr_none.cpp
@@ -26,12 +26,11 @@ TEST(small_roundtrip)
 {
   const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
 
-  XXH64_state_t* checksum = XXH64_createState();
-  XXH64_reset(checksum, 0);
+  Checksum checksum;
 
   FILE* f = fopen("data.uncompressed", "w");
   struct compressor* compr_none = compressor_from_type(COMPR_TYPE_NONE);
-  struct compr_state* c_state = compr_none->init(f, -1, checksum);
+  struct compr_state* c_state = compr_none->init(f, -1, &checksum);
   CHECK(c_state);
 
   CHECK(compr_none->write(c_state, "foobar", 6));
@@ -39,12 +38,12 @@ TEST(small_roundtrip)
   CHECK(compr_none->free(c_state));
   fclose(f);
 
-  CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+  CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
 
-  XXH64_reset(checksum, 0);
+  checksum.reset();
   f = fopen("data.uncompressed", "r");
   struct decompressor* decompr_none = decompressor_from_type(COMPR_TYPE_NONE);
-  struct decompr_state* d_state = decompr_none->init(f, checksum);
+  struct decompr_state* d_state = decompr_none->init(f, &checksum);
   CHECK(d_state);
 
   char buffer[4];
@@ -60,9 +59,7 @@ TEST(small_roundtrip)
   CHECK(!decompr_none->free(d_state));
   fclose(f);
 
-  CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
-
-  XXH64_freeState(checksum);
+  CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
 }
 
 TEST_SUITE_END
diff --git a/unittest/test_compr_zstd.cpp b/unittest/test_compr_zstd.cpp
index 393bb8764..3a4859a9a 100644
--- a/unittest/test_compr_zstd.cpp
+++ b/unittest/test_compr_zstd.cpp
@@ -26,12 +26,11 @@ TEST(small_roundtrip)
 {
   const uint64_t expected_foobar_checksum = 0xa2aa05ed9085aaf9ULL;
 
-  XXH64_state_t* checksum = XXH64_createState();
-  XXH64_reset(checksum, 0);
+  Checksum checksum;
 
   FILE* f = fopen("data.zstd", "w");
   struct compressor* compr_zstd = compressor_from_type(COMPR_TYPE_ZSTD);
-  struct compr_state* c_state = compr_zstd->init(f, -1, checksum);
+  struct compr_state* c_state = compr_zstd->init(f, -1, &checksum);
   CHECK(c_state);
 
   CHECK(compr_zstd->write(c_state, "foobar", 6));
@@ -39,12 +38,12 @@ TEST(small_roundtrip)
   CHECK(compr_zstd->free(c_state));
   fclose(f);
 
-  CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
+  CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
 
-  XXH64_reset(checksum, 0);
+  checksum.reset();
   f = fopen("data.zstd", "r");
   struct decompressor* decompr_zstd = decompressor_from_type(COMPR_TYPE_ZSTD);
-  struct decompr_state* d_state = decompr_zstd->init(f, checksum);
+  struct decompr_state* d_state = decompr_zstd->init(f, &checksum);
   CHECK(d_state);
 
   char buffer[4];
@@ -60,9 +59,7 @@ TEST(small_roundtrip)
   CHECK(!decompr_zstd->free(d_state));
   fclose(f);
 
-  CHECK_INT_EQ(XXH64_digest(checksum), expected_foobar_checksum);
-
-  XXH64_freeState(checksum);
+  CHECK_INT_EQ(checksum.digest(), expected_foobar_checksum);
 }
 
 TEST(large_compressible_roundtrip)