add support for new file signature in the xxhash family

- XXHASH64 fast 64bits signature
- XXH3_64  a bit faster 64bits signature
- XXH3_128 a even faster 128bits signature

diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c
index 2bf4e1be1050b0ffe3d6251ec7b89b552618ccf9..377cbc9060e74131bf847aa881f44be197a8949b 100644 (file)
--- a/bacula/src/dird/backup.c
+++ b/bacula/src/dird/backup.c
@@ -228,6 +228,9 @@ static bool is_checksum_needed_by_fileset(JCR *jcr)
             case '1':           /* SHA1 */
             case '2':           /* SHA256 */
             case '3':           /* SHA512 */
+            case '6':           /* XXHASH64 */
+            case '7':           /* XXH3_64 */
+            case '8':           /* XXH3_128 */
                if (in_block) {
                   Dmsg0(50, "Checksum will be sent to FD\n");
                   return true;

diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c
index 21ad3287c2d082edc35a9967e49c05936f189d66..acf2f9801a5dc7ef575888836cf6a40c2e7755fc 100644 (file)
--- a/bacula/src/dird/catreq.c
+++ b/bacula/src/dird/catreq.c
@@ -780,6 +780,18 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen)
          len = CRYPTO_DIGEST_SHA512_SIZE;
          type = CRYPTO_DIGEST_SHA512;
          break;
+      case STREAM_XXHASH64_DIGEST:
+         len = CRYPTO_DIGEST_XXHASH64_SIZE;
+         type = CRYPTO_DIGEST_XXHASH64;
+         break;
+      case STREAM_XXH3_64_DIGEST:
+         len = CRYPTO_DIGEST_XXH3_64_SIZE;
+         type = CRYPTO_DIGEST_XXH3_64;
+         break;
+      case STREAM_XXH3_128_DIGEST:
+         len = CRYPTO_DIGEST_XXH3_128_SIZE;
+         type = CRYPTO_DIGEST_XXH3_128;
+         break;
       default:
          /* Never reached ... */
          Jmsg(jcr, M_ERROR, 0, _("Catalog error updating file digest. Unsupported digest stream type: %d"),

diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c
index 157fdeabadcb3b3c785d9b280b9bd53ae1a14838..d6d28c2adfbc4bcb7049e11b0245859f21db9b57 100644 (file)
--- a/bacula/src/dird/inc_conf.c
+++ b/bacula/src/dird/inc_conf.c
@@ -191,6 +191,9 @@ struct s_fs_opt FS_options[] = {
    {"Sha256",   INC_KW_DIGEST,       "S2"},
    {"Sha512",   INC_KW_DIGEST,       "S3"},
    {"Sha1",     INC_KW_DIGEST,        "S"},
+   {"Xxhash64", INC_KW_DIGEST,       "S5"},
+   {"Xxh3_64",  INC_KW_DIGEST,       "S6"},
+   {"Xxh3_128", INC_KW_DIGEST,       "S7"},
    {"Gzip",     INC_KW_COMPRESSION,  "Z6"},
    {"Gzip1",    INC_KW_COMPRESSION,  "Z1"},
    {"Gzip2",    INC_KW_COMPRESSION,  "Z2"},

diff --git a/bacula/src/dird/verify.c b/bacula/src/dird/verify.c
index 7f8caa1b19e420c2198b26814508ee478f17415a..040e6a2969ff18698a299b9ef2e22cd9a5a96664 100644 (file)
--- a/bacula/src/dird/verify.c
+++ b/bacula/src/dird/verify.c
@@ -841,6 +841,15 @@ void get_attributes_and_compare_to_catalog(JCR *jcr, JobId_t JobId)
             case '3':                 /* compare SHA512 */
                do_Digest = CRYPTO_DIGEST_SHA512;
                break;
+            case '6':                 /* compare XXHASH64 */
+               do_Digest = CRYPTO_DIGEST_XXHASH64;
+               break;
+            case '7':                 /* compare XXH3_64 */
+               do_Digest = CRYPTO_DIGEST_XXH3_64;
+               break;
+            case '8':                 /* compare XXH3_128 */
+               do_Digest = CRYPTO_DIGEST_XXH3_128;
+               break;
             case ':':
             case 'V':
             default:

diff --git a/bacula/src/filed/accurate.c b/bacula/src/filed/accurate.c
index 6addf5359337b8ad08723ef07b80e62150e7db44..a8242b2c8aa983a89fca4c668a893bfe7e4431c6 100644 (file)
--- a/bacula/src/filed/accurate.c
+++ b/bacula/src/filed/accurate.c
@@ -405,6 +405,18 @@ static int check_checksum_diff(JCR *jcr, FF_PKT *ff_pkt, CurFile *elt)
       } else if (ff_pkt->flags & FO_SHA512) {
          digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
          digest_stream = STREAM_SHA512_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXHASH64) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXHASH64);
+         digest_stream = STREAM_XXHASH64_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXH3_64) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_64);
+         digest_stream = STREAM_XXH3_64_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXH3_128) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_128);
+         digest_stream = STREAM_XXH3_128_DIGEST;
       }
 
       /* Did digest initialization fail? */
@@ -631,6 +643,9 @@ bool accurate_check_file(JCR *jcr, FF_PKT *ff_pkt)
       case '1':                /* compare SHA1 */
       case '2':                /* compare SHA256 */
       case '3':                /* compare SHA512 */
+      case '6':                /* compare XXHASH64 */
+      case '7':                /* compare XXH3_64 */
+      case '8':                /* compare XXH3_128 */
          if (ff_pkt->type != FT_LNKSAVED &&
                (S_ISREG(ff_pkt->statp.st_mode) &&
                 ff_pkt->flags & (FO_MD5|FO_SHA1|FO_SHA256|FO_SHA512))) {

diff --git a/bacula/src/filed/crypto.c b/bacula/src/filed/crypto.c
index c26664284d65633abebff252fe171483dffaf851..425a8caa5ea18ece74818a6d4b66347f93e17eb2 100644 (file)
--- a/bacula/src/filed/crypto.c
+++ b/bacula/src/filed/crypto.c
@@ -96,8 +96,19 @@ bool crypto_setup_digests(bctx_t &bctx)
    } else if (ff_pkt->flags & FO_SHA512) {
       bctx.digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
       bctx.digest_stream = STREAM_SHA512_DIGEST;
-   }
 
+   } else if (ff_pkt->flags & FO_XXHASH64) {
+      bctx.digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXHASH64);
+      bctx.digest_stream = STREAM_XXHASH64_DIGEST;
+
+   } else if (ff_pkt->flags & FO_XXH3_64) {
+      bctx.digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_64);
+      bctx.digest_stream = STREAM_XXH3_64_DIGEST;
+
+   } else if (ff_pkt->flags & FO_XXH3_128) {
+      bctx.digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_128);
+      bctx.digest_stream = STREAM_XXH3_128_DIGEST;
+   }
    /** Did digest initialization fail? */
    if (bctx.digest_stream != STREAM_NONE && bctx.digest == NULL) {
       Jmsg(jcr, M_WARNING, 0, _("%s digest initialization failed\n"),

diff --git a/bacula/src/filed/filed_conf.c b/bacula/src/filed/filed_conf.c
index 3999a68b94fcbd321ce89925aed127f2373167cf..4e340ffc4638bc29242b13f0e4122e5f6de3a4c8 100644 (file)
--- a/bacula/src/filed/filed_conf.c
+++ b/bacula/src/filed/filed_conf.c
@@ -235,6 +235,9 @@ struct s_ct digesttypes[] = {
    {"sha1",        CRYPTO_DIGEST_SHA1},
    {"sha256",      CRYPTO_DIGEST_SHA256},
 //   {"sha512",      CRYPTO_DIGEST_SHA512}, /* Not working yet */
+   {"xxhash64",    CRYPTO_DIGEST_XXHASH64},
+   {"xxh3_128",    CRYPTO_DIGEST_XXH3_128},
+//   {"xxh3_64",     CRYPTO_DIGEST_XXH3_64}, /* Never released, maybe already deprecated ? */
    {NULL,                             0}
 };
 

diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c
index 8d9917d63578f78ec9a684f4025b7db7c0cf7d85..e487f8a23f79ff5f039a1ade9ee169bb6461c2c5 100644 (file)
--- a/bacula/src/filed/job.c
+++ b/bacula/src/filed/job.c
@@ -1985,6 +1985,18 @@ static int set_options(findFOPTS *fo, const char *opts)
             p++;
             break;
 #endif
+         case '5':
+            fo->flags |= FO_XXHASH64;
+            p++;
+            break;
+         case '6':
+            fo->flags |= FO_XXH3_64;
+            p++;
+            break;
+         case '7':
+            fo->flags |= FO_XXH3_128;
+            p++;
+            break;
          default:
             /*
              * If 2 or 3 is seen here, SHA2 is not configured, so

diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c
index 6dc552de4f91bbc8a2cd551f7ab73001ec0e5863..e5258c981a21225cc88f4e9822e169bc0dfc4933 100644 (file)
--- a/bacula/src/filed/restore.c
+++ b/bacula/src/filed/restore.c
@@ -1033,6 +1033,9 @@ void do_restore(JCR *jcr)
       case STREAM_SHA1_DIGEST:
       case STREAM_SHA256_DIGEST:
       case STREAM_SHA512_DIGEST:
+      case STREAM_XXHASH64_DIGEST:
+      case STREAM_XXH3_64_DIGEST:
+      case STREAM_XXH3_128_DIGEST:
          break;
 
       case STREAM_PROGRAM_NAMES:

diff --git a/bacula/src/filed/verify.c b/bacula/src/filed/verify.c
index ae7e886c23c8f7a7f46ff3ab3085b65ae7262e72..a37b847c19150cb7d0a84cc051bfcabc818e7914 100644 (file)
--- a/bacula/src/filed/verify.c
+++ b/bacula/src/filed/verify.c
@@ -238,6 +238,18 @@ static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
       } else if (ff_pkt->flags & FO_SHA512) {
          digest = crypto_digest_new(jcr, CRYPTO_DIGEST_SHA512);
          digest_stream = STREAM_SHA512_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXHASH64) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXHASH64);
+         digest_stream = STREAM_XXHASH64_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXH3_64) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_64);
+         digest_stream = STREAM_XXH3_64_DIGEST;
+
+      } else if (ff_pkt->flags & FO_XXH3_128) {
+         digest = crypto_digest_new(jcr, CRYPTO_DIGEST_XXH3_128);
+         digest_stream = STREAM_XXH3_128_DIGEST;
       }
 
       /* Did digest initialization fail? */
@@ -246,7 +258,7 @@ static int verify_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level)
               stream_to_ascii(digest_stream));
       }
 
-      /* compute MD5 or SHA1 hash */
+      /* compute MD5 or SHA??? or XXH??? hash */
       if (digest) {
          char md[CRYPTO_DIGEST_MAX_SIZE];
          uint32_t size;

diff --git a/bacula/src/filed/verify_vol.c b/bacula/src/filed/verify_vol.c
index 305e674b17e39fa5c0af4768e1b25db37a0f7aff..85b3944366997265a3930e1bd482ae310fc8f6cb 100644 (file)
--- a/bacula/src/filed/verify_vol.c
+++ b/bacula/src/filed/verify_vol.c
@@ -152,7 +152,10 @@ void v_ctx::scan_fileset()
          if ((strchr(fo->VerifyOpts, '1') != NULL) ||
              (strchr(fo->VerifyOpts, '2') != NULL) ||
              (strchr(fo->VerifyOpts, '3') != NULL) ||
-             (strchr(fo->VerifyOpts, '5') != NULL))
+             (strchr(fo->VerifyOpts, '5') != NULL) ||
+             (strchr(fo->VerifyOpts, '6') != NULL) ||
+             (strchr(fo->VerifyOpts, '7') != NULL) ||
+             (strchr(fo->VerifyOpts, '8') != NULL))
          {
             check_chksum = true;
          } 
@@ -173,6 +176,18 @@ void v_ctx::scan_fileset()
             digesttype = CRYPTO_DIGEST_SHA512;
             return;
          }
+         if (fo->flags & FO_XXHASH64) {
+            digesttype = CRYPTO_DIGEST_XXHASH64;
+            return;
+         }
+         if (fo->flags & FO_XXH3_64) {
+            digesttype = CRYPTO_DIGEST_XXH3_64;
+            return;
+         }
+         if (fo->flags & FO_XXH3_128) {
+            digesttype = CRYPTO_DIGEST_XXH3_128;
+            return;
+         }
       }
    }
    digesttype = CRYPTO_DIGEST_NONE;
@@ -508,6 +523,21 @@ void do_verify_volume(JCR *jcr)
          digest_code = "SHA512";
          break;
 
+      case STREAM_XXHASH64_DIGEST:
+         bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_XXHASH64_SIZE, true);
+         digest_code = "XXHASH64";
+         break;
+
+      case STREAM_XXH3_64_DIGEST:
+         bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_XXH3_64_SIZE, true);
+         digest_code = "XXH3_64";
+         break;
+
+      case STREAM_XXH3_128_DIGEST:
+         bin_to_base64(digest, sizeof(digest), (char *)bmsg->rbuf, CRYPTO_DIGEST_XXH3_128_SIZE, true);
+         digest_code = "XXH3_128";
+         break;
+
       default:
          *digest = 0;
          break;

diff --git a/bacula/src/fileopts.h b/bacula/src/fileopts.h
index 8cc30d0876ecf1a2f4ba89d21423679bce67e468..191f6c299d7c8c14e497b59b8784936247c19a9a 100644 (file)
--- a/bacula/src/fileopts.h
+++ b/bacula/src/fileopts.h
@@ -65,4 +65,8 @@
 #define FO_OFFSETS       (1<<30)      /* Keep I/O file offsets */
 #define FO_DEDUPLICATION (1ULL<<31)   /* Do deduplication */
 
+#define FO_XXHASH64      (1ULL<<32)   /* Do XXHASH64 checksum */
+#define FO_XXH3_64       (1ULL<<33)   /* Do XXH3_H64 checksum */
+#define FO_XXH3_128      (1ULL<<34)   /* Do XXH3_128 checksum */
+
 #endif /* __BFILEOPTSS_H */

diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c
index 2477be0381323f131b4c6b3b7a45118c7f7aa068..77bfad029c4afd14b7eb97542fa830a4d3d45da3 100644 (file)
--- a/bacula/src/findlib/bfile.c
+++ b/bacula/src/findlib/bfile.c
@@ -128,6 +128,12 @@ const char *stream_to_ascii(int stream)
          return _("SHA256 digest");
       case STREAM_SHA512_DIGEST:
          return _("SHA512 digest");
+      case STREAM_XXHASH64_DIGEST:
+         return _("XXHASH64 digest");
+      case STREAM_XXH3_64_DIGEST:
+         return _("XXH3_64 digest");
+      case STREAM_XXH3_128_DIGEST:
+         return _("XXH3_128 digest");
       case STREAM_SIGNED_DIGEST:
          return _("Signed digest");
       case STREAM_ENCRYPTED_FILE_DATA:

diff --git a/bacula/src/lib/crypto.c b/bacula/src/lib/crypto.c
index e8195e15105606a33b74139126501d4843316865..d6c64b3ef28175282d9fd7b78111599236da5606 100644 (file)
--- a/bacula/src/lib/crypto.c
+++ b/bacula/src/lib/crypto.c
@@ -132,6 +132,7 @@
 #ifdef HAVE_OPENSSL /* How about OpenSSL? */
 
 #include "openssl-compat.h"
+#include "xxhash.h"
 
 /* ASN.1 Declarations */
 #define BACULA_ASN1_VERSION 0
@@ -582,20 +583,46 @@ void crypto_keypair_free(X509_KEYPAIR *keypair)
    free(keypair);
 }
 
-/*
- * Create a new message digest context of the specified type
- *  Returns: A pointer to a DIGEST object on success.
- *           NULL on failure.
- */
-DIGEST *crypto_digest_new(JCR *jcr, crypto_digest_t type)
+/* called by crypto_digest_new() to create an xxhash digest context */
+static DIGEST *xxhash_crypto_digest_new(JCR *jcr, crypto_digest_t type)
 {
-   DIGEST *digest;
-   const EVP_MD *md = NULL; /* Quell invalid uninitialized warnings */
+   void *state;
 
-   if (!crypto_check_digest(jcr, type)) {
+   switch (type) {
+   case CRYPTO_DIGEST_XXHASH64:
+      state = (void*)XXH64_createState();
+      XXH64_reset((XXH64_state_t *)state, 0);
+      break;
+   case CRYPTO_DIGEST_XXH3_64:
+      state = (void*)XXH3_createState();
+      XXH3_64bits_reset((XXH3_state_t *)state);
+      break;
+   case CRYPTO_DIGEST_XXH3_128:
+      state = (void*)XXH3_createState();
+      XXH3_128bits_reset((XXH3_state_t *)state);
+      break;
+   default:
+      Jmsg1(jcr, M_ERROR, 0, _("Unsupported digest type: %d\n"), type);
       return NULL;
    }
 
+   DIGEST *digest;
+
+   digest = (DIGEST *)malloc(sizeof(DIGEST));
+   digest->type = type;
+   digest->jcr = jcr;
+   digest->ctx = (EVP_MD_CTX *)state; // TODO remove the trans-typage
+   Dmsg1(150, "crypto_digest_new jcr=%p\n", jcr);
+
+   return digest;
+}
+
+/* called by crypto_digest_new() to create an openssl digest context */
+static DIGEST *openssl_crypto_digest_new(JCR *jcr, crypto_digest_t type)
+{
+   DIGEST *digest;
+   const EVP_MD *md = NULL; /* Quell invalid uninitialized warnings */
+
    digest = (DIGEST *)malloc(sizeof(DIGEST));
    digest->type = type;
    digest->jcr = jcr;
@@ -644,6 +671,22 @@ err:
    return NULL;
 }
 
+/*
+ * Create a new message digest context of the specified type
+ *  Returns: A pointer to a DIGEST object on success.
+ *           NULL on failure.
+ */
+DIGEST *crypto_digest_new(JCR *jcr, crypto_digest_t type)
+{
+   if (!crypto_check_digest(jcr, type)) {
+      return NULL;
+   }
+   if (IS_XXHASH_DIGEST(type)) {
+      return xxhash_crypto_digest_new(jcr, type);
+   }
+   return openssl_crypto_digest_new(jcr, type);
+}
+
 /*
  * Hash length bytes of data into the provided digest context.
  * Returns: true on success
@@ -651,12 +694,35 @@ err:
  */
 bool crypto_digest_update(DIGEST *digest, const uint8_t *data, uint32_t length)
 {
-   if (EVP_DigestUpdate(digest->ctx, data, length) == 0) {
-      Dmsg0(150, "digest update failed\n");
-      openssl_post_errors(digest->jcr, M_ERROR, _("OpenSSL digest update failed"));
-      return false;
+   if (IS_XXHASH_DIGEST(digest->type)) {
+      int ret;
+      switch (digest->type) {
+      case CRYPTO_DIGEST_XXHASH64:
+         ret = XXH64_update((XXH64_state_t *)digest->ctx, data, length);
+         break;
+      case CRYPTO_DIGEST_XXH3_64:
+         ret = XXH3_64bits_update((XXH3_state_t *)digest->ctx, data, length);
+         break;
+      case CRYPTO_DIGEST_XXH3_128:
+         ret = XXH3_128bits_update((XXH3_state_t *)digest->ctx, data, length);
+         break;
+      default:
+         Dmsg1(150, "unknown digest %d\n", digest->type);
+         ret = XXH_ERROR;
+         break;
+      }
+      if (ret != XXH_OK) {
+         Dmsg0(150, "digest update failed\n");
+      }
+      return (ret == XXH_OK);
    } else {
-      return true;
+      if (EVP_DigestUpdate((EVP_MD_CTX *)digest->ctx, data, length) == 0) {
+         Dmsg0(150, "digest update failed\n");
+         openssl_post_errors(digest->jcr, M_ERROR, _("OpenSSL digest update failed"));
+         return false;
+      } else {
+         return true;
+      }
    }
 }
 
@@ -669,12 +735,41 @@ bool crypto_digest_update(DIGEST *digest, const uint8_t *data, uint32_t length)
  */
 bool crypto_digest_finalize(DIGEST *digest, uint8_t *dest, uint32_t *length)
 {
-   if (!EVP_DigestFinal(digest->ctx, dest, (unsigned int *)length)) {
-      Dmsg0(150, "digest finalize failed\n");
-      openssl_post_errors(digest->jcr, M_ERROR, _("OpenSSL digest finalize failed"));
-      return false;
+   if (IS_XXHASH_DIGEST(digest->type)) {
+      bool ret = true;
+      XXH64_hash_t val64;
+      XXH128_hash_t val128;   /* this is a struct */
+      switch (digest->type) {
+      case CRYPTO_DIGEST_XXHASH64:
+         *length = CRYPTO_DIGEST_XXHASH64_SIZE;
+         val64 = XXH64_digest((XXH64_state_t *)digest->ctx);
+         XXH64_canonicalFromHash((XXH64_canonical_t*)dest, val64);
+         break;
+      case CRYPTO_DIGEST_XXH3_64:
+         *length = CRYPTO_DIGEST_XXH3_64_SIZE;
+         val64 = XXH3_64bits_digest((XXH3_state_t *)digest->ctx);
+         XXH64_canonicalFromHash((XXH64_canonical_t*)dest, val64);
+      case CRYPTO_DIGEST_XXH3_128:
+         *length = CRYPTO_DIGEST_XXH3_128_SIZE;
+         val128 = XXH3_128bits_digest((XXH3_state_t *)digest->ctx);
+         XXH128_canonicalFromHash((XXH128_canonical_t *)dest, val128);
+         break;
+      default:
+         Dmsg1(150, "unknown digest %d\n", digest->type);
+         ret = false;
+         break;
+      }
+      return ret;
    } else {
-      return true;
+      unsigned int v;
+      if (!EVP_DigestFinal(digest->ctx, dest, &v)) {
+         Dmsg0(150, "digest finalize failed\n");
+         openssl_post_errors(digest->jcr, M_ERROR, _("OpenSSL digest finalize failed"));
+         return false;
+      } else {
+         *length = v;
+         return true;
+      }
    }
 }
 
@@ -683,8 +778,27 @@ bool crypto_digest_finalize(DIGEST *digest, uint8_t *dest, uint32_t *length)
  */
 void crypto_digest_free(DIGEST *digest)
 {
-  EVP_MD_CTX_free(digest->ctx);
-  free(digest);
+   switch (digest->type) {
+   case CRYPTO_DIGEST_MD5:
+   case CRYPTO_DIGEST_SHA1:
+#ifdef HAVE_SHA2
+   case CRYPTO_DIGEST_SHA256:
+   case CRYPTO_DIGEST_SHA512:
+#endif
+      EVP_MD_CTX_free((EVP_MD_CTX *)digest->ctx);
+      break;
+   case CRYPTO_DIGEST_XXHASH64:
+      XXH64_freeState((XXH64_state_t *)digest->ctx);
+      break;
+   case CRYPTO_DIGEST_XXH3_64:
+   case CRYPTO_DIGEST_XXH3_128:
+      XXH3_freeState((XXH3_state_t *)digest->ctx);
+      break;
+   default:
+      Dmsg1(150, "UNKNOWN digest %d !!!\n", digest->type);
+      break;
+   }
+   free(digest);
 }
 
 /*
@@ -1533,6 +1647,12 @@ const char *crypto_digest_name(DIGEST *digest)
       return "SHA256";
    case CRYPTO_DIGEST_SHA512:
       return "SHA512";
+   case CRYPTO_DIGEST_XXHASH64:
+      return "XXHASH64";
+   case CRYPTO_DIGEST_XXH3_64:
+      return "XXH3_64";
+   case CRYPTO_DIGEST_XXH3_128:
+      return "XXH3_128";
    case CRYPTO_DIGEST_NONE:
       return "None";
    default:
@@ -1556,6 +1676,12 @@ crypto_digest_t crypto_digest_stream_type(int stream)
       return CRYPTO_DIGEST_SHA256;
    case STREAM_SHA512_DIGEST:
       return CRYPTO_DIGEST_SHA512;
+   case STREAM_XXHASH64_DIGEST:
+      return CRYPTO_DIGEST_XXHASH64;
+   case STREAM_XXH3_64_DIGEST:
+      return CRYPTO_DIGEST_XXH3_64;
+   case STREAM_XXH3_128_DIGEST:
+      return CRYPTO_DIGEST_XXH3_128;
    default:
       return CRYPTO_DIGEST_NONE;
    }

diff --git a/bacula/src/lib/crypto.h b/bacula/src/lib/crypto.h
index d0e34b7ed654d3aba617c23274a787722d3901a4..efac78fd915cc29e167463ba23d192ffe8271b33 100644 (file)
--- a/bacula/src/lib/crypto.h
+++ b/bacula/src/lib/crypto.h
@@ -64,9 +64,13 @@ typedef enum {
    CRYPTO_DIGEST_MD5 = 1,
    CRYPTO_DIGEST_SHA1 = 2,
    CRYPTO_DIGEST_SHA256 = 3,
-   CRYPTO_DIGEST_SHA512 = 4
+   CRYPTO_DIGEST_SHA512 = 4,
+   CRYPTO_DIGEST_XXHASH64 = 5,
+   CRYPTO_DIGEST_XXH3_128 = 6,
+   CRYPTO_DIGEST_XXH3_64 = 7 /* Never released, maybe already deprecated ? */
 } crypto_digest_t;
 
+#define IS_XXHASH_DIGEST(type) (CRYPTO_DIGEST_XXHASH64<=(type) && (type)<=CRYPTO_DIGEST_XXH3_128)
 
 #ifdef HAVE_SHA2
 # define CRYPTO_DIGEST_DEFAULT CRYPTO_DIGEST_SHA256
@@ -100,6 +104,9 @@ typedef enum {
 #define CRYPTO_DIGEST_SHA1_SIZE 20    /* 160 bits */
 #define CRYPTO_DIGEST_SHA256_SIZE 32  /* 256 bits */
 #define CRYPTO_DIGEST_SHA512_SIZE 64  /* 512 bits */
+#define CRYPTO_DIGEST_XXHASH64_SIZE 8 /*  64bits */
+#define CRYPTO_DIGEST_XXH3_64_SIZE   8 /*  64bits */
+#define CRYPTO_DIGEST_XXH3_128_SIZE 16 /* 128bits */
 
 /* Maximum Message Digest Size */
 #ifdef HAVE_OPENSSL

diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c
index 8bed0750f3b28328258438423a7f735024529f89..c91f6f7252e788fd9e99df0695421504be39e606 100644 (file)
--- a/bacula/src/stored/bextract.c
+++ b/bacula/src/stored/bextract.c
@@ -722,6 +722,9 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
    case STREAM_SHA1_DIGEST:
    case STREAM_SHA256_DIGEST:
    case STREAM_SHA512_DIGEST:
+   case STREAM_XXHASH64_DIGEST:
+   case STREAM_XXH3_64_DIGEST:
+   case STREAM_XXH3_128_DIGEST:
       break;
 
    case STREAM_SIGNED_DIGEST:

diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c
index 4a2b6c0f5640fc0e0fbfd304766003aff04e623c..80f9f011233b862ea07200ecf37047307ed71403 100644 (file)
--- a/bacula/src/stored/bscan.c
+++ b/bacula/src/stored/bscan.c
@@ -946,6 +946,30 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
       update_digest_record(db, digest, rec, CRYPTO_DIGEST_SHA512);
       break;
 
+   case STREAM_XXHASH64_DIGEST:
+      bin_to_base64(digest, sizeof(digest), (char *)rec->data, CRYPTO_DIGEST_XXHASH64_SIZE, true);
+      if (verbose > 1) {
+         Pmsg1(000, _("Got XXHASH64 record: %s\n"), digest);
+      }
+      update_digest_record(db, digest, rec, CRYPTO_DIGEST_XXHASH64);
+      break;
+
+   case STREAM_XXH3_64_DIGEST:
+      bin_to_base64(digest, sizeof(digest), (char *)rec->data, CRYPTO_DIGEST_XXH3_64_SIZE, true);
+      if (verbose > 1) {
+         Pmsg1(000, _("Got XXH3_64 record: %s\n"), digest);
+      }
+      update_digest_record(db, digest, rec, CRYPTO_DIGEST_XXH3_64);
+      break;
+
+   case STREAM_XXH3_128_DIGEST:
+      bin_to_base64(digest, sizeof(digest), (char *)rec->data, CRYPTO_DIGEST_XXH3_128_SIZE, true);
+      if (verbose > 1) {
+         Pmsg1(000, _("Got XXH3_128 record: %s\n"), digest);
+      }
+      update_digest_record(db, digest, rec, CRYPTO_DIGEST_XXH3_128);
+      break;
+
    case STREAM_ENCRYPTED_SESSION_DATA:
       // TODO landonf: Investigate crypto support in bscan
       if (verbose > 1) {

diff --git a/bacula/src/stored/record_util.c b/bacula/src/stored/record_util.c
index bd24d896b9dac1efdc5e5712ff47a3b9dca73eee..a9c80a54913cf963d5a6bb1bcc7e8ee50dc1a316 100644 (file)
--- a/bacula/src/stored/record_util.c
+++ b/bacula/src/stored/record_util.c
@@ -163,6 +163,12 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
          return "contADATA-RECORD-HEADER";
       case STREAM_FILEEVENT:
          return _("FileEvent");
+      case STREAM_XXHASH64_DIGEST:
+         return "contXXHASH64";
+      case STREAM_XXH3_64_DIGEST:
+         return "contXXH3_63";
+      case STREAM_XXH3_128_DIGEST:
+         return "contXXH3_128";
       default:
          sprintf(buf, "%d", -stream);
          return buf;
@@ -242,6 +248,12 @@ const char *stream_to_ascii(char *buf, int stream, int fi)
       return "ADATA-RECORD-HEADER";
    case STREAM_FILEEVENT:
       return _("FileEvent");
+   case STREAM_XXHASH64_DIGEST:
+      return "XXHASH64";
+   case STREAM_XXH3_64_DIGEST:
+      return "XXH3_63";
+   case STREAM_XXH3_128_DIGEST:
+      return "XXH3_128";
    default:
       sprintf(buf, "%d", stream);
       return buf;

diff --git a/bacula/src/streams.h b/bacula/src/streams.h
index dc155f582b655817d2da4f5b29fbec8e9da7804e..226fb8706c4d11841ffa417e51a7d48d141fa2c2 100644 (file)
--- a/bacula/src/streams.h
+++ b/bacula/src/streams.h
@@ -107,6 +107,9 @@
 #define STREAM_PLUGIN_META_CATALOG             36    /* Plugin metadata (to be stored in catalog) for file being backed up */
 #define STREAM_UNIX_ATTRIBUTE_UPDATE           37    /* File's updated metadata */
 #define STREAM_FILEEVENT                       38    /* FileEvent associated with the current object */
+#define STREAM_XXHASH64_DIGEST                 39    /* XXHASH64 digest for the file */
+#define STREAM_XXH3_64_DIGEST                  40    /* XXH3_64 digest for the file */
+#define STREAM_XXH3_128_DIGEST                 41    /* XXH3_128 digest for the file */
 
 #define STREAM_ADATA_BLOCK_HEADER             200    /* Adata block header */
 #define STREAM_ADATA_RECORD_HEADER            201    /* Adata record header */