Add ZSTD compression support

diff --git a/bacula/src/ch.h b/bacula/src/ch.h
index e6ba94034120d52dcc36b0af01e32082c46a4b65..3d5f0b4804de7e7f0f1afb1f7327bcf179559d01 100644 (file)
--- a/bacula/src/ch.h
+++ b/bacula/src/ch.h
@@ -31,6 +31,7 @@
 #define COMPRESS_NONE  0x4e4f4e45  /* used for incompressible block */
 #define COMPRESS_GZIP  0x475a4950
 #define COMPRESS_LZO1X 0x4c5a4f58
+#define COMPRESS_ZSTD  0x4c5a4f59
 
 /*
  * Compression header version

diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c
index 0a685c54cdac4e3f0391d31e5dd6b65e76a03ef6..a43f0354fbf784994653a895a024cbb9ed01e854 100644 (file)
--- a/bacula/src/dird/inc_conf.c
+++ b/bacula/src/dird/inc_conf.c
@@ -202,6 +202,10 @@ struct s_fs_opt FS_options[] = {
    {"Gzip8",    INC_KW_COMPRESSION,  "Z8"},
    {"Gzip9",    INC_KW_COMPRESSION,  "Z9"},
    {"Lzo",      INC_KW_COMPRESSION,  "Zo"},
+   {"zstd",     INC_KW_COMPRESSION,  "Zs"},
+   {"zstd1",    INC_KW_COMPRESSION,  "Zr"},
+   {"zstd10",   INC_KW_COMPRESSION,  "Zs"},
+   {"zstd19",   INC_KW_COMPRESSION,  "Zt"},
    {"blowfish", INC_KW_ENCRYPTION,    "B"},   /* ***FIXME*** not implemented */
    {"3des",     INC_KW_ENCRYPTION,    "3"},   /* ***FIXME*** not implemented */
    {"Storage",  INC_KW_DEDUP,        "d1"},

diff --git a/bacula/src/filed/Makefile.in b/bacula/src/filed/Makefile.in
index 29101ba41e9ff7361e40cdcf9f4fe419a138f0f5..499557a91f27f6355e783848f293cbae3ba48f89 100644 (file)
--- a/bacula/src/filed/Makefile.in
+++ b/bacula/src/filed/Makefile.in
@@ -55,6 +55,8 @@ FDLIBS = @FDLIBS@          # extra libs for File daemon
 ZLIBS = @ZLIBS@
 LZO_LIBS = @LZO_LIBS@
 LZO_INC= @LZO_INC@
+ZSTD_LIBS = @ZSTD_LIBS@
+ZSTD_INC = @ZSTD_INC@
 
 # extra items for linking on Win32
 WIN32OBJS = win32/winmain.o win32/winlib.a win32/winres.res
@@ -69,7 +71,7 @@ WIN32LIBS = $(@WIN32@)
 # inference rules
 .c.o:
        @echo "Compiling $<"
-       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $<
+       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $<
 #-------------------------------------------------------------------------
 all: Makefile @WIN32@ bacula-fd @STATIC_FD@ bfdjson
        @echo "==== Make of filed is good ===="
@@ -77,15 +79,15 @@ all: Makefile @WIN32@ bacula-fd @STATIC_FD@ bfdjson
 
 bacgpfs.o: bacgpfs.c bacgpfs.h
        @echo "Compiling $<"
-       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $(GPFS_CFLAGS) $<
+       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $(GPFS_CFLAGS) $<
 
 bacl.o: bacl.c bacgpfs.o
        @echo "Compiling $<"
-       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $<
+       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $<
 
 bxattr.o: bxattr.c bacgpfs.o
        @echo "Compiling $<"
-       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $<
+       $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $<
 
 win32/winlib.a:
        @if test -f win32/Makefile -a "${GMAKE}" != "none"; then \
@@ -116,18 +118,18 @@ bacula-fd:  Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../
        @echo "Linking $@ ..."
        $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(SVROBJS) \
          $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \
-         $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(IOKITLIBS)
+         $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) $(IOKITLIBS)
 
 bfdjson:  Makefile $(JSONOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@
        @echo "Linking $@ ..."
        $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(JSONOBJS) \
          $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \
-         $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS)
+         $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS)
 
 static-bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@
        $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -L../findlib -o $@ $(SVROBJS) \
           $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \
-          $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS)
+          $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS)
        strip $@
 
 Makefile: $(srcdir)/Makefile.in $(topdir)/config.status
@@ -194,7 +196,7 @@ depend:
        @$(MV) Makefile Makefile.bak
        @$(SED) "/^# DO NOT DELETE:/,$$ d" Makefile.bak > Makefile
        @$(ECHO) "# DO NOT DELETE: nice dependency list follows" >> Makefile
-       @$(CXX) -S -M $(CPPFLAGS) $(XINC) $(LZO_INC) $(AFS_CFLAGS) $(GPFS_CFLAGS) -I$(srcdir) -I$(basedir) *.c >> Makefile
+       @$(CXX) -S -M $(CPPFLAGS) $(XINC) $(LZO_INC) $(ZSTD_INC) $(AFS_CFLAGS) $(GPFS_CFLAGS) -I$(srcdir) -I$(basedir) *.c >> Makefile
        @if test -f Makefile ; then \
            $(RMF) Makefile.bak; \
        else \

diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c
index 51034fb4ad9a5b8c4380d6cc256212493d175696..abedc64a29f084a75d86c9e6a38aa04b373075e2 100644 (file)
--- a/bacula/src/filed/backup.c
+++ b/bacula/src/filed/backup.c
@@ -27,17 +27,8 @@
 #include "filed.h"
 #include "backup.h"
 
-#ifdef HAVE_LZO
-const bool have_lzo = true;
-#else
-const bool have_lzo = false;
-#endif
-
-#ifdef HAVE_LIBZ
-const bool have_libz = true;
-#else
-const bool have_libz = false;
-#endif
+/* Get have_lzo, have_zstd, ... */
+#include "compress.h"
 
 /* Forward referenced functions */
 int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level);
@@ -48,6 +39,7 @@ static void close_vss_backup_session(JCR *jcr);
 static bool send_resource_fork(bctx_t &bctx);
 #endif
 static bool setup_compression(bctx_t &bctx);
+static bool do_zstd_compression(bctx_t &bctx);
 static bool do_lzo_compression(bctx_t &bctx);
 static bool do_libz_compression(bctx_t &bctx);
 
@@ -146,6 +138,11 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
    }
 #endif
 
+#ifdef HAVE_ZSTD
+   ZSTD_CCtx *pZSTD = ZSTD_createCCtx();
+   jcr->ZSTD_compress_workset = pZSTD;
+#endif
+
    if (!crypto_session_start(jcr)) {
       return false;
    }
@@ -228,6 +225,10 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr)
       bfree_and_null(jcr->LZO_compress_workset);
    }
 
+#ifdef HAVE_ZSTD
+   ZSTD_freeCCtx((ZSTD_CCtx*)jcr->ZSTD_compress_workset);
+#endif
+   
    crypto_session_end(jcr);
 
    bdelete_and_null(jcr->sd_packet_mgr);
@@ -943,6 +944,10 @@ bool process_and_send_data(bctx_t &bctx)
       goto err;
    }
 
+   if (have_zstd && !do_zstd_compression(bctx)) {
+      goto err;
+   }
+
    /**
     * Note, here we prepend the current record length to the beginning
     *  of the encrypted data. This is because both sparse and compression
@@ -1251,17 +1256,21 @@ static bool setup_compression(bctx_t &bctx)
 {
    JCR *jcr = bctx.jcr;
 
-#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
+#if defined(HAVE_LIBZ) || defined(HAVE_LZO) || defined(HAVE_ZSTD)
    bctx.compress_len = 0;
    bctx.max_compress_len = 0;
    bctx.cbuf = NULL;
- #ifdef HAVE_LIBZ
+   bctx.cbuf2 = NULL;
+   memset(&bctx.ch, 0, sizeof(comp_stream_header));
+
+# ifdef HAVE_LIBZ
    int zstat;
 
    if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_GZIP) {
       if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
          bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE;
          bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+
       } else {
          bctx.cbuf = (unsigned char *)jcr->compress_buf;
          bctx.max_compress_len = jcr->compress_buf_size; /* set max length */
@@ -1285,16 +1294,18 @@ static bool setup_compression(bctx_t &bctx)
          }
       }
    }
- #endif
- #ifdef HAVE_LZO
-   memset(&bctx.ch, 0, sizeof(comp_stream_header));
-   bctx.cbuf2 = NULL;
+# endif
 
+   /* cbuf points to the destination compression header
+    * cbuf2 points to the destination compression data
+    */
+# ifdef HAVE_LZO
    if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_LZO1X) {
       if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
          bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE;
          bctx.cbuf2 = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
          bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+
       } else {
          bctx.cbuf = (unsigned char *)jcr->compress_buf;
          bctx.cbuf2 = (unsigned char *)jcr->compress_buf + sizeof(comp_stream_header);
@@ -1305,7 +1316,25 @@ static bool setup_compression(bctx_t &bctx)
       bctx.wbuf = jcr->compress_buf;    /* compressed output here */
       bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
    }
- #endif
+# endif
+# ifdef HAVE_ZSTD
+   if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_ZSTD) {
+      if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) {
+         bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE;
+         bctx.cbuf2 = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header);
+         bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE;
+
+      } else {
+         bctx.cbuf = (unsigned char *)jcr->compress_buf;
+         bctx.cbuf2 = (unsigned char *)jcr->compress_buf + sizeof(comp_stream_header);
+         bctx.max_compress_len = jcr->compress_buf_size; /* set max length */
+      }
+      bctx.ch.magic = COMPRESS_ZSTD;
+      bctx.ch.version = COMP_HEAD_VERSION;
+      bctx.wbuf = jcr->compress_buf;    /* compressed output here */
+      bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */
+   }
+# endif
 #endif
    return true;
 }
@@ -1453,6 +1482,53 @@ static bool do_lzo_compression(bctx_t &bctx)
    return true;
 }
 
+static bool do_zstd_compression(bctx_t &bctx)
+{
+#ifdef HAVE_ZSTD
+   JCR *jcr = bctx.jcr;
+   BSOCK *sd = bctx.sd;
+
+   /** Do compression if turned on */
+   if (bctx.ff_pkt->flags & FO_COMPRESS && bctx.ff_pkt->Compress_algo == COMPRESS_ZSTD) {
+      if (jcr->ZSTD_compress_workset == NULL) {
+         Jmsg(jcr, M_FATAL, 0, _("Compression ZSTD error. Unable to allocate the ZSTD context\n"));
+         return false;
+      }
+      ser_declare;
+      ser_begin(bctx.cbuf, sizeof(comp_stream_header));
+
+      Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", bctx.cbuf, bctx.rbuf, sd->msglen);
+      size_t ret = ZSTD_compressCCtx((ZSTD_CCtx*) jcr->ZSTD_compress_workset,
+                                     bctx.cbuf2, bctx.max_compress_len,
+                                     bctx.rbuf, sd->msglen,
+                                     10);
+      
+      if (ZSTD_isError(ret)) {
+         /** this should NEVER happen */
+         Jmsg(jcr, M_FATAL, 0, _("Compression ZSTD error: %d\n"), ret);
+         jcr->setJobStatus(JS_ErrorTerminated);
+         return false;
+
+      }
+
+      bctx.compress_len = ret;
+      /* complete header */
+      ser_uint32(COMPRESS_ZSTD);
+      ser_uint32(bctx.compress_len);
+      ser_uint16(bctx.ch.level);
+      ser_uint16(bctx.ch.version);
+
+      Dmsg2(400, "ZSTD compressed len=%d uncompressed len=%d\n", bctx.compress_len,
+            sd->msglen);
+
+      bctx.compress_len += sizeof(comp_stream_header); /* add size of header */
+      sd->msglen = bctx.compress_len;      /* set compressed length */
+      bctx.cipher_input_len = bctx.compress_len;
+   }
+#endif
+   return true;
+}
+
 /*
  * Do in place strip of path
  */

diff --git a/bacula/src/filed/compress.h b/bacula/src/filed/compress.h
new file mode 100644 (file)
index 0000000..09276d7
--- /dev/null
+++ b/bacula/src/filed/compress.h
@@ -0,0 +1,42 @@
+/*
+   Bacula(R) - The Network Backup Solution
+
+   Copyright (C) 2000-2023 Kern Sibbald
+
+   The original author of Bacula is Kern Sibbald, with contributions
+   from many others, a complete list can be found in the file AUTHORS.
+
+   You may use this file and others of this release according to the
+   license defined in the LICENSE file, which includes the Affero General
+   Public License, v3.0 ("AGPLv3") and some additional permissions and
+   terms pursuant to its AGPLv3 Section 7.
+
+   This notice must be preserved when any source code is
+   conveyed and/or propagated.
+
+   Bacula(R) is a registered trademark of Kern Sibbald.
+*/
+
+#ifndef COMPRESS_H
+#define COMPRESS_H
+
+#ifdef HAVE_LZO
+const bool have_lzo = true;
+#else
+const bool have_lzo = false;
+#endif
+
+#ifdef HAVE_ZSTD
+const bool have_zstd = true;
+#else
+const bool have_zstd = false;
+#endif
+
+#ifdef HAVE_LIBZ
+static const char *zlib_strerror(int stat);
+const bool have_libz = true;
+#else
+const bool have_libz = false;
+#endif
+
+#endif  // COMPRESS_H

diff --git a/bacula/src/filed/filed.h b/bacula/src/filed/filed.h
index 950ce6c3fd2a2ebdf00c6cd64a0a04236cf70440..5858e09b06794b4e1e28da44eb86cf7d0dc28893 100644 (file)
--- a/bacula/src/filed/filed.h
+++ b/bacula/src/filed/filed.h
@@ -59,6 +59,9 @@
 #else
 #define uLongf uint32_t
 #endif
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+#endif
 #ifdef HAVE_LZO
 #include <lzo/lzoconf.h>
 #include <lzo/lzo1x.h>

diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c
index 1d6620f389d1fd1968dc48147b412b799845b683..fcbe3ab07e3842ad3c067f433b29be4771e787fc 100644 (file)
--- a/bacula/src/filed/job.c
+++ b/bacula/src/filed/job.c
@@ -2075,6 +2075,22 @@ static int set_options(findFOPTS *fo, const char *opts)
             fo->Compress_algo = COMPRESS_LZO1X;
             fo->Compress_level = 1; /* not used with LZO */
          }
+         else if (*p >= 'r' && *p <= 't') {
+            fo->flags |= FO_COMPRESS;
+            fo->Compress_algo = COMPRESS_ZSTD;
+            switch(*p) {
+            case 'r':
+               fo->Compress_level = 1;
+               break;
+            case 't':
+               fo->Compress_level = 19;
+               break;
+            case 's':
+            default:
+               fo->Compress_level = 10;
+               break;
+            }
+         }
          break;
       case 'd':                 /* Deduplication 0=none 1=Storage 2=Local */
          p++;                   /* skip d */

diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c
index 57181bfc2219230c3f8bf4f8bc62637420ec2611..69e067665878ff9d3ef1e2af389c088d8d5e82a3 100644 (file)
--- a/bacula/src/filed/restore.c
+++ b/bacula/src/filed/restore.c
@@ -62,18 +62,8 @@ const bool have_xattr = false;
 /* Data received from Storage Daemon */
 static char rec_header[] = "rechdr %ld %ld %ld %ld %ld";
 
-/* Forward referenced functions */
-#if   defined(HAVE_LIBZ)
-static const char *zlib_strerror(int stat);
-const bool have_libz = true;
-#else
-const bool have_libz = false;
-#endif
-#ifdef HAVE_LZO
-const bool have_lzo = true;
-#else
-const bool have_lzo = false;
-#endif
+/* Get have_lzo, have_zstd, ... */
+#include "compress.h"
 
 static void deallocate_cipher(r_ctx &rctx);
 static void deallocate_fork_cipher(r_ctx &rctx);
@@ -400,7 +390,7 @@ void do_restore(JCR *jcr)
    jcr->buf_size = sd->msglen;
 
    /* use the same buffer size to decompress both gzip and lzo */
-   if (have_libz || have_lzo) {
+   if (have_libz || have_lzo || have_zstd) {
       uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100;
       jcr->compress_buf = get_memory(compress_buf_size);
       jcr->compress_buf_size = compress_buf_size;
@@ -411,12 +401,17 @@ void do_restore(JCR *jcr)
    fdmsg->start_read_sock();
    bmessage *bmsg = fdmsg->new_msg(); /* get a message, to exchange with fdmsg */
 
-#ifdef HAVE_LZO
-   if (lzo_init() != LZO_E_OK) {
-      Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n"));
-      goto get_out;
+   if (have_zstd) {
+      ZSTD_DCtx *pZSTD = ZSTD_createDCtx();
+      jcr->ZSTD_decompress_workset = pZSTD;
+   }
+
+   if (have_lzo) {
+      if (lzo_init() != LZO_E_OK) {
+         Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n"));
+         goto get_out;
+      }
    }
-#endif
 
    if (have_crypto) {
       rctx.cipher_ctx.buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE);
@@ -1221,6 +1216,9 @@ ok_out:
       jcr->compress_buf = NULL;
       jcr->compress_buf_size = 0;
    }
+#ifdef HAVE_ZSTD
+   ZSTD_freeDCtx((ZSTD_DCtx_s*)jcr->ZSTD_decompress_workset);
+#endif
 
 #ifdef HAVE_ACL
    if (jcr->bacl) {
@@ -1312,7 +1310,7 @@ bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *le
 
 bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length)
 {
-#if defined(HAVE_LZO) || defined(HAVE_LIBZ)
+#if defined(HAVE_LZO) || defined(HAVE_LIBZ) || defined(HAVE_ZSTD)
    char ec1[50];                   /* Buffer printing huge values */
 #endif
 
@@ -1327,6 +1325,9 @@ bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length)
       const unsigned char *cbuf;
       int r, real_compress_len;
 #endif
+#ifdef HAVE_ZSTD
+      unsigned long long rSize;
+#endif
 
       /* read compress header */
       unser_declare;
@@ -1376,6 +1377,32 @@ bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length)
             *length = compress_len;
             Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1));
             return true;
+#endif
+#ifdef HAVE_ZSTD
+         case COMPRESS_ZSTD:
+            compress_len = jcr->compress_buf_size;
+            cbuf = (const unsigned char*)*data + sizeof(comp_stream_header);
+            real_compress_len = *length - sizeof(comp_stream_header);
+            Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length);
+            rSize = ZSTD_getFrameContentSize(cbuf, real_compress_len);
+            if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+               Qmsg(jcr, M_ERROR, 0, _("ZSTD uncompression error on file %s. Invalid content size. ERR=%d\n"),
+                    jcr->last_fname, rSize);
+               return false;
+            }
+            jcr->compress_buf = check_pool_memory_size(jcr->compress_buf, rSize);
+            rSize = ZSTD_decompressDCtx((ZSTD_DCtx*)jcr->ZSTD_decompress_workset,
+                                        (unsigned char *)jcr->compress_buf, compress_len, cbuf, real_compress_len);
+
+            if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+               Qmsg(jcr, M_ERROR, 0, _("ZSTD uncompression error on file %s. Decompress error. ERR=%d\n"),
+                    jcr->last_fname, rSize);
+               return false;
+            }
+            *data = jcr->compress_buf;
+            *length = compress_len = rSize;
+            Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1));
+            return true;
 #endif
          default:
             Qmsg(jcr, M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic);

diff --git a/bacula/src/filed/status.c b/bacula/src/filed/status.c
index 7b30cef8b44f0008f786c8acdbc717b93be12b33..317ea923a832b61efb385b0a0240f19c978ba4a4 100644 (file)
--- a/bacula/src/filed/status.c
+++ b/bacula/src/filed/status.c
@@ -64,11 +64,8 @@ void output_status(STATUS_PKT *sp)
    list_terminated_jobs(sp);    /* defined in lib/status.h */
 }
 
-#if defined(HAVE_LZO)
-static const bool have_lzo = true;
-#else
-static const bool have_lzo = false;
-#endif
+/* Get have_lzo, have_zstd, ... */
+#include "compress.h"
 
 /* from filed_conf.c */
 extern s_ct ciphertypes[];
@@ -201,13 +198,14 @@ static void  list_status_header(STATUS_PKT *sp)
                  p_SetCurrentDirectoryA?"":"!",
                  p_SetCurrentDirectoryW?"":"!");
       sendit(msg.c_str(), len, sp);
-      len = Mmsg(msg, " %sGCDA,%sGCDW,%sGVPNW,%sGVNFVMPW,%sLZO,%sEFS\n",
+      len = Mmsg(msg, " %sGCDA,%sGCDW,%sGVPNW,%sGVNFVMPW,%sLZO,%sEFS,%sZSTD\n",
                  p_GetCurrentDirectoryA?"":"!",
                  p_GetCurrentDirectoryW?"":"!",
                  p_GetVolumePathNameW?"":"!",
                  p_GetVolumeNameForVolumeMountPointW?"":"!",
                  have_lzo?"":"!",
-                 (BEEF>0)?"":"!");
+                 (BEEF>0)?"":"!",
+                 have_zstd?"":"!");
       sendit(msg.c_str(), len, sp);
    }
 #endif

diff --git a/bacula/src/filed/verify_vol.c b/bacula/src/filed/verify_vol.c
index 793a2ad75d5fb6b989da9beed5695c8205c89146..5be5fae932cb19b294796312ec2fe6373ff9568b 100644 (file)
--- a/bacula/src/filed/verify_vol.c
+++ b/bacula/src/filed/verify_vol.c
@@ -42,6 +42,12 @@ const bool have_lzo = true;
 const bool have_lzo = false;
 #endif
 
+#ifdef HAVE_ZSTD
+const bool have_zstd = true;
+#else
+const bool have_zstd = false;
+#endif
+
 /* Context used during Verify Data job. We use it in the
  * verify loop to compute checksums and check attributes.
  */
@@ -293,7 +299,7 @@ void do_verify_volume(JCR *jcr)
    jcr->buf_size = sd->msglen;
 
    /* use the same buffer size to decompress both gzip and lzo */
-   if (have_libz || have_lzo) {
+   if (have_libz || have_lzo || have_zstd) {
       uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100;
       jcr->compress_buf = get_memory(compress_buf_size);
       jcr->compress_buf_size = compress_buf_size;

diff --git a/bacula/src/findlib/attribs.c b/bacula/src/findlib/attribs.c
index 405c4afbac3312ba6d1be98acf86ef75d42afda7..82bb542af895321bca7c77949a1a621531148697 100644 (file)
--- a/bacula/src/findlib/attribs.c
+++ b/bacula/src/findlib/attribs.c
@@ -227,7 +227,7 @@ int select_data_stream(FF_PKT *ff_pkt)
    /*
     * Handle compression and encryption options
     */
-#if defined(HAVE_LIBZ) || defined(HAVE_LZO)
+#if defined(HAVE_LIBZ) || defined(HAVE_LZO) || defined(HAVE_ZSTD)
    if (ff_pkt->flags & FO_COMPRESS) {
       #ifdef HAVE_LIBZ
          if(ff_pkt->Compress_algo == COMPRESS_GZIP) {
@@ -252,8 +252,8 @@ int select_data_stream(FF_PKT *ff_pkt)
             }
          }
       #endif
-      #ifdef HAVE_LZO
-         if(ff_pkt->Compress_algo == COMPRESS_LZO1X) {
+      #if defined(HAVE_LZO) || defined(HAVE_ZSTD)
+         if(ff_pkt->Compress_algo == COMPRESS_LZO1X || ff_pkt->Compress_algo == COMPRESS_ZSTD) {
             switch (stream) {
             case STREAM_WIN32_DATA:
                   stream = STREAM_WIN32_COMPRESSED_DATA;

diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c
index 7e67984f5fe379c47e9be21ab54aea7d013822db..b4f03f8ea28a16498ad87c976572c4abd53153a4 100644 (file)
--- a/bacula/src/findlib/bfile.c
+++ b/bacula/src/findlib/bfile.c
@@ -373,7 +373,7 @@ bool is_restore_stream_supported(int stream)
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
 #endif
-#ifndef HAVE_LZO
+#if !defined(HAVE_LZO) && !defined(HAVE_ZSTD)
    case STREAM_COMPRESSED_DATA:
    case STREAM_SPARSE_COMPRESSED_DATA:
    case STREAM_WIN32_COMPRESSED_DATA:
@@ -391,7 +391,7 @@ bool is_restore_stream_supported(int stream)
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
 #endif
-#ifdef HAVE_LZO
+#if defined(HAVE_LZO) || defined(HAVE_ZSTD)
    case STREAM_COMPRESSED_DATA:
    case STREAM_SPARSE_COMPRESSED_DATA:
    case STREAM_WIN32_COMPRESSED_DATA:
@@ -415,7 +415,7 @@ bool is_restore_stream_supported(int stream)
    case STREAM_ENCRYPTED_FILE_GZIP_DATA:
    case STREAM_ENCRYPTED_WIN32_DATA:
    case STREAM_ENCRYPTED_WIN32_GZIP_DATA:
-#ifdef HAVE_LZO
+#if defined(HAVE_LZO) || defined(HAVE_ZSTD)
    case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA:
    case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA:
 #endif
@@ -925,7 +925,7 @@ bool is_restore_stream_supported(int stream)
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
 #endif
-#ifndef HAVE_LZO
+#if !defined(HAVE_LZO) && !defined(HAVE_ZSTD)
    case STREAM_COMPRESSED_DATA:
    case STREAM_SPARSE_COMPRESSED_DATA:
    case STREAM_WIN32_COMPRESSED_DATA:
@@ -944,7 +944,7 @@ bool is_restore_stream_supported(int stream)
    case STREAM_SPARSE_GZIP_DATA:
    case STREAM_WIN32_GZIP_DATA:
 #endif
-#ifdef HAVE_LZO
+#if defined(HAVE_LZO) || defined(HAVE_ZSTD)
    case STREAM_COMPRESSED_DATA:
    case STREAM_SPARSE_COMPRESSED_DATA:
    case STREAM_WIN32_COMPRESSED_DATA:

diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h
index 66535ffca4b009a1497894858e3fc996e0d71e1f..50ab23a3f3474c97851e8fe8e5bbe4a39425baca 100644 (file)
--- a/bacula/src/jcr.h
+++ b/bacula/src/jcr.h
@@ -453,6 +453,8 @@ public:
    int32_t compress_buf_size;         /* Length of compression buffer */
    void *pZLIB_compress_workset;      /* zlib compression session data */
    void *LZO_compress_workset;        /* lzo compression session data */
+   void *ZSTD_compress_workset;       /* zstd compression session data */
+   void *ZSTD_decompress_workset;     /* zstd decompression session data */
    int32_t replace;                   /* Replace options */
    int32_t buf_size;                  /* length of buffer */
    FF_PKT *ff;                        /* Find Files packet */

diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in
index 064f607f642a17ce4fa13488538b2e59e4e1d502..7237354124b7e1371b9de20da632a9b900c6dfd3 100644 (file)
--- a/bacula/src/stored/Makefile.in
+++ b/bacula/src/stored/Makefile.in
@@ -144,6 +144,8 @@ CAP_LIBS = @CAP_LIBS@
 ZLIBS=@ZLIBS@
 LZO_LIBS= @LZO_LIBS@
 LZO_INC= @LZO_INC@
+ZSTD_LIBS = @ZSTD_LIBS@
+ZSTD_INC = @ZSTD_INC@
 TOKYOCABINET_LIBS = @TOKYOCABINET_LIBS@
 TOKYOCABINET_INC = @TOKYOCABINET_INC@
 
@@ -302,11 +304,11 @@ bls:      Makefile $(BLSOBJS) libbacsd.la drivers  ../findlib/libbacfind$(DEFAULT_ARC
 bextract.o: bextract.c
        @echo "Compiling $<"
        $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \
-          -I$(basedir) $(DINCLUDE) $(CFLAGS) $(LZO_INC) $<
+          -I$(basedir) $(DINCLUDE) $(CFLAGS) $(LZO_INC) $(ZSTD_INC) $<
 
 bextract: Makefile $(BEXTOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE)
        @echo "Compiling $<"
-       $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZO_LIBS) \
+       $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZO_LIBS) $(ZSTD_LIBS) \
           $(SD_LIBS) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS)
 
 bscan.o: bscan.c

diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c
index b8d9710d9fd9584f647d570678e2e7ba89afe03e..64a49eae102ea2a7cb7bc073ffc99713f5c82acc 100644 (file)
--- a/bacula/src/stored/bextract.c
+++ b/bacula/src/stored/bextract.c
@@ -29,6 +29,11 @@
 #include "ch.h"
 #include "findlib/find.h"
 
+#ifdef HAVE_ZSTD
+#include <zstd.h>
+ZSTD_DCtx *ZSTD_decompress_workset = NULL;
+#endif
+
 #ifdef HAVE_LZO
 #include <lzo/lzoconf.h>
 #include <lzo/lzo1x.h>
@@ -240,6 +245,9 @@ int main (int argc, char *argv[])
       Pmsg1(000, _("%d Win32 data or Win32 gzip data stream records. Ignored.\n"),
          win32_data_msg);
    }
+#ifdef HAVE_ZSTD
+   ZSTD_freeDCtx(ZSTD_decompress_workset);
+#endif
    term_include_exclude_files(ff);
    term_find_files(ff);
    return 0;
@@ -669,7 +677,39 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
                Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len);
                break;
 #endif
-            default:
+#ifdef HAVE_ZSTD
+            case COMPRESS_ZSTD:
+            {
+               if (!ZSTD_decompress_workset) {
+                  ZSTD_decompress_workset = ZSTD_createDCtx();
+               }
+               compress_len = compress_buf_size;
+               cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header);
+               real_compress_len = wsize - sizeof(comp_stream_header);
+               Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize);
+
+               unsigned long long rSize = ZSTD_getFrameContentSize(compress_buf, real_compress_len);
+               if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+                  Emsg1(M_ERROR, 0, _("ZSTD uncompression error. ERR=%d\n"), rSize);
+                  extract = false;
+                  goto bail_out;
+               }
+               compress_buf = check_pool_memory_size(compress_buf, rSize);
+               rSize = ZSTD_decompressDCtx(ZSTD_decompress_workset, (unsigned char *)compress_buf, compress_len, cbuf, real_compress_len);
+
+               if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) {
+                  Emsg1(M_ERROR, 0, _("ZSTD uncompression error. ERR=%d\n"), rSize);
+                  goto bail_out;
+               }
+               Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total);
+               store_data(rec->Stream, &bfd, compress_buf, compress_len);
+               total += compress_len;
+               fileAddr += compress_len;
+               Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len);
+            }
+            break;
+#endif
+          default:
                Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic);
                extract = false;
                goto bail_out;

diff --git a/bacula/src/win32/Makefile.inc.in b/bacula/src/win32/Makefile.inc.in
index 458edc1159045e8ffad5d5c8216abf3ddab7ac6c..c754edc4381cc1b6749ced2fb372e85b1050ae67 100644 (file)
--- a/bacula/src/win32/Makefile.inc.in
+++ b/bacula/src/win32/Makefile.inc.in
@@ -131,6 +131,9 @@ LIBS_ZLIB := \
 LIBS_LZO := \
         $(DEPKGS)/lib/liblzo2.a
 
+LIBS_ZSTD := \
+        $(DEPKGS)/lib/libzstd.a
+
 LIBS_MYSQL := \
         $(DEPKGS)/lib/libmysql.a
 
@@ -161,6 +164,7 @@ HAVES := \
         -DHAVE_MINGW \
         -DHAVE_ZLIB_H \
         -DHAVE_LZO \
+        -DHAVE_ZSTD \
         -DHAVE_LIBZ \
         -DHAVE_CRYPTO \
         -DHAVE_OPENSSL \

diff --git a/bacula/src/win32/compat/mingwconfig.h b/bacula/src/win32/compat/mingwconfig.h
index 251fdbfbdb515809b21559ed20872097bc30ccff..1ff114cb93e72adf7ad47aeffcf54124b927cd94 100644 (file)
--- a/bacula/src/win32/compat/mingwconfig.h
+++ b/bacula/src/win32/compat/mingwconfig.h
@@ -76,6 +76,9 @@
 /* Define if you have zlib */
 #define HAVE_LIBZ 1
 
+/* Define if you have ZSTD */
+#define HAVE_ZSTD 1
+
 /* Define if you have lzo lib */
 #define HAVE_LZO 1
 

diff --git a/regress/scripts/new-test-bacula-dir.conf.in b/regress/scripts/new-test-bacula-dir.conf.in
index a8520f245fca8865f97edf86e00a02fa96b8e614..c0bc9be1fed3f4edbbab773254936dbe6f50e47c 100644 (file)
--- a/regress/scripts/new-test-bacula-dir.conf.in
+++ b/regress/scripts/new-test-bacula-dir.conf.in
@@ -271,6 +271,34 @@ Job {
   Maximum Concurrent Jobs = 10
 }
 
+Job {
+  Name = "ZSTDTest"
+  Type = Backup
+  Client=@hostname@-fd
+  FileSet="ZSTDSet"
+  Storage = File
+  Messages = Standard
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+  Write Bootstrap = "@working_dir@/NightlySave.bsr"
+  Max Run Time = 30min
+  SpoolData=yes
+}
+
+Job {
+  Name = "SparseZSTDTest"
+  Type = Backup
+  Client=@hostname@-fd
+  FileSet="SparseZSTDSet"
+  Storage = File
+  Messages = Standard
+  Pool = Default
+  Write Bootstrap = "@working_dir@/NightlySave.bsr"
+  Max Run Time = 30min
+  SpoolData=yes
+  Maximum Concurrent Jobs = 10
+}
+
 Job {
   Name = "FIFOTest"
   Type = Backup
@@ -590,6 +618,17 @@ FileSet {
   }
 }
 
+FileSet {
+  Name = "ZSTDSet"
+  Include {
+    Options {
+      signature=MD5
+      compression=ZSTD
+    }
+    File = <@tmpdir@/file-list
+  }
+}
+
 FileSet {
   Name = "DeltaSet"
   Include {
@@ -637,6 +676,18 @@ FileSet {
   }
 }
 
+FileSet {
+  Name = "SparseZSTDSet"
+  Include {
+    Options {
+      signature=MD5
+      compression=ZSTD
+      sparse=yes
+    }
+    File = <@tmpdir@/file-list
+  }
+}
+
 FileSet {
   Name = "MonsterFileSet"
   Include {