From: Eric Bollengier Date: Tue, 12 Oct 2021 15:28:58 +0000 (+0200) Subject: Add ZSTD compression support X-Git-Tag: Beta-15.0.0~831 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=0fb00ea135afbd461bd8d9ea70b51e06132e85ed;p=thirdparty%2Fbacula.git Add ZSTD compression support --- diff --git a/bacula/src/ch.h b/bacula/src/ch.h index e6ba94034..3d5f0b480 100644 --- a/bacula/src/ch.h +++ b/bacula/src/ch.h @@ -31,6 +31,7 @@ #define COMPRESS_NONE 0x4e4f4e45 /* used for incompressible block */ #define COMPRESS_GZIP 0x475a4950 #define COMPRESS_LZO1X 0x4c5a4f58 +#define COMPRESS_ZSTD 0x4c5a4f59 /* * Compression header version diff --git a/bacula/src/dird/inc_conf.c b/bacula/src/dird/inc_conf.c index 0a685c54c..a43f0354f 100644 --- a/bacula/src/dird/inc_conf.c +++ b/bacula/src/dird/inc_conf.c @@ -202,6 +202,10 @@ struct s_fs_opt FS_options[] = { {"Gzip8", INC_KW_COMPRESSION, "Z8"}, {"Gzip9", INC_KW_COMPRESSION, "Z9"}, {"Lzo", INC_KW_COMPRESSION, "Zo"}, + {"zstd", INC_KW_COMPRESSION, "Zs"}, + {"zstd1", INC_KW_COMPRESSION, "Zr"}, + {"zstd10", INC_KW_COMPRESSION, "Zs"}, + {"zstd19", INC_KW_COMPRESSION, "Zt"}, {"blowfish", INC_KW_ENCRYPTION, "B"}, /* ***FIXME*** not implemented */ {"3des", INC_KW_ENCRYPTION, "3"}, /* ***FIXME*** not implemented */ {"Storage", INC_KW_DEDUP, "d1"}, diff --git a/bacula/src/filed/Makefile.in b/bacula/src/filed/Makefile.in index 29101ba41..499557a91 100644 --- a/bacula/src/filed/Makefile.in +++ b/bacula/src/filed/Makefile.in @@ -55,6 +55,8 @@ FDLIBS = @FDLIBS@ # extra libs for File daemon ZLIBS = @ZLIBS@ LZO_LIBS = @LZO_LIBS@ LZO_INC= @LZO_INC@ +ZSTD_LIBS = @ZSTD_LIBS@ +ZSTD_INC = @ZSTD_INC@ # extra items for linking on Win32 WIN32OBJS = win32/winmain.o win32/winlib.a win32/winres.res @@ -69,7 +71,7 @@ WIN32LIBS = $(@WIN32@) # inference rules .c.o: @echo "Compiling $<" - $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< + $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< #------------------------------------------------------------------------- all: Makefile @WIN32@ bacula-fd @STATIC_FD@ bfdjson @echo "==== Make of filed is good ====" @@ -77,15 +79,15 @@ all: Makefile @WIN32@ bacula-fd @STATIC_FD@ bfdjson bacgpfs.o: bacgpfs.c bacgpfs.h @echo "Compiling $<" - $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $(GPFS_CFLAGS) $< + $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $(GPFS_CFLAGS) $< bacl.o: bacl.c bacgpfs.o @echo "Compiling $<" - $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $< + $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $(AFS_CFLAGS) $< bxattr.o: bxattr.c bacgpfs.o @echo "Compiling $<" - $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< + $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(WCFLAGS) $(CPPFLAGS) $(LZO_INC) $(ZSTD_INC) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) $< win32/winlib.a: @if test -f win32/Makefile -a "${GMAKE}" != "none"; then \ @@ -116,18 +118,18 @@ bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../ @echo "Linking $@ ..." $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(SVROBJS) \ $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ - $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(IOKITLIBS) + $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) $(IOKITLIBS) bfdjson: Makefile $(JSONOBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@ @echo "Linking $@ ..." $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(JSONOBJS) \ $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ - $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) + $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) static-bacula-fd: Makefile $(SVROBJS) ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @WIN32@ $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -static -L../lib -L../findlib -o $@ $(SVROBJS) \ $(WIN32LIBS) $(FDLIBS) $(ZLIBS) -lbacfind -lbaccfg -lbac -lm $(LIBS) \ - $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) + $(DLIB) $(WRAPLIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) $(CAP_LIBS) $(AFS_LIBS) $(LZO_LIBS) $(ZSTD_LIBS) strip $@ Makefile: $(srcdir)/Makefile.in $(topdir)/config.status @@ -194,7 +196,7 @@ depend: @$(MV) Makefile Makefile.bak @$(SED) "/^# DO NOT DELETE:/,$$ d" Makefile.bak > Makefile @$(ECHO) "# DO NOT DELETE: nice dependency list follows" >> Makefile - @$(CXX) -S -M $(CPPFLAGS) $(XINC) $(LZO_INC) $(AFS_CFLAGS) $(GPFS_CFLAGS) -I$(srcdir) -I$(basedir) *.c >> Makefile + @$(CXX) -S -M $(CPPFLAGS) $(XINC) $(LZO_INC) $(ZSTD_INC) $(AFS_CFLAGS) $(GPFS_CFLAGS) -I$(srcdir) -I$(basedir) *.c >> Makefile @if test -f Makefile ; then \ $(RMF) Makefile.bak; \ else \ diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 51034fb4a..abedc64a2 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -27,17 +27,8 @@ #include "filed.h" #include "backup.h" -#ifdef HAVE_LZO -const bool have_lzo = true; -#else -const bool have_lzo = false; -#endif - -#ifdef HAVE_LIBZ -const bool have_libz = true; -#else -const bool have_libz = false; -#endif +/* Get have_lzo, have_zstd, ... */ +#include "compress.h" /* Forward referenced functions */ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level); @@ -48,6 +39,7 @@ static void close_vss_backup_session(JCR *jcr); static bool send_resource_fork(bctx_t &bctx); #endif static bool setup_compression(bctx_t &bctx); +static bool do_zstd_compression(bctx_t &bctx); static bool do_lzo_compression(bctx_t &bctx); static bool do_libz_compression(bctx_t &bctx); @@ -146,6 +138,11 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) } #endif +#ifdef HAVE_ZSTD + ZSTD_CCtx *pZSTD = ZSTD_createCCtx(); + jcr->ZSTD_compress_workset = pZSTD; +#endif + if (!crypto_session_start(jcr)) { return false; } @@ -228,6 +225,10 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) bfree_and_null(jcr->LZO_compress_workset); } +#ifdef HAVE_ZSTD + ZSTD_freeCCtx((ZSTD_CCtx*)jcr->ZSTD_compress_workset); +#endif + crypto_session_end(jcr); bdelete_and_null(jcr->sd_packet_mgr); @@ -943,6 +944,10 @@ bool process_and_send_data(bctx_t &bctx) goto err; } + if (have_zstd && !do_zstd_compression(bctx)) { + goto err; + } + /** * Note, here we prepend the current record length to the beginning * of the encrypted data. This is because both sparse and compression @@ -1251,17 +1256,21 @@ static bool setup_compression(bctx_t &bctx) { JCR *jcr = bctx.jcr; -#if defined(HAVE_LIBZ) || defined(HAVE_LZO) +#if defined(HAVE_LIBZ) || defined(HAVE_LZO) || defined(HAVE_ZSTD) bctx.compress_len = 0; bctx.max_compress_len = 0; bctx.cbuf = NULL; - #ifdef HAVE_LIBZ + bctx.cbuf2 = NULL; + memset(&bctx.ch, 0, sizeof(comp_stream_header)); + +# ifdef HAVE_LIBZ int zstat; if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_GZIP) { if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) { bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE; bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE; + } else { bctx.cbuf = (unsigned char *)jcr->compress_buf; bctx.max_compress_len = jcr->compress_buf_size; /* set max length */ @@ -1285,16 +1294,18 @@ static bool setup_compression(bctx_t &bctx) } } } - #endif - #ifdef HAVE_LZO - memset(&bctx.ch, 0, sizeof(comp_stream_header)); - bctx.cbuf2 = NULL; +# endif + /* cbuf points to the destination compression header + * cbuf2 points to the destination compression data + */ +# ifdef HAVE_LZO if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_LZO1X) { if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) { bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE; bctx.cbuf2 = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header); bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE; + } else { bctx.cbuf = (unsigned char *)jcr->compress_buf; bctx.cbuf2 = (unsigned char *)jcr->compress_buf + sizeof(comp_stream_header); @@ -1305,7 +1316,25 @@ static bool setup_compression(bctx_t &bctx) bctx.wbuf = jcr->compress_buf; /* compressed output here */ bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ } - #endif +# endif +# ifdef HAVE_ZSTD + if ((bctx.ff_pkt->flags & FO_COMPRESS) && bctx.ff_pkt->Compress_algo == COMPRESS_ZSTD) { + if ((bctx.ff_pkt->flags & FO_SPARSE) || (bctx.ff_pkt->flags & FO_OFFSETS)) { + bctx.cbuf = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE; + bctx.cbuf2 = (unsigned char *)jcr->compress_buf + OFFSET_FADDR_SIZE + sizeof(comp_stream_header); + bctx.max_compress_len = jcr->compress_buf_size - OFFSET_FADDR_SIZE; + + } else { + bctx.cbuf = (unsigned char *)jcr->compress_buf; + bctx.cbuf2 = (unsigned char *)jcr->compress_buf + sizeof(comp_stream_header); + bctx.max_compress_len = jcr->compress_buf_size; /* set max length */ + } + bctx.ch.magic = COMPRESS_ZSTD; + bctx.ch.version = COMP_HEAD_VERSION; + bctx.wbuf = jcr->compress_buf; /* compressed output here */ + bctx.cipher_input = (uint8_t *)jcr->compress_buf; /* encrypt compressed data */ + } +# endif #endif return true; } @@ -1453,6 +1482,53 @@ static bool do_lzo_compression(bctx_t &bctx) return true; } +static bool do_zstd_compression(bctx_t &bctx) +{ +#ifdef HAVE_ZSTD + JCR *jcr = bctx.jcr; + BSOCK *sd = bctx.sd; + + /** Do compression if turned on */ + if (bctx.ff_pkt->flags & FO_COMPRESS && bctx.ff_pkt->Compress_algo == COMPRESS_ZSTD) { + if (jcr->ZSTD_compress_workset == NULL) { + Jmsg(jcr, M_FATAL, 0, _("Compression ZSTD error. Unable to allocate the ZSTD context\n")); + return false; + } + ser_declare; + ser_begin(bctx.cbuf, sizeof(comp_stream_header)); + + Dmsg3(400, "cbuf=0x%x rbuf=0x%x len=%u\n", bctx.cbuf, bctx.rbuf, sd->msglen); + size_t ret = ZSTD_compressCCtx((ZSTD_CCtx*) jcr->ZSTD_compress_workset, + bctx.cbuf2, bctx.max_compress_len, + bctx.rbuf, sd->msglen, + 10); + + if (ZSTD_isError(ret)) { + /** this should NEVER happen */ + Jmsg(jcr, M_FATAL, 0, _("Compression ZSTD error: %d\n"), ret); + jcr->setJobStatus(JS_ErrorTerminated); + return false; + + } + + bctx.compress_len = ret; + /* complete header */ + ser_uint32(COMPRESS_ZSTD); + ser_uint32(bctx.compress_len); + ser_uint16(bctx.ch.level); + ser_uint16(bctx.ch.version); + + Dmsg2(400, "ZSTD compressed len=%d uncompressed len=%d\n", bctx.compress_len, + sd->msglen); + + bctx.compress_len += sizeof(comp_stream_header); /* add size of header */ + sd->msglen = bctx.compress_len; /* set compressed length */ + bctx.cipher_input_len = bctx.compress_len; + } +#endif + return true; +} + /* * Do in place strip of path */ diff --git a/bacula/src/filed/compress.h b/bacula/src/filed/compress.h new file mode 100644 index 000000000..09276d727 --- /dev/null +++ b/bacula/src/filed/compress.h @@ -0,0 +1,42 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2023 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. +*/ + +#ifndef COMPRESS_H +#define COMPRESS_H + +#ifdef HAVE_LZO +const bool have_lzo = true; +#else +const bool have_lzo = false; +#endif + +#ifdef HAVE_ZSTD +const bool have_zstd = true; +#else +const bool have_zstd = false; +#endif + +#ifdef HAVE_LIBZ +static const char *zlib_strerror(int stat); +const bool have_libz = true; +#else +const bool have_libz = false; +#endif + +#endif // COMPRESS_H diff --git a/bacula/src/filed/filed.h b/bacula/src/filed/filed.h index 950ce6c3f..5858e09b0 100644 --- a/bacula/src/filed/filed.h +++ b/bacula/src/filed/filed.h @@ -59,6 +59,9 @@ #else #define uLongf uint32_t #endif +#ifdef HAVE_ZSTD +#include +#endif #ifdef HAVE_LZO #include #include diff --git a/bacula/src/filed/job.c b/bacula/src/filed/job.c index 1d6620f38..fcbe3ab07 100644 --- a/bacula/src/filed/job.c +++ b/bacula/src/filed/job.c @@ -2075,6 +2075,22 @@ static int set_options(findFOPTS *fo, const char *opts) fo->Compress_algo = COMPRESS_LZO1X; fo->Compress_level = 1; /* not used with LZO */ } + else if (*p >= 'r' && *p <= 't') { + fo->flags |= FO_COMPRESS; + fo->Compress_algo = COMPRESS_ZSTD; + switch(*p) { + case 'r': + fo->Compress_level = 1; + break; + case 't': + fo->Compress_level = 19; + break; + case 's': + default: + fo->Compress_level = 10; + break; + } + } break; case 'd': /* Deduplication 0=none 1=Storage 2=Local */ p++; /* skip d */ diff --git a/bacula/src/filed/restore.c b/bacula/src/filed/restore.c index 57181bfc2..69e067665 100644 --- a/bacula/src/filed/restore.c +++ b/bacula/src/filed/restore.c @@ -62,18 +62,8 @@ const bool have_xattr = false; /* Data received from Storage Daemon */ static char rec_header[] = "rechdr %ld %ld %ld %ld %ld"; -/* Forward referenced functions */ -#if defined(HAVE_LIBZ) -static const char *zlib_strerror(int stat); -const bool have_libz = true; -#else -const bool have_libz = false; -#endif -#ifdef HAVE_LZO -const bool have_lzo = true; -#else -const bool have_lzo = false; -#endif +/* Get have_lzo, have_zstd, ... */ +#include "compress.h" static void deallocate_cipher(r_ctx &rctx); static void deallocate_fork_cipher(r_ctx &rctx); @@ -400,7 +390,7 @@ void do_restore(JCR *jcr) jcr->buf_size = sd->msglen; /* use the same buffer size to decompress both gzip and lzo */ - if (have_libz || have_lzo) { + if (have_libz || have_lzo || have_zstd) { uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100; jcr->compress_buf = get_memory(compress_buf_size); jcr->compress_buf_size = compress_buf_size; @@ -411,12 +401,17 @@ void do_restore(JCR *jcr) fdmsg->start_read_sock(); bmessage *bmsg = fdmsg->new_msg(); /* get a message, to exchange with fdmsg */ -#ifdef HAVE_LZO - if (lzo_init() != LZO_E_OK) { - Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n")); - goto get_out; + if (have_zstd) { + ZSTD_DCtx *pZSTD = ZSTD_createDCtx(); + jcr->ZSTD_decompress_workset = pZSTD; + } + + if (have_lzo) { + if (lzo_init() != LZO_E_OK) { + Jmsg(jcr, M_FATAL, 0, _("LZO init failed\n")); + goto get_out; + } } -#endif if (have_crypto) { rctx.cipher_ctx.buf = get_memory(CRYPTO_CIPHER_MAX_BLOCK_SIZE); @@ -1221,6 +1216,9 @@ ok_out: jcr->compress_buf = NULL; jcr->compress_buf_size = 0; } +#ifdef HAVE_ZSTD + ZSTD_freeDCtx((ZSTD_DCtx_s*)jcr->ZSTD_decompress_workset); +#endif #ifdef HAVE_ACL if (jcr->bacl) { @@ -1312,7 +1310,7 @@ bool sparse_data(JCR *jcr, BFILE *bfd, uint64_t *addr, char **data, uint32_t *le bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length) { -#if defined(HAVE_LZO) || defined(HAVE_LIBZ) +#if defined(HAVE_LZO) || defined(HAVE_LIBZ) || defined(HAVE_ZSTD) char ec1[50]; /* Buffer printing huge values */ #endif @@ -1327,6 +1325,9 @@ bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length) const unsigned char *cbuf; int r, real_compress_len; #endif +#ifdef HAVE_ZSTD + unsigned long long rSize; +#endif /* read compress header */ unser_declare; @@ -1376,6 +1377,32 @@ bool decompress_data(JCR *jcr, int32_t stream, char **data, uint32_t *length) *length = compress_len; Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1)); return true; +#endif +#ifdef HAVE_ZSTD + case COMPRESS_ZSTD: + compress_len = jcr->compress_buf_size; + cbuf = (const unsigned char*)*data + sizeof(comp_stream_header); + real_compress_len = *length - sizeof(comp_stream_header); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, *length); + rSize = ZSTD_getFrameContentSize(cbuf, real_compress_len); + if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) { + Qmsg(jcr, M_ERROR, 0, _("ZSTD uncompression error on file %s. Invalid content size. ERR=%d\n"), + jcr->last_fname, rSize); + return false; + } + jcr->compress_buf = check_pool_memory_size(jcr->compress_buf, rSize); + rSize = ZSTD_decompressDCtx((ZSTD_DCtx*)jcr->ZSTD_decompress_workset, + (unsigned char *)jcr->compress_buf, compress_len, cbuf, real_compress_len); + + if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) { + Qmsg(jcr, M_ERROR, 0, _("ZSTD uncompression error on file %s. Decompress error. ERR=%d\n"), + jcr->last_fname, rSize); + return false; + } + *data = jcr->compress_buf; + *length = compress_len = rSize; + Dmsg2(200, "Write uncompressed %d bytes, total before write=%s\n", compress_len, edit_uint64(jcr->JobBytes, ec1)); + return true; #endif default: Qmsg(jcr, M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); diff --git a/bacula/src/filed/status.c b/bacula/src/filed/status.c index 7b30cef8b..317ea923a 100644 --- a/bacula/src/filed/status.c +++ b/bacula/src/filed/status.c @@ -64,11 +64,8 @@ void output_status(STATUS_PKT *sp) list_terminated_jobs(sp); /* defined in lib/status.h */ } -#if defined(HAVE_LZO) -static const bool have_lzo = true; -#else -static const bool have_lzo = false; -#endif +/* Get have_lzo, have_zstd, ... */ +#include "compress.h" /* from filed_conf.c */ extern s_ct ciphertypes[]; @@ -201,13 +198,14 @@ static void list_status_header(STATUS_PKT *sp) p_SetCurrentDirectoryA?"":"!", p_SetCurrentDirectoryW?"":"!"); sendit(msg.c_str(), len, sp); - len = Mmsg(msg, " %sGCDA,%sGCDW,%sGVPNW,%sGVNFVMPW,%sLZO,%sEFS\n", + len = Mmsg(msg, " %sGCDA,%sGCDW,%sGVPNW,%sGVNFVMPW,%sLZO,%sEFS,%sZSTD\n", p_GetCurrentDirectoryA?"":"!", p_GetCurrentDirectoryW?"":"!", p_GetVolumePathNameW?"":"!", p_GetVolumeNameForVolumeMountPointW?"":"!", have_lzo?"":"!", - (BEEF>0)?"":"!"); + (BEEF>0)?"":"!", + have_zstd?"":"!"); sendit(msg.c_str(), len, sp); } #endif diff --git a/bacula/src/filed/verify_vol.c b/bacula/src/filed/verify_vol.c index 793a2ad75..5be5fae93 100644 --- a/bacula/src/filed/verify_vol.c +++ b/bacula/src/filed/verify_vol.c @@ -42,6 +42,12 @@ const bool have_lzo = true; const bool have_lzo = false; #endif +#ifdef HAVE_ZSTD +const bool have_zstd = true; +#else +const bool have_zstd = false; +#endif + /* Context used during Verify Data job. We use it in the * verify loop to compute checksums and check attributes. */ @@ -293,7 +299,7 @@ void do_verify_volume(JCR *jcr) jcr->buf_size = sd->msglen; /* use the same buffer size to decompress both gzip and lzo */ - if (have_libz || have_lzo) { + if (have_libz || have_lzo || have_zstd) { uint32_t compress_buf_size = jcr->buf_size + 12 + ((jcr->buf_size+999) / 1000) + 100; jcr->compress_buf = get_memory(compress_buf_size); jcr->compress_buf_size = compress_buf_size; diff --git a/bacula/src/findlib/attribs.c b/bacula/src/findlib/attribs.c index 405c4afba..82bb542af 100644 --- a/bacula/src/findlib/attribs.c +++ b/bacula/src/findlib/attribs.c @@ -227,7 +227,7 @@ int select_data_stream(FF_PKT *ff_pkt) /* * Handle compression and encryption options */ -#if defined(HAVE_LIBZ) || defined(HAVE_LZO) +#if defined(HAVE_LIBZ) || defined(HAVE_LZO) || defined(HAVE_ZSTD) if (ff_pkt->flags & FO_COMPRESS) { #ifdef HAVE_LIBZ if(ff_pkt->Compress_algo == COMPRESS_GZIP) { @@ -252,8 +252,8 @@ int select_data_stream(FF_PKT *ff_pkt) } } #endif - #ifdef HAVE_LZO - if(ff_pkt->Compress_algo == COMPRESS_LZO1X) { + #if defined(HAVE_LZO) || defined(HAVE_ZSTD) + if(ff_pkt->Compress_algo == COMPRESS_LZO1X || ff_pkt->Compress_algo == COMPRESS_ZSTD) { switch (stream) { case STREAM_WIN32_DATA: stream = STREAM_WIN32_COMPRESSED_DATA; diff --git a/bacula/src/findlib/bfile.c b/bacula/src/findlib/bfile.c index 7e67984f5..b4f03f8ea 100644 --- a/bacula/src/findlib/bfile.c +++ b/bacula/src/findlib/bfile.c @@ -373,7 +373,7 @@ bool is_restore_stream_supported(int stream) case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #endif -#ifndef HAVE_LZO +#if !defined(HAVE_LZO) && !defined(HAVE_ZSTD) case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: @@ -391,7 +391,7 @@ bool is_restore_stream_supported(int stream) case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #endif -#ifdef HAVE_LZO +#if defined(HAVE_LZO) || defined(HAVE_ZSTD) case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: @@ -415,7 +415,7 @@ bool is_restore_stream_supported(int stream) case STREAM_ENCRYPTED_FILE_GZIP_DATA: case STREAM_ENCRYPTED_WIN32_DATA: case STREAM_ENCRYPTED_WIN32_GZIP_DATA: -#ifdef HAVE_LZO +#if defined(HAVE_LZO) || defined(HAVE_ZSTD) case STREAM_ENCRYPTED_FILE_COMPRESSED_DATA: case STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA: #endif @@ -925,7 +925,7 @@ bool is_restore_stream_supported(int stream) case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #endif -#ifndef HAVE_LZO +#if !defined(HAVE_LZO) && !defined(HAVE_ZSTD) case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: @@ -944,7 +944,7 @@ bool is_restore_stream_supported(int stream) case STREAM_SPARSE_GZIP_DATA: case STREAM_WIN32_GZIP_DATA: #endif -#ifdef HAVE_LZO +#if defined(HAVE_LZO) || defined(HAVE_ZSTD) case STREAM_COMPRESSED_DATA: case STREAM_SPARSE_COMPRESSED_DATA: case STREAM_WIN32_COMPRESSED_DATA: diff --git a/bacula/src/jcr.h b/bacula/src/jcr.h index 66535ffca..50ab23a3f 100644 --- a/bacula/src/jcr.h +++ b/bacula/src/jcr.h @@ -453,6 +453,8 @@ public: int32_t compress_buf_size; /* Length of compression buffer */ void *pZLIB_compress_workset; /* zlib compression session data */ void *LZO_compress_workset; /* lzo compression session data */ + void *ZSTD_compress_workset; /* zstd compression session data */ + void *ZSTD_decompress_workset; /* zstd decompression session data */ int32_t replace; /* Replace options */ int32_t buf_size; /* length of buffer */ FF_PKT *ff; /* Find Files packet */ diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in index 064f607f6..723735412 100644 --- a/bacula/src/stored/Makefile.in +++ b/bacula/src/stored/Makefile.in @@ -144,6 +144,8 @@ CAP_LIBS = @CAP_LIBS@ ZLIBS=@ZLIBS@ LZO_LIBS= @LZO_LIBS@ LZO_INC= @LZO_INC@ +ZSTD_LIBS = @ZSTD_LIBS@ +ZSTD_INC = @ZSTD_INC@ TOKYOCABINET_LIBS = @TOKYOCABINET_LIBS@ TOKYOCABINET_INC = @TOKYOCABINET_INC@ @@ -302,11 +304,11 @@ bls: Makefile $(BLSOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARC bextract.o: bextract.c @echo "Compiling $<" $(NO_ECHO)$(CXX) $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) \ - -I$(basedir) $(DINCLUDE) $(CFLAGS) $(LZO_INC) $< + -I$(basedir) $(DINCLUDE) $(CFLAGS) $(LZO_INC) $(ZSTD_INC) $< bextract: Makefile $(BEXTOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_ARCHIVE_TYPE) ../lib/libbaccfg$(DEFAULT_ARCHIVE_TYPE) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) @echo "Compiling $<" - $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZO_LIBS) \ + $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(BEXTOBJS) $(DLIB) $(ZLIBS) $(LZO_LIBS) $(ZSTD_LIBS) \ $(SD_LIBS) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) bscan.o: bscan.c diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c index b8d9710d9..64a49eae1 100644 --- a/bacula/src/stored/bextract.c +++ b/bacula/src/stored/bextract.c @@ -29,6 +29,11 @@ #include "ch.h" #include "findlib/find.h" +#ifdef HAVE_ZSTD +#include +ZSTD_DCtx *ZSTD_decompress_workset = NULL; +#endif + #ifdef HAVE_LZO #include #include @@ -240,6 +245,9 @@ int main (int argc, char *argv[]) Pmsg1(000, _("%d Win32 data or Win32 gzip data stream records. Ignored.\n"), win32_data_msg); } +#ifdef HAVE_ZSTD + ZSTD_freeDCtx(ZSTD_decompress_workset); +#endif term_include_exclude_files(ff); term_find_files(ff); return 0; @@ -669,7 +677,39 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len); break; #endif - default: +#ifdef HAVE_ZSTD + case COMPRESS_ZSTD: + { + if (!ZSTD_decompress_workset) { + ZSTD_decompress_workset = ZSTD_createDCtx(); + } + compress_len = compress_buf_size; + cbuf = (const unsigned char*) wbuf + sizeof(comp_stream_header); + real_compress_len = wsize - sizeof(comp_stream_header); + Dmsg2(200, "Comp_len=%d msglen=%d\n", compress_len, wsize); + + unsigned long long rSize = ZSTD_getFrameContentSize(compress_buf, real_compress_len); + if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) { + Emsg1(M_ERROR, 0, _("ZSTD uncompression error. ERR=%d\n"), rSize); + extract = false; + goto bail_out; + } + compress_buf = check_pool_memory_size(compress_buf, rSize); + rSize = ZSTD_decompressDCtx(ZSTD_decompress_workset, (unsigned char *)compress_buf, compress_len, cbuf, real_compress_len); + + if (rSize == ZSTD_CONTENTSIZE_ERROR || rSize == ZSTD_CONTENTSIZE_UNKNOWN) { + Emsg1(M_ERROR, 0, _("ZSTD uncompression error. ERR=%d\n"), rSize); + goto bail_out; + } + Dmsg2(100, "Write uncompressed %d bytes, total before write=%d\n", compress_len, total); + store_data(rec->Stream, &bfd, compress_buf, compress_len); + total += compress_len; + fileAddr += compress_len; + Dmsg2(100, "Compress len=%d uncompressed=%d\n", rec->data_len, compress_len); + } + break; +#endif + default: Emsg1(M_ERROR, 0, _("Compression algorithm 0x%x found, but not supported!\n"), comp_magic); extract = false; goto bail_out; diff --git a/bacula/src/win32/Makefile.inc.in b/bacula/src/win32/Makefile.inc.in index 458edc115..c754edc43 100644 --- a/bacula/src/win32/Makefile.inc.in +++ b/bacula/src/win32/Makefile.inc.in @@ -131,6 +131,9 @@ LIBS_ZLIB := \ LIBS_LZO := \ $(DEPKGS)/lib/liblzo2.a +LIBS_ZSTD := \ + $(DEPKGS)/lib/libzstd.a + LIBS_MYSQL := \ $(DEPKGS)/lib/libmysql.a @@ -161,6 +164,7 @@ HAVES := \ -DHAVE_MINGW \ -DHAVE_ZLIB_H \ -DHAVE_LZO \ + -DHAVE_ZSTD \ -DHAVE_LIBZ \ -DHAVE_CRYPTO \ -DHAVE_OPENSSL \ diff --git a/bacula/src/win32/compat/mingwconfig.h b/bacula/src/win32/compat/mingwconfig.h index 251fdbfbd..1ff114cb9 100644 --- a/bacula/src/win32/compat/mingwconfig.h +++ b/bacula/src/win32/compat/mingwconfig.h @@ -76,6 +76,9 @@ /* Define if you have zlib */ #define HAVE_LIBZ 1 +/* Define if you have ZSTD */ +#define HAVE_ZSTD 1 + /* Define if you have lzo lib */ #define HAVE_LZO 1 diff --git a/regress/scripts/new-test-bacula-dir.conf.in b/regress/scripts/new-test-bacula-dir.conf.in index a8520f245..c0bc9be1f 100644 --- a/regress/scripts/new-test-bacula-dir.conf.in +++ b/regress/scripts/new-test-bacula-dir.conf.in @@ -271,6 +271,34 @@ Job { Maximum Concurrent Jobs = 10 } +Job { + Name = "ZSTDTest" + Type = Backup + Client=@hostname@-fd + FileSet="ZSTDSet" + Storage = File + Messages = Standard + Pool = Default + Maximum Concurrent Jobs = 10 + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes +} + +Job { + Name = "SparseZSTDTest" + Type = Backup + Client=@hostname@-fd + FileSet="SparseZSTDSet" + Storage = File + Messages = Standard + Pool = Default + Write Bootstrap = "@working_dir@/NightlySave.bsr" + Max Run Time = 30min + SpoolData=yes + Maximum Concurrent Jobs = 10 +} + Job { Name = "FIFOTest" Type = Backup @@ -590,6 +618,17 @@ FileSet { } } +FileSet { + Name = "ZSTDSet" + Include { + Options { + signature=MD5 + compression=ZSTD + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "DeltaSet" Include { @@ -637,6 +676,18 @@ FileSet { } } +FileSet { + Name = "SparseZSTDSet" + Include { + Options { + signature=MD5 + compression=ZSTD + sparse=yes + } + File = <@tmpdir@/file-list + } +} + FileSet { Name = "MonsterFileSet" Include {