From 8efb7e6d67db1a040e0cb7f07343a4bbd861a064 Mon Sep 17 00:00:00 2001 From: Alain Spineux Date: Wed, 25 Mar 2020 18:43:51 +0100 Subject: [PATCH] Move dedup code into dedup1 directory - separate dedup2 and dedup1 code - Inherit DedupEngine1 from DedupEngine --- bacula/src/dird/protos.h | 2 - bacula/src/lib/org_lib_dedup.h | 1 - bacula/src/stored/Makefile.in | 31 ++++--- bacula/src/stored/append.c | 1 + bacula/src/stored/bls.c | 1 - bacula/src/stored/dedup_interface.h | 122 ++++++++-------------------- bacula/src/stored/dedupstored.h | 66 --------------- bacula/src/stored/read.c | 2 +- bacula/src/stored/stored.h | 2 +- 9 files changed, 52 insertions(+), 176 deletions(-) delete mode 100644 bacula/src/stored/dedupstored.h diff --git a/bacula/src/dird/protos.h b/bacula/src/dird/protos.h index adf26a0af..ea175fcd2 100644 --- a/bacula/src/dird/protos.h +++ b/bacula/src/dird/protos.h @@ -390,5 +390,3 @@ bool catreq_get_pool_info(JCR *jcr, BSOCK *bs); # define mark_access_denied(a) #endif -/* dedup_util.c */ -bool is_dedup_ref(DEV_RECORD *rec, bool lazy); diff --git a/bacula/src/lib/org_lib_dedup.h b/bacula/src/lib/org_lib_dedup.h index 4f1a7e0de..04361696e 100644 --- a/bacula/src/lib/org_lib_dedup.h +++ b/bacula/src/lib/org_lib_dedup.h @@ -65,6 +65,5 @@ int bhash_info(int hash_id, const char **hash_name); bool is_deduplicable_stream(int stream); bool is_client_rehydration_friendly_stream(int stream); -void dedup_get_limits(int64_t *nofile, int64_t *memlock); #endif /* ORG_LIB_DEDUP_H */ diff --git a/bacula/src/stored/Makefile.in b/bacula/src/stored/Makefile.in index c01ff18ba..f35f415d1 100644 --- a/bacula/src/stored/Makefile.in +++ b/bacula/src/stored/Makefile.in @@ -110,9 +110,7 @@ CLOUDTESTOBJS = cloud_test.o $(SDCORE_OBJS) CLOUDCLIOBJS = bcloud.o $(SDCORE_OBJS) $(CLOUD_ALL_LOBJS) DEDUP_SRCS = \ - bitarray.c dde_cmd.c dde_status.c dde_vacuum.c \ - dedupstored.c dedupengine.c dde_scrub.c \ - dedup_dev.c + dedup_dev.c dedupengine.c dedup_interface.c DEDUP_OBJS = $(DEDUP_SRCS:.c=.o) DEDUP_LOBJS = $(DEDUP_SRCS:.c=.lo) @@ -139,9 +137,6 @@ LIBBACSD_LOBJS = $(LIBBACSD_SRCS:.c=.lo) LIBBACSD_LT_RELEASE = @LIBBAC_LT_RELEASE@ -TUNEDDEOBJS = bitarray.o dedupengine.o dde_scrub.o tune-dde.o - - # these are the objects that are changed by the .configure process EXTRAOBJS = @OBJLIST@ @@ -241,10 +236,14 @@ bacula-sd-cloud-gs-driver.la: Makefile $(CLOUD_GENERIC_LOBJS) bacula-sd-cloud-swift-driver.la: Makefile $(CLOUD_GENERIC_LOBJS) $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(CLOUD_GENERIC_LOBJS) -o $@ -R $(libdir) -rpath $(libdir) -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) -bacula-sd-dedup-driver.la: Makefile $(DEDUP_LOBJS) - $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(DEDUP_LOBJS) -o $@ -rpath $(libdir) -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) \ - $(TOKYOCABINET_LIBS) - +dedup1/libdedup1.la: + (cd dedup1 && make libdedup1.la || exit 1) + +bacula-sd-dedup-driver.la: Makefile $(DEDUP_LOBJS) dedup1/libdedup1.la + @echo "Making $@ ..." + $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(DEDUP_LOBJS) dedup1/libdedup1.la -o $@ -rpath $(plugindir) -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) \ + $(TOKYOCABINET_LIBS) + bacula-sd-aligned-driver.la: Makefile $(ALIGNED_LOBJS) $(LIBTOOL_LINK) $(CXX) $(LDFLAGS) -shared $(ALIGNED_LOBJS) -o $@ -rpath $(plugindir) \ -module -export-dynamic -release $(LIBBACSD_LT_RELEASE) @@ -329,12 +328,6 @@ bcopy: Makefile $(COPYOBJS) libbacsd.la drivers ../findlib/libbacfind$(DEFAULT_A $(LIBTOOL_LINK) $(CXX) $(TTOOL_LDFLAGS) $(LDFLAGS) -L../lib -L../findlib -o $@ $(COPYOBJS) \ $(SD_LIBS) -lm $(LIBS) $(GETTEXT_LIBS) $(OPENSSL_LIBS) -tune-dde: Makefile $(TUNEDDEOBJS) ../lib/libbac$(DEFAULT_ARCHIVE_TYPE) - @echo "Linking $@ ..." - $(LIBTOOL_LINK) $(CXX) $(WLDFLAGS) $(LDFLAGS) -L../lib -o $@ $(TUNEDDEOBJS) $(ZLIBS) \ - -lbac -lm $(DLIB) $(LIBS) $(TOKYOCABINET_LIBS) \ - $(GETTEXT_LIBS) - cloud_parts_test: Makefile cloud_parts.c $(RMF) cloud_parts.o $(CXX) -DTEST_PROGRAM $(DEFS) $(DEBUG) -c $(CPPFLAGS) -I$(srcdir) -I$(basedir) $(DINCLUDE) $(CFLAGS) cloud_parts.c @@ -464,7 +457,7 @@ install-aligned: bacula-sd-aligned-driver.la $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-sd-aligned-driver$(DEFAULT_SHARED_OBJECT_TYPE) $(DESTDIR)$(plugindir) $(RMF) $(DESTDIR)$(plugindir)/bacula-sd-aligned-driver.la -install-dedup: bacula-sd-dedup-driver.la +install-dedup: bacula-sd-dedup-driver.la $(MKDIR) $(DESTDIR)$(plugindir) $(LIBTOOL_INSTALL) $(INSTALL_PROGRAM) bacula-sd-dedup-driver$(DEFAULT_SHARED_OBJECT_TYPE) $(DESTDIR)$(plugindir) $(RMF) $(DESTDIR)$(plugindir)/bacula-sd-dedup-driver.la @@ -484,10 +477,12 @@ libtool-clean: @find . -name '*.lo' -print | xargs $(LIBTOOL_CLEAN) $(RMF) @$(RMF) -r .libs _libs @$(RMF) *.la + (cd dedup1 && make libtool-clean) clean: libtool-clean @$(RMF) bacula-sd stored bls bextract bpool btape shmfree core core.* a.out *.o *.bak *~ *.intpro *.extpro 1 2 3 @$(RMF) bscan bsdjson bcopy static-bacula-sd acsls-changer + (cd dedup1 && make clean) realclean: clean @$(RMF) tags bacula-sd.conf acsls-changer.conf @@ -495,6 +490,7 @@ realclean: clean distclean: realclean if test $(srcdir) = .; then $(MAKE) realclean; fi (cd $(srcdir); $(RMF) Makefile) + (cd dedup1 && make distclean) devclean: realclean if test $(srcdir) = .; then $(MAKE) realclean; fi @@ -517,6 +513,7 @@ depend: $(MV) Makefile.bak Makefile; \ echo " ======= Something went wrong with make depend. ======="; \ fi + (cd dedup1 && make depend) # ----------------------------------------------------------------------- # DO NOT DELETE: nice dependency list follows diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c index 4f463b406..722865027 100644 --- a/bacula/src/stored/append.c +++ b/bacula/src/stored/append.c @@ -25,6 +25,7 @@ #include "stored.h" #include "prepare.h" +bool is_dedup_server_side(DEVICE *dev, int32_t stream, uint64_t stream_len); /* Responses sent to the File daemon */ static char OK_data[] = "3000 OK data\n"; diff --git a/bacula/src/stored/bls.c b/bacula/src/stored/bls.c index 5fa792f57..1bb3fa128 100644 --- a/bacula/src/stored/bls.c +++ b/bacula/src/stored/bls.c @@ -26,7 +26,6 @@ #include "bacula.h" #include "stored.h" -#include "dedupstored.h" #include "findlib/find.h" #include "lib/cmd_parser.h" diff --git a/bacula/src/stored/dedup_interface.h b/bacula/src/stored/dedup_interface.h index 3e8bafae1..e8667b6fb 100644 --- a/bacula/src/stored/dedup_interface.h +++ b/bacula/src/stored/dedup_interface.h @@ -20,96 +20,44 @@ #ifndef DEDUP_INTERFACE_H_ #define DEDUP_INTERFACE_H_ -class BufferedMsgSD: public DDEConnectionBackup, public BufferedMsgBase +bool is_dedup_server_side(DEVICE *dev, int32_t stream, uint64_t stream_len); +bool is_dedup_ref(DEV_RECORD *rec, bool lazy); +void list_dedupengines(char *cmd, STATUS_PKT *sp); +void dedup_get_limits(int64_t *nofile, int64_t *memlock); +bool dedup_parse_filter(char *fltr); +void dedup_filter_record(int verbose, DCR *dcr, DEV_RECORD *rec, char *dedup_msg, int len); + +/* Interface between DEDUP and SD */ +class DedupStoredInterfaceBase { public: - POOL_MEM block; // used for BNET_CMD_STO_BLOCK - BufferedMsgSD(DedupEngine *dedupengine, JCR *jcr, BSOCK *sock, const char *a_rec_header, int32_t bufsize, int capacity, int con_capacity); - virtual ~BufferedMsgSD(); - virtual void *do_read_sock_thread(void); - virtual int commit(POOLMEM *&errmsg, uint32_t jobid) - { return DDEConnectionBackup::Commit(errmsg, jobid); }; - virtual bool dedup_store_chunk(DEV_RECORD *rec, const char *rbuf, int rbuflen, char *dedup_ref_buf, char *wdedup_ref_buf, POOLMEM *&errmsg); -}; - - -/* - * DedupStoredInterface is the JCR->dedup component in charge of handling DEDUP - * and REHYDRATION on the Storage side - * - * When doing backup, only the dedup part is in use, the rehydration part is not - * used but don't take any resources. The same is true for the opposite. - * - */ -class DedupStoredInterface: public DedupStoredInterfaceBase, DDEConnectionRestore -{ -public: - enum { di_rehydration_srv=1 }; - JCR *jcr; - -private: - bool _is_rehydration_srvside; - bool _is_thread_started; - POOLMEM *msgbuf; - POOLMEM *eblock; - -public: - - pthread_mutex_t mutex; - pthread_cond_t cond; - - bool emergency_exit; - bool is_eod_sent; /* end of backup, EOD has been sent */ - - /* deduplication */ - - /* rehydration */ - int client_rec_capacity; /* size of the buffer client on the other side, */ - /* don't send more rec before an ACK */ - int64_t sent_rec_count, recv_ack_count; /* keep count of rec sent and received ACK */ - pthread_t rehydration_thread; - - bool do_checksum_after_rehydration; /* compute checksum after the rehydration ? */ - bool use_index_for_recall; /* Use index to restore a block (or use the volume information) */ - int rehydra_check_hash; /* copied from DedupEngine */ - POOL_MEM m_errmsg; /* error buffer used in record_rehydration */ - - struct DedupReference *circular_ref; - int circular_ref_count; /* number of ref in the circular buffer */ - int circular_ref_pos; /* where to store the next ref sent to the "other" */ - int circular_ref_search; /* pos of the last hit, start searching from here */ - - DedupStoredInterface(JCR *jcr, DedupEngine *dedupengine); - ~DedupStoredInterface(); - - /* rehydration / restore */ - int start_rehydration(); /* Start rehydration thread */ - void *wait_rehydration(bool emergency=false); /* Stop rehydration thread */ - void *do_rehydration_thread(void); /* Actual thread startup function */ - int handle_rehydration_command(BSOCK *fd); - bool wait_flowcontrol_rehydration(int free_rec_count, int timeoutms); - bool do_flowcontrol_rehydration(int free_rec_count, int retry_timeoutms=250); - - void warn_rehydration_eod(); /* tell rehydration than EOD was sent */ - int record_rehydration(DCR *dcr, DEV_RECORD *rec, char *buf, POOLMEM *&errmsg, bool despite_of_error, int *chunk_size); - int add_circular_buf(DCR *dcr, DEV_RECORD *rec); - - /* Some tools like bextract may want to test the checksum after a rehydration. */ - void set_checksum_after_rehydration(bool val) { - do_checksum_after_rehydration = val; - }; - - /* Use the index during restore, or the volume information */ - void set_use_index_for_recall(bool val) { - use_index_for_recall = val; - }; - - POOLMEM *get_msgbuf() { return msgbuf; }; - bool is_rehydration_srvside() { return _is_rehydration_srvside; }; - bool is_thread_started() { return _is_thread_started; }; - void set_rehydration_srvside() { _is_rehydration_srvside = true; }; - void unset_rehydration_srvside() { _is_rehydration_srvside = false; }; + DedupStoredInterfaceBase(JCR *jcr, DedupEngine *dedupengine) {}; + virtual ~DedupStoredInterfaceBase() {}; + + // deduplication + virtual int start_deduplication() {return -1;}; + virtual void *wait_deduplication(bool emergency=false) {return NULL;}; + virtual void *do_deduplication_thread(void){return NULL;}; + + // rehydration + virtual int start_rehydration(){return -1;}; + virtual void *wait_rehydration(bool emergency=false){return NULL;}; + virtual void *do_rehydration_thread(void){return NULL;}; + virtual int handle_rehydration_command(BSOCK *fd){return -1;}; + virtual bool wait_flowcontrol_rehydration(int free_rec_count, int timeoutms){return false;}; + virtual bool do_flowcontrol_rehydration(int free_rec_count, int retry_timeoutms=250){return false;}; + virtual void warn_rehydration_eod() {}; + + virtual int record_rehydration(DCR *dcr, DEV_RECORD *rec, char *buf, POOLMEM *&errmsg, bool despite_of_error, int *chunk_size){return -1;}; + virtual int add_circular_buf(DCR *dcr, DEV_RECORD *rec){return -1;}; + virtual void set_checksum_after_rehydration(bool val) {}; + virtual void set_use_index_for_recall(bool val) {}; + + virtual POOLMEM *get_msgbuf() { return NULL; }; + virtual void unset_rehydration_srvside() { return; }; + virtual bool is_rehydration_srvside() { return false; }; + virtual bool is_thread_started() { return false; }; }; diff --git a/bacula/src/stored/dedupstored.h b/bacula/src/stored/dedupstored.h deleted file mode 100644 index bef4b5ddb..000000000 --- a/bacula/src/stored/dedupstored.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - Bacula(R) - The Network Backup Solution - - Copyright (C) 2000-2020 Kern Sibbald - - The original author of Bacula is Kern Sibbald, with contributions - from many others, a complete list can be found in the file AUTHORS. - - You may use this file and others of this release according to the - license defined in the LICENSE file, which includes the Affero General - Public License, v3.0 ("AGPLv3") and some additional permissions and - terms pursuant to its AGPLv3 Section 7. - - This notice must be preserved when any source code is - conveyed and/or propagated. - - Bacula(R) is a registered trademark of Kern Sibbald. -*/ - -#ifndef DEDUPSTORED_H_ -#define DEDUPSTORED_H_ - -bool is_dedup_server_side(DEVICE *dev, int32_t stream, uint64_t stream_len); -bool is_dedup_ref(DEV_RECORD *rec, bool lazy); -void list_dedupengines(char *cmd, STATUS_PKT *sp); -void dedup_get_limits(int64_t *nofile, int64_t *memlock); -bool dedup_parse_filter(char *fltr); -void dedup_filter_record(int verbose, DCR *dcr, DEV_RECORD *rec, char *dedup_msg, int len); - -class DedupEngine; - -/* Interface between DEDUP and SD */ -class DedupStoredInterfaceBase -{ -public: - - DedupStoredInterfaceBase(JCR *jcr, DedupEngine *dedupengine) {}; - virtual ~DedupStoredInterfaceBase() {}; - - // deduplication - virtual int start_deduplication() {return -1;}; - virtual void *wait_deduplication(bool emergency=false) {return NULL;}; - virtual void *do_deduplication_thread(void){return NULL;}; - - // rehydration - virtual int start_rehydration(){return -1;}; - virtual void *wait_rehydration(bool emergency=false){return NULL;}; - virtual void *do_rehydration_thread(void){return NULL;}; - virtual int handle_rehydration_command(BSOCK *fd){return -1;}; - virtual bool wait_flowcontrol_rehydration(int free_rec_count, int timeoutms){return false;}; - virtual bool do_flowcontrol_rehydration(int free_rec_count, int retry_timeoutms=250){return false;}; - virtual void warn_rehydration_eod() {}; - - virtual int record_rehydration(DCR *dcr, DEV_RECORD *rec, char *buf, POOLMEM *&errmsg, bool despite_of_error, int *chunk_size){return -1;}; - virtual int add_circular_buf(DCR *dcr, DEV_RECORD *rec){return -1;}; - virtual void set_checksum_after_rehydration(bool val) {}; - virtual void set_use_index_for_recall(bool val) {}; - - virtual POOLMEM *get_msgbuf() { return NULL; }; - virtual void unset_rehydration_srvside() { return; }; - virtual bool is_rehydration_srvside() { return false; }; - virtual bool is_thread_started() { return false; }; - -}; - -#endif /* DEDUPSTORED_H_ */ diff --git a/bacula/src/stored/read.c b/bacula/src/stored/read.c index c60f5b34d..5c2e2957b 100644 --- a/bacula/src/stored/read.c +++ b/bacula/src/stored/read.c @@ -25,7 +25,7 @@ #include "bacula.h" #include "stored.h" -#include "dedupstored.h" +#include "dedup_interface.h" /* Forward referenced subroutines */ static bool read_record_cb(DCR *dcr, DEV_RECORD *rec); static bool mac_record_cb(DCR *dcr, DEV_RECORD *rec); diff --git a/bacula/src/stored/stored.h b/bacula/src/stored/stored.h index 1ca1d6807..c2e650a25 100644 --- a/bacula/src/stored/stored.h +++ b/bacula/src/stored/stored.h @@ -73,7 +73,7 @@ const int sd_dbglvl = 300; #include "vol_mgr.h" #include "reserve.h" #include "protos.h" -#include "dedupstored.h" +#include "dedup_interface.h" #ifdef HAVE_LIBZ #include /* compression headers */ #else -- 2.47.3