From: Radosław Korzeniewski Date: Thu, 7 Oct 2021 15:32:28 +0000 (+0200) Subject: pluginlib: Add dynamic file backup to Include{} in metaplugin. X-Git-Tag: Beta-15.0.0~859 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=315ab54e9328054d394e7e1350af2a49d3f0f56b;p=thirdparty%2Fbacula.git pluginlib: Add dynamic file backup to Include{} in metaplugin. Metaplugin now support dynamic Include{} block generation with new `INCLUDE:/path/to/file` command. Now the backend can ask Bacula to include additional files/directories in the backup job saved by Bacula Core as it would be the standard user config. These files can be saved with stripped location path which is useful for some resources mounted locally (NFS) to remove local mount point from the file name. This Path Strip feature can be configured at compile time with `ADDINCLUDESTRIPOPTION` variable, setup globally during `BackupStart` phase of the protocol for entire job or by file to file basis with `STRIP:` command. --- diff --git a/bacula/src/plugins/fd/Makefile.inc.in b/bacula/src/plugins/fd/Makefile.inc.in index 70fb71e25..f26c41714 100644 --- a/bacula/src/plugins/fd/Makefile.inc.in +++ b/bacula/src/plugins/fd/Makefile.inc.in @@ -39,8 +39,9 @@ PLUGINLIBDIR = $(FDPLUGDIR)/pluginlib UNITTESTSOBJ = $(LIBDIR)/unittests.lo LIBBACOBJ = $(LIBDIR)/libbac.la +INIOBJ = $(LIBDIR)/ini.lo PLUGINLIBOBJ = $(PLUGINLIBDIR)/pluginlib.lo -METAPLUGINOBJ = $(PLUGINLIBOBJ) $(PLUGINLIBDIR)/ptcomm.lo $(PLUGINLIBDIR)/metaplugin.lo $(PLUGINLIBDIR)/metaplugin_attributes.lo +METAPLUGINOBJ = $(PLUGINLIBOBJ) $(PLUGINLIBDIR)/ptcomm.lo $(PLUGINLIBDIR)/metaplugin.lo $(PLUGINLIBDIR)/metaplugin_attributes.lo $(PLUGINLIBDIR)/metaplugin_accurate.lo $(PLUGINLIBDIR)/metaplugin_metadata.lo EXTRA_INSTALL_TARGET = @FD_PLUGIN_INSTALL@ @@ -52,6 +53,9 @@ $(UNITTESTSOBJ): $(LIBBACOBJ): $(MAKE) -C $(LIBDIR) libbac.la +$(INIOBJ): + $(MAKE) -C $(LIBDIR) ini.lo + $(PLUGINLIBDIR)/test_metaplugin_backend.lo: $(MAKE) -C $(PLUGINLIBDIR) test_metaplugin_backend.lo @@ -67,6 +71,12 @@ $(PLUGINLIBDIR)/metaplugin.lo: $(PLUGINLIBDIR)/metaplugin.cpp $(PLUGINLIBDIR)/me $(PLUGINLIBDIR)/metaplugin_attributes.lo: $(PLUGINLIBDIR)/metaplugin_attributes.cpp $(PLUGINLIBDIR)/metaplugin_attributes.h $(MAKE) -C $(PLUGINLIBDIR) metaplugin_attributes.lo +$(PLUGINLIBDIR)/metaplugin_accurate.lo: $(PLUGINLIBDIR)/metaplugin_accurate.cpp $(PLUGINLIBDIR)/metaplugin_accurate.h + $(MAKE) -C $(PLUGINLIBDIR) metaplugin_accurate.lo + +$(PLUGINLIBDIR)/metaplugin_metadata.lo: $(PLUGINLIBDIR)/metaplugin_metadata.cpp $(PLUGINLIBDIR)/metaplugin_metadata.h + $(MAKE) -C $(PLUGINLIBDIR) metaplugin_metadata.lo + $(PLUGINLIBDIR)/iso8601.lo: $(PLUGINLIBDIR)/iso8601.cpp $(PLUGINLIBDIR)/iso8601.h $(MAKE) -C $(PLUGINLIBDIR) iso8601.lo diff --git a/bacula/src/plugins/fd/pluginlib/Makefile b/bacula/src/plugins/fd/pluginlib/Makefile index ad5f36c90..410365776 100644 --- a/bacula/src/plugins/fd/pluginlib/Makefile +++ b/bacula/src/plugins/fd/pluginlib/Makefile @@ -27,19 +27,22 @@ SMARTALISTSRC = smartalist.h SMARTPTRSRC = smartptr.h SMARTMUTEXSRC = smartmutex.h SMARTLOCKSRC = smartlock.h -METAPLUGINSRC = metaplugin.cpp metaplugin.h metaplugin_attributes.cpp metaplugin_attributes.h $(SMARTMUTEXSRC) $(SMARTLOCKSRC) $(COMMCTXSRC) +METAPLUGINSRC = metaplugin.cpp metaplugin.h metaplugin_attributes.cpp metaplugin_attributes.h metaplugin_accurate.cpp metaplugin_accurate.h metaplugin_metadata.cpp metaplugin_metadata.h $(SMARTMUTEXSRC) $(SMARTLOCKSRC) $(COMMCTXSRC) METAPLUGINOBJ = $(filter %.lo,$(METAPLUGINSRC:.cpp=.lo)) -PLUGINLIBSTEST = pluginlib_test.cpp $(PLUGINLIBSSRC) $(UNITTESTSOBJ) -PLUGINLIBSTESTOBJ = $(filter %.lo,$(PLUGINLIBSTEST:.cpp=.lo)) -ISO8601TEST = iso8601_test.cpp $(ISO8601SRC) $(UNITTESTSOBJ) -ISO8601TESTOBJ = $(filter %.lo,$(ISO8601TEST:.cpp=.lo)) -COMMCTXTEST = commctx_test.cpp $(COMMCTXSRC) $(SMARTALISTSRC) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) -COMMCTXTESTOBJ = $(filter %.lo,$(COMMCTXTEST:.cpp=.lo)) -SMARTALISTTEST = smartalist_test.cpp $(SMARTALISTSRC) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) -SMARTALISTTESTOBJ = $(filter %.lo,$(SMARTALISTTEST:.cpp=.lo)) -SMARTPTRTEST = smartptr_test.cpp $(SMARTPTRSRC) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) -SMARTPTRTESTOBJ = $(filter %.lo,$(SMARTPTRTEST:.cpp=.lo)) +PLUGINLIBSTEST = pluginlib_test.cpp $(PLUGINLIBSSRC) +PLUGINLIBSTESTOBJ = $(filter %.lo,$(PLUGINLIBSTEST:.cpp=.lo)) $(INIOBJ) $(UNITTESTSOBJ) +ISO8601TEST = iso8601_test.cpp $(ISO8601SRC) +ISO8601TESTOBJ = $(filter %.lo,$(ISO8601TEST:.cpp=.lo)) $(UNITTESTSOBJ) +COMMCTXTEST = commctx_test.cpp $(COMMCTXSRC) $(SMARTALISTSRC) +COMMCTXTESTOBJ = $(filter %.lo,$(COMMCTXTEST:.cpp=.lo)) $(INIOBJ) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) +SMARTALISTTEST = smartalist_test.cpp $(SMARTALISTSRC) +SMARTALISTTESTOBJ = $(filter %.lo,$(SMARTALISTTEST:.cpp=.lo)) $(INIOBJ) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) +SMARTPTRTEST = smartptr_test.cpp $(SMARTPTRSRC) +SMARTPTRTESTOBJ = $(filter %.lo,$(SMARTPTRTEST:.cpp=.lo)) $(INIOBJ) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) +SMARTLOCKTEST = smartlock_test.cpp $(SMARTLOCKSRC) +SMARTLOCKTESTOBJ = $(filter %.lo,$(SMARTLOCKTEST:.cpp=.lo)) $(INIOBJ) $(PLUGINLIBSOBJ) $(UNITTESTSOBJ) + METAPLUGINTEST = metaplugin_test.cpp $(METAPLUGINSRC) METAPLUGINTESTOBJ = $(filter %.lo,$(METAPLUGINTEST:.cpp=.lo)) $(PTCOMMOBJ) $(PLUGINLIBSOBJ) $(METAPLUGINOBJ) $(UNITTESTSOBJ) diff --git a/bacula/src/plugins/fd/pluginlib/metaplugin.cpp b/bacula/src/plugins/fd/pluginlib/metaplugin.cpp index 78914b15f..12253ccdb 100644 --- a/bacula/src/plugins/fd/pluginlib/metaplugin.cpp +++ b/bacula/src/plugins/fd/pluginlib/metaplugin.cpp @@ -20,8 +20,8 @@ * @file metaplugin.cpp * @author Radosław Korzeniewski (radoslaw@korzeniewski.net) * @brief This is a Bacula metaplugin interface. - * @version 2.1.0 - * @date 2020-12-23 + * @version 3.0.0 + * @date 2021-10-07 * * @copyright Copyright (c) 2021 All rights reserved. * IP transferred to Bacula Systems according to agreement. @@ -29,6 +29,7 @@ #include "metaplugin.h" #include "metaplugin_attributes.h" +#include "metaplugin_accurate.h" #include #include #include @@ -782,6 +783,7 @@ bRC METAPLUGIN::send_parameters(bpContext *ctx, char *command) "regress_standard_error_backup", "regress_cancel_backup", "regress_cancel_restore", + "regress_working_dir", NULL, }; #endif @@ -842,7 +844,8 @@ bRC METAPLUGIN::send_parameters(bpContext *ctx, char *command) } // now send accurate parameter if requested and available - if (ACCURATEPLUGINPARAMETER && accurate_mode) { + if (ACCURATEPLUGINPARAMETER && accurate_mode) + { pm_strcpy(cmd, "Accurate=1\n"); rc = backend.ctx->write_command(ctx, cmd); if (rc < 0) { @@ -851,6 +854,17 @@ bRC METAPLUGIN::send_parameters(bpContext *ctx, char *command) } } +#ifdef DEVELOPER + const char * regress_working_dir; + bfuncs->getBaculaValue(ctx, bVarWorkingDir, ®ress_working_dir); + Mmsg(cmd, "regress_working_dir=%s\n", regress_working_dir); + rc = backend.ctx->write_command(ctx, cmd); + if (rc < 0) { + /* error */ + return bRC_Error; + } +#endif + // signal end of parameters block backend.ctx->signal_eod(ctx); /* ack Params command */ @@ -883,11 +897,30 @@ bRC METAPLUGIN::send_startjob(bpContext *ctx, const char *command) return bRC_Error; } - if (!backend.ctx->read_ack(ctx)){ - strip_trailing_newline(cmd.c_str()); - DMSG(ctx, DERROR, "Wrong backend response to %s command.\n", cmd.c_str()); - JMSG(ctx, backend.ctx->jmsg_err_level(), "Wrong backend response to %s command.\n", cmd.c_str()); - return bRC_Error; + // if (!backend.ctx->read_ack(ctx)){ + // strip_trailing_newline(cmd.c_str()); + // DMSG(ctx, DERROR, "Wrong backend response to %s command.\n", cmd.c_str()); + // JMSG(ctx, backend.ctx->jmsg_err_level(), "Wrong backend response to %s command.\n", cmd.c_str()); + // return bRC_Error; + // } + + int32_t status; + while ((status = backend.ctx->read_command(ctx, cmd)) != 0) + { + if (status < 0) + { + // handle error + strip_trailing_newline(cmd.c_str()); + DMSG(ctx, DERROR, "Wrong backend response to %s command.\n", cmd.c_str()); + JMSG(ctx, backend.ctx->jmsg_err_level(), "Wrong backend response to %s command.\n", cmd.c_str()); + return bRC_Error; + } + + if (status > 0 && scan_parameter_int(cmd, "STRIP:", strip_path_option)) + { + DMSG1(ctx, DINFO, "set strip path = %d\n", strip_path_option); + continue; + } } return bRC_OK; @@ -1702,44 +1735,64 @@ bRC METAPLUGIN::perform_read_metacommands(bpContext *ctx) // loop on metadata from backend or EOD which means no more files to backup while (true) { - if (backend.ctx->read_command(ctx, cmd) > 0){ + if (backend.ctx->read_command(ctx, cmd) > 0) + { /* yup, should read FNAME, ACL or XATTR from backend, check which one */ DMSG(ctx, DDEBUG, "read_command(1): %s\n", cmd.c_str()); - if (scan_parameter_str(cmd, "FNAME:", fname)){ + if (scan_parameter_str(cmd, "FNAME:", fname)) + { /* got FNAME: */ nextfile = true; object = FileObject; return bRC_OK; } - if (scan_parameter_str(cmd, "PLUGINOBJ:", fname)){ + if (scan_parameter_str(cmd, "PLUGINOBJ:", fname)) + { /* got Plugin Object header */ nextfile = true; object = PluginObject; // pluginobject = true; return bRC_OK; } - if (scan_parameter_str(cmd, "RESTOREOBJ:", fname)){ + if (scan_parameter_str(cmd, "RESTOREOBJ:", fname)) + { /* got Restore Object header */ nextfile = true; object = RestoreObject; // restoreobject = true; return bRC_OK; } - if (scan_parameter_str(cmd, "CHECK:", fname)){ + if (scan_parameter_str(cmd, "CHECK:", fname)) + { /* got accurate check query */ - perform_accurate_check(ctx); + metaplugin::accurate::perform_accurate_check(ctx, backend.ctx, fname, accurate_mode, accurate_mode_err); continue; } - if (scan_parameter_str(cmd, "CHECKGET:", fname)){ + if (scan_parameter_str(cmd, "CHECKGET:", fname)) + { /* got accurate get query */ - perform_accurate_check_get(ctx); + metaplugin::accurate::perform_accurate_check_get(ctx, backend.ctx, fname, accurate_mode, accurate_mode_err); continue; } - if (scan_parameter_str(cmd, "ACCEPT:", fname)){ + if (scan_parameter_str(cmd, "ACCEPT:", fname)) + { /* got AcceptFile() query */ perform_accept_file(ctx); continue; } + if (scan_parameter_str(cmd, "INCLUDE:", fname)) + { + /* got AddInclude() request */ + perform_addinclude(ctx); + continue; + } + int split_nr = -1; + if (scan_parameter_int(cmd, "STRIP:", split_nr)) + { + /* got Split Option change request */ + perform_change_split_option(ctx, split_nr); + continue; + } if (bstrcmp(cmd.c_str(), "ACL")){ /* got ACL header */ perform_read_acl(ctx); @@ -1775,171 +1828,85 @@ bRC METAPLUGIN::perform_read_metacommands(bpContext *ctx) return bRC_Error; } +struct bacula_ctx { + JCR *jcr; /* jcr for plugin */ + bRC rc; /* last return code */ + bool disabled; /* set if plugin disabled */ + bool restoreFileStarted; + bool createFileCalled; + bool cancelCalled; /* true if the plugin got the cancel event */ + void *exclude; /* pointer to exclude files */ + void *include; /* pointer to include/exclude files */ +}; + /** - * @brief Respond to the file index query command from backend. + * @brief Perform AddInclude() for change current FileSet. * * @param ctx bpContext - for Bacula debug and jobinfo messages - * @return bRC bRC_OK when success, bRC_Error if not + * @return bRC always return bRC_OK */ -bRC METAPLUGIN::perform_file_index_query(bpContext *ctx) +bRC METAPLUGIN::perform_addinclude(bpContext *ctx) { - POOL_MEM cmd(PM_FNAME); - int32_t fileindex; + if (!new_include_created) + { + DMSG0(ctx, DDEBUG, "perform_addinclude():create new Include\n"); + bfuncs->NewInclude(ctx); + new_include_created = true; - getBaculaVar(bVarFileIndex, (void *)&fileindex); - Mmsg(cmd, "%d\n", fileindex); - if (backend.ctx->write_command(ctx, cmd) < 0){ - /* error */ - return bRC_Error; + if (strip_path_option > 0) + { + POOL_MEM tmp; + Mmsg(tmp, "fP%d:", strip_path_option); + DMSG1(ctx, DDEBUG, "perform_addinclude():addoption:\"%s\"\n", tmp.c_str()); + bfuncs->AddOptions(ctx, tmp.c_str()); // Force onefs=no and strip base path + } } + DMSG1(ctx, DDEBUG, "perform_addinclude():%s\n", fname.c_str()); + bfuncs->AddInclude(ctx, fname.c_str()); + pm_strcpy(fname, NULL); + return bRC_OK; } /** - * @brief + * @brief Changes a current split option value and forces new Include creation. * * @param ctx bpContext - for Bacula debug and jobinfo messages - * @return bRC bRC_OK when success, bRC_Error if not + * @param nr a new split option to set + * @return bRC bRC always return bRC_OK */ -bRC METAPLUGIN::perform_accurate_check(bpContext *ctx) +bRC METAPLUGIN::perform_change_split_option(bpContext *ctx, int nr) { - if (strlen(fname.c_str()) == 0){ - // input variable is not valid - return bRC_Error; - } - - DMSG0(ctx, DDEBUG, "perform_accurate_check()\n"); - - POOL_MEM cmd(PM_FNAME); - struct save_pkt sp; - memset(&sp, 0, sizeof(sp)); - - // supported sequence is `STAT` followed by `TSTAMP` - if (backend.ctx->read_command(ctx, cmd) < 0) { - // error - return bRC_Error; - } - - metaplugin::attributes::Status status = metaplugin::attributes::read_scan_stat_command(ctx, cmd, &sp); - if (status == metaplugin::attributes::Status_OK) { - if (backend.ctx->read_command(ctx, cmd) < 0) { - // error - return bRC_Error; - } - - status = metaplugin::attributes::read_scan_tstamp_command(ctx, cmd, &sp); - if (status == metaplugin::attributes::Status_OK) { - // success we can perform accurate check for stat packet - bRC rc = bRC_OK; // return 'OK' as a default - if (accurate_mode) { - sp.fname = fname.c_str(); - rc = checkChanges(&sp); - } else { - if (!accurate_mode_err) { - DMSG0(ctx, DERROR, "Backend CHECK command require accurate mode on!\n"); - JMSG0(ctx, M_ERROR, "Backend CHECK command require accurate mode on!\n"); - accurate_mode_err = true; - } - } - - POOL_MEM checkstatus(PM_NAME); - Mmsg(checkstatus, "%s\n", rc == bRC_Seen ? "SEEN" : "OK"); - DMSG1(ctx, DINFO, "perform_accurate_check(): %s", checkstatus.c_str()); - - if (!backend.ctx->write_command(ctx, checkstatus)) { - DMSG0(ctx, DERROR, "Cannot send checkChanges() response to backend\n"); - JMSG0(ctx, backend.ctx->jmsg_err_level(), "Cannot send checkChanges() response to backend\n"); - return bRC_Error; - } - - return bRC_OK; - } - } else { - // check possible errors - switch (status) - { - case metaplugin::attributes::Invalid_File_Type: - JMSG2(ctx, M_ERROR, "Invalid file type: %c for %s\n", sp.type, fname.c_str()); - return bRC_Error; - - case metaplugin::attributes::Invalid_Stat_Packet: - JMSG1(ctx, backend.ctx->jmsg_err_level(), "Invalid stat packet: %s\n", cmd.c_str()); - return bRC_Error; - default: - break; - } - // future extension for `ATTR` command - // ... + nr = nr < 0 ? 0 : nr; + if (nr != strip_path_option) + { + DMSG2(ctx, DDEBUG, "perform_change_split_option():%d -> %d\n", strip_path_option, nr); + strip_path_option = nr; + new_include_created = false; } - return bRC_Error; + return bRC_OK; } /** - * @brief Perform accurate query check and resturn accurate data to backend. + * @brief Respond to the file index query command from backend. * * @param ctx bpContext - for Bacula debug and jobinfo messages * @return bRC bRC_OK when success, bRC_Error if not */ -bRC METAPLUGIN::perform_accurate_check_get(bpContext *ctx) +bRC METAPLUGIN::perform_file_index_query(bpContext *ctx) { POOL_MEM cmd(PM_FNAME); + int32_t fileindex; - if (strlen(fname.c_str()) == 0){ - // input variable is not valid + getBaculaVar(bVarFileIndex, (void *)&fileindex); + Mmsg(cmd, "%d\n", fileindex); + if (backend.ctx->write_command(ctx, cmd) < 0){ + /* error */ return bRC_Error; } - DMSG0(ctx, DDEBUG, "perform_accurate_check_get()\n"); - - if (!accurate_mode) { - // the job is not accurate, so no accurate data will be available at all - pm_strcpy(cmd, "NOACCJOB\n"); - if (!backend.ctx->signal_error(ctx, cmd)) { - DMSG0(ctx, DERROR, "Cannot send 'No Accurate Job' info to backend\n"); - JMSG0(ctx, backend.ctx->jmsg_err_level(), "Cannot send 'No Accurate Job' info to backend\n"); - return bRC_Error; - } - return bRC_OK; - } - - accurate_attribs_pkt attribs; - memset(&attribs, 0, sizeof(attribs)); - - attribs.fname = fname.c_str(); - bRC rc = getAccurateAttribs(&attribs); - - struct restore_pkt rp; - - switch (rc) - { - case bRC_Seen: - memcpy(&rp.statp, &attribs.statp, sizeof(rp.statp)); - rp.type = FT_MASK; // This is a special metaplugin protocol hack - // because the current Bacula accurate code does - // not handle FileType on catalog attributes, yet. - // STAT:... - metaplugin::attributes::make_stat_command(ctx, cmd, &rp); - backend.ctx->write_command(ctx, cmd); - - // TSTAMP:... - if (metaplugin::attributes::make_tstamp_command(ctx, cmd, &rp) == metaplugin::attributes::Status_OK) { - backend.ctx->write_command(ctx, cmd); - DMSG(ctx, DINFO, "createFile:%s", cmd.c_str()); - } - - break; - default: - pm_strcpy(cmd, "UNAVAIL\n"); - if (!backend.ctx->write_command(ctx, cmd)) { - DMSG0(ctx, DERROR, "Cannot send 'UNAVAIL' response to backend\n"); - JMSG0(ctx, backend.ctx->jmsg_err_level(), "Cannot send 'UNAVAIL' response to backend\n"); - return bRC_Error; - } - break; - } - return bRC_OK; } diff --git a/bacula/src/plugins/fd/pluginlib/metaplugin.h b/bacula/src/plugins/fd/pluginlib/metaplugin.h index 3e2c3057a..ce1978347 100644 --- a/bacula/src/plugins/fd/pluginlib/metaplugin.h +++ b/bacula/src/plugins/fd/pluginlib/metaplugin.h @@ -21,7 +21,7 @@ * @author Radosław Korzeniewski (radoslaw@korzeniewski.net) * @brief This is a Bacula metaplugin interface. * @version 3.0.0 - * @date 2021-08-20 + * @date 2021-10-07 * * @copyright Copyright (c) 2021 All rights reserved. * IP transferred to Bacula Systems according to agreement. @@ -69,6 +69,7 @@ extern const char *PLUGINAPI; /// the plugin api string which sho extern const char *BACKEND_CMD; /// a backend execution command path extern const int32_t CUSTOMCANCELSLEEP; /// custom wait time for backend between USR1 and terminate procedures extern const bool ACCURATEPLUGINPARAMETER; /// accurate parameter for plugin parameter +extern const int ADDINCLUDESTRIPOPTION; /// setup precompiled include path strip option /// defines if metaplugin should handle local filesystem restore with Bacula Core functions /// `false` means metaplugin will redirect local restore to backend @@ -158,12 +159,11 @@ public: openerror(false), object(FileObject), objectsent(false), - // pluginobject(false), - // pluginobjectsent(false), - // restoreobject(false), readacl(false), readxattr(false), skipextract(false), + new_include_created(false), + strip_path_option(ADDINCLUDESTRIPOPTION), last_type(0), fname(PM_FNAME), lname(PM_FNAME), @@ -223,12 +223,11 @@ private: bool openerror; // show if "openfile" was unsuccessful OBJECT object; // bool objectsent; // set when startBackupFile handled object and endBackupFile has to check for nextfile - // bool pluginobject; // set when got PLUGINOBJ: command - // bool pluginobjectsent; // set when startBackupFile handled plugin object and endBackupFile has to check for nextfile - // bool restoreobject; // set when got RESTOREOBJ: command bool readacl; // got ACL data from backend bool readxattr; // got XATTR data from backend bool skipextract; // got SKIP response from backend, so we should artificially skip it for backend + bool new_include_created; // when NewInclude() was executed + int strip_path_option; // int32_t last_type; // contains the last restore file type COMMCTX backend; // the backend context list for multiple backend execution for a single job POOL_MEM fname; // current file name to backup (grabbed from backend) @@ -271,7 +270,6 @@ private: bRC perform_write_data(bpContext *ctx, struct io_pkt *io); bRC perform_write_end(bpContext *ctx, struct io_pkt *io); bRC perform_read_metacommands(bpContext *ctx); - bRC perform_read_fstatdata(bpContext *ctx, struct save_pkt *sp); bRC perform_read_pluginobject(bpContext *ctx, struct save_pkt *sp); bRC perform_read_restoreobject(bpContext *ctx, struct save_pkt *sp); bRC perform_read_acl(bpContext *ctx); @@ -280,9 +278,9 @@ private: bRC perform_write_xattr(bpContext *ctx, const xacl_pkt * xacl); bRC perform_read_metadata_info(bpContext *ctx, metadata_type type, struct save_pkt *sp); bRC perform_file_index_query(bpContext *ctx); - bRC perform_accurate_check(bpContext *ctx); - bRC perform_accurate_check_get(bpContext *ctx); bRC perform_accept_file(bpContext *ctx); + bRC perform_addinclude(bpContext *ctx); + bRC perform_change_split_option(bpContext *ctx, int nr); // bRC perform_write_metadata_info(bpContext *ctx, struct meta_pkt *mp); metadata_type scan_metadata_type(bpContext *ctx, const POOL_MEM &cmd); const char *prepare_metadata_type(metadata_type type); diff --git a/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.cpp b/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.cpp new file mode 100644 index 000000000..2b7e008df --- /dev/null +++ b/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.cpp @@ -0,0 +1,208 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2020 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. + */ +/** + * @file metaplugin_accurate.cpp + * @author Radosław Korzeniewski (radoslaw@korzeniewski.net) + * @brief This is an Accurate Mode handling subroutines for metaplugin. + * @version 1.0.0 + * @date 2021-09-20 + * + * @copyright Copyright (c) 2021 All rights reserved. IP transferred to Bacula Systems according to agreement. + */ + +#include "metaplugin_accurate.h" +#include "metaplugin_attributes.h" + + +/* + * libbac uses its own sscanf implementation which is not compatible with + * libc implementation, unfortunately. + * use bsscanf for Bacula sscanf flavor + */ +#ifdef sscanf +#undef sscanf +#endif + +namespace metaplugin +{ +namespace accurate +{ + /** + * @brief Perform a backend Accurate Mode query. + * + * @param ctx bpContext - for Bacula debug and jobinfo messages + * @param ptcomm backend communication class - the current context + * @param fname a file name to handle + * @param accurate_mode the current state of accurate mode + * @param accurate_mode_err when accurate mode error handled + * @return bRC bRC_OK when success, bRC_Error if not + */ + bRC perform_accurate_check(bpContext *ctx, PTCOMM *ptcomm, POOL_MEM &fname, bool accurate_mode, bool &accurate_mode_err) + { + if (strlen(fname.c_str()) == 0){ + // input variable is not valid + return bRC_Error; + } + + DMSG0(ctx, DDEBUG, "perform_accurate_check()\n"); + + POOL_MEM cmd(PM_FNAME); + struct save_pkt sp; + memset(&sp, 0, sizeof(sp)); + + // supported sequence is `STAT` followed by `TSTAMP` + if (ptcomm->read_command(ctx, cmd) < 0) { + // error + return bRC_Error; + } + + metaplugin::attributes::Status status = metaplugin::attributes::read_scan_stat_command(ctx, cmd, &sp); + if (status == metaplugin::attributes::Status_OK) { + if (ptcomm->read_command(ctx, cmd) < 0) { + // error + return bRC_Error; + } + + status = metaplugin::attributes::read_scan_tstamp_command(ctx, cmd, &sp); + if (status == metaplugin::attributes::Status_OK) { + // success we can perform accurate check for stat packet + bRC rc = bRC_OK; // return 'OK' as a default + if (accurate_mode) { + sp.fname = fname.c_str(); + rc = checkChanges(&sp); + } else { + if (!accurate_mode_err) { + DMSG0(ctx, DERROR, "Backend CHECK command require accurate mode on!\n"); + JMSG0(ctx, M_ERROR, "Backend CHECK command require accurate mode on!\n"); + accurate_mode_err = true; + } + } + + POOL_MEM checkstatus(PM_NAME); + Mmsg(checkstatus, "%s\n", rc == bRC_Seen ? "SEEN" : "OK"); + DMSG1(ctx, DINFO, "perform_accurate_check(): %s", checkstatus.c_str()); + + if (!ptcomm->write_command(ctx, checkstatus)) { + DMSG0(ctx, DERROR, "Cannot send checkChanges() response to backend\n"); + JMSG0(ctx, ptcomm->jmsg_err_level(), "Cannot send checkChanges() response to backend\n"); + return bRC_Error; + } + + return bRC_OK; + } + } else { + // check possible errors + switch (status) + { + case metaplugin::attributes::Invalid_File_Type: + JMSG2(ctx, M_ERROR, "Invalid file type: %c for %s\n", sp.type, fname.c_str()); + return bRC_Error; + + case metaplugin::attributes::Invalid_Stat_Packet: + JMSG1(ctx, ptcomm->jmsg_err_level(), "Invalid stat packet: %s\n", cmd.c_str()); + return bRC_Error; + default: + break; + } + // future extension for `ATTR` command + // ... + } + + return bRC_Error; + } + + /** + * @brief Perform a backend Accurate Get Mode query. + * + * @param ctx bpContext - for Bacula debug and jobinfo messages + * @param ptcomm backend communication class - the current context + * @param fname a file name to handle + * @param accurate_mode the current state of accurate mode + * @param accurate_mode_err when accurate mode error handled + * @return bRC bRC_OK when success, bRC_Error if not + */ + bRC perform_accurate_check_get(bpContext *ctx, PTCOMM *ptcomm, POOL_MEM &fname, bool accurate_mode, bool &accurate_mode_err) + { + POOL_MEM cmd(PM_FNAME); + + if (strlen(fname.c_str()) == 0){ + // input variable is not valid + return bRC_Error; + } + + DMSG0(ctx, DDEBUG, "perform_accurate_check_get()\n"); + + if (!accurate_mode) + { + DMSG0(ctx, DERROR, "Backend CHECKGET command require accurate mode on!\n"); + JMSG0(ctx, M_ERROR, "Backend CHECKGET command require accurate mode on!\n"); + accurate_mode_err = true; + + // the job is not accurate, so no accurate data will be available at all + pm_strcpy(cmd, "NOACCJOB\n"); + if (!ptcomm->signal_error(ctx, cmd)) { + DMSG0(ctx, DERROR, "Cannot send 'No Accurate Job' info to backend\n"); + JMSG0(ctx, ptcomm->jmsg_err_level(), "Cannot send 'No Accurate Job' info to backend\n"); + return bRC_Error; + } + + return bRC_OK; + } + + accurate_attribs_pkt attribs; + memset(&attribs, 0, sizeof(attribs)); + + attribs.fname = fname.c_str(); + bRC rc = getAccurateAttribs(&attribs); + + struct restore_pkt rp; + + switch (rc) + { + case bRC_Seen: + memcpy(&rp.statp, &attribs.statp, sizeof(rp.statp)); + rp.type = FT_MASK; // This is a special metaplugin protocol hack + // because the current Bacula accurate code does + // not handle FileType on catalog attributes, yet. + // STAT:... + metaplugin::attributes::make_stat_command(ctx, cmd, &rp); + ptcomm->write_command(ctx, cmd); + + // TSTAMP:... + if (metaplugin::attributes::make_tstamp_command(ctx, cmd, &rp) == metaplugin::attributes::Status_OK) { + ptcomm->write_command(ctx, cmd); + DMSG(ctx, DINFO, "createFile:%s", cmd.c_str()); + } + + break; + default: + pm_strcpy(cmd, "UNAVAIL\n"); + if (!ptcomm->write_command(ctx, cmd)) { + DMSG0(ctx, DERROR, "Cannot send 'UNAVAIL' response to backend\n"); + JMSG0(ctx, ptcomm->jmsg_err_level(), "Cannot send 'UNAVAIL' response to backend\n"); + return bRC_Error; + } + break; + } + + return bRC_OK; + } + +} // namespace accurate +} // namespace metaplugin diff --git a/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.h b/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.h new file mode 100644 index 000000000..6399db0c4 --- /dev/null +++ b/bacula/src/plugins/fd/pluginlib/metaplugin_accurate.h @@ -0,0 +1,46 @@ +/* + Bacula(R) - The Network Backup Solution + + Copyright (C) 2000-2020 Kern Sibbald + + The original author of Bacula is Kern Sibbald, with contributions + from many others, a complete list can be found in the file AUTHORS. + + You may use this file and others of this release according to the + license defined in the LICENSE file, which includes the Affero General + Public License, v3.0 ("AGPLv3") and some additional permissions and + terms pursuant to its AGPLv3 Section 7. + + This notice must be preserved when any source code is + conveyed and/or propagated. + + Bacula(R) is a registered trademark of Kern Sibbald. + */ +/** + * @file metaplugin_accurate.h + * @author Radosław Korzeniewski (radoslaw@korzeniewski.net) + * @brief This is an Accurate Mode handling subroutines for metaplugin. + * @version 1.0.0 + * @date 2021-09-20 + * + * @copyright Copyright (c) 2021 All rights reserved. IP transferred to Bacula Systems according to agreement. + */ + +#ifndef _METAPLUGIN_ACCURATE_H_ +#define _METAPLUGIN_ACCURATE_H_ + +#include "pluginlib.h" +#include "ptcomm.h" + + +namespace metaplugin +{ +namespace accurate +{ + bRC perform_accurate_check(bpContext *ctx, PTCOMM *ptcomm, POOL_MEM &fname, bool accurate_mode, bool &accurate_mode_err); + bRC perform_accurate_check_get(bpContext *ctx, PTCOMM *ptcomm, POOL_MEM &fname, bool accurate_mode, bool &accurate_mode_err); + +} // namespace accurate +} // namespace metaplugin + +#endif // _METAPLUGIN_ACCURATE_H_ diff --git a/bacula/src/plugins/fd/pluginlib/pluginlib.cpp b/bacula/src/plugins/fd/pluginlib/pluginlib.cpp index f4d7944d3..54837684c 100644 --- a/bacula/src/plugins/fd/pluginlib/pluginlib.cpp +++ b/bacula/src/plugins/fd/pluginlib/pluginlib.cpp @@ -542,19 +542,6 @@ bool parse_param(bool ¶m, const char *pname, const char *name, const char *v return false; } -/* - * - * - * in: - * param - a pointer to - * pname - - * name - - * value - - * out: - * True if parameter was parsed - * False if it was not the parameter required - */ - /** * @brief Setup Plugin parameter for integer from string value. * @@ -651,7 +638,8 @@ bool parse_param_add_str(alist &list, const char *pname, const char *name, const */ bool scan_parameter_str(const char * cmd, const char *prefix, POOL_MEM ¶m) { - if (prefix != NULL){ + if (prefix != NULL) + { int len = strlen(prefix); if (strncmp(cmd, prefix, len) == 0) { @@ -665,6 +653,19 @@ bool scan_parameter_str(const char * cmd, const char *prefix, POOL_MEM ¶m) return false; } +bool scan_parameter_int(const char * cmd, const char *prefix, int ¶m) +{ + POOL_MEM tmp; + + if (scan_parameter_str(cmd, prefix, tmp)) + { + param = atoi(tmp.c_str()); + return true; + } + + return false; +} + // ensure error message is terminated with newline and terminated with standard c-string nul void scan_and_terminate_str(POOL_MEM &buf, int msglen) { diff --git a/bacula/src/plugins/fd/pluginlib/pluginlib.h b/bacula/src/plugins/fd/pluginlib/pluginlib.h index bd46d01c2..2f24ebe1f 100644 --- a/bacula/src/plugins/fd/pluginlib/pluginlib.h +++ b/bacula/src/plugins/fd/pluginlib/pluginlib.h @@ -241,6 +241,8 @@ bool parse_param_add_str(alist &list, const char *pname, const char *name, const bool scan_parameter_str(const char * cmd, const char *prefix, POOL_MEM ¶m); inline bool scan_parameter_str(const POOL_MEM &cmd, const char *prefix, POOL_MEM ¶m) { return scan_parameter_str(cmd.c_str(), prefix, param); } +bool scan_parameter_int(const char *cmd, const char *prefix, int ¶m); +inline bool scan_parameter_int(const POOL_MEM &cmd, const char *prefix, int ¶m) { return scan_parameter_int(cmd.c_str(), prefix, param); } void scan_and_terminate_str(POOL_MEM &buf, int msglen); diff --git a/bacula/src/plugins/fd/pluginlib/pluginlib_test.cpp b/bacula/src/plugins/fd/pluginlib/pluginlib_test.cpp index b4de2a604..c75110c2d 100644 --- a/bacula/src/plugins/fd/pluginlib/pluginlib_test.cpp +++ b/bacula/src/plugins/fd/pluginlib/pluginlib_test.cpp @@ -130,9 +130,10 @@ int main() POOL_MEM param(PM_NAME); const char *prefix = "FNAME:"; const char *fname1 = "/etc/passwd"; - pm_strcpy(cmd1, prefix); - pm_strcat(cmd1, fname1); - pm_strcat(cmd1, "\n"); + Mmsg(cmd1, "%s%s\n", prefix, fname1); + // pm_strcpy(cmd1, prefix); + // pm_strcat(cmd1, fname1); + // pm_strcat(cmd1, "\n"); ok(scan_parameter_str(cmd1, prefix, param), "check scan parameter str match"); ok(bstrcmp(param.c_str(), fname1) , "check scan parameter str param"); nok(scan_parameter_str(cmd1, "prefix", param), "check scan parameter str not match"); @@ -150,20 +151,28 @@ int main() ok(bstrcmp(param.c_str(), fname1) , "check scan parameter for char* str param"); nok(scan_parameter_str(cmd2, "prefix", param), "check scan parameter for char* str not match"); + int value = -1; + snprintf(cmd2, 256, "%s10\n", prefix); + ok(scan_parameter_int(cmd2, prefix, value), "test scan_parameter_int"); + ok(value == 10, "test scan_parameter_int value"); + value = -1; + nok(scan_parameter_int(cmd2, "prefix", value), "test scan_parameter_int not match"); + ok(value == -1, "test scan_parameter_int value unchanged"); + const vectstruct testvect1[] = { - { "", false, "checking empty" }, - { "/", false, "checking slash" }, - { "other", false, "checking other" }, - { "/tmp", true, "checking local" }, - { "/tmp/restore", true, "checking local" }, + {"", false, "checking empty"}, + {"/", false, "checking slash"}, + {"other", false, "checking other"}, + {"/tmp", true, "checking local"}, + {"/tmp/restore", true, "checking local"}, #ifdef HAVE_WIN32 - { "c:", true, "checking local win32" }, - { "d:/", true, "checking local win32" }, - { "E:/", true, "checking local win32" }, - { "F:/test", true, "checking local win32" }, - { "g:/test/restore", true, "checking local win32" }, + {"c:", true, "checking local win32"}, + {"d:/", true, "checking local win32"}, + {"E:/", true, "checking local win32"}, + {"F:/test", true, "checking local win32"}, + {"g:/test/restore", true, "checking local win32"}, #endif - { NULL, false, NULL }, + {NULL, false, NULL}, }; for (int i = 0; testvect1[i].path != NULL; i++) @@ -197,7 +206,6 @@ int main() scan_and_terminate_str(ebuf, testvect2[i].msglen); ok(memcmp(ebuf.c_str(), testvect2[i].output, testvect2[i].len) == 0, testvect2[i].descr); } - // scan_and_terminate_str return report(); } diff --git a/bacula/src/plugins/fd/pluginlib/test_metaplugin_backend.c b/bacula/src/plugins/fd/pluginlib/test_metaplugin_backend.c index afbd464bf..31ebf58d1 100644 --- a/bacula/src/plugins/fd/pluginlib/test_metaplugin_backend.c +++ b/bacula/src/plugins/fd/pluginlib/test_metaplugin_backend.c @@ -74,6 +74,8 @@ bool regress_cancel_restore = false; bool Job_Level_Incremental = false; +char working_directory[4096]; // it should be no more + #define BUFLEN 4096 #define BIGBUFLEN 131072 @@ -865,6 +867,30 @@ void perform_backup() if (strncmp(buf, "UNAVAIL", 7) == 0) { write_plugin('I', "TEST CHECK nonexistentok"); } + + // check addinclude() + snprintf(buf, BIGBUFLEN, "%s/passwd", working_directory); + creat(buf, 0600); + snprintf(buf, BIGBUFLEN, "INCLUDE:%s/passwd", working_directory); + write_plugin('C', buf); + snprintf(buf, BIGBUFLEN, "%s/group", working_directory); + creat(buf, 0600); + snprintf(buf, BIGBUFLEN, "INCLUDE:%s/group", working_directory); + write_plugin('C', buf); + + write_plugin('C', "STRIP:3\n"); + snprintf(buf, BIGBUFLEN, "%s/tmp", working_directory); + mkdir(buf, 0700); + snprintf(buf, BIGBUFLEN, "%s/tmp/bacula", working_directory); + mkdir(buf, 0700); + snprintf(buf, BIGBUFLEN, "%s/tmp/bacula/test", working_directory); + mkdir(buf, 0700); + snprintf(buf, BIGBUFLEN, "%s/tmp/bacula/test/split", working_directory); + mkdir(buf, 0700); + snprintf(buf, BIGBUFLEN, "%s/tmp/bacula/test/split/file", working_directory); + creat(buf, 0600); + snprintf(buf, BIGBUFLEN, "INCLUDE:%s/tmp/bacula/test/split", working_directory); + write_plugin('C', buf); } // backup if full or not seen @@ -1563,6 +1589,7 @@ int main(int argc, char** argv) { exit(EXIT_BACKEND_NOMEMORY); } + working_directory[0] = 0; mypid = getpid(); snprintf(buf, 4096, "%s/%s_backend_%d.log", LOGDIR, PLUGINNAME, mypid); logfd = open(buf, O_CREAT|O_TRUNC|O_WRONLY, 0640); @@ -1678,6 +1705,11 @@ int main(int argc, char** argv) { strcpy(query, buf); continue; } + if (sscanf(buf, "regress_working_dir=%s\n", buf) == 1) + { + strcpy(working_directory, buf); + continue; + } } if (regress_cancel_restore || regress_cancel_backup) { if (signal(SIGUSR1, catch_function) == SIG_ERR){ @@ -1701,23 +1733,33 @@ int main(int argc, char** argv) { goto Term; } - signal_eod(); + if (Job_Level_Incremental) + { + write_plugin('C', "STRIP:1\n"); + } + + signal_eod(); // ACK of the `BackupStart` phase is always required! /* check what kind of Job we have */ buf[len] = 0; - if (strcmp(buf, "BackupStart\n") == 0){ + if (strcmp(buf, "BackupStart\n") == 0) + { perform_backup(); } else - if (strcmp(buf, "EstimateStart\n") == 0){ + if (strcmp(buf, "EstimateStart\n") == 0) + { perform_estimate(); } else - if (strcmp(buf, "ListingStart\n") == 0){ + if (strcmp(buf, "ListingStart\n") == 0) + { perform_listing(listing); } else - if (strcmp(buf, "QueryStart\n") == 0){ + if (strcmp(buf, "QueryStart\n") == 0) + { perform_queryparam(query); } else - if (strcmp(buf, "RestoreStart\n") == 0){ + if (strcmp(buf, "RestoreStart\n") == 0) + { perform_restore(); } diff --git a/regress/scripts/metaplugin-protocol-tests.sh b/regress/scripts/metaplugin-protocol-tests.sh index 702c4b5c5..25632a89a 100755 --- a/regress/scripts/metaplugin-protocol-tests.sh +++ b/regress/scripts/metaplugin-protocol-tests.sh @@ -206,6 +206,7 @@ wait status client=$CLIENT messages llist jobid=8 +list files jobid=8 @output quit END_OF_DATA @@ -528,10 +529,16 @@ fi RET=$(grep "jobstatus:" ${cwd}/tmp/log10.out | awk '{print $2}') SEEN=$(grep -c "SEEN" ${cwd}/tmp/log10.out) NONEXIST=$(grep -c "nonexistentok" ${cwd}/tmp/log10.out) -if [ "x$RET" != "xT" ] || [ "$SEEN" -ne 1 ] || [ "$NONEXIST" -ne 2 ] +STRIP1=$(grep "passwd" ${cwd}/tmp/log10.out | awk '{print $2}') +STRIP2=$(grep "bacula/test/split/file" ${cwd}/tmp/log10.out | awk '{print $2}') +GENIUE1="${cwd}/working/passwd" +GENIUE2="${cwd}/working/tmp/bacula/test/split/file" +if [ "x$RET" != "xT" ] || [ "$SEEN" -ne 1 ] || [ "$NONEXIST" -ne 2 ] || [ "$STRIP1" = "$GENIUE1" ] || [ "$STRIP2" = "$GENIUE2" ] then echo "log10" "$RET" "$SEEN" "$NONEXIST" bstat=$((bstat+512)) + echo "$GENIUE1" >> ${cwd}/tmp/log10.out + echo "$GENIUE2" >> ${cwd}/tmp/log10.out fi LFILE1=$(grep "drwxr-xr-x" ${cwd}/tmp/llog1.out | grep -c containers)