From: Eric Bollengier <eric@baculasystems.com>
Date: Mon, 30 May 2022 08:59:55 +0000 (+0200)
Subject: Add VirtualFull plugin support
X-Git-Tag: Beta-15.0.0~570
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c5a890ebd2fbc37560203f5c91aba58d0436bc0d;p=thirdparty%2Fbacula.git

Add VirtualFull plugin support
---

diff --git a/bacula/src/cats/bdb.h b/bacula/src/cats/bdb.h
index ad1bf6eda..b1b8bf1d7 100644
--- a/bacula/src/cats/bdb.h
+++ b/bacula/src/cats/bdb.h
@@ -54,6 +54,7 @@ typedef enum
 #define DBL_ALL_FILES    (1<<1)    /* Return all files including deleted ones */
 #define DBL_DELETED      (1<<2)    /* Return only deleted files */
 #define DBL_USE_MD5      (1<<3)    /* Include md5 */
+#define DBL_USE_OBJ      (1<<4)    /* Include RestoreObjects */
 
 /* Turn the num to a bit field */
 #define DB_ACL_BIT(x) (1<<x)
diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c
index 610d0bb97..d8a799e03 100644
--- a/bacula/src/cats/sql_cmds.c
+++ b/bacula/src/cats/sql_cmds.c
@@ -1087,3 +1087,43 @@ const char *regexp_value[] = {
    /* SQLite */
    regexp_value_default
 };
+
+/* Here we choose the last Object for a list of Job, we check if the object points to a place
+ * that still exists. We take the last version of them
+ */
+
+/* In the MySQL version, we cannot use DISTINCT ON(), so, when we have our MAX(JobTDate) (last record), we
+ * need to find back the other values by joining back to Object. Of course, if we have multiple objects
+ * with the same kind of attributes for the same job, it can cause a problem.
+ */
+const char *copy_object_default =
+   "INSERT INTO Object (JobId, Path, Filename, PluginName, ObjectCategory, ObjectType, ObjectName, ObjectSource, ObjectUUID, "
+       " ObjectSize, ObjectStatus, ObjectCount) "
+   // If we rename a VM for example, the Path and the Filename will stay, but not the Name or maybe the PluginName
+   "SELECT %lu, A.Path, A.Filename, Object.PluginName, A.ObjectCategory, A.ObjectType, Object.ObjectName, Object.ObjectSource, Object.ObjectUUID, Object.ObjectSize, Object.ObjectStatus, Object.ObjectCount "
+     "FROM (SELECT O.Path, O.Filename, O.ObjectCategory, O.ObjectType, MAX(J.JobTDate) AS JobTDate "
+             "FROM Object AS O JOIN Path USING (Path) JOIN Job AS J USING (JobId) WHERE J.JobId IN (%s) AND O.Path = Path.Path AND "
+             "EXISTS (SELECT 1 FROM File WHERE O.JobId = File.JobId AND Path.PathId = File.PathId AND (O.Filename = File.Filename OR O.Filename = '')) "
+            "GROUP BY O.Path, O.Filename, O.ObjectCategory, O.ObjectType) AS A "
+     "JOIN Job USING (JobTDate) JOIN Object ON (Job.JobId = Object.JobId AND Object.Path = A.Path AND Object.Filename = A.Filename AND A.ObjectCategory = Object.ObjectCategory AND A.ObjectType = Object.ObjectType) WHERE Job.JobId IN (%s)";
+
+const char *copy_object[] = {
+   /* MySQL */
+   copy_object_default,
+
+   /* PostgreSQL */
+   "INSERT INTO Object (JobId, Path, Filename, PluginName, ObjectCategory, ObjectType, ObjectName, ObjectSource, ObjectUUID, "
+       " ObjectSize, ObjectStatus, ObjectCount) "
+   // If we rename a VM for example, the Path and the Filename will stay, but not the Name or maybe the PluginName
+   "SELECT DISTINCT ON (Object.Path, Object.Filename, ObjectCategory, ObjectType) "
+     " %lu, Object.Path, Object.Filename, PluginName, ObjectCategory, ObjectType, ObjectName, ObjectSource, ObjectUUID, "
+     " ObjectSize, ObjectStatus, ObjectCount "
+   "FROM Object JOIN Path USING (Path) JOIN Job USING (JobId) "
+  "WHERE Job.JobId IN (%s) AND Object.Path = Path.Path "
+   // In some plugins, the filename is empty but the associated record in the DB doesn't exist
+    "AND EXISTS (SELECT 1 FROM File WHERE Object.JobId = File.JobId AND Path.PathId = File.PathId AND (Object.Filename = File.Filename OR Object.Filename = '')) "
+   "ORDER BY Object.Path, Object.Filename, ObjectCategory, ObjectType, JobTDate DESC",
+
+   /* SQLite */
+   copy_object_default
+};
diff --git a/bacula/src/cats/sql_cmds.h b/bacula/src/cats/sql_cmds.h
index 2f78896bd..634697fd1 100644
--- a/bacula/src/cats/sql_cmds.h
+++ b/bacula/src/cats/sql_cmds.h
@@ -114,3 +114,4 @@ extern const char CATS_IMP_EXP *sum_volumes_bytes;
 extern const char CATS_IMP_EXP *get_volume_size;
 extern const char CATS_IMP_EXP *escape_char_value[];
 extern const char CATS_IMP_EXP *regexp_value[];
+extern const char CATS_IMP_EXP *copy_object[];
diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c
index 4e514dda5..f5004cd7f 100644
--- a/bacula/src/cats/sql_get.c
+++ b/bacula/src/cats/sql_get.c
@@ -1465,6 +1465,10 @@ static void strip_md5(char *q)
    while ((p = strstr(p, ", MD5"))) {
       memset(p, ' ', 5 * sizeof(char));
    }
+   p = q;
+   while ((p = strstr(p, ", '' AS MD5"))) {
+      memset(p, ' ', 11 * sizeof(char));
+   }
 }
 
 /**
@@ -1477,6 +1481,10 @@ static void strip_md5(char *q)
  * 3) Join the result to file table to get fileindex, jobid and lstat information
  *
  * TODO: See if we can do the SORT only if needed (as an argument)
+ *
+ * In addition, find all objects for the given jobs, and keep all versions.
+ * For Object, the FileIndex is not stored, so the code is prepared but not yet
+ * fonctionnal.  It is done via an "manual" procedure waiting for FileIndex
  */
 bool BDB::bdb_get_file_list(JCR *jcr, char *jobids, int opts,
                       DB_RESULT_HANDLER *result_handler, void *ctx)
@@ -1500,6 +1508,7 @@ bool BDB::bdb_get_file_list(JCR *jcr, char *jobids, int opts,
    }
    POOL_MEM buf(PM_MESSAGE);
    POOL_MEM buf2(PM_MESSAGE);
+   POOL_MEM buf3(PM_MESSAGE);
    if (opts & DBL_USE_DELTA) {
       Mmsg(buf2, select_recent_version_with_basejob_and_delta[bdb_get_type_index()],
            jobids, jobids, jobids, jobids);
@@ -1509,24 +1518,32 @@ bool BDB::bdb_get_file_list(JCR *jcr, char *jobids, int opts,
            jobids, jobids, jobids, jobids);
    }
 
+   /* During virtual full, we want to include Objects in the BSR */
+   if (opts & DBL_USE_OBJ) {
+      Mmsg(buf3,
+           "UNION (SELECT ObjectName AS Path, PluginName AS Filename, FileIndex, JobId, '' AS LStat, 0 AS DeltaSeq, '' AS MD5, JobTDate "
+           "FROM Job JOIN RestoreObject USING (JobId) "
+           "WHERE JobId IN (%s) ORDER BY JobTDate ASC, FileIndex ASC) ", jobids);
+   }
    /* bsr code is optimized for JobId sorted, with Delta, we need to get
     * them ordered by date. JobTDate and JobId can be mixed if using Copy
     * or Migration
     */
    Mmsg(buf,
-"SELECT Path.Path, T1.Filename, T1.FileIndex, T1.JobId, LStat, DeltaSeq, MD5 "
- "FROM ( %s ) AS T1 "
- "JOIN Path ON (Path.PathId = T1.PathId) %s "
-"ORDER BY T1.JobTDate, FileIndex ASC",/* Return sorted by JobTDate */
-                                      /* FileIndex for restore code */
-        buf2.c_str(), type);
+"SELECT Path, Filename, FileIndex, JobId, LStat, DeltaSeq, MD5, JobTDate "
+"FROM (("
+ "SELECT Path.Path, T1.Filename, T1.FileIndex, T1.JobId, LStat, DeltaSeq, MD5, JobTDate "
+  "FROM ( %s ) AS T1 "
+  "JOIN Path ON (Path.PathId = T1.PathId) %s "
+ ") %s ) AS U1 ORDER BY JobTDate, FileIndex ASC",/* Return sorted by JobTDate */
+                                               /* FileIndex for restore code */
+        buf2.c_str(), type, buf3.c_str());
 
    if (!(opts & DBL_USE_MD5)) {
       strip_md5(buf.c_str());
    }
 
-   Dmsg1(100, "q=%s\n", buf.c_str());
-
+   Dmsg1(50|DT_SQL, "q=%s\n", buf.c_str());
    return bdb_big_sql_query(buf.c_str(), result_handler, ctx);
 }
 
diff --git a/bacula/src/dird/backup.c b/bacula/src/dird/backup.c
index fca7177d2..fa66cb79e 100644
--- a/bacula/src/dird/backup.c
+++ b/bacula/src/dird/backup.c
@@ -174,7 +174,7 @@ static int accurate_list_handler(void *ctx, int num_fields, char **row)
 
    /* sending with checksum */
    if (jcr->use_accurate_chksum
-       && num_fields == 7
+       && num_fields > 6
        && row[6][0] /* skip checksum = '0' */
        && row[6][1])
    {
diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c
index 1f46c3f65..663b881c4 100644
--- a/bacula/src/dird/catreq.c
+++ b/bacula/src/dird/catreq.c
@@ -571,8 +571,8 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen)
     */
 
    Dmsg1(400, "UpdCat msg=%s\n", msg);
-   Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%d reclen=%d\n",
-      VolSessionId, VolSessionTime, FileIndex, Stream, reclen);
+   Dmsg5(400, "UpdCat VolSessId=%d VolSessT=%d FI=%d Strm=%s reclen=%d\n",
+      VolSessionId, VolSessionTime, FileIndex, stream_to_ascii(Stream), reclen);
 
    if (Stream == STREAM_UNIX_ATTRIBUTES ||
        Stream == STREAM_UNIX_ATTRIBUTES_EX ||
diff --git a/bacula/src/dird/ua_run.c b/bacula/src/dird/ua_run.c
index 032ee7fd4..d014d93e3 100644
--- a/bacula/src/dird/ua_run.c
+++ b/bacula/src/dird/ua_run.c
@@ -970,7 +970,7 @@ static int plugin_display_options(UAContext *ua, JCR *jcr, ConfigFile *ini)
 
 configure_again:
    ua->send_msg(_("Plugin Restore Options\n"));
-   ua->send_msg(_("Option               Current Value        Default Value\n"));
+   ua->send_msg(_("Option                         Current Value        Default Value\n"));
    for (nb=0; nb < MAX_INI_ITEMS && ini->items[nb].name ; nb++) {
 
       if (ini->items[nb].found) {
@@ -989,7 +989,7 @@ configure_again:
 
       Mmsg(tmp, "%s:", ini->items[nb].name);
 
-      Mmsg(prompt, "%-20s %-20s ",
+      Mmsg(prompt, "%-30s %-20s ",
            tmp.c_str(), ini->edit);
 
       if (ini->items[nb].default_value) {
diff --git a/bacula/src/dird/vbackup.c b/bacula/src/dird/vbackup.c
index 4ebe1b352..6a206706e 100644
--- a/bacula/src/dird/vbackup.c
+++ b/bacula/src/dird/vbackup.c
@@ -85,6 +85,23 @@ bool do_vbackup_init(JCR *jcr)
    return true;
 }
 
+/* TODO: Workaround waiting to get the FileIndex in the Object table. With this trick, a bscan will
+ * not re-create the objects in the catalog, So the object selection during the restore process
+ * is important.
+ */
+static bool copy_object_list(JCR *jcr, const char *jobids, uint32_t JobId)
+{
+   /* The batch session is not used anymore at this point */
+   db_lock(jcr->db_batch);
+   Mmsg(jcr->db_batch->cmd, copy_object[db_get_type_index(jcr->db_batch)], JobId, jobids, jobids);
+   if (!db_sql_query(jcr->db_batch, jcr->db_batch->cmd, NULL, NULL)) {
+      db_unlock(jcr->db_batch);
+      return false;
+   }
+   db_unlock(jcr->db_batch);
+   return true;
+}
+
 /*
  * Do a virtual backup, which consolidates all previous backups into
  *  a sort of synthetic Full.
@@ -366,6 +383,11 @@ _("This Job is not an Accurate backup so is not equivalent to a Full backup.\n")
       return false;
    }
 
+   if (!copy_object_list(jcr, jobids.list, jcr->JobId))
+   {
+      Jmsg(jcr, M_ERROR, 0, "Unable to copy objects ERR=%s\n", jcr->db_batch->errmsg);
+   }
+
    if (jcr->JobStatus != JS_Terminated) {
       return false;
    }
@@ -579,6 +601,7 @@ int insert_bootstrap_handler(void *ctx, int num_fields, char **row)
 
    JobId = str_to_int64(row[3]);
    FileIndex = str_to_int64(row[2]);
+   Dmsg2(10, "insert bootstrap(%ld, %d)\n", JobId, FileIndex);
    add_findex(bsr_list, JobId, FileIndex);
    return 0;
 }
@@ -605,7 +628,7 @@ static bool create_bootstrap_file(JCR *jcr, char *jobids)
     * however, if it's an incremental/differential, we should not restore
     * the files from the Full, so we keep the deleted records
     */
-   int opt = DBL_USE_DELTA;
+   int opt = DBL_USE_DELTA | DBL_USE_OBJ;
    if (jcr->jr.JobLevel != 'F') {
       opt |= DBL_ALL_FILES;
    }
diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c
index 4c9c76348..236c40d14 100644
--- a/bacula/src/stored/append.c
+++ b/bacula/src/stored/append.c
@@ -452,7 +452,7 @@ bool send_attrs_to_dir(JCR *jcr, DEV_RECORD *rec)
          if (are_attributes_spooled(jcr)) {
             dir->set_spooling();
          }
-         Dmsg1(850, "Send attributes to dir. FI=%d\n", rec->FileIndex);
+         Dmsg1(100, "Send attributes to dir. FI=%d\n", rec->FileIndex);
          if (!dir_update_file_attributes(jcr->dcr, rec)) {
             Jmsg(jcr, M_FATAL, 0, _("Error updating file attributes. ERR=%s\n"),
                dir->bstrerror());
diff --git a/bacula/src/stored/vbackup.c b/bacula/src/stored/vbackup.c
index 8d2f31a7c..56412094e 100644
--- a/bacula/src/stored/vbackup.c
+++ b/bacula/src/stored/vbackup.c
@@ -308,7 +308,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec)
       goto bail_out;
    }
    jcr->JobBytes += rec->data_len;   /* increment bytes this job */
-   Dmsg5(500, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
+   Dmsg5(200, "wrote_record JobId=%d FI=%s SessId=%d Strm=%s len=%d\n",
       jcr->JobId,
       FI_to_ascii(buf1, rec->FileIndex), rec->VolSessionId,
       stream_to_ascii(buf2, rec->Stream, rec->FileIndex), rec->data_len);