k8s: Add backup_mode parameter, add resiliance when snapshot backup is not compatible...

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
index 47166ec01afc84ed56fddd783e7949d5c1fd3cb1..3f9ae82fdcf3aaf19434cc7748c8b7095ba05ae7 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
@@ -39,8 +39,10 @@ BA_MODE_ERROR = "Invalid annotations for Pod: {namespace}/{podname}. Backup Mode
 BA_EXEC_STDOUT = "{}:{}"
 BA_EXEC_STDERR = "{} Error:{}"
 BA_EXEC_ERROR = "Pod Container execution: {}"
-POD_BACKUP_SELECTED = "The backup mode selected to the pvc `{}` is `{}`"
-CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC = "The pvc `{}` is not compatible with snapshot backup, changing mode to standard. Only pvc with storage that they use CSI driver are compatible."
+POD_BACKUP_SELECTED = "The selected backup mode in pod to the pvc `{}` is `{}`"
+CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC = "The pvc `{}` is not compatible with snapshot backup, changing mode to clone. Only pvc with storage that they use CSI driver are compatible."
+PVC_BACKUP_MODE_APPLIED_INFO = "The pvc `{}` will be backup with {} mode."
+RETRY_BACKUP_WITH_STANDARD_MODE = "If the clone backup is empty. It will try again to do a backup using standard mode."
 
 class BackupJob(EstimationJob):
     """
@@ -53,8 +55,10 @@ class BackupJob(EstimationJob):
     def __init__(self, plugin, params):
         super().__init__(plugin, params, BACKUP_START_PACKET)
         _label = params.get('labels', None)
+        self.fs_backup_mode = BaculaBackupMode.process_param(params.get("backup_mode", BaculaBackupMode.Snapshot)) # Fileset backup mode defined.
         if _label is not None:
             self._io.send_info(BACKUP_PARAM_LABELS.format(_label))
+        self._io.send_info("The selected default backup mode to do pvc backup is `{}`.".format(self.fs_backup_mode))
 
     def execution_loop(self):
         super().processing_loop(estimate=False)
@@ -128,24 +132,36 @@ class BackupJob(EstimationJob):
             return False
         return True
     
-    def process_pvcdata(self, namespace, pvcdata, backup_with_pod = False):
+    def process_pvcdata(self, namespace, pvcdata, backup_with_pod = False, retry_backup = False):
         status = None
         vsnapshot = None
         is_cloned = False
         cloned_pvc_name = None
+        # For retry if clone backup is incompatible.
+        orig_pvcdata = pvcdata
         # Detect if pvcdata is compatible with snapshots
-        if not backup_with_pod:
-            logging.debug('Backup without pod')
-            vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata.get('name'))
+        if not backup_with_pod and not retry_backup:
+            logging.debug('Backup mode {} of pvc {} without pod:'.format(self.fs_backup_mode, pvcdata.get('name')))
+            if self.fs_backup_mode == BaculaBackupMode.Snapshot:
+                logging.debug('Snapshot is activated')
+                vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata.get('name'))
+                self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Snapshot))
+
+            if (vsnapshot is None and self.fs_backup_mode != BaculaBackupMode.Standard) or self.fs_backup_mode == BaculaBackupMode.Clone:
+                if self.fs_backup_mode != BaculaBackupMode.Clone:
+                    self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcdata.get('name')))
+                self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Clone))
+                cloned_pvc_name = self.create_pvcclone(namespace, pvcdata.get('name'))
+                cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, cloned_pvc_name)
+                logging.debug('Cloned pvc fi:{}'.format(cloned_pvc.get('fi')))
+                cloned_pvc.get('fi').set_name(pvcdata.get('fi').name)
+                pvcdata = cloned_pvc
+                is_cloned = True
+
+            if self.fs_backup_mode == BaculaBackupMode.Standard:
+                self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Standard))
         logging.debug('Process_pvcdata (Backup_job): {} --- {}'.format(vsnapshot, pvcdata))
-        # if vsnapshot is None:
-        #     self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcdata.get('name')))
-        #     cloned_pvc_name = self.create_pvcclone(namespace, pvcdata.get('name'))
-        #     cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, cloned_pvc_name)
-        #     logging.debug('Cloned pvc fi:{}'.format(cloned_pvc.get('fi')))
-        #     cloned_pvc.get('fi').set_name(pvcdata.get('fi').name)
-        #     pvcdata = cloned_pvc
-        #     is_cloned = True
+
         if self.prepare_bacula_pod(pvcdata, namespace=namespace, mode='backup'):
             super()._estimate_file(pvcdata)     # here to send info about pvcdata to plugin
             status = self.__backup_pvcdata(namespace=namespace)
@@ -158,6 +174,14 @@ class BackupJob(EstimationJob):
             self.handle_delete_vsnapshot_backup(namespace, vsnapshot, pvcdata)
         if is_cloned:
             self.delete_pvcclone(namespace, cloned_pvc_name)
+        # It retries when the backup is not with pod annotation (it is controlled in their function)
+        if not backup_with_pod and not retry_backup and not self.backup_clone_compatibility:
+            # It is important set to True before recall the process.
+            self.backup_clone_compatibility = True # We only try once
+
+            self._io.send_info(RETRY_BACKUP_WITH_STANDARD_MODE)
+
+            status = self.process_pvcdata(namespace, orig_pvcdata, backup_with_pod, True)
         return status
 
     def handle_pod_container_exec_command(self, corev1api, namespace, pod, runjobparam, failonerror=False):
@@ -222,36 +246,45 @@ class BackupJob(EstimationJob):
         logging.debug("iterate over requested vols for backup: {}".format(requestedvolumes))
         for pvc in requestedvolumes:
             pvcname = pvc
+            original_pvc = self._plugin.get_pvcdata_namespaced(namespace, pvcname)
             vsnapshot = None
             logging.debug("handling vol before backup: {}".format(pvcname))
             self._io.send_info(POD_BACKUP_SELECTED.format(pvcname, backupmode))
-            # self._io.send_info("The pvc `{}` will be backedup with {} mode".format(pvcname, backupmode))
-            if backupmode == BaculaBackupMode.Clone:
-                # snapshot if requested
-                pvcname = self.create_pvcclone(namespace, pvcname)
-                if pvcname is None:
-                    # error
-                    logging.error("create_pvcclone failed!")
-                    return False
 
             if backupmode == BaculaBackupMode.Snapshot:
                 logging.debug('Snapshot mode chosen')
                 vsnapshot, pvc_from_vsnap = self.handle_create_vsnapshot_backup(namespace, pvcname)
                 logging.debug("The vsnapshot created from pvc {} is: {}".format(pvcname, vsnapshot))
                 logging.debug("The pvc create from vsnapshot {} is: {}. FI: {}".format(vsnapshot, pvc_from_vsnap, pvc_from_vsnap.get('fi')))
-                if vsnapshot == None:
+                if vsnapshot is None:
                     logging.debug(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname))
                     # backupmode = BaculaBackupMode.Clone
                     self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname))
+                    backupmode = BaculaBackupMode.Clone
                 else:
                     pvc = pvc_from_vsnap
                     pvcname = pvc_from_vsnap.get("name")
 
+            if backupmode == BaculaBackupMode.Clone:
+                pvcname = self.create_pvcclone(namespace, pvcname)
+                cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, pvcname)
+                if pvcname is None:
+                    # error
+                    logging.error("create_pvcclone failed!")
+                    return False
+                else:
+                    logging.debug('Original_pvc: {}'.format(original_pvc.get('fi')))
+                    logging.debug('Cloned_pvc->fi: {}'.format(cloned_pvc.get('fi')))
+                    cloned_pvc.get('fi').set_name(original_pvc.get('fi').name)
+                    pvc = cloned_pvc
+
             logging.debug("handling vol after snapshot/clone: {}".format(pvcname))
             handledvolumes.append({
                 'pvcname': pvcname,
                 'pvc': pvc,
-                'vsnapshot': vsnapshot
+                'vsnapshot': vsnapshot,
+                'backupmode': backupmode,
+                'original_pvc': original_pvc
                 })
 
         failonerror = BoolParam.handleParam(pod.get(BaculaAnnotationsClass.RunAfterSnapshotonError), False)     # the default is ignore errors
@@ -276,22 +309,26 @@ class BackupJob(EstimationJob):
             else:
                 # Modify the name in FileInfo because we need save the file like original name 
                 # and not the new pvc (from vsnapshot) name.
-                if volumes.get('vsnapshot') is not None:
+                if volumes.get('backupmode') == BaculaBackupMode.Snapshot or volumes.get('backupmode') == BaculaBackupMode.Clone:
                     logging.debug('We change the name of FileInfo to adapt the original pvc name with the new pvc name')
                     pvcdata.get('fi').set_name(pvc.get('fi').name)
 
                 if len(pvcdata) > 0:
                     status = self.process_pvcdata(namespace, pvcdata, True)
+                    # Control the compatibility of snapshot or clone. If it is not compatible, retry again with standard backup mode.
+                    logging.debug('Call again process_pvcdata from process_pod_pvcdata? {}'.format(self.backup_clone_compatibility))
+                    if not self.backup_clone_compatibility:
+                        status = self.process_pvcdata(namespace, volumes['original_pvc'], True)
 
         # iterate on requested volumes for delete snap
         logging.debug("iterate over requested vols for delete snap: {}".format(handledvolumes))
         for volumes in handledvolumes:
             pvcname = volumes['pvcname']
             logging.debug("Should remove this pvc: {}".format(pvcname))
-            if backupmode == BaculaBackupMode.Clone:
+            if volumes.get('backupmode') == BaculaBackupMode.Clone:
                 # snapshot delete if snapshot requested
                 status = self.delete_pvcclone(namespace, pvcname)
-            if backupmode == BaculaBackupMode.Snapshot and volumes['vsnapshot'] is not None:
+            if volumes.get('backupmode') == BaculaBackupMode.Snapshot and volumes['vsnapshot'] is not None:
                 status = self.handle_delete_vsnapshot_backup(namespace, volumes['vsnapshot'], volumes['pvc'])
         logging.debug("Finish removing pvc clones and vsnapshots. Status {}".format(status))
 

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
index 88b4c07931d23debcc08ce82535c8d2c390a016a..848f17ebc77c2fe3f33bbcefde2bffbdc73fdf0d 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
@@ -45,7 +45,7 @@ POD_EXIST_ERR = "Job already running in '{namespace}' namespace. Check logs or d
 TAR_STDERR_UNKNOWN = "Unknown error. You should check Pod logs for possible explanation."
 PLUGINPORT_VALUE_ERR = "Cannot use provided pluginport={port} option. Used default!"
 FDPORT_VALUE_ERR = "Cannot use provided fdport={port} option. Used default!"
-POD_YAML_PREPARED_INFO = "Prepare backup Pod with: {image} <{pullpolicy}> {pluginhost}:{pluginport}"
+POD_YAML_PREPARED_INFO = "Prepare bacula-backup Pod with: {image} <{pullpolicy}> {pluginhost}:{pluginport}"
 POD_YAML_PREPARED_INFO_NODE = "Prepare Bacula Pod on: {nodename} with: {image} <{pullpolicy}> {pluginhost}:{pluginport}"
 CANNOT_CREATE_BACKUP_POD_ERR = "Cannot create backup pod. Err={}"
 CANNOT_REMOVE_BACKUP_POD_ERR = "Cannot remove backup pod. Err={}"
@@ -58,6 +58,7 @@ CANNOT_REMOVE_PVC_CLONE_ERR = "Cannot remove PVC snapshot. Err={}"
 CANNOT_REMOVE_VSNAPSHOT_ERR = "Unable to remove volume snapshot {vsnapshot}! Please you must remove it manually."
 CANNOT_START_CONNECTIONSERVER = "Cannot start ConnectionServer. Err={}"
 
+WARNING_CLONED_PVC_WAS_NOT_WORKED = "As clone backup is empty. It will retry again to do a backup with standard mode."
 VSNAPSHOT_BACKUP_COMPATIBLE_INFO = "The pvc `{}` is compatible with volume snapshot backup. Doing backup with this technology."
 PVC_FROM_SNAPSHOT_CREATED = "The pvc `{}` was created from volume snapshot from pvc `{}`."
 CREATING_PVC_FROM_VSNAPSHOT = "Creating pvc from volume snapshot of pvc `{}`."
@@ -93,10 +94,13 @@ class JobPodBacula(Job, metaclass=ABCMeta):
         self.tarexitcode = None
         self.backupimage = params.get('baculaimage', BACULABACKUPIMAGE)
         self.imagepullpolicy = ImagePullPolicy.process_param(params.get('imagepullpolicy'))
+        self.backup_clone_compatibility = True
 
     def handle_pod_logs(self, connstream):
         logmode = ''
         self.tarstderr = ''
+        file_count = 0
+        bytes_count = 0
         with connstream.makefile(mode='r') as fd:
             self.tarexitcode = fd.readline().strip()
             logging.debug('handle_pod_logs:tarexitcode:{}'.format(self.tarexitcode))
@@ -118,7 +122,20 @@ class JobPodBacula(Job, metaclass=ABCMeta):
                     continue
                 elif logmode == 'list':
                     # no listing feature yet
+                    file_count += 1
+                    try:
+                        file_props = data.split()
+                        logging.debug('Data Split:{}'.format(','.join(file_props)))
+                        bytes_count += int(file_props[2]) 
+                    except Exception as e:
+                        logging.exception(e)
                     continue
+        logging.debug('Bytes/files in backup: {}/{}'.format(bytes_count, file_count))
+        logging.debug('Type of job:' + str(self._params.get('type')))
+        if self._params.get('type') == 'b' and bytes_count == 0 and file_count < 3:
+            self._io.send_non_fatal_error(WARNING_CLONED_PVC_WAS_NOT_WORKED)
+            self.backup_clone_compatibility = False
+
 
     def handle_pod_data_recv(self, connstream):
         while True:

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
index af50ec6a4c375e148d5e8e5bc256ec3cbf586b81..c33bd94563733f528db6f8821397c5ab27c26765 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
@@ -55,12 +55,13 @@ class BaculaBackupMode(object):
         Returns:
             str: backup mode normalized to consts, `None` when error
         """
-        if mode is not None:
-            mode = mode.lower()
-            for p in BaculaBackupMode.params:
-                if p == mode:
-                    return p
-        return None
+        if mode is None:
+            return None
+        mode = mode.lower()
+        for p in BaculaBackupMode.params:
+            if p == mode:
+                return p
+        raise Exception('This backup mode `{}` is not supported. Only supported: snapshot, clone or standard.'.format(mode))
 
 
 class BaculaAnnotationsClass(object):

diff --git a/bacula/src/plugins/fd/kubernetes-backend/setup.py b/bacula/src/plugins/fd/kubernetes-backend/setup.py
index 4e96137669055b83d97f7af4001b838413fe9cbe..97bad4b2a3f25de468483feaa7bbaf9358a2dc40 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/setup.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/setup.py
@@ -24,7 +24,7 @@ if sys.version_info < (3, 0):
 
 setup(
     name='baculak8s',
-    version='2.1.1',
+    version='2.2.0',
     author='Radoslaw Korzeniewski, Francisco Manuel Garcia Botella',
     author_email='radekk@korzeniewski.net, francisco.garcia@baculasystems.com',
     packages=find_packages(exclude=('tests', 'tests.*')),

diff --git a/bacula/src/plugins/fd/kubernetes-fd.c b/bacula/src/plugins/fd/kubernetes-fd.c
index 1a55dbfbd3e25d9037bc6331066ca97b6b3c8b83..331806814ea2f27445db84ad2cd6d72000b7997f 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-fd.c
+++ b/bacula/src/plugins/fd/kubernetes-fd.c
@@ -21,8 +21,8 @@
  * @author Radosław Korzeniewski (radoslaw@korzeniewski.net)
  * Modified by: Francisco Manuel Garcia Botella (francisco.garcia@baculasystems.com)
  * @brief This is a Bacula Kubernetes Plugin with metaplugin interface.
- * @version 2.1.0
- * @date 2023-07-31
+ * @version 2.2.0
+ * @date 2024-02-20
  *
  * @copyright Copyright (c) 2021 All rights reserved.
  *            IP transferred to Bacula Systems according to agreement.
@@ -33,8 +33,8 @@
 /* Plugin Info definitions */
 const char *PLUGIN_LICENSE       = "Bacula AGPLv3";
 const char *PLUGIN_AUTHOR        = "Radoslaw Korzeniewski, Francisco Manuel Garcia Botella";
-const char *PLUGIN_DATE          = "July 2023";
-const char *PLUGIN_VERSION       = "2.1.0"; // TODO: should synchronize with kubernetes-fd.json
+const char *PLUGIN_DATE          = "February 2024";
+const char *PLUGIN_VERSION       = "2.2.0"; // TODO: should synchronize with kubernetes-fd.json
 const char *PLUGIN_DESCRIPTION   = "Bacula Kubernetes Plugin";
 
 /* Plugin compile time variables */

diff --git a/bacula/src/plugins/fd/kubernetes-fd.h b/bacula/src/plugins/fd/kubernetes-fd.h
index 0aae5797e633f0064fe8df5cac6f455739bd1ccf..33f0cc2e47c7690c306e2fe811eb34bef04fccc6 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-fd.h
+++ b/bacula/src/plugins/fd/kubernetes-fd.h
@@ -68,6 +68,7 @@ const char * valid_params[] =
    "ssl_ca_cert",
    "timeout",
    "debug",
+   "backup_mode",
    "namespace",
    "ns",
    "persistentvolume",