k8s: Add snapshot support feature in pod annotations and fix minor bugs.

diff --git a/bacula/scripts/kubernetes-bacula-backup/baculatar.py b/bacula/scripts/kubernetes-bacula-backup/baculatar.py
index 5ddc8467a2010533bf2c4ed7beb6174c2cb29ed8..1eeeb5ccc318f66d165c718b2cec1b30b20c4dae 100644 (file)
--- a/bacula/scripts/kubernetes-bacula-backup/baculatar.py
+++ b/bacula/scripts/kubernetes-bacula-backup/baculatar.py
@@ -89,6 +89,8 @@ class BaculaConnection(object):
         ))
 
     def authenticate(self):
+        logging.debug('Authenticate')
+        logging.debug(self.conn)
         if self.conn is not None:
             # first prepare the hello string
             self.conn.send(bytes('{strlen:03}:Hello:{job}'.format(job=self.jobname, strlen=len(self.jobname) + 7), 'ascii'))
@@ -116,6 +118,12 @@ class BaculaConnection(object):
                 except OSError as e:
                     logging.debug(e)
                     time.sleep(1)
+                    logging.debug('Connected OSError')
+                except Exception as e:
+                    logging.debug(e)
+                    time.sleep(1)
+                    logging.debug('Connected Exception')
+                    break
                 else:
                     logging.info('Connected.')
                     break
@@ -178,6 +186,7 @@ class BaculaConnection(object):
             logging.error('Timeout waiting {} for tar to proceed. Terminate!'.format(self.conntimeout))
             self.tarproc.terminate()
         # send execution logs
+        logging.debug('Final_execute')
         self.connect()
         logging.info('tar exit status:{}'.format(exitcode))
         self.conn.send("{}\n".format(exitcode).encode())
@@ -199,17 +208,22 @@ class BaculaConnection(object):
         self.final_execute()
 
     def execute_backup(self):
+        logging.debug('Execute_backup')
         self.prepare_execute()
+        logging.debug('Executed_prepare_execute')
         if os.path.isdir('/backup'):
+            logging.debug('Isdir')
             self.tarproc = subprocess.Popen([TARCMD, '-cvvf', TARPIPE, '-C', '/backup', '.'],
                                             stderr=self.terr,
                                             stdout=self.tout)
             self.sendfile(TARPIPE)
             self.final_execute()
         else:
+            logging.debug('Nodir')
             self.err_no_dir_found()
 
     def execute_restore(self):
+        logging.debug('Execute_restore')
         self.prepare_execute()
         if os.path.isdir('/restore'):
             self.tarproc = subprocess.Popen([TARCMD, '-xvvf', TARPIPE, '-C', '/restore'],

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
index 0edb2f3fbef1eafa99508dbf1ab31e7db3b1fbea..4842dd6302d5902352ffc5a7c204ddd3b6aa025a 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
@@ -38,7 +38,8 @@ BA_MODE_ERROR = "Invalid annotations for Pod: {namespace}/{podname}. Backup Mode
 BA_EXEC_STDOUT = "{}:{}"
 BA_EXEC_STDERR = "{} Error:{}"
 BA_EXEC_ERROR = "Pod Container execution: {}"
-
+POD_BACKUP_SELECTED = "The backup mode selected to the pvc `{}` is `{}`"
+CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC = "The pvc `{}` is not compatible with snapshot backup, changing mode to standard. Only pvc with storage that they use CSI driver are compatible."
 
 class BackupJob(EstimationJob):
     """
@@ -126,10 +127,12 @@ class BackupJob(EstimationJob):
             return False
         return True
 
-    def process_pvcdata(self, namespace, pvcdata):
+    def process_pvcdata(self, namespace, pvcdata, backup_with_pod = False):
         status = None
+        vsnapshot = None
         # Detect if pvcdata is compatible with snapshots
-        vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata)
+        if not backup_with_pod:
+            vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata.get('name'))
 
         logging.debug('Process_pvcdata (Backup_job): {} {}'.format(vsnapshot, pvcdata))
         if self.prepare_bacula_pod(pvcdata, namespace=namespace, mode='backup'):
@@ -140,7 +143,8 @@ class BackupJob(EstimationJob):
                 self.handle_tarstderr()
             self.handle_delete_pod(namespace=namespace)
         # Both prepare_bacula_pod fails or not, we must remove snapshot and pvc
-        self.handle_delete_vsnapshot_backup(namespace, vsnapshot, pvcdata)
+        if not backup_with_pod:
+            self.handle_delete_vsnapshot_backup(namespace, vsnapshot, pvcdata)
         return status
 
     def handle_pod_container_exec_command(self, corev1api, namespace, pod, runjobparam, failonerror=False):
@@ -202,21 +206,38 @@ class BackupJob(EstimationJob):
         handledvolumes = []
 
         # iterate on requested volumes for shapshot
-        logging.debug("iterate over requested vols for snapshot: {}".format(requestedvolumes))
+        logging.debug("iterate over requested vols for backup: {}".format(requestedvolumes))
         for pvc in requestedvolumes:
             pvcname = pvc
-            logging.debug("handling vol before snapshot: {}".format(pvcname))
-            if backupmode == BaculaBackupMode.Snapshot:
+            vsnapshot = None
+            logging.debug("handling vol before backup: {}".format(pvcname))
+            self._io.send_info(POD_BACKUP_SELECTED.format(pvcname, backupmode))
+            # self._io.send_info("The pvc `{}` will be backedup with {} mode".format(pvcname, backupmode))
+            if backupmode == BaculaBackupMode.Clone:
                 # snapshot if requested
                 pvcname = self.create_pvcclone(namespace, pvcname)
                 if pvcname is None:
                     # error
                     logging.error("create_pvcclone failed!")
                     return False
-            logging.debug("handling vol after snapshot: {}".format(pvcname))
+
+            if backupmode == BaculaBackupMode.Snapshot:
+                vsnapshot, pvc_from_vsnap = self.handle_create_vsnapshot_backup(namespace, pvcname)
+                logging.debug("The vsnapshot created from pvc {} is: {}".format(pvcname, vsnapshot))
+                logging.debug("The pvc create from vsnapshot {} is: {}".format(vsnapshot, pvc_from_vsnap))
+                if vsnapshot == None:
+                    logging.debug(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname))
+                    # backupmode = BaculaBackupMode.Clone
+                    self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname))
+                else:
+                    pvc = pvc_from_vsnap.get("name")
+                    pvcname = pvc_from_vsnap.get("name")
+
+            logging.debug("handling vol after snapshot/clone: {}".format(pvcname))
             handledvolumes.append({
                 'pvcname': pvcname,
                 'pvc': pvc,
+                'vsnapshot': vsnapshot
                 })
 
         failonerror = BoolParam.handleParam(pod.get(BaculaAnnotationsClass.RunAfterSnapshotonError), False)     # the default is ignore errors
@@ -241,16 +262,19 @@ class BackupJob(EstimationJob):
                 logging.debug('PVCDATA:{}:{}'.format(pvc, pvcdata))
                 logging.debug('PVCDATA FI.name:{}'.format(pvcdata.get('fi').name))
                 if len(pvcdata) > 0:
-                    status = self.process_pvcdata(namespace, pvcdata)
+                    status = self.process_pvcdata(namespace, pvcdata, True)
 
         # iterate on requested volumes for delete snap
         logging.debug("iterate over requested vols for delete snap: {}".format(handledvolumes))
         for volumes in handledvolumes:
             pvcname = volumes['pvcname']
-
-            if backupmode == BaculaBackupMode.Snapshot:
+            logging.debug("Should remove this pvc: {}".format(pvcname))
+            if backupmode == BaculaBackupMode.Clone:
                 # snapshot delete if snapshot requested
                 status = self.delete_pvcclone(namespace, pvcname)
+            if backupmode == BaculaBackupMode.Snapshot and volumes['vsnapshot'] is not None:
+                status = self.handle_delete_vsnapshot_backup(namespace, volumes['vsnapshot'], volumes['pvc'])
+        logging.debug("Finish removing pvc clones and vsnapshots. Status {}".format(status))
 
         failonerror = BoolParam.handleParam(pod.get(BaculaAnnotationsClass.RunAfterJobonError), False)     # the default is ignore errors
         # here we execute remote command after Pod backup

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
index 9be602dc89a84f15444e7656d4e4b6df0c7cad0f..231cc512e8d9d436be279f83e0aba3a2fa9f1260 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
@@ -139,11 +139,11 @@ class EstimationJob(JobPodBacula):
                 # We can define what pvc backup in two places: fileset and pod annotation. So, we save the pod annotations to we don't backup two times the same.
                 pvc_backed={}
                 # here we have a list of pods which are anotated
-                if podsannotated is not None:
+                if podsannotated is not None and self._plugin.config.get('pvcdata') is not None:
                     if isinstance(podsannotated, dict) and podsannotated.get('exception'):
                         self._handle_error(PODS_LIST_ERROR.format(parse_json_descr(podsannotated)))
                     else:
-                        if not estimate:
+                        if len(podsannotated) > 0 and not estimate:
                             self._io.send_info(PROCESSING_PODBACKUP_PHASE_START_INFO)
                         for pod in podsannotated:
                             logging.debug('PODDATA:{}'.format(pod))
@@ -161,6 +161,7 @@ class EstimationJob(JobPodBacula):
                                     self._io.send_info(PROCESSING_PODBACKUP_START_INFO.format(namespace=nsname,
                                                                                               podname=podname))
                                 status = self.process_pod_pvcdata(nsname, pod, pvcnames)
+                                logging.debug("Status in processing_loop: {}".format(status))
                                 if status is None:
                                     for pvc_name in pvcnames.split(','):
                                         pvc_backed[pvc_name] = 'error' 
@@ -172,7 +173,7 @@ class EstimationJob(JobPodBacula):
                                         pvc_backed[pvc_name] = 'ok'
                                     self._io.send_info(PROCESSING_PODBACKUP_FINISH_INFO.format(namespace=nsname,
                                                                                                podname=podname))
-                        if not estimate:
+                        if len(podsannotated) > 0 and not estimate:
                             self._io.send_info(PROCESSING_PODBACKUP_PHASE_FINISH_INFO)
                             logging.debug("PVCs backed through pod annotations: {}".format(pvc_backed))
                 
@@ -192,7 +193,7 @@ class EstimationJob(JobPodBacula):
                                     self._io.send_info(SKIP_PVCDATA_SECOND_BACKUP_INFO.format(pvc))
                                     continue
                                 elif pvc_backed.get(pvc) == 'error':
-                                    self._io.send_info(SECOND_TRY_PVCDATA_INFO.format(pvc))
+                                    self._io.send_warning(SECOND_TRY_PVCDATA_INFO.format(pvc))
                                 self._io.send_info(PROCESSING_PVCDATA_START_INFO.format(pvc=pvc))
                             status = self.process_pvcdata(nsname, pvcdata)
                             if status is None:

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
index 296388b0c3ceaab6b65c701f6e6a91699b3904f5..6cb8486e6cee1022dccb91834f2c18a98066a6cf 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py
@@ -50,7 +50,7 @@ POD_YAML_PREPARED_INFO_NODE = "Prepare Bacula Pod on: {nodename} with: {image} <
 CANNOT_CREATE_BACKUP_POD_ERR = "Cannot create backup pod. Err={}"
 CANNOT_REMOVE_BACKUP_POD_ERR = "Cannot remove backup pod. Err={}"
 PVCDATA_GET_ERROR = "Cannot get PVC Data object Err={}"
-PVCCLONE_YAML_PREPARED_INFO = "Prepare snapshot: {namespace}/{snapname} storage: {storage} capacity: {capacity}"
+PVCCLONE_YAML_PREPARED_INFO = "Prepare clone: {namespace}/{snapname} storage: {storage} capacity: {capacity}"
 CANNOT_CREATE_PVC_CLONE_ERR = "Cannot create PVC snapshot. Err={}"
 CANNOT_CREATE_VSNAPSHOT_ERR = "Cannot create Volume Snapshot. Err={}"
 CANNOT_CREATE_PVC_SNAPSHOT_ERR = "Cannot create PVC from Volume Snapshot. Err={}"
@@ -335,21 +335,22 @@ class JobPodBacula(Job, metaclass=ABCMeta):
 
         return clonename
 
-    def handle_create_vsnapshot_backup(self, namespace, pvcdata):
+    def handle_create_vsnapshot_backup(self, namespace, pvcname):
         """
         Manage operations to create snapshot and new pvc from this snapshot to do backup. 
         """
+        pvc = self._plugin.get_pvcdata_namespaced(namespace, pvcname)
         # Check if pvc is compatible with vsnapshot
-        if not self._plugin.check_vsnapshot_compatibility(pvcdata.get('storage_class_name')):
-            return None, pvcdata
+        if not self._plugin.check_pvc_compatiblity_with_vsnapshot(namespace, pvc.get('name')):
+            return None, pvc
 
-        self._io.send_info(VSNAPSHOT_BACKUP_COMPATIBLE_INFO.format(pvcdata.get('name')))
-        vsnapshot = self._plugin.create_vsnapshot(namespace, pvcdata.get('name'))
+        self._io.send_info(VSNAPSHOT_BACKUP_COMPATIBLE_INFO.format(pvc.get('name')))
+        vsnapshot = self._plugin.create_vsnapshot(namespace, pvc.get('name'))
         if isinstance(vsnapshot, dict) and 'error' in vsnapshot:
             self._handle_error(CANNOT_CREATE_VSNAPSHOT_ERR.format(parse_json_descr(vsnapshot)))
             return None, None
         # Create pvc from volume snapshot
-        new_pvc = self._plugin.create_pvc_from_vsnapshot(namespace, pvcdata)
+        new_pvc = self._plugin.create_pvc_from_vsnapshot(namespace, pvc)
         if isinstance(new_pvc, dict) and 'error' in new_pvc:
             self._handle_error(CANNOT_CREATE_PVC_SNAPSHOT_ERR.format(parse_json_descr(new_pvc)))
             return None, None
@@ -357,12 +358,16 @@ class JobPodBacula(Job, metaclass=ABCMeta):
         return vsnapshot, new_pvc
 
     def handle_delete_vsnapshot_backup(self, namespace, vsnapshot, pvcdata):
-        logging.debug('handle_delete_vsnapshot: {}/{}'.format(namespace, vsnapshot))
+        logging.debug('handle_delete_vsnapshot: {}/{}/{}'.format(namespace, vsnapshot, pvcdata))
         if vsnapshot is None:
             return None
-        response = self._plugin.remove_pvcclone(namespace, pvcdata.get('name'))
+        pvc_name = pvcdata
+        if type(pvcdata) == dict:
+            pvc_name = pvcdata.get('name')
+        response = self._plugin.remove_pvcclone(namespace, pvc_name)
         if isinstance(response, dict) and "error" in response:
-            return self._handle_error(CANNOT_REMOVE_PVC_CLONE_ERR.format(vsnapshot=vsnapshot.get('name')))
+            return self._handle_error(CANNOT_REMOVE_PVC_CLONE_ERR.format(pvc_name))
         response = self._plugin.remove_vsnapshot(namespace, vsnapshot.get('name'))
         if isinstance(response, dict) and "error" in response:
             return self._handle_error(CANNOT_REMOVE_VSNAPSHOT_ERR.format(vsnapshot=vsnapshot.get('name')))
+        return True

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
index ca8c6eb7891f5d3e622f1ac0e6b297499cb3a054..af50ec6a4c375e148d5e8e5bc256ec3cbf586b81 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py
@@ -41,8 +41,9 @@ class BaculaBackupMode(object):
     This is a class to manage snapshot mode.
     """
     Snapshot = 'snapshot'
+    Clone = 'clone'
     Standard = 'standard'
-    params = (Snapshot, Standard)
+    params = (Snapshot, Clone, Standard)
 
     @staticmethod
     def process_param(mode):

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
index f3eb78c2fb78f744a0e0e7fb93e1857e280a777a..f37b5572b745840d230760daaf12a818ab118b26 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
@@ -80,7 +80,7 @@ class KubernetesPlugin(Plugin):
                 # pvcdata without value
                 _pvcdata = True
             else:
-                _pvcdata = [_pvcdata]
+                _pvcdata = _pvcdata.split(',')
 
         self.config = {
             'config_file': params.get("config", None),
@@ -533,9 +533,13 @@ class KubernetesPlugin(Plugin):
             return self.__restore_k8s_object(file_info, file_content_source)
 
     # TODO: export/move all checks into k8sbackend
-    def check_vsnapshot_compatibility(self, storage_class_name):
+    def check_storage_compatibility_with_vsnapshot(self, storage_class_name):
         return SNAPSHOT_DRIVER_COMPATIBLE in storage_class_name
 
+    def check_pvc_compatiblity_with_vsnapshot(self, namespace, pvc_name):
+        pvc = self.get_pvcdata_namespaced(namespace, pvc_name)
+        return self.check_storage_compatibility_with_vsnapshot(pvc.get('storage_class_name'))
+
     def _check_config_map(self, file_info):
         return self.__exec_check_object(
             lambda: self.corev1api.read_namespaced_config_map(k8sfile2objname(file_info.name), file_info.namespace))
@@ -881,6 +885,7 @@ class KubernetesPlugin(Plugin):
         response = self.__execute(lambda: self.corev1api.delete_namespaced_persistent_volume_claim(
             clonename, namespace, grace_period_seconds=0,
             propagation_policy='Foreground'))
+        logging.debug("Response: {}".format(response))
         if isinstance(response, dict) and "error" in response:
             return response
         r1 = self.get_pvcdata_namespaced(namespace, clonename)

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/util/sslserver.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/util/sslserver.py
index d3b9e77ad3d439332e287d07c6f5bda609c8210c..dcad860f12aa7875fc2b454337eab7050e22f8db 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/util/sslserver.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/util/sslserver.py
@@ -199,6 +199,12 @@ class ConnectionServer(object):
             process_client_data(self.connstream)
         finally:
             logging.debug('ConnectionServer:Finish - disconnect.')
-            self.connstream.shutdown(socket.SHUT_RDWR)
+            try:
+                self.connstream.shutdown(socket.SHUT_RDWR)
+            except Exception as e:
+                logging.debug('throw exception when disconnect.')
+                logging.error(e)
+            logging.debug('ConnectionServer:Finish - shutdown.')
             self.connstream.close()
+            logging.debug('ConnectionServer:Finish - closed.')
         return {}

diff --git a/regress/scripts/kubernetes-plugin-test-bacula-dir.conf.in b/regress/scripts/kubernetes-plugin-test-bacula-dir.conf.in
index a58ab360d4023e9a56c25386df6c0217420f4174..8ef48b95c30c024188ee75aabe595f2ad59df7be 100644 (file)
--- a/regress/scripts/kubernetes-plugin-test-bacula-dir.conf.in
+++ b/regress/scripts/kubernetes-plugin-test-bacula-dir.conf.in
@@ -192,7 +192,7 @@ Job {
 FileSet {
   Name = "TestPluginKubernetesSet4"
   Include { Options { signature=SHA1 }
-    Plugin = "@LPLUG@ namespace=plugintest persistentvolume=@PV_CSI1@ @BACKUP_PROXY_ARGS@"
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@"
   }
 }
 Job {
@@ -261,6 +261,297 @@ Job {
   FileSet = TestPluginKubernetesSet22
 }
 
+FileSet {
+  Name = "TestPluginKubernetesSet30"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest persistentvolume=@PV_CSI1@ @BACKUP_PROXY_ARGS@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest30"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet30
+}
+
+FileSet {
+  Name = "TestPluginKubernetesSet31"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=@BACKUP_ONLY_PVC@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest31"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet31
+}
+
+#
+# Jobs with csi-driver to check combinations in pod annotations 
+#
+
+# 40. Check annotation:
+# mode: standard
+# one vol: plugintest-persistent-volume-claim-csi
+FileSet {
+  Name = "TestPluginKubernetesSet40"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest40"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet40
+}
+
+# 41. Check annotation:
+# mode: standard
+# two vols: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+FileSet {
+  Name = "TestPluginKubernetesSet41"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest41"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet41
+}
+
+# 50. Check annotation:
+# mode: snapshot
+# one vol no compatible: plugintest-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet50"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest50"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet50
+}
+
+# 51. Check annotation:
+# mode: snapshot
+# one vol compatible: plugintest-persistent-volume-claim-csi
+FileSet {
+  Name = "TestPluginKubernetesSet51"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest51"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet51
+}
+
+# 52. Check annotation:
+# mode: snapshot
+# two vol compatible: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+FileSet {
+  Name = "TestPluginKubernetesSet52"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest52"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet52
+}
+
+# 53. Check annotation:
+# mode: snapshot
+# one vol compatible and another not: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet53"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest53"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet53
+}
+
+# 54. Check annotation:
+# mode: snapshot
+# one vol not compatible and another yes: plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi
+FileSet {
+  Name = "TestPluginKubernetesSet54"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest54"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet54
+}
+
+# 60. Check annotation:
+# mode: clone
+# one vol compatible: plugintest-persistent-volume-claim-csi
+FileSet {
+  Name = "TestPluginKubernetesSet60"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest60"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet60
+}
+
+# 61. Check annotation:
+# mode: clone
+# one vol no compatible: plugintest-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet61"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest61"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet61
+}
+
+# 62. Check annotation:
+# mode: clone
+# two vols compatible: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+FileSet {
+  Name = "TestPluginKubernetesSet62"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest62"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet62
+}
+
+# 63. Check annotation:
+# mode: clone
+# one vol compatible and another not: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet63"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest63"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet63
+}
+
+# 64. Check annotation:
+# mode: clone
+# one vol not compatible and another yes: plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi
+FileSet {
+  Name = "TestPluginKubernetesSet64"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest64"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet64
+}
+
+# 70. Check pvcdata:
+# one vol (standard): plugintest-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet70"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest70"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet70
+}
+
+# 71. Check pvcdata:
+# two vols (standard): plugintest-persistent-volume-claim,plugintest3-persistent-volume-claim
+FileSet {
+  Name = "TestPluginKubernetesSet71"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@=plugintest-persistent-volume-claim,plugintest3-persistent-volume-claim"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest71"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet71
+}
+
+# 72. Check pvcdata:
+# any vol (all vols will be backed)
+FileSet {
+  Name = "TestPluginKubernetesSet72"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest72"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet72
+}
+
+# 73. Check pvcdata:
+# any vol, one pod with annotation standard
+FileSet {
+  Name = "TestPluginKubernetesSet73"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest73"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet73
+}
+
+# 74. Check pvcdata:
+# any vol, one pod with annotation clone
+FileSet {
+  Name = "TestPluginKubernetesSet74"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest74"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet74
+}
+
+# 75. Check pvcdata:
+# any vol, one pod with annotation snapshot
+FileSet {
+  Name = "TestPluginKubernetesSet75"
+  Include { Options { signature=SHA1 }
+    Plugin = "@LPLUG@ namespace=plugintest @BACKUP_PROXY_ARGS@"
+  }
+}
+Job {
+  Name = "PluginKubernetesTest75"
+  JobDefs = Default
+  FileSet = TestPluginKubernetesSet75
+}
 
 # List of files to be backed up
 FileSet {

diff --git a/regress/scripts/kubernetes-plugintest-csi-driver.yaml b/regress/scripts/kubernetes-plugintest-csi-driver.yaml
index b81716b30ed734067d1529d590ca739cb949a206..2176708579f094cbbe6e270ab63b357f9cd6628c 100644 (file)
--- a/regress/scripts/kubernetes-plugintest-csi-driver.yaml
+++ b/regress/scripts/kubernetes-plugintest-csi-driver.yaml
@@ -13,8 +13,56 @@ metadata:
   name: plugintest-persistent-volume-claim-csi
 spec:
   accessModes:
-  - ReadWriteOnce
+    - ReadWriteOnce
   resources:
     requests:
       storage: 1Gi
-  storageClassName: csi-hostpath-sc
\ No newline at end of file
+  storageClassName: csi-hostpath-sc
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: plugintest-persistent-volume-claim-csi-2
+  namespace: plugintest
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: csi-hostpath-sc
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: plugintest-annotations-test
+  namespace: plugintest
+  labels:
+    app: plugintest
+    environment: production
+spec:
+  hostname: plugintest-3-csi
+  subdomain: plugintest-subdomain
+  containers:
+    - image: busybox:1.28
+      command:
+        - sleep
+        - "3600"
+      name: plugintest
+      volumeMounts:
+        - name: plugintest-pvc-persistent-storage-csi
+          mountPath: /data-csi
+        - name: plugintest-pvc-persistent-storage-csi-2
+          mountPath: /data-csi-2
+        - name: plugintest-pvc-persistent-storage-standard
+          mountPath: /data
+  volumes:
+    - name: plugintest-pvc-persistent-storage-csi
+      persistentVolumeClaim:
+        claimName: plugintest-persistent-volume-claim-csi
+    - name: plugintest-pvc-persistent-storage-csi-2
+      persistentVolumeClaim:
+        claimName: plugintest-persistent-volume-claim-csi-2
+    - name: plugintest-pvc-persistent-storage-standard
+      persistentVolumeClaim:
+        claimName: plugintest-persistent-volume-claim

diff --git a/regress/scripts/kubernetes-plugintest.yaml b/regress/scripts/kubernetes-plugintest.yaml
index 80941a252cf2a8b8a7b15b9fbac75976d3bebf2d..fc4bea684e253201763b4fa6c4b0d2b4b6bb3611 100644 (file)
--- a/regress/scripts/kubernetes-plugintest.yaml
+++ b/regress/scripts/kubernetes-plugintest.yaml
@@ -77,6 +77,20 @@ spec:
       storage: 1Gi
 ---
 apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: plugintest3-persistent-volume-claim
+  namespace: plugintest
+  labels:
+    app: plugintest
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
 kind: Pod
 metadata:
   name: plugintest1
@@ -119,6 +133,35 @@ spec:
       - "3600"
     name: plugintest
 ---
+apiVersion: v1
+kind: Pod
+metadata:
+  name: plugintest3
+  namespace: plugintest
+  labels:
+    app: plugintest
+    environment: production
+    # tier: frontend
+  annotations:
+    bacula/backup.mode: standard
+    bacula/backup.volumes: plugintest3-persistent-volume-claim
+spec:
+  hostname: plugintest-3
+  subdomain: plugintest-subdomain
+  containers:
+  - image: busybox:1.28
+    command:
+      - sleep
+      - "3600"
+    name: plugintest
+    volumeMounts:
+      - name: plugintest3-persistent-storage
+        mountPath: /data
+  volumes:
+    - name: plugintest3-persistent-storage
+      persistentVolumeClaim:
+        claimName: plugintest3-persistent-volume-claim
+---
 apiVersion: apps/v1
 kind: ReplicaSet
 metadata:

diff --git a/regress/scripts/regress-utils.sh b/regress/scripts/regress-utils.sh
index 980f43eda3f990013a5dc91daba1a43967d5865c..a3e46827bdbfb3baef6057288c062c658ba0d122 100755 (executable)
--- a/regress/scripts/regress-utils.sh
+++ b/regress/scripts/regress-utils.sh
@@ -313,11 +313,11 @@ fi
 }
 
 #
-# Check the expected string on any log file
+# Check the expected string in any log file
 #
 # in:
 # $1 - type of log. Values allowed: b,r,e,l
-# $2 - the string to seach into logfile
+# $2 - the string to search into logfile
 # $3 - a test number to examine which means we will check log${ltest}.out logfile
 check_regress_string_in_log()
 {
@@ -337,6 +337,94 @@ check_regress_string_in_log()
    fi
 }
 
+#
+# Check expected string is not in any log file
+#
+# in:
+# $1 - type of log. Values allowed: b,r,e,l
+# $2 - the string to search into logfile
+# $3 - a test number to examine which means we will check log${ltest}.out logfile
+check_regress_string_not_in_log()
+{
+   type_log=$1
+   to_search=$2
+   n_test=$3
+   log_file=${tmp}/${type_log}log${n_test}.out
+   grep "${to_search}" ${log_file} > /dev/null
+   F=$?
+   if [ $F -ne 1 ]
+   then
+      error_stat="${type_log}stat"
+      declare -g "${error_stat}"=$((${!error_stat}+1))
+      return 1
+   else
+      return 0
+   fi
+}
+
+#
+# Get a value of parameter
+#
+# in:
+# $1 - type of log. Values allowed: b,r,e,l
+# $2 - the parameter to get value. For example: 'FD Bytes Written', 'SD Bytes Written', 'jobbytes'
+# $3 - a test number to examine which means we will check log${ltest}.out logfile
+#
+# Example of call: To get value of parameter "SD Bytes Written" in test 5
+# get_value_of_parameter_in_log "b" "SD Bytes Written" 5
+get_value_of_parameter_in_log()
+{
+   type_log=$1
+   job_parameter=$2
+   n_test=$3
+   # echo -e "get_value_of_parameter_in_log:381 - Input: ${type_log} ${job_parameter} ${n_test}\n"
+   log_file=${tmp}/${type_log}log${n_test}.out
+   # Catch up the line with job_parameter, remove spaces and ',' and get value after ':'.
+   parameter_value=$(grep "${job_parameter}:" ${log_file} | sed 's/ //g' | sed 's/,//g' | awk -F'[: (]+' '{print $2}')
+   echo $parameter_value
+}
+#
+#
+# Check a value of parameter if greater, equal or less than input value
+#
+# in:
+# $1 - type of log. Values allowed: b,r,e,l
+# $2 - if operator without '-'. For example: 'gt' to greater than, 'ge' to greater than or equal, 'lt' to less than
+# $3 - the parameter to check. For example: 'FD Bytes Written', 'SD Bytes Written', 'jobbytes'
+# $4 - number to check. Can use *KB, *MB format. Example: 5, 50KB, 100MB 
+# $5 - a test number to examine which means we will check log${ltest}.out logfile
+#
+# Example of call: To check if a backup job wrote more than 100MB in test 5
+# check_regress_number_in_log "b" "gt" "SD Bytes Written" "100MB" 5
+check_regress_number_in_log()
+{
+   type_log=$1
+   operator="-"$2
+   job_parameter=$3
+   number=$4
+   n_test=$5
+   # printf "regress-utils.sh:420 - Input: %s;%s;%s;%s;%s\n" "${type_log}" "$operator" "${job_parameter}" "${number}" "${n_test}"
+   
+   parameter_value=$(get_value_of_parameter_in_log "${type_log}" "${job_parameter}" "${n_test}")
+
+   if [[ $number == *KB ]]; then
+      number=$(echo $number | tr -d 'KB' | awk '{print $1 * 1000}')
+   elif [[ $number == *MB ]]; then
+      number=$(echo $number | tr -d 'MB' | awk '{print $1 * 1000 * 1000}')
+   elif [[ $number == *GB ]]; then
+      number=$(echo $number | tr -d 'GB' | awk '{print $1 * 1000 * 1000 * 1000}')
+   fi
+   # printf "regress-utils.sh:420 - %s;%s;%s\n" "${parameter_value}" "$operator" "${number}"
+   if [ "${parameter_value}" $operator "${number}"  ]
+   then
+      return 0
+   else
+      error_stat="${type_log}stat"
+      declare -g "${error_stat}"=$((${!error_stat}+1))
+      return 1
+   fi
+}
+
 #
 # This is a simple common function which start a fresh, new local slapd
 # available on ldap://localhost:3890

diff --git a/regress/tests/kubernetes-plugin-tests b/regress/tests/kubernetes-plugin-tests
index 65c0a675c6ec5f2bbab4b4f30f8051f760ba2ffd..d868c9aa9827f02146dfa82968a6f2457eda01e8 100755 (executable)
--- a/regress/tests/kubernetes-plugin-tests
+++ b/regress/tests/kubernetes-plugin-tests
@@ -12,55 +12,129 @@
 #  resources to handle test and a available "plugintest" namespace.
 #
 
-# $ kubectl apply -f plugintest.yaml
+# $ kubectl apply -f kubernetes-plugintest.yaml
 # namespace/plugintest created
 # secret/plugintest-secrets created
 # configmap/plugintest-configmap created
 # service/plugintest-subdomain created
 # service/plugintest-nginx-service created
 # persistentvolumeclaim/plugintest-persistent-volume-claim created
+# persistentvolumeclaim/plugintest3-persistent-volume-claim created
 # pod/plugintest1 created
 # pod/plugintest2 created
+# pod/plugintest3 created
 # replicaset.apps/plugintest-frontend created
 # deployment.apps/plugintest-nginx-deployment created
 # statefulset.apps/plugintest-nginx-web created
+
+# $ kubectl apply -f kubernetes-plugintest-csi-driver.yaml 
+# storageclass.storage.k8s.io/csi-hostpath-sc configured
+# persistentvolumeclaim/plugintest-persistent-volume-claim-csi created
+
 # $ kubectl -n plugintest get pods -o wide
-# NAME                                           READY   STATUS    RESTARTS   AGE     IP          NODE                                     NOMINATED NODE   READINESS GATES
-# plugintest-frontend-twqkj                      1/1     Running   0          3m33s   10.0.2.30   gke-c-j6f8g-default-pool-5892648d-g32p   <none>           <none>
-# plugintest-frontend-w5z6s                      1/1     Running   0          3m33s   10.0.1.26   gke-c-j6f8g-default-pool-5892648d-50s4   <none>           <none>
-# plugintest-frontend-zvl98                      1/1     Running   0          3m33s   10.0.0.41   gke-c-j6f8g-default-pool-5892648d-3f8l   <none>           <none>
-# plugintest-nginx-deployment-557d8579b5-9r4z4   1/1     Running   0          3m32s   10.0.0.42   gke-c-j6f8g-default-pool-5892648d-3f8l   <none>           <none>
-# plugintest-nginx-deployment-557d8579b5-dmn4f   1/1     Running   0          3m32s   10.0.1.27   gke-c-j6f8g-default-pool-5892648d-50s4   <none>           <none>
-# plugintest-nginx-deployment-557d8579b5-kmbjs   1/1     Running   0          3m32s   10.0.0.43   gke-c-j6f8g-default-pool-5892648d-3f8l   <none>           <none>
-# plugintest-nginx-web-0                         1/1     Running   0          3m32s   10.0.1.28   gke-c-j6f8g-default-pool-5892648d-50s4   <none>           <none>
-# plugintest-nginx-web-1                         1/1     Running   0          3m9s    10.0.2.32   gke-c-j6f8g-default-pool-5892648d-g32p   <none>           <none>
-# plugintest-nginx-web-2                         1/1     Running   0          2m46s   10.0.0.44   gke-c-j6f8g-default-pool-5892648d-3f8l   <none>           <none>
-# plugintest1                                    1/1     Running   0          3m34s   10.0.2.31   gke-c-j6f8g-default-pool-5892648d-g32p   <none>           <none>
-# plugintest2                                    1/1     Running   0          3m33s   10.0.2.29   gke-c-j6f8g-default-pool-5892648d-g32p   <none>           <none>
+# NAME                                          READY   STATUS    RESTARTS   AGE   IP              NODE         NOMINATED NODE   READINESS GATES
+# plugintest-frontend-4mpdj                     1/1     Running   0          78s   10.244.10.217   test-local   <none>           <none>
+# plugintest-frontend-f85pr                     1/1     Running   0          78s   10.244.10.214   test-local   <none>           <none>
+# plugintest-frontend-hg5ft                     1/1     Running   0          78s   10.244.10.211   test-local   <none>           <none>
+# plugintest-nginx-deployment-568dfdd98-bclmb   1/1     Running   0          78s   10.244.10.216   test-local   <none>           <none>
+# plugintest-nginx-deployment-568dfdd98-hbz8w   1/1     Running   0          78s   10.244.10.218   test-local   <none>           <none>
+# plugintest-nginx-deployment-568dfdd98-xxkz2   1/1     Running   0          78s   10.244.10.215   test-local   <none>           <none>
+# plugintest-nginx-web-0                        1/1     Running   0          78s   10.244.10.219   test-local   <none>           <none>
+# plugintest-nginx-web-1                        1/1     Running   0          66s   10.244.10.220   test-local   <none>           <none>
+# plugintest-nginx-web-2                        1/1     Running   0          62s   10.244.10.221   test-local   <none>           <none>
+# plugintest1                                   1/1     Running   0          78s   10.244.10.210   test-local   <none>           <none>
+# plugintest2                                   1/1     Running   0          78s   10.244.10.212   test-local   <none>           <none>
+# plugintest3                                   1/1     Running   0          78s   10.244.10.213   test-local   <none>           <none>
+
 # $ kubectl -n plugintest get svc -o wide
-# NAME                       TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)    AGE     SELECTOR
-# plugintest-nginx-service   ClusterIP   None          <none>        80/TCP     3m43s   app=plugintest-nginx-web
-# plugintest-subdomain       ClusterIP   10.3.244.64   <none>        1234/TCP   3m43s   name=plugintest
+# NAME                       TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)    AGE     SELECTOR
+# plugintest-nginx-service   ClusterIP   None             <none>        80/TCP     2m26s   app=plugintest-nginx-web
+# plugintest-subdomain       ClusterIP   10.100.124.195   <none>        1234/TCP   2m26s   name=plugintest
+
 # $ kubectl -n plugintest get pvc -o wide
-# NAME                                         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
-# plugintest-persistent-volume-claim           Bound    pvc-ca02fc8e-dea9-11e9-8dea-42010a8e0025   1Gi        RWO            standard       6m18s
-# plugintest-www-data-plugintest-nginx-web-0   Bound    pvc-cb881652-dea9-11e9-8dea-42010a8e0025   1Gi        RWO            standard       6m16s
-# plugintest-www-data-plugintest-nginx-web-1   Bound    pvc-d93ae99d-dea9-11e9-8dea-42010a8e0025   1Gi        RWO            standard       5m53s
-# plugintest-www-data-plugintest-nginx-web-2   Bound    pvc-e690586f-dea9-11e9-8dea-42010a8e0025   1Gi        RWO            standard       5m30s
+# NAME                                         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS      AGE     VOLUMEMODE
+# plugintest-persistent-volume-claim           Bound    pvc-781fa285-9cc4-449c-b3e2-850c704aabe4   1Gi        RWO            standard          2m59s   Filesystem
+# plugintest-persistent-volume-claim-csi       Bound    pvc-47265f3d-68ca-4fd7-9c55-af93f31f8a72   1Gi        RWO            csi-hostpath-sc   2m41s   Filesystem
+# plugintest-www-data-plugintest-nginx-web-0   Bound    pvc-60dd219e-4259-4149-befe-3b2230eb4dba   1Gi        RWO            standard          2m59s   Filesystem
+# plugintest-www-data-plugintest-nginx-web-1   Bound    pvc-5b666e3b-f728-41d2-a13c-2a31667b9136   1Gi        RWO            standard          2m47s   Filesystem
+# plugintest-www-data-plugintest-nginx-web-2   Bound    pvc-58464dd1-7884-4985-9ee8-6c9591f45d3c   1Gi        RWO            standard          2m43s   Filesystem
+# plugintest3-persistent-volume-claim          Bound    pvc-4dc9815f-f736-42fd-ae3e-3569ecb83e86   1Gi        RWO            standard          2m59s   Filesystem
+
 # $ kubectl -n plugintest get rs -o wide
-# NAME                                     DESIRED   CURRENT   READY   AGE     CONTAINERS                 IMAGES                                 SELECTOR
-# plugintest-frontend                      3         3         3       6m56s   plugintest-frontend-test   gcr.io/google_samples/gb-frontend:v3   tier=frontend
-# plugintest-nginx-deployment-557d8579b5   3         3         3       6m55s   plugintest-nginx           nginx:1.7.9                            app=plugintest-deployment,pod-template-hash=557d8579b5
+# NAME                                    DESIRED   CURRENT   READY   AGE     CONTAINERS                 IMAGES                                 SELECTOR
+# plugintest-frontend                     3         3         3       3m21s   plugintest-frontend-test   gcr.io/google_samples/gb-frontend:v3   tier=frontend
+# plugintest-nginx-deployment-568dfdd98   3         3         3       3m21s   plugintest-nginx           nginx:latest                           app=plugintest-deployment,pod-template-hash=568dfdd98
+
 # $ kubectl -n plugintest get sts -o wide
-# NAME                   READY   AGE   CONTAINERS             IMAGES
-# plugintest-nginx-web   3/3     7m    plugintest-nginx-web   k8s.gcr.io/nginx-slim:0.8
+# NAME                   READY   AGE     CONTAINERS             IMAGES
+# plugintest-nginx-web   3/3     3m48s   plugintest-nginx-web   k8s.gcr.io/nginx-slim:0.8
 
 TestName="kubernetes-plugin-test"
 JobName="PluginKubernetesTest"
 FileSetName="TestPluginKubernetesSet"
+
 . scripts/functions
 . scripts/regress-utils.sh
 
+
+#
+# Check a value of parameter if greater and less than input values. 
+#
+# in:
+# $1 - type of log. Values allowed: b,r,e,l
+# $2 - the parameter to check. For example: 'FD Bytes Written', 'SD Bytes Written', 'jobbytes'
+# $3 - number to use to operation greater than. Can use *KB, *MB format. Example: 5, 50KB, 100MB 
+# $4 - number to use to operation less than. Can use *KB, *MB format. Example: 5, 50KB, 100MB 
+# $5 - a test number to examine which means we will check log${ltest}.out logfile
+check_regress_size_backup() {
+   type_log=$1
+   param=$2
+   gt_value=$3
+   lt_value=$4
+   n_test=$5
+
+   bytes_written_value=$(get_value_of_parameter_in_log $type_log "${param}" "${n_test}")
+   
+   check_regress_number_in_log $type_log "gt" "${param}" "${gt_value}" "${n_test}"
+   F=$?
+   printf "Result expected bytes backed up ;%s; \n" "${F}"
+   printf "%s%s\n" " -> Nº bytes of this backup (${gt_value} < ${bytes_written_value}): " $(regress_test_result ${F})
+   
+   check_regress_number_in_log $type_log "lt" "${param}" "${lt_value}" "${n_test}"
+   F=$?
+   printf "Result expected bytes backed up ;%s; \n" "${F}"
+   printf "%s%s\n" " -> Nº bytes of this backup (${bytes_written_value} < ${lt_value}): " $(regress_test_result ${F})
+}
+
+#
+# Setup pod annotations in kubernetes
+#
+# in:
+# $1: pod_name
+# $2: mode to annotate
+# $3: volumes to annotate
+set_up_k8s_annotations() {
+   pod=$1
+   mode_value=$2
+   vols_value=$3
+   BACKUP_MODE_ANN=bacula/backup.mode
+   BACKUP_VOL_ANN=bacula/backup.volumes
+   
+   # --- SetUp
+   ${KUBECTL} annotate pod $pod ${BACKUP_MODE_ANN}=${mode_value} --overwrite > /dev/null
+   ${KUBECTL} annotate pod $pod ${BACKUP_VOL_ANN}=${vols_value} --overwrite > /dev/null
+}
+
+end_set_up_k8s_annotations() {
+   pod=$1
+   BACKUP_MODE_ANN=bacula/backup.mode
+   BACKUP_VOL_ANN=bacula/backup.volumes
+
+   # Remove annotation in pod.
+   ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_MODE_ANN}- > /dev/null
+   ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_VOL_ANN}- > /dev/null
+}
+
 export debug=1
 scripts/cleanup
 scripts/copy-kubernetes-plugin-confs
@@ -127,6 +201,69 @@ do
 done
 printf "\b"
 
+printf "Refill data in pvcs...\n"
+POD_NAME="plugintest1"
+POD_PATH="/data"
+# Command to create a file of 150M inside pvc
+DD_CMD="dd if=/dev/urandom of=${POD_PATH}/file15MB bs=1M count=15"
+# Exec command inside pod.
+${KUBECTL} exec -it $POD_NAME -- /bin/ash -c "$DD_CMD"
+
+sleep 3
+
+printf "File of 15MB created in pod ${POD_NAME}\n"
+
+for N in `seq 0 2`
+do
+   POD_NAME="plugintest-nginx-web-${N}"
+   POD_PATH="/usr/share/nginx/html"
+   FILE_SIZE=$((10 * ${N}))
+   # Command to create a file of 1N0M inside pvc
+   DD_CMD="dd if=/dev/urandom of=${POD_PATH}/file${FILE_SIZE}MB bs=1M count=${FILE_SIZE}"
+   ${KUBECTL} exec -it $POD_NAME -- /bin/bash -c "$DD_CMD"
+   sleep 3
+   printf "File of ${FILE_SIZE}MB created in pod ${POD_NAME}\n"
+done
+
+POD_NAME="plugintest3"
+DD_CMD="dd if=/dev/urandom of=/data/file30MB bs=1M count=30"
+
+# Exec command inside pod.
+${KUBECTL} exec -it $POD_NAME -- /bin/ash -c "$DD_CMD"
+
+sleep 3
+
+printf "File of 30MB created in pod ${POD_NAME}\n"
+
+if [ $KUBE_PROXY_BACKUP_TEST -ne 0 ]
+then
+   POD_NAME="plugintest-annotations-test"
+   POD_PATH="/data"
+   # Command to create a file of 10M inside pvc
+   DD_CMD="dd if=/dev/urandom of=${POD_PATH}/file10MB bs=1M count=10"
+   # Exec command inside pod.
+   ${KUBECTL} exec -it $POD_NAME -- /bin/ash -c "$DD_CMD"
+
+   sleep 3
+   POD_PATH="/data-csi"
+   # Command to create a file of 20M inside pvc
+   DD_CMD="dd if=/dev/urandom of=${POD_PATH}/file20MB bs=1M count=20"
+   # Exec command inside pod.
+   ${KUBECTL} exec -it $POD_NAME -- /bin/ash -c "$DD_CMD"
+
+   sleep 3
+   POD_PATH="/data-csi-2"
+   # Command to create a file of 30M inside pvc
+   DD_CMD="dd if=/dev/urandom of=${POD_PATH}/file30MB bs=1M count=30"
+   # Exec command inside pod.
+   ${KUBECTL} exec -it $POD_NAME -- /bin/ash -c "$DD_CMD"
+
+   sleep 3
+fi
+
+
+printf "End refill data in pvcs...\n"
+
 # wait abit to objects to populate.
 sleep 30
 
@@ -136,6 +273,7 @@ ${KUBECTL} get ns -o name > ${tmp}/allns.log
 ${KUBECTL} get pv -o name > ${tmp}/allpv.log
 PV1=`${KUBECTL} -n plugintest get pvc/plugintest-persistent-volume-claim -o go-template='{{.spec.volumeName}}'`
 PV_CSI1=`${KUBECTL} -n plugintest get pvc/plugintest-persistent-volume-claim-csi -o go-template='{{.spec.volumeName}}'`
+BACKUP_ONLY_PVC="plugintest-persistent-volume-claim"
 BACKUP_PROXY_ARGS=""
 
 if [ $KUBE_PROXY_BACKUP_TEST -ne 0 ]
@@ -150,8 +288,8 @@ then
       fi
    fi
    BACKUP_PROXY_CERTS="fdkeyfile=$KUBE_FD_KEY_FILE fdcertfile=$KUBE_FD_CERT_FILE"
-   BACKUP_PROXY_ARGS="pvcdata baculaimage=${KUBE_BACULA_IMAGE} pluginhost=${KUBE_PROXY_POD_PLUGIN_HOST}"
-   BACKUP_PROXY_ARGS="$BACKUP_PROXY_ARGS $BACKUP_PROXY_CERTS"
+   BACKUP_PROXY_WITHOUT_PVC="baculaimage=${KUBE_BACULA_IMAGE} pluginhost=${KUBE_PROXY_POD_PLUGIN_HOST} $BACKUP_PROXY_CERTS"
+   BACKUP_PROXY_ARGS="$BACKUP_PROXY_WITHOUT_PVC pvcdata"
 fi
 
 
@@ -165,6 +303,8 @@ echo "s%@LPLUG@%${LPLUG}%" > ${out_sed}
 echo "s%@PV1@%${PV1}%" >> ${out_sed}
 echo "s%@PV_CSI1@%${PV_CSI1}%" >> ${out_sed}
 echo "s%@BACKUP_PROXY_ARGS@%${BACKUP_PROXY_ARGS}%" >> ${out_sed}
+echo "s%@BACKUP_PROXY_WITHOUT_PVC@%${BACKUP_PROXY_WITHOUT_PVC}%" >> ${out_sed}
+echo "s%@BACKUP_ONLY_PVC@%${BACKUP_ONLY_PVC}%" >> ${out_sed}
 
 sed -i -f ${out_sed} ${conf}/bacula-dir.conf
 
@@ -197,7 +337,10 @@ then
    F=1
    ((estat++))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results estimate test ${TEST}:\n"
+printf "%s%s\n" " -> Estimated all objects: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 # then estimate with data
 TEST=1
@@ -212,7 +355,10 @@ then
    F=1
    ((estat++))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results estimate test ${TEST}:\n"
+printf "%s%s\n" " -> Estimated with data: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 ((TEST++))
 do_regress_estimate_test ${TEST}
@@ -225,7 +371,10 @@ then
    F=1
    ((estat++))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results estimate test ${TEST}:\n"
+printf "%s%s\n" " -> Estimated objects: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 ((TEST++))
 do_regress_estimate_test ${TEST}
@@ -238,7 +387,10 @@ then
    F=1
    ((estat++))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results estimate test ${TEST}:\n"
+printf "%s%s\n" " -> Estimated objects: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 
 # listing tests goes to estimate tests
@@ -252,7 +404,10 @@ then
    F=1
    estat=$((estat+1))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results listing test ${TEST}:\n"
+printf "%s%s\n" " -> All: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 allns=`cat ${tmp}/allns.log | wc -l`
 ((TEST++))
@@ -265,7 +420,10 @@ then
    F=1
    estat=$((estat+1))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results listing test ${TEST}:\n"
+printf "%s%s\n" " -> Namespaces: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 allpv=`cat ${tmp}/allpv.log | wc -l`
 ((TEST++))
@@ -278,7 +436,10 @@ then
    F=1
    estat=$((estat+1))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Results listing test ${TEST}:\n"
+printf "%s%s\n" " -> Persistent volumes: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 #
 # now do backups
@@ -290,21 +451,509 @@ do
    do_regress_backup_test ${TEST}
    check_regress_backup_statusT ${TEST}
    F=$?
-   regress_test_result ${F}
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
 done
 
+# Check backup pvcdata in pod annotations without pvcdata parameter in fileset
+TEST=4
+expected_string_in_log="plugintest3-persistent-volume-claim.tar"
+
+do_regress_backup_test ${TEST}
+check_regress_backup_statusT ${TEST}
+F=$?
+printf "%s\n" "--------"
+printf "Results backup test ${TEST}:\n"
+printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+
+check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+F=$?
+printf "%s%s\n" " -> Backup pvc not specified in fileset but yes in pod annotations: " $(regress_test_result ${F})
+printf "%s\n" "--------"
+
 # second, backup with proxy backup if it's activated
 if [ $KUBE_PROXY_BACKUP_TEST -ne 0 ]
 then
    expected_string_in_log="is compatible with volume snapshot backup"
-   TEST=4
+   TEST=30
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> Use volume snapshot tecnology: " $(regress_test_result ${F})
+   expected_string_in_log="because it did previously with a pod"
+   check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> Skip volume backup when it was done in pod annotations: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
+
+   # Test if you define one pvc to backup, only backup this pvc.
+   TEST=31
+   
    do_regress_backup_test ${TEST}
    check_regress_backup_statusT ${TEST}
    F=$?
-   echo "Check backup statusT:" $(regress_test_result ${F})
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   expected_string_in_log="Finish backup volume claim: plugintest-persistent-volume-claim"
    check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
    F=$?
-   echo "Check use volume snapshot tecnology: " $(regress_test_result ${F})
+   printf "Result expected FinishBackup ;%s; \n" "${F}"
+   printf "%s%s\n" " -> Backup only one defined pvcdata(plugintest-persistent-volume-claim): " $(regress_test_result ${F})
+   not_expected_string_in_log="Start backup volume claim: plugintest-www-data-plugintest-nginx-web-0"
+   check_regress_string_not_in_log "b" "${not_expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> Backup only one defined pvcdata: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
+
+   #
+   # Test with csi-driver to check combinations in pod annotations 
+   #
+
+   # Remove other pods annotations
+   POD_WITH_ANNOTATIONS=plugintest3
+   BACKUP_MODE_ANN=bacula/backup.mode
+   BACKUP_VOL_ANN=bacula/backup.volumes
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_MODE_ANN}- > /dev/null
+   # ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_VOL_ANN}- > /dev/null
+
+   POD_WITH_ANNOTATIONS=plugintest-annotations-test
+   BYTES_WRITTEN_PARAM="SD Bytes Written"
+   # 40. Check annotation:
+   # mode: standard
+   # one vol: plugintest-persistent-volume-claim-csi
+   TEST=40
+   BACKUP_MODE_ANN_VALUE=standard
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=20MB
+   LT_VALUE=30MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 41. Check annotation:
+   # mode: standard
+   # two vols: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   TEST=41
+   BACKUP_MODE_ANN_VALUE=standard
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=50MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+   
+   # 50. Check annotation:
+   # mode: snapshot
+   # one vol (no compatible): plugintest-persistent-volume-claim
+   TEST=50
+   BACKUP_MODE_ANN_VALUE=snapshot
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+
+   expected_string_in_log="is not compatible with snapshot"
+   check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> PVC is not compatible with snapshot mode: " $(regress_test_result ${F})
+
+   GT_VALUE=25MB
+   LT_VALUE=30MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 51. Check annotation:
+   # mode: snapshot
+   # one vol(comp): plugintest-persistent-volume-claim-csi
+   TEST=51
+   BACKUP_MODE_ANN_VALUE=standard
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+      
+   GT_VALUE=20MB
+   LT_VALUE=30MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+   
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 52. Check annotation:
+   # mode: snapshot
+   # two vols(comp): plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   TEST=52
+   BACKUP_MODE_ANN_VALUE=standard
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+      
+   GT_VALUE=50MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 53. Check annotation:
+   # mode: snapshot
+   # one vol comp and another not: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim
+   TEST=53
+   BACKUP_MODE_ANN_VALUE=snapshot
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   expected_string_in_log="is not compatible with snapshot"
+   check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> PVC is not compatible with snapshot mode: " $(regress_test_result ${F})
+   
+   GT_VALUE=45MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 54. Check annotation:
+   # mode: snapshot
+   # one vol not comp and another yes: plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi
+   TEST=54
+   BACKUP_MODE_ANN_VALUE=snapshot
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim,plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   expected_string_in_log="is not compatible with snapshot"
+   check_regress_string_in_log "b" "${expected_string_in_log}" ${TEST}
+   F=$?
+   printf "%s%s\n" " -> PVC is not compatible with snapshot mode: " $(regress_test_result ${F})
+
+   GT_VALUE=45MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 60. Check annotation:
+   # mode: clone
+   # one vol: plugintest-persistent-volume-claim-csi
+   TEST=60
+   BACKUP_MODE_ANN_VALUE=clone
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+
+   GT_VALUE=20MB
+   LT_VALUE=30MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" ${GT_VALUE} ${LT_VALUE} ${TEST}
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # # 61. Check annotation:
+   # # mode: clone
+   # # one vol (no comp): plugintest-persistent-volume-claim
+   # TEST=61
+   # BACKUP_MODE_ANN_VALUE=clone
+   # BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim
+   # # --- SetUp
+   # ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_MODE_ANN}=${BACKUP_MODE_ANN_VALUE} --overwrite > /dev/null
+   # ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_VOL_ANN}=${BACKUP_VOLS_ANN_VALUE} --overwrite > /dev/null
+   # # ---
+
+   # do_regress_backup_test ${TEST}
+   # check_regress_backup_statusT ${TEST}
+   # F=$?
+   # printf "%s\n" "--------"
+   # printf "Results backup test ${TEST}:\n"
+   # printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+
+   # BYTES_WRITTEN_VALUE=$(get_value_of_parameter_in_log "b" "${BYTES_WRITTEN_PARAM}" "${TEST}")
+   # GT_VALUE=20MB
+   # check_regress_number_in_log "b" "gt" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${TEST}"
+   # F=$?
+   # printf "Result expected bytes backed up ;%s; \n" "${F}"
+   # printf "%s%s\n" " -> Nº bytes of this backup (${BACKUP_MODE_ANN_VALUE} mode) one vol annotated inside pod (${GT_VALUE} < ${BYTES_WRITTEN_VALUE}): " $(regress_test_result ${F})
+   # LT_VALUE=30MB
+   # check_regress_number_in_log "b" "lt" "${BYTES_WRITTEN_PARAM}" "${LT_VALUE}" "${TEST}"
+   # F=$?
+   # printf "Result expected bytes backed up ;%s; \n" "${F}"
+   # printf "%s%s\n" " -> Nº bytes of this backup (${BACKUP_MODE_ANN_VALUE} mode) one vol annotated inside pod (${BYTES_WRITTEN_VALUE} < ${LT_VALUE}): " $(regress_test_result ${F})
+
+   # printf "%s\n" "--------"
+
+   # 62. Check annotation:
+   # mode: clone
+   # two vols: plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   TEST=62
+   BACKUP_MODE_ANN_VALUE=clone
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi,plugintest-persistent-volume-claim-csi-2
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=50MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # --- After all
+   ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_MODE_ANN}- > /dev/null
+   ${KUBECTL} annotate pod ${POD_WITH_ANNOTATIONS} ${BACKUP_VOL_ANN}- > /dev/null
+   # --- After All annotations
+
+   # 70. Check pvcdata in fileset:
+   # pvcdata
+   # one vol: plugintest-persistent-volume-claim
+   TEST=70
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=25MB
+   LT_VALUE=30MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 71. Check pvcdata in fileset:
+   # pvcdata
+   # two vols: plugintest-persistent-volume-claim,plugintest3-persistent-volume-claim
+   TEST=71
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=55MB
+   LT_VALUE=60MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 72. Check pvcdata in fileset:
+   # pvcdata
+   # all vols
+   TEST=72
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=140MB
+   LT_VALUE=150MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 73. Check pvcdata in fileset one annotation standard:
+   # pvcdata
+   # all vols
+   TEST=73
+   BACKUP_MODE_ANN_VALUE=standard
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=140MB
+   LT_VALUE=150MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+
+   printf "%s\n" "--------"
+
+   # 74. Check pvcdata in fileset one annotation standard:
+   # pvcdata
+   # all vols
+   TEST=74
+   BACKUP_MODE_ANN_VALUE=clone
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=140MB
+   LT_VALUE=150MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
+
+   # 75. Check pvcdata in fileset one annotation standard:
+   # pvcdata
+   # all vols
+   TEST=75
+   BACKUP_MODE_ANN_VALUE=snapshot
+   BACKUP_VOLS_ANN_VALUE=plugintest-persistent-volume-claim-csi
+   # --- SetUp
+   set_up_k8s_annotations "${POD_WITH_ANNOTATIONS}" "${BACKUP_MODE_ANN_VALUE}" "${BACKUP_VOLS_ANN_VALUE}"
+   # ---
+
+   do_regress_backup_test ${TEST}
+   check_regress_backup_statusT ${TEST}
+   F=$?
+   printf "%s\n" "--------"
+   printf "Results backup test ${TEST}:\n"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   
+   GT_VALUE=140MB
+   LT_VALUE=150MB
+   check_regress_size_backup "b" "${BYTES_WRITTEN_PARAM}" "${GT_VALUE}" "${LT_VALUE}" "${TEST}"
+
+   # --- EndSetup
+   end_set_up_k8s_annotations ${POD_WITH_ANNOTATIONS}
+   # ---
+   printf "%s\n" "--------"
 fi
 
 # now, backup with warnings
@@ -313,7 +962,10 @@ do
    do_regress_backup_test ${TEST}
    check_regress_backup_statusW ${TEST}
    F=$?
-   regress_test_result ${F}
+   printf "%s\n" "--------"
+   printf "Result backup test ${TEST}:"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
 done
 
 # now, backup failed to test
@@ -322,7 +974,10 @@ do
    do_regress_backup_test ${TEST}
    check_regress_backup_statusE ${TEST}
    F=$?
-   regress_test_result ${F}
+   printf "%s\n" "--------"
+   printf "Result failed backup test ${TEST}:"
+   printf "%s%s\n" " -> StatusE: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
 done
 
 #do_regress_backup_test 1 "incremental"
@@ -347,7 +1002,10 @@ then
    F=1
    dstat=$((dstat+1))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Result restore test ${TEST}:"
+printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 ((TEST++))
 ${KUBECTL} -n plugintest delete secret/plugintest-secrets 2>&1 > ${tmp}/rlog${TEST}.out
@@ -363,7 +1021,10 @@ then
    F=1
    dstat=$((dstat+1))
 fi
-regress_test_result ${F}
+printf "%s\n" "--------"
+printf "Result restore test ${TEST}:"
+printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+printf "%s\n" "--------"
 
 ((TEST++))
 ${KUBECTL} delete ns/plugintest 2>&1 > ${tmp}/rlog${TEST}.out
@@ -401,7 +1062,10 @@ then
       F=1
       dstat=$((dstat+1))
    fi
-   regress_test_result ${F}
+   printf "%s\n" "--------"
+   printf "Result restore test ${TEST}:"
+   printf "%s%s\n" " -> StatusT: " $(regress_test_result ${F})
+   printf "%s\n" "--------"
 fi