k8s: Avoid pvcs backup when the pvc status is Terminating

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
index 3f9ae82fdcf3aaf19434cc7748c8b7095ba05ae7..a1e51bb7d4f5331baf86dd242ccb82bd00b1de0c 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py
@@ -187,7 +187,7 @@ class BackupJob(EstimationJob):
     def handle_pod_container_exec_command(self, corev1api, namespace, pod, runjobparam, failonerror=False):
         podname = pod.get('name')
         containers = pod.get('containers')
-        logging.debug("pod {} containers: {}".format(podname, containers))
+        logging.debug("[{}] pod {} containers: {}".format(runjobparam, podname, containers))
         # now check if run before job
         container, command = BaculaAnnotationsClass.handle_run_job_container_command(pod.get(runjobparam))
         if container is not None:
@@ -251,6 +251,12 @@ class BackupJob(EstimationJob):
             logging.debug("handling vol before backup: {}".format(pvcname))
             self._io.send_info(POD_BACKUP_SELECTED.format(pvcname, backupmode))
 
+            # Check if pvc has status: 'Terminating'. Because in this state, the backup raise error.
+            if self._plugin.pvc_is_terminating(namespace, original_pvc):
+                logging.debug("Skip pvc. Cause Terminating status")
+                self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvcname))
+                continue
+
             if backupmode == BaculaBackupMode.Snapshot:
                 logging.debug('Snapshot mode chosen')
                 vsnapshot, pvc_from_vsnap = self.handle_create_vsnapshot_backup(namespace, pvcname)

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
index 231cc512e8d9d436be279f83e0aba3a2fa9f1260..36ee66c5325981c980f79f52637e6f251c1dcb08 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py
@@ -193,14 +193,23 @@ class EstimationJob(JobPodBacula):
                                     self._io.send_info(SKIP_PVCDATA_SECOND_BACKUP_INFO.format(pvc))
                                     continue
                                 elif pvc_backed.get(pvc) == 'error':
+                                    if self._plugin.pvc_is_terminating(nsname, pvcdata):
+                                        logging.debug("Skip pvc. Cause Terminating status")
+                                        self._io.send_warning("Skip pvc of second try `{}` because it is in Terminating status.".format(pvc))
+                                        continue
                                     self._io.send_warning(SECOND_TRY_PVCDATA_INFO.format(pvc))
                                 self._io.send_info(PROCESSING_PVCDATA_START_INFO.format(pvc=pvc))
+                                if self._plugin.pvc_is_terminating(nsname, pvcdata):
+                                    logging.debug("Skip pvc. Cause Terminating status")
+                                    self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvc))
+                                    continue
                             status = self.process_pvcdata(nsname, pvcdata)
                             if status is None:
                                 # None means unable to prepare listening service during backup
                                 break
                             if not estimate and status:
                                 self._io.send_info(PROCESSING_PVCDATA_STOP_INFO.format(pvc=pvc))
+                        logging.debug("Finish pvcdatalist")
 
     def _estimate_file(self, data):
         logging.debug('{}'.format(data))
@@ -223,6 +232,10 @@ class EstimationJob(JobPodBacula):
         # iterate on requested pvc
         logging.debug("process_pod_pvcdata in Estimate mode")
         for pvc in pvcnames.split(','):
+            if self._plugin.pvc_is_terminating(namespace, pvc):
+                logging.debug("Skip pvc. Cause Terminating status")
+                self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvc))
+                continue
             # get pvcdata for this volume
             pvcdata = self._plugin.get_pvcdata_namespaced(namespace, pvc)
             if isinstance(pvcdata, dict) and 'exception' in pvcdata:

diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
index dc04c8b106b10cbba5e1744c3dd7fc686890b270..339a7fe226136ea46a9cc03a06df03cdca9c83b6 100644 (file)
--- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
+++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py
@@ -596,6 +596,11 @@ class KubernetesPlugin(Plugin):
             lambda: self.corev1api.read_namespaced_persistent_volume_claim(k8sfile2objname(file_info.name),
                                                                            file_info.namespace))
 
+    def _check_persistentvolume_claim_status(self, namespace, pvc_name):
+        return self.__exec_check_object(
+            lambda: self.corev1api.read_namespaced_persistent_volume_claim_status(pvc_name, namespace)
+        )
+
     def _check_persistentvolume(self, file_info):
         return self.__exec_check_object(
             lambda: self.corev1api.read_persistent_volume(k8sfile2objname(file_info.name)))
@@ -896,6 +901,27 @@ class KubernetesPlugin(Plugin):
         logging.debug("pvc_isready:status:{}".format(status))
         return status.phase == 'Bound'
 
+    # Check if the pvc is terminating status.
+    # This can be checked if the property 'metadata'>'deletion_timestamp' is not None
+    def pvc_is_terminating(self, namespace, pvc):
+        if not isinstance(pvc, dict):
+            raise Exception('Error when try to get pvc status. PVC must be a `dict`')
+        logging.debug('PVC is terminating status?. Namespace:{}.\nPVC Name:{}'.format(namespace,pvc.get('name')))
+
+        pvc_status = self._check_persistentvolume_claim_status(namespace, pvc.get('name'))
+        logging.debug('Pvc status:{}'.format(pvc_status))
+        try:
+            deletion_timestamp = pvc_status.metadata.deletion_timestamp
+            logging.debug('Deletion_Timestamp:{}'.format(deletion_timestamp))
+            if deletion_timestamp is not None:
+                return True
+            return False
+        except Exception as ex:
+            logging.debug('Exception ocurrs:{}'.format(ex))
+            logging.exception(ex)
+        logging.error('Had a error when try to get deletion_timestamp of pvc status')
+        return True
+
     def remove_backup_pod(self, namespace, podname=BACULABACKUPPODNAME):
         logging.debug('remove_backup_pod')
         response = self.__execute(lambda: self.corev1api.delete_namespaced_pod(