From: Francisco Manuel Garcia Botella Date: Tue, 14 May 2024 11:10:06 +0000 (+0200) Subject: k8s: Avoid pvcs backup when the pvc status is Terminating X-Git-Tag: Release-15.0.3~58 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=c78a1ef0f51eac09156f38d92999df1a50941a8d;p=thirdparty%2Fbacula.git k8s: Avoid pvcs backup when the pvc status is Terminating --- diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py index 3f9ae82fd..a1e51bb7d 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py @@ -187,7 +187,7 @@ class BackupJob(EstimationJob): def handle_pod_container_exec_command(self, corev1api, namespace, pod, runjobparam, failonerror=False): podname = pod.get('name') containers = pod.get('containers') - logging.debug("pod {} containers: {}".format(podname, containers)) + logging.debug("[{}] pod {} containers: {}".format(runjobparam, podname, containers)) # now check if run before job container, command = BaculaAnnotationsClass.handle_run_job_container_command(pod.get(runjobparam)) if container is not None: @@ -251,6 +251,12 @@ class BackupJob(EstimationJob): logging.debug("handling vol before backup: {}".format(pvcname)) self._io.send_info(POD_BACKUP_SELECTED.format(pvcname, backupmode)) + # Check if pvc has status: 'Terminating'. Because in this state, the backup raise error. + if self._plugin.pvc_is_terminating(namespace, original_pvc): + logging.debug("Skip pvc. Cause Terminating status") + self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvcname)) + continue + if backupmode == BaculaBackupMode.Snapshot: logging.debug('Snapshot mode chosen') vsnapshot, pvc_from_vsnap = self.handle_create_vsnapshot_backup(namespace, pvcname) diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py index 231cc512e..36ee66c53 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/estimation_job.py @@ -193,14 +193,23 @@ class EstimationJob(JobPodBacula): self._io.send_info(SKIP_PVCDATA_SECOND_BACKUP_INFO.format(pvc)) continue elif pvc_backed.get(pvc) == 'error': + if self._plugin.pvc_is_terminating(nsname, pvcdata): + logging.debug("Skip pvc. Cause Terminating status") + self._io.send_warning("Skip pvc of second try `{}` because it is in Terminating status.".format(pvc)) + continue self._io.send_warning(SECOND_TRY_PVCDATA_INFO.format(pvc)) self._io.send_info(PROCESSING_PVCDATA_START_INFO.format(pvc=pvc)) + if self._plugin.pvc_is_terminating(nsname, pvcdata): + logging.debug("Skip pvc. Cause Terminating status") + self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvc)) + continue status = self.process_pvcdata(nsname, pvcdata) if status is None: # None means unable to prepare listening service during backup break if not estimate and status: self._io.send_info(PROCESSING_PVCDATA_STOP_INFO.format(pvc=pvc)) + logging.debug("Finish pvcdatalist") def _estimate_file(self, data): logging.debug('{}'.format(data)) @@ -223,6 +232,10 @@ class EstimationJob(JobPodBacula): # iterate on requested pvc logging.debug("process_pod_pvcdata in Estimate mode") for pvc in pvcnames.split(','): + if self._plugin.pvc_is_terminating(namespace, pvc): + logging.debug("Skip pvc. Cause Terminating status") + self._io.send_warning("Skip pvc `{}` because it is in Terminating status.".format(pvc)) + continue # get pvcdata for this volume pvcdata = self._plugin.get_pvcdata_namespaced(namespace, pvc) if isinstance(pvcdata, dict) and 'exception' in pvcdata: diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py index dc04c8b10..339a7fe22 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/kubernetes_plugin.py @@ -596,6 +596,11 @@ class KubernetesPlugin(Plugin): lambda: self.corev1api.read_namespaced_persistent_volume_claim(k8sfile2objname(file_info.name), file_info.namespace)) + def _check_persistentvolume_claim_status(self, namespace, pvc_name): + return self.__exec_check_object( + lambda: self.corev1api.read_namespaced_persistent_volume_claim_status(pvc_name, namespace) + ) + def _check_persistentvolume(self, file_info): return self.__exec_check_object( lambda: self.corev1api.read_persistent_volume(k8sfile2objname(file_info.name))) @@ -896,6 +901,27 @@ class KubernetesPlugin(Plugin): logging.debug("pvc_isready:status:{}".format(status)) return status.phase == 'Bound' + # Check if the pvc is terminating status. + # This can be checked if the property 'metadata'>'deletion_timestamp' is not None + def pvc_is_terminating(self, namespace, pvc): + if not isinstance(pvc, dict): + raise Exception('Error when try to get pvc status. PVC must be a `dict`') + logging.debug('PVC is terminating status?. Namespace:{}.\nPVC Name:{}'.format(namespace,pvc.get('name'))) + + pvc_status = self._check_persistentvolume_claim_status(namespace, pvc.get('name')) + logging.debug('Pvc status:{}'.format(pvc_status)) + try: + deletion_timestamp = pvc_status.metadata.deletion_timestamp + logging.debug('Deletion_Timestamp:{}'.format(deletion_timestamp)) + if deletion_timestamp is not None: + return True + return False + except Exception as ex: + logging.debug('Exception ocurrs:{}'.format(ex)) + logging.exception(ex) + logging.error('Had a error when try to get deletion_timestamp of pvc status') + return True + def remove_backup_pod(self, namespace, podname=BACULABACKUPPODNAME): logging.debug('remove_backup_pod') response = self.__execute(lambda: self.corev1api.delete_namespaced_pod(