From: francisco.garcia Date: Tue, 5 Mar 2024 22:48:13 +0000 (+0100) Subject: k8s: Add backup_mode parameter, add resiliance when snapshot backup is not compatible... X-Git-Tag: Release-15.0.2~22 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=1b9ce29e0fa0b8df6e9195c89e5d5cf0db1500f4;p=thirdparty%2Fbacula.git k8s: Add backup_mode parameter, add resiliance when snapshot backup is not compatible and clone backup fails and minor fixes --- diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py index 47166ec01..3f9ae82fd 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/backup_job.py @@ -39,8 +39,10 @@ BA_MODE_ERROR = "Invalid annotations for Pod: {namespace}/{podname}. Backup Mode BA_EXEC_STDOUT = "{}:{}" BA_EXEC_STDERR = "{} Error:{}" BA_EXEC_ERROR = "Pod Container execution: {}" -POD_BACKUP_SELECTED = "The backup mode selected to the pvc `{}` is `{}`" -CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC = "The pvc `{}` is not compatible with snapshot backup, changing mode to standard. Only pvc with storage that they use CSI driver are compatible." +POD_BACKUP_SELECTED = "The selected backup mode in pod to the pvc `{}` is `{}`" +CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC = "The pvc `{}` is not compatible with snapshot backup, changing mode to clone. Only pvc with storage that they use CSI driver are compatible." +PVC_BACKUP_MODE_APPLIED_INFO = "The pvc `{}` will be backup with {} mode." +RETRY_BACKUP_WITH_STANDARD_MODE = "If the clone backup is empty. It will try again to do a backup using standard mode." class BackupJob(EstimationJob): """ @@ -53,8 +55,10 @@ class BackupJob(EstimationJob): def __init__(self, plugin, params): super().__init__(plugin, params, BACKUP_START_PACKET) _label = params.get('labels', None) + self.fs_backup_mode = BaculaBackupMode.process_param(params.get("backup_mode", BaculaBackupMode.Snapshot)) # Fileset backup mode defined. if _label is not None: self._io.send_info(BACKUP_PARAM_LABELS.format(_label)) + self._io.send_info("The selected default backup mode to do pvc backup is `{}`.".format(self.fs_backup_mode)) def execution_loop(self): super().processing_loop(estimate=False) @@ -128,24 +132,36 @@ class BackupJob(EstimationJob): return False return True - def process_pvcdata(self, namespace, pvcdata, backup_with_pod = False): + def process_pvcdata(self, namespace, pvcdata, backup_with_pod = False, retry_backup = False): status = None vsnapshot = None is_cloned = False cloned_pvc_name = None + # For retry if clone backup is incompatible. + orig_pvcdata = pvcdata # Detect if pvcdata is compatible with snapshots - if not backup_with_pod: - logging.debug('Backup without pod') - vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata.get('name')) + if not backup_with_pod and not retry_backup: + logging.debug('Backup mode {} of pvc {} without pod:'.format(self.fs_backup_mode, pvcdata.get('name'))) + if self.fs_backup_mode == BaculaBackupMode.Snapshot: + logging.debug('Snapshot is activated') + vsnapshot, pvcdata = self.handle_create_vsnapshot_backup(namespace, pvcdata.get('name')) + self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Snapshot)) + + if (vsnapshot is None and self.fs_backup_mode != BaculaBackupMode.Standard) or self.fs_backup_mode == BaculaBackupMode.Clone: + if self.fs_backup_mode != BaculaBackupMode.Clone: + self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcdata.get('name'))) + self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Clone)) + cloned_pvc_name = self.create_pvcclone(namespace, pvcdata.get('name')) + cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, cloned_pvc_name) + logging.debug('Cloned pvc fi:{}'.format(cloned_pvc.get('fi'))) + cloned_pvc.get('fi').set_name(pvcdata.get('fi').name) + pvcdata = cloned_pvc + is_cloned = True + + if self.fs_backup_mode == BaculaBackupMode.Standard: + self._io.send_info(PVC_BACKUP_MODE_APPLIED_INFO.format(pvcdata.get('name'), BaculaBackupMode.Standard)) logging.debug('Process_pvcdata (Backup_job): {} --- {}'.format(vsnapshot, pvcdata)) - # if vsnapshot is None: - # self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcdata.get('name'))) - # cloned_pvc_name = self.create_pvcclone(namespace, pvcdata.get('name')) - # cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, cloned_pvc_name) - # logging.debug('Cloned pvc fi:{}'.format(cloned_pvc.get('fi'))) - # cloned_pvc.get('fi').set_name(pvcdata.get('fi').name) - # pvcdata = cloned_pvc - # is_cloned = True + if self.prepare_bacula_pod(pvcdata, namespace=namespace, mode='backup'): super()._estimate_file(pvcdata) # here to send info about pvcdata to plugin status = self.__backup_pvcdata(namespace=namespace) @@ -158,6 +174,14 @@ class BackupJob(EstimationJob): self.handle_delete_vsnapshot_backup(namespace, vsnapshot, pvcdata) if is_cloned: self.delete_pvcclone(namespace, cloned_pvc_name) + # It retries when the backup is not with pod annotation (it is controlled in their function) + if not backup_with_pod and not retry_backup and not self.backup_clone_compatibility: + # It is important set to True before recall the process. + self.backup_clone_compatibility = True # We only try once + + self._io.send_info(RETRY_BACKUP_WITH_STANDARD_MODE) + + status = self.process_pvcdata(namespace, orig_pvcdata, backup_with_pod, True) return status def handle_pod_container_exec_command(self, corev1api, namespace, pod, runjobparam, failonerror=False): @@ -222,36 +246,45 @@ class BackupJob(EstimationJob): logging.debug("iterate over requested vols for backup: {}".format(requestedvolumes)) for pvc in requestedvolumes: pvcname = pvc + original_pvc = self._plugin.get_pvcdata_namespaced(namespace, pvcname) vsnapshot = None logging.debug("handling vol before backup: {}".format(pvcname)) self._io.send_info(POD_BACKUP_SELECTED.format(pvcname, backupmode)) - # self._io.send_info("The pvc `{}` will be backedup with {} mode".format(pvcname, backupmode)) - if backupmode == BaculaBackupMode.Clone: - # snapshot if requested - pvcname = self.create_pvcclone(namespace, pvcname) - if pvcname is None: - # error - logging.error("create_pvcclone failed!") - return False if backupmode == BaculaBackupMode.Snapshot: logging.debug('Snapshot mode chosen') vsnapshot, pvc_from_vsnap = self.handle_create_vsnapshot_backup(namespace, pvcname) logging.debug("The vsnapshot created from pvc {} is: {}".format(pvcname, vsnapshot)) logging.debug("The pvc create from vsnapshot {} is: {}. FI: {}".format(vsnapshot, pvc_from_vsnap, pvc_from_vsnap.get('fi'))) - if vsnapshot == None: + if vsnapshot is None: logging.debug(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname)) # backupmode = BaculaBackupMode.Clone self._io.send_info(CHANGE_BACKUP_MODE_FOR_INCOMPATIBLITY_PVC.format(pvcname)) + backupmode = BaculaBackupMode.Clone else: pvc = pvc_from_vsnap pvcname = pvc_from_vsnap.get("name") + if backupmode == BaculaBackupMode.Clone: + pvcname = self.create_pvcclone(namespace, pvcname) + cloned_pvc = self._plugin.get_pvcdata_namespaced(namespace, pvcname) + if pvcname is None: + # error + logging.error("create_pvcclone failed!") + return False + else: + logging.debug('Original_pvc: {}'.format(original_pvc.get('fi'))) + logging.debug('Cloned_pvc->fi: {}'.format(cloned_pvc.get('fi'))) + cloned_pvc.get('fi').set_name(original_pvc.get('fi').name) + pvc = cloned_pvc + logging.debug("handling vol after snapshot/clone: {}".format(pvcname)) handledvolumes.append({ 'pvcname': pvcname, 'pvc': pvc, - 'vsnapshot': vsnapshot + 'vsnapshot': vsnapshot, + 'backupmode': backupmode, + 'original_pvc': original_pvc }) failonerror = BoolParam.handleParam(pod.get(BaculaAnnotationsClass.RunAfterSnapshotonError), False) # the default is ignore errors @@ -276,22 +309,26 @@ class BackupJob(EstimationJob): else: # Modify the name in FileInfo because we need save the file like original name # and not the new pvc (from vsnapshot) name. - if volumes.get('vsnapshot') is not None: + if volumes.get('backupmode') == BaculaBackupMode.Snapshot or volumes.get('backupmode') == BaculaBackupMode.Clone: logging.debug('We change the name of FileInfo to adapt the original pvc name with the new pvc name') pvcdata.get('fi').set_name(pvc.get('fi').name) if len(pvcdata) > 0: status = self.process_pvcdata(namespace, pvcdata, True) + # Control the compatibility of snapshot or clone. If it is not compatible, retry again with standard backup mode. + logging.debug('Call again process_pvcdata from process_pod_pvcdata? {}'.format(self.backup_clone_compatibility)) + if not self.backup_clone_compatibility: + status = self.process_pvcdata(namespace, volumes['original_pvc'], True) # iterate on requested volumes for delete snap logging.debug("iterate over requested vols for delete snap: {}".format(handledvolumes)) for volumes in handledvolumes: pvcname = volumes['pvcname'] logging.debug("Should remove this pvc: {}".format(pvcname)) - if backupmode == BaculaBackupMode.Clone: + if volumes.get('backupmode') == BaculaBackupMode.Clone: # snapshot delete if snapshot requested status = self.delete_pvcclone(namespace, pvcname) - if backupmode == BaculaBackupMode.Snapshot and volumes['vsnapshot'] is not None: + if volumes.get('backupmode') == BaculaBackupMode.Snapshot and volumes['vsnapshot'] is not None: status = self.handle_delete_vsnapshot_backup(namespace, volumes['vsnapshot'], volumes['pvc']) logging.debug("Finish removing pvc clones and vsnapshots. Status {}".format(status)) diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py index 88b4c0793..848f17ebc 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/jobs/job_pod_bacula.py @@ -45,7 +45,7 @@ POD_EXIST_ERR = "Job already running in '{namespace}' namespace. Check logs or d TAR_STDERR_UNKNOWN = "Unknown error. You should check Pod logs for possible explanation." PLUGINPORT_VALUE_ERR = "Cannot use provided pluginport={port} option. Used default!" FDPORT_VALUE_ERR = "Cannot use provided fdport={port} option. Used default!" -POD_YAML_PREPARED_INFO = "Prepare backup Pod with: {image} <{pullpolicy}> {pluginhost}:{pluginport}" +POD_YAML_PREPARED_INFO = "Prepare bacula-backup Pod with: {image} <{pullpolicy}> {pluginhost}:{pluginport}" POD_YAML_PREPARED_INFO_NODE = "Prepare Bacula Pod on: {nodename} with: {image} <{pullpolicy}> {pluginhost}:{pluginport}" CANNOT_CREATE_BACKUP_POD_ERR = "Cannot create backup pod. Err={}" CANNOT_REMOVE_BACKUP_POD_ERR = "Cannot remove backup pod. Err={}" @@ -58,6 +58,7 @@ CANNOT_REMOVE_PVC_CLONE_ERR = "Cannot remove PVC snapshot. Err={}" CANNOT_REMOVE_VSNAPSHOT_ERR = "Unable to remove volume snapshot {vsnapshot}! Please you must remove it manually." CANNOT_START_CONNECTIONSERVER = "Cannot start ConnectionServer. Err={}" +WARNING_CLONED_PVC_WAS_NOT_WORKED = "As clone backup is empty. It will retry again to do a backup with standard mode." VSNAPSHOT_BACKUP_COMPATIBLE_INFO = "The pvc `{}` is compatible with volume snapshot backup. Doing backup with this technology." PVC_FROM_SNAPSHOT_CREATED = "The pvc `{}` was created from volume snapshot from pvc `{}`." CREATING_PVC_FROM_VSNAPSHOT = "Creating pvc from volume snapshot of pvc `{}`." @@ -93,10 +94,13 @@ class JobPodBacula(Job, metaclass=ABCMeta): self.tarexitcode = None self.backupimage = params.get('baculaimage', BACULABACKUPIMAGE) self.imagepullpolicy = ImagePullPolicy.process_param(params.get('imagepullpolicy')) + self.backup_clone_compatibility = True def handle_pod_logs(self, connstream): logmode = '' self.tarstderr = '' + file_count = 0 + bytes_count = 0 with connstream.makefile(mode='r') as fd: self.tarexitcode = fd.readline().strip() logging.debug('handle_pod_logs:tarexitcode:{}'.format(self.tarexitcode)) @@ -118,7 +122,20 @@ class JobPodBacula(Job, metaclass=ABCMeta): continue elif logmode == 'list': # no listing feature yet + file_count += 1 + try: + file_props = data.split() + logging.debug('Data Split:{}'.format(','.join(file_props))) + bytes_count += int(file_props[2]) + except Exception as e: + logging.exception(e) continue + logging.debug('Bytes/files in backup: {}/{}'.format(bytes_count, file_count)) + logging.debug('Type of job:' + str(self._params.get('type'))) + if self._params.get('type') == 'b' and bytes_count == 0 and file_count < 3: + self._io.send_non_fatal_error(WARNING_CLONED_PVC_WAS_NOT_WORKED) + self.backup_clone_compatibility = False + def handle_pod_data_recv(self, connstream): while True: diff --git a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py index af50ec6a4..c33bd9456 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py +++ b/bacula/src/plugins/fd/kubernetes-backend/baculak8s/plugins/k8sbackend/baculaannotations.py @@ -55,12 +55,13 @@ class BaculaBackupMode(object): Returns: str: backup mode normalized to consts, `None` when error """ - if mode is not None: - mode = mode.lower() - for p in BaculaBackupMode.params: - if p == mode: - return p - return None + if mode is None: + return None + mode = mode.lower() + for p in BaculaBackupMode.params: + if p == mode: + return p + raise Exception('This backup mode `{}` is not supported. Only supported: snapshot, clone or standard.'.format(mode)) class BaculaAnnotationsClass(object): diff --git a/bacula/src/plugins/fd/kubernetes-backend/setup.py b/bacula/src/plugins/fd/kubernetes-backend/setup.py index 4e9613766..97bad4b2a 100644 --- a/bacula/src/plugins/fd/kubernetes-backend/setup.py +++ b/bacula/src/plugins/fd/kubernetes-backend/setup.py @@ -24,7 +24,7 @@ if sys.version_info < (3, 0): setup( name='baculak8s', - version='2.1.1', + version='2.2.0', author='Radoslaw Korzeniewski, Francisco Manuel Garcia Botella', author_email='radekk@korzeniewski.net, francisco.garcia@baculasystems.com', packages=find_packages(exclude=('tests', 'tests.*')), diff --git a/bacula/src/plugins/fd/kubernetes-fd.c b/bacula/src/plugins/fd/kubernetes-fd.c index 1a55dbfbd..331806814 100644 --- a/bacula/src/plugins/fd/kubernetes-fd.c +++ b/bacula/src/plugins/fd/kubernetes-fd.c @@ -21,8 +21,8 @@ * @author Radosław Korzeniewski (radoslaw@korzeniewski.net) * Modified by: Francisco Manuel Garcia Botella (francisco.garcia@baculasystems.com) * @brief This is a Bacula Kubernetes Plugin with metaplugin interface. - * @version 2.1.0 - * @date 2023-07-31 + * @version 2.2.0 + * @date 2024-02-20 * * @copyright Copyright (c) 2021 All rights reserved. * IP transferred to Bacula Systems according to agreement. @@ -33,8 +33,8 @@ /* Plugin Info definitions */ const char *PLUGIN_LICENSE = "Bacula AGPLv3"; const char *PLUGIN_AUTHOR = "Radoslaw Korzeniewski, Francisco Manuel Garcia Botella"; -const char *PLUGIN_DATE = "July 2023"; -const char *PLUGIN_VERSION = "2.1.0"; // TODO: should synchronize with kubernetes-fd.json +const char *PLUGIN_DATE = "February 2024"; +const char *PLUGIN_VERSION = "2.2.0"; // TODO: should synchronize with kubernetes-fd.json const char *PLUGIN_DESCRIPTION = "Bacula Kubernetes Plugin"; /* Plugin compile time variables */ diff --git a/bacula/src/plugins/fd/kubernetes-fd.h b/bacula/src/plugins/fd/kubernetes-fd.h index 0aae5797e..33f0cc2e4 100644 --- a/bacula/src/plugins/fd/kubernetes-fd.h +++ b/bacula/src/plugins/fd/kubernetes-fd.h @@ -68,6 +68,7 @@ const char * valid_params[] = "ssl_ca_cert", "timeout", "debug", + "backup_mode", "namespace", "ns", "persistentvolume",