From: Norbert Bizet Date: Fri, 27 Aug 2021 10:05:07 +0000 (-0400) Subject: cloud: Add generic aws_cloud_driver X-Git-Tag: Beta-15.0.0~792 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=f26dd91cd44b3c290aa84575718b426560b7c925;p=thirdparty%2Fbacula.git cloud: Add generic aws_cloud_driver --- diff --git a/bacula/scripts/Makefile.in b/bacula/scripts/Makefile.in index 1dd9a74fb..f359c6b7a 100755 --- a/bacula/scripts/Makefile.in +++ b/bacula/scripts/Makefile.in @@ -93,6 +93,25 @@ uninstall: (cd $(DESTDIR)$(scriptdir); $(RMF) btraceback.mdb) (cd $(DESTDIR)$(scriptdir); $(RMF) breload) (cd $(DESTDIR)$(sbindir); $(RMF) btraceback) + (cd $(DESTDIR)$(sbindir); $(RMF) generic_cloud_driver) + (cd $(DESTDIR)$(sbindir); $(RMF) aws_cloud_driver) + + +aws_cloud_driver.C: aws_cloud_driver + $(CYTHON) --embed -o aws_cloud_driver.C aws_cloud_driver + +aws_cloud_driver.bin: aws_cloud_driver.C + $(CC) -o aws_cloud_driver.bin aws_cloud_driver.C $(CYTHON_INC) $(CYTHON_LIBS) + strip aws_cloud_driver.bin + +install-aws-cloud: aws_cloud_driver.bin + $(INSTALL_SCRIPT) aws_cloud_driver.bin $(DESTDIR)$(plugindir)/aws_cloud_driver + +install-generic-cloud: + $(INSTALL_SCRIPT) generic_cloud_driver $(DESTDIR)$(plugindir)/generic_cloud_driver + +install-regress-drivers: + $(INSTALL_SCRIPT) aws_cloud_driver $(DESTDIR)$(plugindir)/aws_cloud_driver Makefile: Makefile.in cd $(topdir) \ diff --git a/bacula/scripts/aws_cloud_driver.in b/bacula/scripts/aws_cloud_driver.in new file mode 100755 index 000000000..2fbd0285f --- /dev/null +++ b/bacula/scripts/aws_cloud_driver.in @@ -0,0 +1,282 @@ +#!/usr/bin/env python2 +# -*- coding: utf-8 -*- + +# Bacula® - The Network Backup Solution + +# Copyright (C) 2000-2020 Bacula Systems SA +# All rights reserved. +# +# The main author of Bacula is Kern Sibbald, with contributions from many +# others, a complete list can be found in the file AUTHORS. +# +# Licensees holding a valid Bacula Systems SA license may use this file +# and others of this release in accordance with the proprietary license +# agreement provided in the LICENSE file. Redistribution of any part of +# this release is not permitted. +# +# Bacula® is a registered trademark of Kern Sibbald. +# +# +# Routines for aws S3 cloud driver +# +# Written by Norbert Bizet, August MMXVIII +# +# + +import sys, os, json, time +from subprocess import Popen, PIPE, call +from multiprocessing import Pool, cpu_count +from inspect import stack +import logging + +def vol_ls(): + try: + logging.info("enter vol_ls") + proc = Popen(["aws", "s3", "ls", + cloud_path], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("vol_ls proc communicate output:{0} , err:{1}".format(output,err)) + # sort out only exception errors since progress is reported into stderr + if "Exception" in err: + logging.error("vol_ls got error {0}".format(err)) + sys.stderr.write(err) + if output: + output = "\n".join(list(filter(None, [line.rsplit(' ',2)[2].strip() for line in output.replace(cloud_path, '').replace('/','').splitlines()]))) + # forward out stds + logging.info("vol_ls got ouput") + logging.info("vol_ls outputing {0}".format(output)) + sys.stdout.write(output) + return proc.returncode + except OSError as o: + logging.exception("vol_ls OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("vol_ls ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("vol_ls Generic exception") + sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3])) + return 1 + +def ls(): + try: + logging.info("enter ls") + proc = Popen(["aws", "s3", "ls", + os.path.join(cloud_path, volume, part)], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("ls proc communicate output:{0} , err:{1}".format(output,err)) + # sort out only exception errors since progress is reported into stderr + if "Exception" in err: + logging.error("ls got error {0}".format(err)) + sys.stderr.write(err) + if output: + logging.info("ls got ouput") + # parse and format output + #mtime + mtimes = [int(time.mktime(time.strptime(line.rsplit(' ',2)[0].strip(),'%Y-%m-%d %H:%M:%S'))) for line in output.splitlines()] + #size + sizes = [line.rsplit(' ',2)[1].strip() for line in output.splitlines()] + #names + names = [line.rsplit(' ',2)[2].strip() for line in output.splitlines()] + output = "\n".join(["name:{0},mtime:{1},size:{2}".format(n,t,s) for n, t, s in zip(names, mtimes, sizes)]) + # forward out stds + logging.info("ls outputing {0}".format(output)) + sys.stdout.write(output) + return proc.returncode + except OSError as o: + logging.exception("ls OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("ls ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("ls Generic exception") + sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3])) + return 1 + +def download(): + try: + logging.info("enter download") + return call(["aws", "s3", "cp", + os.path.join(cloud_path, volume, part), "-", "--only-show-errors"]) + except OSError as o: + logging.exception("download OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\n".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("download ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\n".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("download Generic exception") + sys.stderr.write("{1} Generic exception: {0}\n".format(sys.exc_info()[0], stack()[0][3])) + return 1 + +def delete(): + try: + logging.info("enter delete") + proc = Popen(["aws", "s3", "rm", + os.path.join(cloud_path, volume, part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("delete proc communicate output:{0} , err:{1}".format(output,err)) + # sort out only exception errors since progress is reported into stderr (yuck!) + if "Exception" in err: + logging.error("delete got error {0}".format(err)) + sys.stderr.write(err) + if output: + logging.info("delete got ouput {0}".format(output)) + sys.stdout.write(output) + return proc.returncode + except OSError as o: + logging.exception("delete OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("delete ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("delete Generic exception") + sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3])) + return 1 + +def upload(): + try: + logging.info("enter upload") + proc = Popen(["aws", "s3", "cp", + "-", os.path.join(cloud_path, volume, part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("upload proc communicate output:{0} , err:{1}".format(output,err)) + # sort out only exception errors since progress is reported into stderr (yuck!) + if "Exception" in err: + sys.stderr.write(err) + proc = Popen(["aws", "s3", "ls", + os.path.join(cloud_path, volume, part)], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("ls proc communicate output:{0} , err:{1}".format(output,err)) + # sort out only exception errors since progress is reported into stderr + if "Exception" in err: + logging.error("ls got error {0}".format(err)) + sys.stderr.write(err) + if output: + logging.info("ls got ouput") + # parse and format output + #mtime + mtimes = [int(time.mktime(time.strptime(line.rsplit(' ',2)[0].strip(),'%Y-%m-%d %H:%M:%S'))) for line in output.splitlines() if line.endswith(part)] + #size + sizes = [line.rsplit(' ',2)[1].strip() for line in output.splitlines() if line.endswith(part)] + #names + names = [line.rsplit(' ',2)[2].strip() for line in output.splitlines() if line.endswith(part)] + output = "\n".join(["name:{0},mtime:{1},size:{2}".format(n,t,s) for n, t, s in zip(names, mtimes, sizes)]) + # forward out stds + logging.info("ls outputing {0}".format(output)) + sys.stdout.write(output) + return proc.returncode + except OSError as o: + logging.exception("upload OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("upload ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("upload Generic exception") + sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3])) + return 1 + +def move(): + try: + logging.info("enter move from {0} to {1}".format(os.path.join(cloud_path, volume, part), os.path.join(cloud_path, volume, local_part))) + proc = Popen(["aws", "s3", "mv", + os.path.join(cloud_path, volume, part), + os.path.join(cloud_path, volume, local_part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True) + output,err = proc.communicate() + logging.debug("move proc communicate output:{0} , err:{1}".format(output,err)) + ret = proc.returncode + if not err: + sys.stdout.write("{0}\0".format(local_part)) + return 0 + else: + if err.find("No URLs matched") != -1: + logging.debug("move cant find source {0}. OK.".format(part)) + return 0 + else: + logging.error("move got error {0}".format(err)) + sys.stderr.write("{0}\0".format(err.split("\n\n")[1].split("\n")[0])) + return ret + except NameError as c: + logging.exception("move NameError exception") + if (err.find("CommandException: No URLs matched")): + logging.debug("move cant find source {0}. OK.".format(part)) + return 0 + except OSError as o: + logging.exception("move OSError exception") + sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3])) + except ValueError as v: + logging.exception("move ValueError exception") + sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3])) + except: + logging.exception("move Generic exception") + sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3])) + return 1 + + +if __name__ == '__main__': + # initialize the return code with a weird value + ret = 10 + + if ('CLOUD_DEBUG' in os.environ) and os.environ['CLOUD_DEBUG']: + logging.basicConfig(filename='@working_dir@/aws_cloud_driver.log', level=logging.DEBUG, filemode='a', format='%(asctime)-15s-%(process)d- %(levelname)s -%(message)s') + + try: + if len(sys.argv) < 3: + sys.stderr.write("google_driver: invalid number of parameters\0.") + ret = 1 + + fct=sys.argv[1] + volume=sys.argv[2] + if len(sys.argv) > 3: + part=sys.argv[3] + else: + part='' + if len(sys.argv) > 4: + local_part=sys.argv[4] + else: + local_part='*None*' + logging.info("--{1} {2} {3} {4} --".format(sys.argv[0], fct, volume, part, local_part)) + + bucket=os.environ['CLOUD_BUCKET'] + cloud_path = "s3://{0}".format(bucket) + access_key=os.environ['CLOUD_ACCESS_KEY'] + secret_key=os.environ['CLOUD_SECRET_KEY'] + region=os.environ['CLOUD_REGION'] + protocol=os.environ['CLOUD_PROTOCOL'] + uri_type=os.environ['CLOUD_URI_TYPE'] + + logging.info("bucket {0}, cloud_path {1}, access_key {2}, secret_key {3}, region {4}, protocol {5}, uri_type {6}".format(bucket, cloud_path, access_key, secret_key, region, protocol, uri_type)) + + switcher = { + "vol_ls":vol_ls, + "ls":ls, + "download":download, + "delete":delete, + "upload":upload, + "move":move + } + + if sys.argv[1] in switcher: + ret = switcher[sys.argv[1]]() + if ret is 0: + logging.info("{0} returned {1}".format(fct, ret)) + else: + logging.error("{0} returned {1}".format(fct, ret)) + else: + sys.stderr.write('unsupported function {0}\n'.format(sys.argv[1])) + ret = 3 + + except KeyError as k: + logging.exception("main KeyError exception") + sys.stderr.write("{0} Unknown key: {1}\n".format(sys.argv[0], k)) + ret = 4 + except: + logging.exception("main Generic exception") + sys.stderr.write("{0} Generic exception: {1}\n".format(sys.argv[0], sys.exc_info()[0])) + ret = 5 + + #only once outside of the try-catch statement + sys.exit(ret)