--- /dev/null
+#!/usr/bin/env python2
+# -*- coding: utf-8 -*-
+
+# Bacula® - The Network Backup Solution
+
+# Copyright (C) 2000-2020 Bacula Systems SA
+# All rights reserved.
+#
+# The main author of Bacula is Kern Sibbald, with contributions from many
+# others, a complete list can be found in the file AUTHORS.
+#
+# Licensees holding a valid Bacula Systems SA license may use this file
+# and others of this release in accordance with the proprietary license
+# agreement provided in the LICENSE file. Redistribution of any part of
+# this release is not permitted.
+#
+# Bacula® is a registered trademark of Kern Sibbald.
+#
+#
+# Routines for aws S3 cloud driver
+#
+# Written by Norbert Bizet, August MMXVIII
+#
+#
+
+import sys, os, json, time
+from subprocess import Popen, PIPE, call
+from multiprocessing import Pool, cpu_count
+from inspect import stack
+import logging
+
+def vol_ls():
+ try:
+ logging.info("enter vol_ls")
+ proc = Popen(["aws", "s3", "ls",
+ cloud_path], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("vol_ls proc communicate output:{0} , err:{1}".format(output,err))
+ # sort out only exception errors since progress is reported into stderr
+ if "Exception" in err:
+ logging.error("vol_ls got error {0}".format(err))
+ sys.stderr.write(err)
+ if output:
+ output = "\n".join(list(filter(None, [line.rsplit(' ',2)[2].strip() for line in output.replace(cloud_path, '').replace('/','').splitlines()])))
+ # forward out stds
+ logging.info("vol_ls got ouput")
+ logging.info("vol_ls outputing {0}".format(output))
+ sys.stdout.write(output)
+ return proc.returncode
+ except OSError as o:
+ logging.exception("vol_ls OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("vol_ls ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("vol_ls Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+def ls():
+ try:
+ logging.info("enter ls")
+ proc = Popen(["aws", "s3", "ls",
+ os.path.join(cloud_path, volume, part)], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("ls proc communicate output:{0} , err:{1}".format(output,err))
+ # sort out only exception errors since progress is reported into stderr
+ if "Exception" in err:
+ logging.error("ls got error {0}".format(err))
+ sys.stderr.write(err)
+ if output:
+ logging.info("ls got ouput")
+ # parse and format output
+ #mtime
+ mtimes = [int(time.mktime(time.strptime(line.rsplit(' ',2)[0].strip(),'%Y-%m-%d %H:%M:%S'))) for line in output.splitlines()]
+ #size
+ sizes = [line.rsplit(' ',2)[1].strip() for line in output.splitlines()]
+ #names
+ names = [line.rsplit(' ',2)[2].strip() for line in output.splitlines()]
+ output = "\n".join(["name:{0},mtime:{1},size:{2}".format(n,t,s) for n, t, s in zip(names, mtimes, sizes)])
+ # forward out stds
+ logging.info("ls outputing {0}".format(output))
+ sys.stdout.write(output)
+ return proc.returncode
+ except OSError as o:
+ logging.exception("ls OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("ls ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("ls Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+def download():
+ try:
+ logging.info("enter download")
+ return call(["aws", "s3", "cp",
+ os.path.join(cloud_path, volume, part), "-", "--only-show-errors"])
+ except OSError as o:
+ logging.exception("download OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\n".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("download ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\n".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("download Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\n".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+def delete():
+ try:
+ logging.info("enter delete")
+ proc = Popen(["aws", "s3", "rm",
+ os.path.join(cloud_path, volume, part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("delete proc communicate output:{0} , err:{1}".format(output,err))
+ # sort out only exception errors since progress is reported into stderr (yuck!)
+ if "Exception" in err:
+ logging.error("delete got error {0}".format(err))
+ sys.stderr.write(err)
+ if output:
+ logging.info("delete got ouput {0}".format(output))
+ sys.stdout.write(output)
+ return proc.returncode
+ except OSError as o:
+ logging.exception("delete OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("delete ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("delete Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+def upload():
+ try:
+ logging.info("enter upload")
+ proc = Popen(["aws", "s3", "cp",
+ "-", os.path.join(cloud_path, volume, part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("upload proc communicate output:{0} , err:{1}".format(output,err))
+ # sort out only exception errors since progress is reported into stderr (yuck!)
+ if "Exception" in err:
+ sys.stderr.write(err)
+ proc = Popen(["aws", "s3", "ls",
+ os.path.join(cloud_path, volume, part)], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("ls proc communicate output:{0} , err:{1}".format(output,err))
+ # sort out only exception errors since progress is reported into stderr
+ if "Exception" in err:
+ logging.error("ls got error {0}".format(err))
+ sys.stderr.write(err)
+ if output:
+ logging.info("ls got ouput")
+ # parse and format output
+ #mtime
+ mtimes = [int(time.mktime(time.strptime(line.rsplit(' ',2)[0].strip(),'%Y-%m-%d %H:%M:%S'))) for line in output.splitlines() if line.endswith(part)]
+ #size
+ sizes = [line.rsplit(' ',2)[1].strip() for line in output.splitlines() if line.endswith(part)]
+ #names
+ names = [line.rsplit(' ',2)[2].strip() for line in output.splitlines() if line.endswith(part)]
+ output = "\n".join(["name:{0},mtime:{1},size:{2}".format(n,t,s) for n, t, s in zip(names, mtimes, sizes)])
+ # forward out stds
+ logging.info("ls outputing {0}".format(output))
+ sys.stdout.write(output)
+ return proc.returncode
+ except OSError as o:
+ logging.exception("upload OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("upload ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("upload Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+def move():
+ try:
+ logging.info("enter move from {0} to {1}".format(os.path.join(cloud_path, volume, part), os.path.join(cloud_path, volume, local_part)))
+ proc = Popen(["aws", "s3", "mv",
+ os.path.join(cloud_path, volume, part),
+ os.path.join(cloud_path, volume, local_part), "--only-show-errors"], stdout=PIPE, stderr=PIPE, universal_newlines=True)
+ output,err = proc.communicate()
+ logging.debug("move proc communicate output:{0} , err:{1}".format(output,err))
+ ret = proc.returncode
+ if not err:
+ sys.stdout.write("{0}\0".format(local_part))
+ return 0
+ else:
+ if err.find("No URLs matched") != -1:
+ logging.debug("move cant find source {0}. OK.".format(part))
+ return 0
+ else:
+ logging.error("move got error {0}".format(err))
+ sys.stderr.write("{0}\0".format(err.split("\n\n")[1].split("\n")[0]))
+ return ret
+ except NameError as c:
+ logging.exception("move NameError exception")
+ if (err.find("CommandException: No URLs matched")):
+ logging.debug("move cant find source {0}. OK.".format(part))
+ return 0
+ except OSError as o:
+ logging.exception("move OSError exception")
+ sys.stderr.write("{2} OSError({0}): {1}\0".format(o.errno, o.strerror, stack()[0][3]))
+ except ValueError as v:
+ logging.exception("move ValueError exception")
+ sys.stderr.write("{2} Wrong Value({0}): {1}\0".format(v.errno, v.strerror, stack()[0][3]))
+ except:
+ logging.exception("move Generic exception")
+ sys.stderr.write("{1} Generic exception: {0}\0".format(sys.exc_info()[0], stack()[0][3]))
+ return 1
+
+
+if __name__ == '__main__':
+ # initialize the return code with a weird value
+ ret = 10
+
+ if ('CLOUD_DEBUG' in os.environ) and os.environ['CLOUD_DEBUG']:
+ logging.basicConfig(filename='@working_dir@/aws_cloud_driver.log', level=logging.DEBUG, filemode='a', format='%(asctime)-15s-%(process)d- %(levelname)s -%(message)s')
+
+ try:
+ if len(sys.argv) < 3:
+ sys.stderr.write("google_driver: invalid number of parameters\0.")
+ ret = 1
+
+ fct=sys.argv[1]
+ volume=sys.argv[2]
+ if len(sys.argv) > 3:
+ part=sys.argv[3]
+ else:
+ part=''
+ if len(sys.argv) > 4:
+ local_part=sys.argv[4]
+ else:
+ local_part='*None*'
+ logging.info("--{1} {2} {3} {4} --".format(sys.argv[0], fct, volume, part, local_part))
+
+ bucket=os.environ['CLOUD_BUCKET']
+ cloud_path = "s3://{0}".format(bucket)
+ access_key=os.environ['CLOUD_ACCESS_KEY']
+ secret_key=os.environ['CLOUD_SECRET_KEY']
+ region=os.environ['CLOUD_REGION']
+ protocol=os.environ['CLOUD_PROTOCOL']
+ uri_type=os.environ['CLOUD_URI_TYPE']
+
+ logging.info("bucket {0}, cloud_path {1}, access_key {2}, secret_key {3}, region {4}, protocol {5}, uri_type {6}".format(bucket, cloud_path, access_key, secret_key, region, protocol, uri_type))
+
+ switcher = {
+ "vol_ls":vol_ls,
+ "ls":ls,
+ "download":download,
+ "delete":delete,
+ "upload":upload,
+ "move":move
+ }
+
+ if sys.argv[1] in switcher:
+ ret = switcher[sys.argv[1]]()
+ if ret is 0:
+ logging.info("{0} returned {1}".format(fct, ret))
+ else:
+ logging.error("{0} returned {1}".format(fct, ret))
+ else:
+ sys.stderr.write('unsupported function {0}\n'.format(sys.argv[1]))
+ ret = 3
+
+ except KeyError as k:
+ logging.exception("main KeyError exception")
+ sys.stderr.write("{0} Unknown key: {1}\n".format(sys.argv[0], k))
+ ret = 4
+ except:
+ logging.exception("main Generic exception")
+ sys.stderr.write("{0} Generic exception: {1}\n".format(sys.argv[0], sys.exc_info()[0]))
+ ret = 5
+
+ #only once outside of the try-catch statement
+ sys.exit(ret)
projects / thirdparty / bacula.git / commitdiff
