From: Arran Cudbard-Bell Date: Fri, 5 Oct 2018 14:29:50 +0000 (+0700) Subject: Script to upload log files to AWS S3 bucket X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=9cd1fb5be99887d3dff6269b10ebab625720d4fe;p=thirdparty%2Ffreeradius-server.git Script to upload log files to AWS S3 bucket --- diff --git a/scripts/aws_archive.sh b/scripts/aws_archive.sh new file mode 100644 index 00000000000..4248e763d5b --- /dev/null +++ b/scripts/aws_archive.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +# aws_archive.sh AWS Log upload script. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA +# +# Copyright (C) 2018 Arran Cudbard-Bell +# Copyright (C) 2018 The FreeRADIUS Project http://www.freeradius.org + +# Set AWS credentials +: ${AWS_ACCESS_KEY_ID:=} +: ${AWS_SECRET_ACCESS_KEY:=} +: ${AWS_BUCKET:=s3://} + +# Default options +: ${COMPRESS:=false} +: ${EXPIRES:=0} +: ${VERBOSE:=0} +: ${LOG_DIR=/var/log/radius} + +function ERROR +{ + echo "$@" 1>&2; +} + +function INFO +{ + echo "$@" +} + +function DEBUG +{ + if [ $VERBOSE -gt 0 ]; then + echo "$@" + fi +} + +function check_binaries +{ + need_bin="aws lsof find rm" + + if $COMPRESS; then + need_bin+=" bzip2 mktemp" + fi + + if [ "$EXPIRES" -gt 0 ]; then + need_bin+=" date" + fi + + # Check we have everything we need + for bin in ${need_bin}; do + if ! which $bin > /dev/null; then + ERROR "Can't find '${bin}' binary. Ensure it's available in the current PATH" + exit 64 + fi + done +} + +function unix_now +{ + if [ "$PLATFORM" = 'SunOS' ]; then + perl -le 'print time()' + else + date +%s + fi +} + +# Output date $1 days in the future. +function future_date +{ + if [ -z "$1" ]; then + date +%Y-%m-%dT%H:%M:%S + exit + else + # Date works fine on Linux but not on old versions of Solaris + if [ "$PLATFORM" = 'SunOS' ]; then + awk "BEGIN { print strftime(\"%Y-%m-%dT%H:%M:%S\", `expr \`unix_now\` + $1 \* 86400`)}" + else + date -d @`expr \`unix_now\` + $1 \* 86400` +%Y-%m-%dT%H:%M:%S; + fi + fi +} + +function show_help +{ + INFO $(basename $0)" [options]" + INFO " -a Append to existing log files." + INFO " -d Log directory - defaults to \"${LOG_DIR}\"." + INFO " -e Expire log files N days after they're moved to AWS" + INFO " -z bzip files before uploading." + INFO " -v Print debugging information." +} + +while getopts "h?ad:e:zv" opt; do + case "$opt" in + h|\?) + show_help + exit 0 + ;; + + a) + APPEND=true + ;; + + d) + LOG_DIR="$OPTARG" + ;; + + e) + EXPIRES="$OPTARG" + ;; + + v) + VERBOSE=1 + ;; + + z) + COMPRESS=true + ;; + esac +done + +# Check for binaries +check_binaries + +# Get expiry time +if [ $EXPIRES -gt 0 ]; then + EXPIRES_AT="$(future_date ${EXPIRES})" + EXPIRES_ARG=" --expires ${EXPIRES_AT}" +fi + +# Get a temporary directory +if $APPEND; then tmpdir=$(mktemp -d); fi + +# For any log files sitting in the log directory see if they're still in use +# and if they're not, compress them and upload them to AWS. +copied=0 +while read -r in_file; do + expire_arg= + + # The file is still in use, don't move it + if lsof -lnP -- $in_file > /dev/null; then + DEBUG "Skipping ${in_file} (still in use)" + continue + fi + + # Compress the input file + if $COMPRESS; then + if [ "${in_file#*.}" != 'bz2' ]; then + DEBUG "Compressing \"${in_file}\"" + if ! bzip2 "${in_file}"; then + ERROR "Failed compressing ${in_file}, skipping..." + continue + fi + in_file="${in_file}.bz2" + fi + fi + + # Path to our destination in the AWS bucket + aws_file="${AWS_BUCKET}/${in_file}" + + # + # Check if the file already exists in AWS and append to it if it does + # + if $APPEND && aws s3 ls "${aws_file}" > /dev/null; then + DEBUG "Retrieving previous file \"${aws_file}\"" + + tmp_file="${tmpdir}/$(basename \"$aws_file\")" + if ! aws s3 cp "${aws_file}" "${tmp_file}" > /dev/null; then + ERROR "Failed retrieving ${aws_file}, skipping..." + continue + fi + + # Concatenate the two files together. This works for bzip files too + DEBUG "Concatenating \"${in_file}\" with \"${tmp_file}\"" + if ! cat "${in_file}" >> "${tmp_file}" || ! mv "${tmp_file}" "${in_file}"; then + ERROR "Failed merging previous AWS file \"${tmp_file}\" with input file \"${in_file}\"" + continue + fi + fi + + if ! aws s3 mv "${in_file}" "${aws_file}" $EXPIRES_ARG > /dev/null; then + ERROR "aws s3 mv \"${in_file}\" \"${aws_file}\" $EXPIRES_ARG failed, skipping..." + continue + fi + + copied=$((copied + 1)) +done < <(find "$LOG_DIR" -type f) + +# Cleanup our temporary directory +if $APPEND; then rm -rf "$tmpdir"; fi + +if [ $copied -eq 0 ]; then + DEBUG "No files to process" +elif [ ! -z "$EXPIRES_AT" ]; then + DEBUG "Logs will expire at $EXPIRES_AT" +fi + +exit 0