From ff2a6c5a683d4e74e68c026bb59df758d3818baf Mon Sep 17 00:00:00 2001 From: Norbert Bizet Date: Wed, 6 Sep 2023 17:25:31 +0200 Subject: [PATCH] regress: Add cloud tests --- regress/tests/cloud-test-hash | 128 ++++++++++++++++++++++++ regress/tests/cloud-versioned-part-test | 108 ++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100755 regress/tests/cloud-test-hash create mode 100755 regress/tests/cloud-versioned-part-test diff --git a/regress/tests/cloud-test-hash b/regress/tests/cloud-test-hash new file mode 100755 index 000000000..c2a511ded --- /dev/null +++ b/regress/tests/cloud-test-hash @@ -0,0 +1,128 @@ +#!/bin/sh +# +# Copyright (C) 2000-2020 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Cloud test. We run backups and restore with the "truncate cache" command in-between +# +TestName="cloud-test" +JobName=NightlySave +. scripts/functions + +require_cloud + +#config is required for cloud cleanup +scripts/copy-test-confs +scripts/cleanup + +FORCE_FILE_SET=${FORCE_FILE_SET:-"${cwd}/build/src/stored"} +echo "$FORCE_FILE_SET" >${cwd}/tmp/file-list + +export GENERATE_CLOUD_HASH="YES" + +start_test + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumPartSize", "10000000", "Device")' + + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=Vol1 +END_OF_DATA + +# do label +run_bacula + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug level=500 storage +run job=$JobName level=Full yes +wait +list volumes +llist volume=Vol1 +messages +truncate cache volume=Vol1 storage=File +END_OF_DATA + +for i in 1 2 ; do + echo "Running job $i" + run_bconsole + ls -l tmp/Vol* +done + +cat <${cwd}/tmp/bconcmds-form +@# +@# now do a restore +@# +@$out ${cwd}/tmp/log2.out +@#setdebug level=200 client +@#setdebug level=350 storage +truncate cache volume=Vol1 storage=File +@exec "ls -l ${cwd}/tmp/Vol1" +restore where=${cwd}/tmp/bacula-restores storage=File jobid=@jobid@ +mark * +done +yes +wait +messages +sql +select * from JobMedia; + +@$out $tmp/log31.out +cloud list storage=File +@$out $tmp/log3.out +cloud list volume=Vol1 storage=File +quit +END_OF_DATA + +# Restore and check each job +for i in 2; do + rm -rf ${cwd}/tmp/bacula-restores $tmp/log3*out + echo "s%@jobid@%$i%" >${cwd}/tmp/in + sed -f ${cwd}/tmp/in ${cwd}/tmp/bconcmds-form >${cwd}/tmp/bconcmds + run_bconsole + nb=`cat $tmp/log3.out | wc -l` + if [ "$nb" -lt 10 ]; then + print_debug "ERROR: Not enough lines in $tmp/log3.out for the parts list" + estat=1 + fi + grep Vol1 $tmp/log31.out > /dev/null + if [ $? -ne 0 ]; then + print_debug "ERROR: Unable to find the volume list in $tmp/log31.out" + estat=1 + fi +done + +ls -l tmp/Vol* >>${cwd}/tmp/log5.out +ls -l ${cwd}/tmp/Vol1 + +sleep 2 +check_for_zombie_jobs storage=File +stop_bacula + +check_two_logs + +# get hash tags from log1 +upload_hash=$(grep -o 'JobId 2: Vol1\/part.*hash=.*' ${cwd}/tmp/log1.out | sed -n -e 's/^.*hash=//p') +# get hash tags from log2 +download_hash=$(grep -o 'JobId 3: Vol1\/part.*hash=.*' ${cwd}/tmp/log2.out | sed -n -e 's/^.*hash=//p') +if test "$upload_hash" != "$download_hash"; then + print_debug "ERROR: part hashes don't match between upload and download" + print_debug "Uploads:" + print_debug $upload_hash + print_debug "Downloads:" + print_debug $download_hash + estat=1 +fi + +#check_restore_diff +$rscripts/diff.pl -s "$FORCE_FILE_SET" -d "$cwd/tmp/bacula-restores/$FORCE_FILE_SET" +if test $? -ne 0; then + dstat=$? +fi + +end_test diff --git a/regress/tests/cloud-versioned-part-test b/regress/tests/cloud-versioned-part-test new file mode 100755 index 000000000..4a047accf --- /dev/null +++ b/regress/tests/cloud-versioned-part-test @@ -0,0 +1,108 @@ +#!/bin/sh +# +# Copyright (C) 2000-2021 Kern Sibbald +# Copyright (C) 2021-2023 Bacula Systems SA +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test versioning. We run 1 backup, modify part.2 and reupload it -> should get partXXXX.2 versioned part generated +# Test version cleanup. Do a cloud truncate -> should get partXXXX.2 deleted in debug trace +TestName="cloud-versioned-part-test" +JobName=NightlySave +. scripts/functions + +require_cloud + +#config is required for cloud cleanup +scripts/copy-test-confs +scripts/cleanup + +FORCE_FILE_SET=${FORCE_FILE_SET:-"${cwd}/build/src/stored"} +echo "$FORCE_FILE_SET" >${cwd}/tmp/file-list + +start_test + +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumPartSize", "10000000", "Device")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumUploadBandwidth", "4MB/s", "Cloud")' +$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumDownloadBandwidth", "4MB/s", "Cloud")' + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +label storage=File volume=Vol1 +END_OF_DATA + +# do label +run_bacula + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@#setdebug tags=cloud level=50 trace=1 storage +run job=$JobName level=Full yes +wait +list volumes +llist volume=Vol1 +messages +END_OF_DATA + +echo "Running job 1" +run_bconsole + +echo $tmp/Vol1/part.1 >> $tmp/Vol1/part.2 + +cat <$tmp/bconcmds +@output /dev/null +messages +@$out $tmp/log2.out +@#setdebug tags=cloud level=50 trace=1 storage +cloud upload storage=File volume=Vol1 +messages +END_OF_DATA + +echo "Running job 2" +run_bconsole +ls -l tmp/Vol* + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@# +@# now do purge the cloud +@# +@$out $tmp/log4.out +setdebug tags=cloud level=50 trace=1 storage +update volume=Vol1 actiononpurge=Truncate +purge volume=Vol1 +truncate storage=File + +@$out $tmp/log41.out +cloud list storage=File +cloud list volume=Vol1 storage=File +quit +END_OF_DATA + +run_bconsole + +sleep 2 +check_for_zombie_jobs storage=File +stop_bacula + +# must have a versioned part reported in job2 +grep "Vol1/part.2 was present on the cloud and has been versioned to " ${tmp}/log2.out 2>&1 >/dev/null +if test $? -ne 0; then + echo "Error: could not find versioned part" + estat=1 +fi + +# must check if the versioned part has been cleaned up +grep "clean_cloud_volume for Vol1: Unlink file" ${working}/*.trace 2>&1 >/dev/null +if test $? -ne 0; then + echo "Error: could not find versioned part cleanup" + estat=2 +fi + +check_two_logs + +end_test -- 2.47.3