regress: Add cloud tests

diff --git a/regress/tests/cloud-test-hash b/regress/tests/cloud-test-hash
new file mode 100755 (executable)
index 0000000..c2a511d
--- /dev/null
+++ b/regress/tests/cloud-test-hash
@@ -0,0 +1,128 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2020 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Cloud test. We run backups and restore with the "truncate cache" command in-between
+#
+TestName="cloud-test"
+JobName=NightlySave
+. scripts/functions
+
+require_cloud
+
+#config is required for cloud cleanup
+scripts/copy-test-confs
+scripts/cleanup
+
+FORCE_FILE_SET=${FORCE_FILE_SET:-"${cwd}/build/src/stored"}
+echo "$FORCE_FILE_SET" >${cwd}/tmp/file-list
+
+export GENERATE_CLOUD_HASH="YES"
+
+start_test
+
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumPartSize", "10000000", "Device")'
+
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+label storage=File volume=Vol1
+END_OF_DATA
+
+# do label
+run_bacula
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+@#setdebug level=500 storage
+run job=$JobName level=Full yes
+wait
+list volumes
+llist volume=Vol1
+messages
+truncate cache volume=Vol1 storage=File
+END_OF_DATA
+
+for i in 1 2 ; do
+  echo "Running job $i"
+  run_bconsole
+  ls -l tmp/Vol*
+done
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds-form
+@# 
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+@#setdebug level=200 client
+@#setdebug level=350 storage
+truncate cache volume=Vol1 storage=File
+@exec "ls -l ${cwd}/tmp/Vol1"
+restore where=${cwd}/tmp/bacula-restores storage=File jobid=@jobid@
+mark *
+done
+yes
+wait
+messages
+sql
+select * from JobMedia;
+
+@$out $tmp/log31.out
+cloud list storage=File
+@$out $tmp/log3.out
+cloud list volume=Vol1 storage=File
+quit
+END_OF_DATA
+
+# Restore and check each job
+for i in 2; do
+   rm -rf ${cwd}/tmp/bacula-restores $tmp/log3*out
+   echo "s%@jobid@%$i%" >${cwd}/tmp/in
+   sed -f ${cwd}/tmp/in ${cwd}/tmp/bconcmds-form >${cwd}/tmp/bconcmds
+   run_bconsole
+   nb=`cat $tmp/log3.out | wc -l`
+   if [ "$nb" -lt 10 ]; then
+       print_debug "ERROR: Not enough lines in $tmp/log3.out for the parts list"
+       estat=1
+   fi
+   grep Vol1 $tmp/log31.out > /dev/null
+   if [ $? -ne 0 ]; then
+       print_debug "ERROR: Unable to find the volume list in $tmp/log31.out"
+       estat=1
+   fi
+done
+
+ls -l tmp/Vol* >>${cwd}/tmp/log5.out
+ls -l ${cwd}/tmp/Vol1
+
+sleep 2
+check_for_zombie_jobs storage=File 
+stop_bacula
+
+check_two_logs
+
+# get hash tags from log1
+upload_hash=$(grep -o 'JobId 2: Vol1\/part.*hash=.*' ${cwd}/tmp/log1.out | sed -n -e 's/^.*hash=//p')
+# get hash tags from log2
+download_hash=$(grep -o 'JobId 3: Vol1\/part.*hash=.*' ${cwd}/tmp/log2.out | sed -n -e 's/^.*hash=//p')
+if test "$upload_hash" != "$download_hash"; then
+    print_debug "ERROR: part hashes don't match between upload and download"
+    print_debug "Uploads:"
+    print_debug $upload_hash
+    print_debug "Downloads:"
+    print_debug $download_hash
+    estat=1
+fi
+
+#check_restore_diff
+$rscripts/diff.pl -s "$FORCE_FILE_SET" -d "$cwd/tmp/bacula-restores/$FORCE_FILE_SET"
+if test $? -ne 0; then
+    dstat=$?
+fi
+
+end_test

diff --git a/regress/tests/cloud-versioned-part-test b/regress/tests/cloud-versioned-part-test
new file mode 100755 (executable)
index 0000000..4a047ac
--- /dev/null
+++ b/regress/tests/cloud-versioned-part-test
@@ -0,0 +1,108 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# Copyright (C) 2021-2023 Bacula Systems SA
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Test versioning. We run 1 backup, modify part.2 and reupload it -> should get partXXXX.2 versioned part generated
+# Test version cleanup. Do a cloud truncate -> should get partXXXX.2 deleted in debug trace
+TestName="cloud-versioned-part-test"
+JobName=NightlySave
+. scripts/functions
+
+require_cloud
+
+#config is required for cloud cleanup
+scripts/copy-test-confs
+scripts/cleanup
+
+FORCE_FILE_SET=${FORCE_FILE_SET:-"${cwd}/build/src/stored"}
+echo "$FORCE_FILE_SET" >${cwd}/tmp/file-list
+
+start_test
+
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumPartSize", "10000000", "Device")'
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumUploadBandwidth", "4MB/s", "Cloud")'
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumDownloadBandwidth", "4MB/s", "Cloud")'
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+label storage=File volume=Vol1
+END_OF_DATA
+
+# do label
+run_bacula
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+@#setdebug tags=cloud level=50 trace=1 storage
+run job=$JobName level=Full yes
+wait
+list volumes
+llist volume=Vol1
+messages
+END_OF_DATA
+
+echo "Running job 1"
+run_bconsole
+
+echo $tmp/Vol1/part.1 >> $tmp/Vol1/part.2
+
+cat <<END_OF_DATA >$tmp/bconcmds
+@output /dev/null
+messages
+@$out $tmp/log2.out
+@#setdebug tags=cloud level=50 trace=1 storage
+cloud upload storage=File volume=Vol1
+messages
+END_OF_DATA
+
+echo "Running job 2"
+run_bconsole
+ls -l tmp/Vol*
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@# 
+@# now do purge the cloud
+@#
+@$out $tmp/log4.out
+setdebug tags=cloud level=50 trace=1 storage
+update volume=Vol1 actiononpurge=Truncate
+purge volume=Vol1
+truncate storage=File
+
+@$out $tmp/log41.out
+cloud list storage=File
+cloud list volume=Vol1 storage=File
+quit
+END_OF_DATA
+
+run_bconsole
+
+sleep 2
+check_for_zombie_jobs storage=File 
+stop_bacula
+
+# must have a versioned part reported in job2
+grep "Vol1/part.2 was present on the cloud and has been versioned to " ${tmp}/log2.out 2>&1 >/dev/null
+if test $? -ne 0; then
+    echo "Error: could not find versioned part"
+    estat=1
+fi
+
+# must check if the versioned part has been cleaned up
+grep "clean_cloud_volume for Vol1: Unlink file" ${working}/*.trace 2>&1 >/dev/null
+if test $? -ne 0; then
+    echo "Error: could not find versioned part cleanup"
+    estat=2
+fi
+
+check_two_logs
+
+end_test