regress: add tests for storage manager groups

Added following tests:
- store-mngr-busy-dev-test
- store-mngr-connection-err-test
- store-mngr-invalid-dev-test
- store-mngr-leastused-test

diff --git a/regress/tests/store-mngr-busy-dev-test b/regress/tests/store-mngr-busy-dev-test
new file mode 100755 (executable)
index 0000000..ea93b23
--- /dev/null
+++ b/regress/tests/store-mngr-busy-dev-test
@@ -0,0 +1,81 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Simple test to check if storage list is correctly handled when of the devices in the list is busy
+#
+
+TestName="store-mngr-busy-dev"
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-confs
+
+
+# Add two simple jobs (one of them with store group)
+cat <<END_OF_DATA >> $bin/bacula-dir.conf
+Job {
+  Name = "SingleStoreJob"
+  JobDefs = "DefaultJob"
+  Storage = File1
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+}
+Job {
+  Name = "StoreGroupJob"
+  Type = Backup
+  JobDefs = "DefaultJob"
+  Storage = File1, File2
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+}
+END_OF_DATA
+
+# Limit per-device max concurrent jobs to 1
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumConcurrentJobs", "1", "Device")'
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumConcurrentJobs", "10", "Client")'
+
+# Make sure that each storage used has only 1 device
+sed -i "s/Device = FileChgr1-Dev1, FileChgr1-Dev2/Device = FileChgr1-Dev1/g" $conf/bacula-sd.conf
+sed -i "s/Device = FileChgr2-Dev1, FileChgr2-Dev2/Device = FileChgr2-Dev1/g" $conf/bacula-sd.conf
+start_test
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out  ${cwd}/tmp/log1.out
+label volume=TestVolume001 storage=File1 pool=Default slot=1 drive=0
+label volume=TestVolume002 storage=File2 pool=Default slot=1 drive=0
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=StoreGroupJob level=Full yes
+@sleep 3
+list jobs
+wait
+@$out  ${cwd}/tmp/log2.out
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+stop_bacula
+
+# We should see that File1 is used three times (JobId 1,2,3),
+# while JobId 4 uses File2 (because File1 is busy with SingleStoreJobs)
+n_file1=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File1\"" | wc -l`
+if [ $n_file1 -ne 3 ]; then
+   estat=1
+   print_debug "Wrong usage count for File1, expectd 3, got: ${n_file1}"
+fi
+
+
+n_file2=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File2\"" | wc -l`
+if [ $n_file2 -ne 1 ]; then
+   estat=1
+   print_debug "Wrong usage count for File2, expectd 1, got: ${n_file2}"
+fi
+
+end_test

diff --git a/regress/tests/store-mngr-connection-err-test b/regress/tests/store-mngr-connection-err-test
new file mode 100755 (executable)
index 0000000..4a6532a
--- /dev/null
+++ b/regress/tests/store-mngr-connection-err-test
@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Create storage group with unreachable device as a first one on the list.
+# Storage manager should skip it and use next one.
+#
+TestName="storage-mngr-connection-err"
+JobName=backup
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-confs
+
+require_linux
+
+#
+# Zap out any schedule in default conf file so that
+#  it doesn't start during our test
+#
+outf="$tmp/sed_tmp"
+echo "s%  Schedule =%# Schedule =%g" >${outf}
+cp $scripts/bacula-dir.conf $tmp/1
+sed -f ${outf} $tmp/1 >$scripts/bacula-dir.conf
+
+change_jobname BackupClient1 $JobName
+start_test
+
+$bperl -e "add_attribute('$conf/bacula-dir.conf', 'SdConnectTimeout', '5', 'Director')"
+
+cat << EOF >> $conf/bacula-dir.conf
+Storage {
+  Name = File3
+# Do not use "localhost" here
+  Address = localhost                # N.B. Use a fully qualified name here
+  SDPort = 100
+  Password = "xxx/y9VVvh+609xUnhhJBI5aL"
+  Device = FileChgr3
+  Media Type = File3
+  Maximum Concurrent Jobs = 10        # run up to 10 jobs a the same time
+}
+EOF
+
+sed -i "s/Storage = File1/Storage = File3, File2, File1/g" $conf/bacula-dir.conf
+
+cat <<END_OF_DATA >$tmp/bconcmds
+@output /dev/null
+messages
+@$out $tmp/log1.out
+run job=$JobName yes
+wait
+messages
+@$out $tmp/log2.out
+restore where=$tmp/bacula-restores select all done
+yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+n=`cat ${cwd}/tmp/log1.out | grep "Possible storage choices: \"File3, File2, File1\"" | wc -l`
+if [ $n -ne 1 ]; then
+   estat=1
+   print_debug "Did not found list of possible storages in job lob"
+fi
+
+# File3 should be skipped since it's not reachable, File2 should be used instead
+n_file2=`cat ${cwd}/tmp/log1.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File2\"" | wc -l`
+if [ $n_file2 -ne 1 ]; then
+   estat=1
+   print_debug "Wrong usage count for File2, expectd 1, got: ${n_file1}"
+fi
+
+n_file1=`cat ${cwd}/tmp/log1.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File1\"" | wc -l`
+if [ $n_file1 -ne 0 ]; then
+   estat=1
+   print_debug "Wrong usage count for File0, expectd 0, got: ${n_file1}"
+fi
+
+stop_bacula
+
+end_test

diff --git a/regress/tests/store-mngr-invalid-dev-test b/regress/tests/store-mngr-invalid-dev-test
new file mode 100755 (executable)
index 0000000..e55603c
--- /dev/null
+++ b/regress/tests/store-mngr-invalid-dev-test
@@ -0,0 +1,67 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Simple test to check if storage groups are handling invalid device correctly
+#
+
+TestName="store-mngr-invalid-dev"
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-confs
+
+# Add simple job with store group
+cat <<END_OF_DATA >> $bin/bacula-dir.conf
+Job {
+  Name = "StoreGroupJob"
+  Type = Backup
+  JobDefs = "DefaultJob"
+  Storage = File1, File2
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+}
+END_OF_DATA
+
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumConcurrentJobs", "10", "Client")'
+
+# Make File1 contains invalid device
+sed -i "s/Device = FileChgr1/Device = DummyDev/g" $conf/bacula-dir.conf
+
+start_test
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out  ${cwd}/tmp/log1.out
+label volume=TestVolume002 storage=File2 pool=Default slot=1 drive=0
+run job=StoreGroupJob level=Full yes
+run job=StoreGroupJob level=Full yes
+run job=StoreGroupJob level=Full yes
+@sleep 3
+list jobs
+wait
+@$out  ${cwd}/tmp/log2.out
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+stop_bacula
+
+# We should see that File2 is used each time, File1 should not be used at all since it has an invalid device
+n_file1=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File1\"" | wc -l`
+if [ $n_file1 -ne 0 ]; then
+   estat=1
+   print_debug "Wrong usage count for File1, expectd 3, got: ${n_file1}"
+fi
+
+n_file2=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File2\"" | wc -l`
+if [ $n_file2 -ne 3 ]; then
+   estat=1
+   print_debug "Wrong usage count for File2, expectd 1, got: ${n_file2}"
+fi
+
+end_test

diff --git a/regress/tests/store-mngr-leastused-test b/regress/tests/store-mngr-leastused-test
new file mode 100755 (executable)
index 0000000..413db73
--- /dev/null
+++ b/regress/tests/store-mngr-leastused-test
@@ -0,0 +1,101 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Simple test to check if storage list is correctly handled for the LeastUsed policy
+#
+
+TestName="store-mngr-leastused"
+. scripts/functions
+
+scripts/cleanup
+scripts/copy-confs
+
+
+# Add two simple jobs (one of them with store group)
+cat <<END_OF_DATA >> $bin/bacula-dir.conf
+Job {
+  Name = "SingleStoreJob"
+  JobDefs = "DefaultJob"
+  Storage = File1
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+}
+Job {
+  Name = "StoreGroupJob"
+  Type = Backup
+  JobDefs = "DefaultJob"
+  Storage = File1, File2
+  StoragePolicy = LeastUsed
+  Pool = Default
+  Maximum Concurrent Jobs = 10
+}
+END_OF_DATA
+
+# Limit per-device max concurrent jobs to 1
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumConcurrentJobs", "10", "Device")'
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "MaximumConcurrentJobs", "10", "Client")'
+
+# Make sure that each storage used has only 1 device
+sed -i "s/Device = FileChgr1-Dev1, FileChgr1-Dev2/Device = FileChgr1-Dev1/g" $conf/bacula-sd.conf
+sed -i "s/Device = FileChgr2-Dev1, FileChgr2-Dev2/Device = FileChgr2-Dev1/g" $conf/bacula-sd.conf
+start_test
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@output /dev/null
+messages
+@$out  ${cwd}/tmp/log1.out
+label volume=TestVolume001 storage=File1 pool=Default slot=1 drive=0
+label volume=TestVolume002 storage=File2 pool=Default slot=1 drive=0
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=SingleStoreJob level=Full yes
+run job=StoreGroupJob level=Full yes
+@sleep 3
+list jobs
+wait
+@$out  ${cwd}/tmp/log2.out
+messages
+@$out  ${cwd}/tmp/log3.out
+run job=StoreGroupJob level=Full yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+stop_bacula
+
+# We should see that File1 is used 5 times (JobId 1,2,3, 4, 5),
+# while JobId 6 uses File2 (because File1 has 5 concurent jobs, File2 has 0)
+n_file1=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File1\"" | wc -l`
+if [ $n_file1 -ne 5 ]; then
+   estat=1
+   print_debug "Wrong usage count for File1, expectd 5, got: ${n_file1}"
+fi
+
+n_file2=`cat ${cwd}/tmp/log2.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File2\"" | wc -l`
+if [ $n_file2 -ne 1 ]; then
+   estat=1
+   print_debug "Wrong usage count for File2, expectd 1, got: ${n_file2}"
+fi
+
+# This time only File1 should be used (because it's the first on in the storage list and it was not
+# utilized during Job 7 run
+n_file1=`cat ${cwd}/tmp/log3.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File1\"" | wc -l`
+if [ $n_file1 -ne 1 ]; then
+   estat=1
+   print_debug "Wrong usage count for File1, expectd 1, got: ${n_file1}"
+fi
+
+n_file2=`cat ${cwd}/tmp/log3.out | grep "Storage:" | tr -s ' ' | grep "Storage: \"File2\"" | wc -l`
+if [ $n_file2 -ne 0 ]; then
+   estat=1
+   print_debug "Wrong usage count for File2, expectd 0, got: ${n_file2}"
+fi
+
+end_test