regress: add tests for checking the 'o' acccurate option

diff --git a/regress/tests/accurate-only-meta-bextract-test b/regress/tests/accurate-only-meta-bextract-test
new file mode 100755 (executable)
index 0000000..528c34f
--- /dev/null
+++ b/regress/tests/accurate-only-meta-bextract-test
@@ -0,0 +1,127 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Bextract test for the 'o' accurate backup option to backup only file's metadata
+# if file contets did not change.
+#
+
+TestName="accurate-only-meta-bextract-test"
+JobName=backup
+. scripts/functions
+$rscripts/cleanup
+
+copy_test_confs
+cp -f $rscripts/bacula-dir.conf.accurate $conf/bacula-dir.conf
+
+change_jobname BackupClient1 $JobName
+
+rm -f ${cwd}/build/testfile
+
+cat <<END_OF_DATA >>${conf}/bacula-dir.conf
+FileSet {
+ Name = FS_META_TEST
+ Include {
+    Options {
+      Signature = MD5
+      Verify = mc
+      #Accurate = mpin5o
+      Accurate = c5o
+    }
+    File=<${cwd}/tmp/file-list
+ }
+}
+END_OF_DATA
+
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "FileSet", "FS_META_TEST" , "Job", "backup")'
+
+echo "${cwd}/build" >${cwd}/tmp/file-list
+# Add one more file
+echo "testfile" >> ${cwd}/build/testfile
+
+start_test
+
+# Run normal backup
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Default
+messages
+@$out ${cwd}/tmp/log1.out
+run job=$JobName yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+# Sleep for a few seconds, touch all files to change each file's metadata
+sleep 3
+files_count=`find ${cwd}/build -type f | wc -l`
+find ${cwd}/build -type f -exec touch {} + > /dev/null
+# Change some properties of testfile
+chmod 740 ${cwd}/build/testfile
+perm_after=`stat -c %a ${cwd}/build/testfile`
+
+# Run backup for the second time, this time we expect that only metadata part for each file will be
+# backed up, so we expect the backup size to be much smaller than the first one.
+# Do the restore after that and check if metadata is updated (meaning if second backup overwritten files's metadata)
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log2.out
+run job=$JobName yes
+wait
+messages
+@$out ${cwd}/tmp/log3.out
+restore select bootstrap=${cwd}/working/restore.bsr where=${cwd}/tmp/bacula-restores select all done
+no
+wait
+messages
+quit
+END_OF_DATA
+
+run_bconsole
+
+# Check if number of second backup files equals to files touched
+backup_files=`cat ${cwd}/tmp/log2.out | grep 'FD Files Written:' | tr -s ' ' | cut -d ':' -f 2 | sed s/,//`
+if [ ${backup_files} -ne ${files_count} ]; then
+   estat=1
+   print_debug "Wrong number of files backed up: ${backup_files}, expected: ${files_count}"
+fi
+
+mkdir -p ${cwd}/tmp/bacula-restores
+if test "$debug" -eq 1 ; then
+  $bin/bextract -v -b working/restore.bsr -c bin/bacula-sd.conf ${cwd}/tmp ${cwd}/tmp/bacula-restores
+else
+  $bin/bextract -b working/restore.bsr -c bin/bacula-sd.conf ${cwd}/tmp ${cwd}/tmp/bacula-restores 2>&1 >/dev/null
+fi
+rstat=$?
+grep "^  Termination: *Backup OK" ${cwd}/tmp/log1.out 2>&1 >/dev/null
+bstat=$?
+
+# Check if testfile's restored permission are correctly changed
+perm_restored=`stat -c %a ${cwd}/tmp/bacula-restores/${cwd}/build/testfile`
+if [ ${perm_after} -ne ${perm_restored} ]; then
+   estat=1
+   print_debug "Wrong permissions restored: ${perm_restored}, expected: ${perm_after}"
+fi
+
+# Make sure that second backup was only some KB so that only metadata is transferred
+bytes_written_kb=`cat tmp/log2.out | grep 'SD Bytes Written' | grep '(.* KB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+$rscripts/diff.pl -s $cwd/build -d $tmp/bacula-restores/$cwd/build
+if [ $? -ne 0 ]; then
+    print_debug "ERROR: Found difference after restore"
+    rstat=1
+fi
+
+# Cleanup
+rm ${cwd}/build/testfile
+
+stop_bacula
+end_test

diff --git a/regress/tests/accurate-only-meta-test b/regress/tests/accurate-only-meta-test
new file mode 100755 (executable)
index 0000000..afbe2ef
--- /dev/null
+++ b/regress/tests/accurate-only-meta-test
@@ -0,0 +1,116 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Test for the 'o' accurate backup option to backup only file's metadata
+# if file contets did not change.
+#
+
+TestName="accurate-only-meta-test"
+JobName=backup
+. scripts/functions
+$rscripts/cleanup
+
+copy_test_confs
+cp -f $rscripts/bacula-dir.conf.accurate $conf/bacula-dir.conf
+
+change_jobname BackupClient1 $JobName
+
+rm -f ${cwd}/build/testfile
+
+cat <<END_OF_DATA >>${conf}/bacula-dir.conf
+FileSet {
+ Name = FS_META_TEST
+ Include {
+    Options {
+      Signature = MD5
+      Verify = mc
+      #Accurate = mpin5o
+      Accurate = c5o
+    }
+    File=<${cwd}/tmp/file-list
+ }
+}
+END_OF_DATA
+
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "FileSet", "FS_META_TEST" , "Job", "backup")'
+
+echo "${cwd}/build" >${cwd}/tmp/file-list
+# Add one more file
+echo "testfile" >> ${cwd}/build/testfile
+
+start_test
+
+# Run normal backup
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Default
+messages
+@$out ${cwd}/tmp/log1.out
+run job=$JobName yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+# Sleep for a few seconds, touch all files to change each file's metadata
+sleep 3
+files_count=`find ${cwd}/build -type f | wc -l`
+find ${cwd}/build -type f -exec touch {} + > /dev/null
+# Change some properties of testfile
+chmod 740 ${cwd}/build/testfile
+perm_after=`stat -c %a ${cwd}/build/testfile`
+
+# Run backup for the second time, this time we expect that only metadata part for each file will be
+# backed up, so we expect the backup size to be much smaller than the first one.
+# Do the restore after that and check if metadata is updated (meaning if second backup overwritten files's metadata)
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log2.out
+run job=$JobName yes
+wait
+messages
+@$out ${cwd}/tmp/log3.out
+restore select all done yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bconsole
+
+# Check if number of second backup files equals to files touched
+backup_files=`cat ${cwd}/tmp/log2.out | grep 'FD Files Written:' | tr -s ' ' | cut -d ':' -f 2 | sed s/,//`
+if [ ${backup_files} -ne ${files_count} ]; then
+   estat=1
+   print_debug "Wrong number of files backed up: ${backup_files}, expected: ${files_count}"
+fi
+
+# Check if testfile's restored permission are correctly changed
+perm_restored=`stat -c %a ${cwd}/tmp/bacula-restores/${cwd}/build/testfile`
+if [ ${perm_after} -ne ${perm_restored} ]; then
+   estat=1
+   print_debug "Wrong permissions restored: ${perm_restored}, expected: ${perm_after}"
+fi
+
+# Make sure that second backup was only some KB so that only metadata is transferred
+bytes_written_kb=`cat tmp/log2.out | grep 'SD Bytes Written' | grep '(.* KB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+$rscripts/diff.pl -s $cwd/build -d $tmp/bacula-restores/$cwd/build
+if [ $? -ne 0 ]; then
+    print_debug "ERROR: Found difference after restore"
+    rstat=1
+fi
+
+# Cleanup
+rm ${cwd}/build/testfile
+
+stop_bacula
+end_test

diff --git a/regress/tests/accurate-only-meta-virtual-full-test b/regress/tests/accurate-only-meta-virtual-full-test
new file mode 100755 (executable)
index 0000000..3ec448a
--- /dev/null
+++ b/regress/tests/accurate-only-meta-virtual-full-test
@@ -0,0 +1,148 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Test for the 'o' accurate backup (virtual full variant) option to backup only file's metadata
+# if file contets did not change.
+#
+
+TestName="accurate-only-meta-virtual-full-test"
+JobName=backup
+. scripts/functions
+$rscripts/cleanup
+
+scripts/copy-migration-confs
+
+change_jobname NightlySave $JobName
+
+rm -f ${cwd}/build/testfile
+
+# Turn accurate mode on for Job
+$bperl -e "add_attribute('$conf/bacula-dir.conf', 'Accurate', 'Yes', 'Job', 'backup')"
+
+cat <<END_OF_DATA >>${conf}/bacula-dir.conf
+FileSet {
+ Name = FS_META_TEST
+ Include {
+    Options {
+      Signature = MD5
+      Verify = mc
+      Accurate = c5o
+    }
+    File=<${cwd}/tmp/file-list
+ }
+}
+END_OF_DATA
+
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "FileSet", "FS_META_TEST" , "Job", "backup")'
+
+echo "${cwd}/build" >${cwd}/tmp/file-list
+# Add one more file
+dd if=/dev/urandom of=${cwd}/build/testfile count=10 bs=1M
+
+start_test
+
+# Run normal backup
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Default
+label storage=DiskChanger volume=ChangerVolume001 slot=1 Pool=Full drive=0
+label storage=DiskChanger volume=ChangerVolume002 slot=2 Pool=Full drive=0
+messages
+@$out ${cwd}/tmp/log1.out
+run job=$JobName yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+# Sleep for a few seconds, touch all files to change each file's metadata
+sleep 3
+files_count=`find ${cwd}/build -type f | wc -l`
+find ${cwd}/build -type f -exec touch {} + > /dev/null
+# Change some properties of testfile
+chmod 740 ${cwd}/build/testfile
+
+# Run backup for the second time, this time we expect that only metadata part for each file will be
+# backed up, so we expect the backup size to be much smaller than the first one.
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log2.out
+run job=$JobName level=Incremental yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bconsole
+
+# Change permissions once more
+chmod 776 ${cwd}/build/testfile
+perm_after=`stat -c %a ${cwd}/build/testfile`
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log3.out
+run job=$JobName level=Differential yes
+wait
+messages
+@$out ${cwd}/tmp/log4.out
+run job=$JobName level=VirtualFull yes
+wait
+messages
+@$out ${cwd}/tmp/log5.out
+restore select all done yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bconsole
+
+# Check if number of second backup files equals to files touched
+backup_files=`cat ${cwd}/tmp/log2.out | grep 'FD Files Written:' | tr -s ' ' | cut -d ':' -f 2 | sed s/,//`
+if [ ${backup_files} -ne ${files_count} ]; then
+   estat=1
+   print_debug "Wrong number of files backed up: ${backup_files}, expected: ${files_count}"
+fi
+
+# Check if testfile's restored from Vfull permission are correctly changed
+perm_restored=`stat -c %a ${cwd}/tmp/bacula-restores/${cwd}/build/testfile`
+if [ ${perm_after} -ne ${perm_restored} ]; then
+   estat=1
+   print_debug "Wrong permissions restored: ${perm_restored}, expected: ${perm_after}"
+fi
+
+# Make sure that second (incremental) and third (differential) backup was only some KB so that only metadata is transferred
+bytes_written_kb=`cat tmp/log2.out | grep 'SD Bytes Written' | grep '(.* KB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+bytes_written_kb=`cat tmp/log3.out | grep 'SD Bytes Written' | grep '(.* KB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+# For the virtual full we expect size to be expressed in MBs
+bytes_written_kb=`cat tmp/log4.out | grep 'SD Bytes Written' | grep '(.* MB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+$rscripts/diff.pl -s $cwd/build -d $tmp/bacula-restores/$cwd/build
+if [ $? -ne 0 ]; then
+    print_debug "ERROR: Found difference after restore"
+    rstat=1
+fi
+
+# Cleanup
+rm ${cwd}/build/testfile
+
+stop_bacula
+end_test

diff --git a/regress/tests/bscan-accurate-only-meta-test b/regress/tests/bscan-accurate-only-meta-test
new file mode 100755 (executable)
index 0000000..7549875
--- /dev/null
+++ b/regress/tests/bscan-accurate-only-meta-test
@@ -0,0 +1,158 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+# Bscan Test for the 'o' accurate backup option to backup only file's metadata
+# if file contets did not change.
+#
+
+TestName="bscan-accurate-only-meta-test"
+JobName=backup
+. scripts/functions
+$rscripts/cleanup
+
+copy_test_confs
+cp -f $rscripts/bacula-dir.conf.accurate $conf/bacula-dir.conf
+
+change_jobname BackupClient1 $JobName
+
+rm ${cwd}/build/testfile
+
+cat <<END_OF_DATA >>${conf}/bacula-dir.conf
+FileSet {
+ Name = FS_META_TEST
+ Include {
+    Options {
+      Signature = MD5
+      Verify = mc
+      #Accurate = mpin5o
+      Accurate = c5o
+    }
+    File=<${cwd}/tmp/file-list
+ }
+}
+END_OF_DATA
+
+$bperl -e 'add_attribute("$conf/bacula-dir.conf", "FileSet", "FS_META_TEST" , "Job", "backup")'
+
+echo "${cwd}/build" >${cwd}/tmp/file-list
+# Add one more file
+echo "testfile" >> ${cwd}/build/testfile
+
+start_test
+
+# Run normal backup
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+label volume=TestVolume001 storage=File pool=Default
+messages
+@$out ${cwd}/tmp/log1.out
+run job=$JobName yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+# Sleep for a few seconds, touch all files to change each file's metadata
+sleep 3
+files_count=`find ${cwd}/build -type f | wc -l`
+find ${cwd}/build -type f -exec touch {} + > /dev/null
+# Change some properties of testfile
+chmod 777 ${cwd}/build/testfile
+perm_after=`stat -c %a ${cwd}/build/testfile`
+
+# Run backup for the second time, this time we expect that only metadata part for each file will be
+# backed up, so we expect the backup size to be much smaller than the first one.
+# Next purge and delete volume so that it can be bscanned later.
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@$out ${cwd}/tmp/log2.out
+run job=$JobName yes
+wait
+messages
+sql
+@$out ${cwd}/tmp/log3.out
+select count(*) from File;
+
+@$out /dev/null
+@$out ${cwd}/tmp/log4.out
+purge volume=TestVolume001
+delete volume=TestVolume001 yes
+messages
+quit
+END_OF_DATA
+
+run_bconsole
+
+stop_bacula
+
+# Check if number of second backup files equals to files touched
+backup_files=`cat ${cwd}/tmp/log2.out | grep 'FD Files Written:' | tr -s ' ' | cut -d ':' -f 2 | sed s/,//`
+if [ ${backup_files} -ne ${files_count} ]; then
+   estat=1
+   print_debug "Wrong number of files backed up: ${backup_files}, expected: ${files_count}"
+fi
+
+echo "volume=TestVolume001" >tmp/bscan.bsr
+
+bscan_libdbi
+
+# If the database has a password pass it to bscan
+if test "x${db_password}" = "x"; then
+  PASSWD=
+else
+  PASSWD="-P ${db_password}"
+fi
+
+# Run bscan
+if test "$debug" -eq 1 ; then
+  $bin/bscan -w working $BSCANLIBDBI -u ${db_user} -n ${db_name} $PASSWD -m -s -v -b tmp/bscan.bsr -c bin/bacula-sd.conf ${cwd}/tmp
+else
+  $bin/bscan -w working $BSCANLIBDBI -u ${db_user} -n ${db_name} $PASSWD -m -s -v -b tmp/bscan.bsr -c bin/bacula-sd.conf ${cwd}/tmp > tmp/log5.out 2>&1
+fi
+
+# Restore bscanned jobs
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+sql
+@$out ${cwd}/tmp/log6.out
+select count(*) from File;
+
+@$out /dev/null
+@$out ${cwd}/tmp/log7.out
+restore jobid=1,2 select all done yes
+wait
+messages
+quit
+END_OF_DATA
+
+run_bacula
+
+# Check if number if records inserted to the File table by bscan is correct
+diff ${cwd}/tmp/log3.out ${cwd}/tmp/log6.out
+if [ $? -ne 0 ]; then
+   estat=1
+   print_debug "Bscan inserted invalid number of File records! See ${cwd}/tmp/log3.out (expected) and ${cwd}/tmp/log6.out (bsanned)"
+fi
+
+# Check if testfile's restored permission are correctly changed
+perm_restored=`stat -c %a ${cwd}/tmp/bacula-restores/${cwd}/build/testfile`
+if [ ${perm_after} -ne ${perm_restored} ]; then
+   estat=1
+   print_debug "Wrong permissions restored: ${perm_restored}, expected: ${perm_after}"
+fi
+
+# Make sure that second backup was only some KB so that only metadata is transferred
+bytes_written_kb=`cat tmp/log2.out | grep 'SD Bytes Written' | grep '(.* KB)' | wc -l`
+if [ ${bytes_written_kb} -ne 1 ]; then
+   estat=1
+   print_debug "SD Bytes Written not expressed in KB as expected!"
+fi
+
+# Cleanup
+rm ${cwd}/build/testfile
+
+stop_bacula
+end_test