From: Kern Sibbald Date: Wed, 2 Sep 2020 13:37:37 +0000 (+0200) Subject: BEE Backport regress/tests/virtualfull-extreme-concurrency-bug-test X-Git-Tag: Release-11.3.2~1138 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=92a23bb62bba978f799e6bcdc07c57d4bd5688b9;p=thirdparty%2Fbacula.git BEE Backport regress/tests/virtualfull-extreme-concurrency-bug-test This commit is the result of the squash of the following main commits: Author: Eric Bollengier Date: Tue Jul 21 10:28:13 2020 +0200 regress: Add copyright to regress scripts Author: Kern Sibbald Date: Mon Aug 3 08:41:44 2015 +0200 Tweak remove reference to non-existent directory Author: Kern Sibbald Date: Mon Feb 18 09:51:41 2013 +0100 Tweak change customer name Heise to Hot in regress test Author: Kern Sibbald Date: Sun Dec 16 08:29:50 2012 +0100 Tweak fix typo in cp redirect regress test Author: Kern Sibbald Date: Sat Dec 15 08:14:17 2012 +0100 Tweak virtualfull extreme test Author: Kern Sibbald Date: Wed Dec 12 20:39:03 2012 +0100 Add Arno's extreme test --- diff --git a/regress/tests/virtualfull-extreme-concurrency-bug-test b/regress/tests/virtualfull-extreme-concurrency-bug-test new file mode 100755 index 0000000000..24afacb561 --- /dev/null +++ b/regress/tests/virtualfull-extreme-concurrency-bug-test @@ -0,0 +1,261 @@ +#!/bin/sh +# +# Copyright (C) 2000-2020 Kern Sibbald +# License: BSD 2-Clause; see file LICENSE-FOSS +# +# Test for a bug that causes massively parallel virtual full jobs to +# deadlock. First create a lot of backups, then run +# virtual fulls on them. +# +TestName="virtualfull-extreme-concurrency-bug-test" +. scripts/functions + +# Setup conf files +scripts/cleanup +scripts/copy-test-confs +cp scripts/virtualfull-extreme-bacula-dir.conf bin/bacula-dir.conf +cp scripts/migrate-bacula-sd.conf bin/bacula-sd.conf + +# we have 20 distinct jobs. Each of those is run once at full level, +# then four times incremental, each time adding data so we end up with +# many used volumes. +# We should have jobs where several virtual fulls compete for volumes. + + +# Backup Bacula stored+objects directory +# Backup director +rm -rf ${tmpsrc} +mkdir ${tmpsrc} +echo "${tmpsrc}" >${tmp}/file-list +for i in `ls ${src}/src/stored/*` ; do + cp -p $i ${tmpsrc} +done +# Get path to temp source files to restore + +start_test + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out +@# No prelabeled volumes needed since they are automatically created +@#label storage=File slot=0 drive=0 Pool=Hot1 volume=Vol1 +@#list volumes +END_OF_DATA + +# Start Bacula and do label +run_bacula + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log0.out +run job=HotV01 level=Full yes +run job=HotV02 level=Full yes +run job=HotV03 level=Full yes +run job=HotV04 level=Full yes +run job=HotV05 level=Full yes +run job=HotV06 level=Full yes +run job=HotV07 level=Full yes +run job=HotV08 level=Full yes +run job=HotV09 level=Full yes +run job=HotV10 level=Full yes +run job=HotV11 level=Full yes +run job=HotV12 level=Full yes +run job=HotV13 level=Full yes +run job=HotV14 level=Full yes +run job=HotV15 level=Full yes +run job=HotV16 level=Full yes +run job=HotV17 level=Full yes +run job=HotV18 level=Full yes +run job=HotV19 level=Full yes +run job=HotV20 level=Full yes +wait +list volumes +messages +END_OF_DATA + +run_bconsole + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +END_OF_DATA + +# add more files to backup source, so following incrementals +# have something to back up +for a in console filed lib tools ; do + #echo ">>> adding ${a} to backup" + for i in `ls ${src}/src/${a}/*` ; do + cp -p "$i" ${tmpsrc} >/dev/null 2>&1 + done + #echo "now $(ls ${tmpsrc}|wc -l) files to back up" + + cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out a +time +run job=HotV01 level=Incremental yes +run job=HotV02 level=Incremental yes +run job=HotV03 level=Incremental yes +run job=HotV04 level=Incremental yes +run job=HotV05 level=Incremental yes +run job=HotV06 level=Incremental yes +run job=HotV07 level=Incremental yes +run job=HotV08 level=Incremental yes +run job=HotV09 level=Incremental yes +run job=HotV10 level=Incremental yes +run job=HotV11 level=Incremental yes +run job=HotV12 level=Incremental yes +run job=HotV13 level=Incremental yes +run job=HotV14 level=Incremental yes +run job=HotV15 level=Incremental yes +run job=HotV16 level=Incremental yes +run job=HotV17 level=Incremental yes +run job=HotV18 level=Incremental yes +run job=HotV19 level=Incremental yes +run job=HotV20 level=Incremental yes +wait +list volumes +messages +END_OF_DATA +# Run Backup Jobs + run_bconsole +done + +# now do the final virtual full levels. +#echo ">>>> Now the 1st VFull" + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out a +@#setdebug level=100 storage +run job=HotV01 level=VirtualFull yes +run job=HotV02 level=VirtualFull yes +run job=HotV03 level=VirtualFull yes +run job=HotV04 level=VirtualFull yes +run job=HotV05 level=VirtualFull yes +run job=HotV06 level=VirtualFull yes +run job=HotV07 level=VirtualFull yes +run job=HotV08 level=VirtualFull yes +run job=HotV09 level=VirtualFull yes +run job=HotV10 level=VirtualFull yes +run job=HotV11 level=VirtualFull yes +run job=HotV12 level=VirtualFull yes +run job=HotV13 level=VirtualFull yes +run job=HotV14 level=VirtualFull yes +run job=HotV15 level=VirtualFull yes +run job=HotV16 level=VirtualFull yes +run job=HotV17 level=VirtualFull yes +run job=HotV18 level=VirtualFull yes +run job=HotV19 level=VirtualFull yes +run job=HotV20 level=VirtualFull yes +wait +sql +select pool.name as poolname,volumename,volstatus,volbytes,voljobs,lastwritten from Pool,Media where Pool.PoolId=Media.MediaId order by poolname,volumename; + +list jobs +messages +END_OF_DATA + +# Run Virtual Full Jobs +run_bconsole + +# add more files to backup source, so following incrementals +# have something to back up +for a in `seq 1 12` ; do + #echo ">>> adding files to backup: ${a}" + for b in `seq 1 150` ; do + date > "${tmpsrc}/run${a}-${b}.data" + done + #echo "now $(ls ${tmpsrc}|wc -l) files to back up" + + cat <${cwd}/tmp/bconcmds +@$out ${cwd}/tmp/log1.out a +time +run job=HotV01 level=Incremental yes +run job=HotV02 level=Incremental yes +run job=HotV03 level=Incremental yes +run job=HotV04 level=Incremental yes +run job=HotV05 level=Incremental yes +run job=HotV06 level=Incremental yes +run job=HotV07 level=Incremental yes +run job=HotV08 level=Incremental yes +run job=HotV09 level=Incremental yes +run job=HotV10 level=Incremental yes +run job=HotV11 level=Incremental yes +run job=HotV12 level=Incremental yes +run job=HotV13 level=Incremental yes +run job=HotV14 level=Incremental yes +run job=HotV15 level=Incremental yes +run job=HotV16 level=Incremental yes +run job=HotV17 level=Incremental yes +run job=HotV18 level=Incremental yes +run job=HotV19 level=Incremental yes +run job=HotV20 level=Incremental yes +wait +list volumes +messages +END_OF_DATA +# Run Backup Jobs + run_bconsole +done + +# restore file list +ls ${tmpsrc}/* >${tmp}/restore-list + +# now do the final virtual full levels. +#echo ">>>> Now the 2nd VFull" + +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log1.out a +@#setdebug level=100 storage +run job=HotV01 level=VirtualFull yes +run job=HotV02 level=VirtualFull yes +run job=HotV03 level=VirtualFull yes +run job=HotV04 level=VirtualFull yes +run job=HotV05 level=VirtualFull yes +run job=HotV06 level=VirtualFull yes +run job=HotV07 level=VirtualFull yes +run job=HotV08 level=VirtualFull yes +run job=HotV09 level=VirtualFull yes +run job=HotV10 level=VirtualFull yes +run job=HotV11 level=VirtualFull yes +run job=HotV12 level=VirtualFull yes +run job=HotV13 level=VirtualFull yes +run job=HotV14 level=VirtualFull yes +run job=HotV15 level=VirtualFull yes +run job=HotV16 level=VirtualFull yes +run job=HotV17 level=VirtualFull yes +run job=HotV18 level=VirtualFull yes +run job=HotV19 level=VirtualFull yes +run job=HotV20 level=VirtualFull yes +wait +sql +select pool.name as poolname,volumename,volstatus,volbytes,voljobs,lastwritten from Pool,Media where Pool.PoolId=Media.MediaId order by poolname,volumename; + +list jobs +messages +END_OF_DATA + +# Run Virtual Full Jobs +run_bconsole +cat <${cwd}/tmp/bconcmds +@output /dev/null +messages +@$out ${cwd}/tmp/log2.out +restore client=HotV20-fd where=$tmp/bacula-restores select all done yes +wait +messages +END_OF_DATA + +# Do restore +run_bconsole +sleep 2 +stop_bacula + +check_two_logs +check_restore_tmp_build_diff +end_test