From: norbert.bizet <norbert.bizet@baculasystems.com>
Date: Tue, 6 Dec 2022 16:50:07 +0000 (-0500)
Subject: cloud: Fix #9714 introduce max_vol_parts_num variable to limit cloud volumes parts... 
X-Git-Tag: Beta-15.0.0~328
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=26882449af4f5a8ae8f8b49c0228209014bb9b2c;p=thirdparty%2Fbacula.git

cloud: Fix #9714 introduce max_vol_parts_num variable to limit cloud volumes parts count
---

diff --git a/bacula/src/stored/cloud_dev.c b/bacula/src/stored/cloud_dev.c
index a98344ae7..538abe24d 100644
--- a/bacula/src/stored/cloud_dev.c
+++ b/bacula/src/stored/cloud_dev.c
@@ -2309,6 +2309,15 @@ bool cloud_dev::do_size_checks(DCR *dcr, DEV_BLOCK *block)
          return false;
       }
 
+      /* switch volume when max_vol_parts_num is set and reached. Do it before next part is openned to avoid small orphean part at the end of volume */
+      if ((max_vol_parts_num) > 0 && (part >= max_vol_parts_num)) {
+         
+         Dmsg2(dbglvl, "max_vol_parts_num = %d exceeded by partidx= %d. Calling terminate_writing_volume\n", max_vol_parts_num, part);
+         terminate_writing_volume(dcr);
+         dev_errno = ENOSPC;
+         return false;
+      }
+
       if (!open_next_part(dcr)) {
          return false;
       }
diff --git a/bacula/src/stored/dev.h b/bacula/src/stored/dev.h
index 6acf154d0..12a6b6166 100644
--- a/bacula/src/stored/dev.h
+++ b/bacula/src/stored/dev.h
@@ -334,6 +334,7 @@ public:
    uint64_t adata_addr;               /* Next adata write address */
 
    uint64_t max_part_size;            /* max part size */
+   uint64_t max_vol_parts_num;        /* Max number of parts in a cloud volume */
    uint64_t part_size;                /* current part size */
    uint32_t part;                     /* current part number (starts at 0) */
    /* state ST_FREESPACE_OK is set if free_space is valid */
diff --git a/bacula/src/stored/init_dev.c b/bacula/src/stored/init_dev.c
index 769aa5fa5..53f2b678f 100644
--- a/bacula/src/stored/init_dev.c
+++ b/bacula/src/stored/init_dev.c
@@ -317,8 +317,10 @@ void DEVICE::device_generic_init(JCR *jcr, DEVRES *device)
    dev->crypto_device_ctx = NULL;
    if (dev->is_tape()) { /* No parts on tapes */
       dev->max_part_size = 0;
+      dev->max_vol_parts_num = 0;
    } else {
       dev->max_part_size = device->max_part_size;
+      dev->max_vol_parts_num = device->max_vol_parts_num;
    }
 
 #ifndef DEVELOPER
diff --git a/bacula/src/stored/label.c b/bacula/src/stored/label.c
index 8362a1581..1f06bf539 100644
--- a/bacula/src/stored/label.c
+++ b/bacula/src/stored/label.c
@@ -837,6 +837,7 @@ void create_volume_header(DEVICE *dev, const char *VolName,
       dev->VolHdr.VerNum = BaculaS3CloudVersion;
       dev->VolHdr.BlockSize = dev->max_block_size;
       dev->VolHdr.MaxPartSize = dev->max_part_size;
+      dev->VolHdr.MaxVolPartsNum = dev->max_vol_parts_num;
    } else {
       bstrncpy(dev->VolHdr.Id, BaculaId, sizeof(dev->VolHdr.Id));
       dev->VolHdr.VerNum = BaculaTapeVersion;
diff --git a/bacula/src/stored/record.h b/bacula/src/stored/record.h
index 71da5620c..59f7b6957 100644
--- a/bacula/src/stored/record.h
+++ b/bacula/src/stored/record.h
@@ -207,6 +207,7 @@ struct Volume_Label {
 
   /* For Cloud */
   uint64_t  MaxPartSize;              /* Maximum Part Size */
+  uint64_t  MaxVolPartsNum;           /* Maximum Num of parts in a volume */
 
   /* For Volume encryption */
   bool is_vol_encrypted;
diff --git a/bacula/src/stored/stored_conf.c b/bacula/src/stored/stored_conf.c
index 01dda27e0..b0cde618d 100644
--- a/bacula/src/stored/stored_conf.c
+++ b/bacula/src/stored/stored_conf.c
@@ -191,6 +191,7 @@ static RES_ITEM dev_items[] = {
    {"MaximumJobSpoolSize",   store_size64, ITEM(res_dev.max_job_spool_size), 0, 0, 0},
    {"DriveIndex",            store_pint32, ITEM(res_dev.drive_index), 0, 0, 0},
    {"MaximumPartSize",       store_size64, ITEM(res_dev.max_part_size), 0, ITEM_DEFAULT, 0},
+   {"MaximumVolumeParts", store_size64, ITEM(res_dev.max_vol_parts_num), 0, ITEM_DEFAULT, 0},
    {"MountPoint",            store_strname,ITEM(res_dev.mount_point), 0, 0, 0},
    {"MountCommand",          store_strname,ITEM(res_dev.mount_command), 0, 0, 0},
    {"UnmountCommand",        store_strname,ITEM(res_dev.unmount_command), 0, 0, 0},
diff --git a/bacula/src/stored/stored_conf.h b/bacula/src/stored/stored_conf.h
index ebadb8b87..1cdd6de2b 100644
--- a/bacula/src/stored/stored_conf.h
+++ b/bacula/src/stored/stored_conf.h
@@ -266,6 +266,7 @@ public:
    int64_t max_job_spool_size;        /* Max spool size for any single job */
 
    int64_t max_part_size;             /* Max part size */
+   int64_t max_vol_parts_num;          /* Max number of parts in a cloud volume */
    char *mount_point;                 /* Mount point for require mount devices */
    char *mount_command;               /* Mount command */
    char *unmount_command;             /* Unmount command */
diff --git a/regress/tests/cloud-maxvolpartnums-test b/regress/tests/cloud-maxvolpartnums-test
new file mode 100755
index 000000000..4197bc45b
--- /dev/null
+++ b/regress/tests/cloud-maxvolpartnums-test
@@ -0,0 +1,111 @@
+#!/bin/sh
+#
+# Copyright (C) 2000-2021 Kern Sibbald
+# Copyright (C) 2021-2022 Bacula Systems SA
+# License: BSD 2-Clause; see file LICENSE-FOSS
+#
+#  Short elementary Volume span test case for debugging.
+#  Here we create at least two volumes and the data spans the volume.
+#
+
+TestName="cloud-maxvolpartnums-test"
+JobName=MaxVolPartNums
+. scripts/functions
+
+copy_test_confs
+
+FORCE_FILE_SET=${FORCE_FILE_SET:-"${cwd}/build"}
+echo "$FORCE_FILE_SET" >${cwd}/tmp/file-list
+
+MaximumVolumeParts=5
+MaximumPartSize=1000000
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumPartSize", "'$MaximumPartSize'", "Device")'
+$bperl -e 'add_attribute("$conf/bacula-sd.conf", "MaximumVolumeParts", "'$MaximumVolumeParts'", "Device")'
+#$bperl -e 'add_attribute("$conf/bacula-sd.conf", "TruncateCache", "AfterUpload", "Cloud")'
+
+cp ${cwd}/bin/bacula-dir.conf ${cwd}/tmp/1
+sed "s%# Simple Label Format%  Label Format%" ${cwd}/tmp/1 >${cwd}/bin/bacula-dir.conf
+
+change_jobname Simple $JobName
+start_test
+
+cat <<END_OF_SCRIPT >${cwd}/tmp/bconcmds
+@$out /dev/null
+messages
+@$out ${cwd}/tmp/log1.out
+status all
+list pools
+messages
+@#setdebug level=110 storage
+run job=$JobName level=Full storage=File yes
+wait
+list pools
+list volumes
+messages
+END_OF_SCRIPT
+
+run_bacula
+
+for v in ${tmp}/Backup-*; do
+   for p in ${v}/part.*; do
+      # check the part size (must be <= MaximumPartSize)
+      size=$(stat -c "%s" $p);
+      if [ "$size" -gt "$MaximumPartSize" ]; then
+         print_debug "ERROR: part $p is too big $size > $MaximumPartSize";
+         estat=1;
+      fi
+      # check the part index (must be <= MAX_PART)
+      extension="${p##*.}";
+      if [ "$extension" -gt "$MaximumVolumeParts" ]; then
+         print_debug "ERROR: part $p is out-of-range: $extension > $MaximumVolumeParts";
+         estat=2;
+      fi
+   done
+
+# Now truncate every volume from the cache
+volume=$(basename $v);
+
+cat <<END_OF_SCRIPT >${cwd}/tmp/bconcmds
+messages
+truncate cache volume=$volume storage=File
+wait
+messages
+END_OF_SCRIPT
+run_bconsole
+
+done # for each volume
+
+cat <<END_OF_DATA >${cwd}/tmp/bconcmds
+@# 
+@# now do a restore
+@#
+@$out ${cwd}/tmp/log2.out
+@#setdebug level=500 storage=File
+restore where=${cwd}/tmp/bacula-restores select storage=File
+unmark *
+mark *
+pwd
+done
+yes
+wait
+messages
+@$out $tmp/log31.out
+cloud list storage=File
+@$out $tmp/log3.out
+cloud list storage=File
+quit
+END_OF_DATA
+
+run_bconsole
+
+check_for_zombie_jobs storage=File
+stop_bacula
+
+check_two_logs
+# Check single file
+diff -ur ${cwd}/build/$file ${tmp}/bacula-restores${src}
+if test $? -ne 0; then
+   dstat=1
+fi
+#check_restore_diff
+end_test