From: Alain Spineux Date: Tue, 12 May 2020 19:23:06 +0000 (+0200) Subject: BEE Backport bacula/src/stored/stored_conf.h X-Git-Tag: Release-11.3.2~1615 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=cef7eb716898b9f68d49de63d84e4643e84cce19;p=thirdparty%2Fbacula.git BEE Backport bacula/src/stored/stored_conf.h This commit is the result of the squash of the following main commits: Author: Eric Bollengier Date: Wed Feb 5 12:00:41 2020 +0100 Fix *json segfault on OpenBSD In memcpy(&res_all, res, sizeof(res_all)); The res_all is likely bigger than res itself. We copy too much data, and on OpenBSD, we get segfaults. Author: Alain Spineux Date: Wed Nov 20 13:32:52 2019 +0100 dedup: detect and report dedupengine startup failure - 2 cases to handle - cannot load the driver - error at the dedupengine creation time or startup - the error message is stored in the dedup ressourse - see regress/tests/dedup-start-fail-test for more # Conflicts: # bacula/src/stored/dircmd.c Author: Eric Bollengier Date: Fri Nov 15 09:12:29 2019 +0100 cloud: Rename TransferRetentionDays to TransferRetention (expressed as a time) Author: Norbert Bizet Date: Mon Oct 14 05:35:57 2019 -0400 cloud: handle restore params in Cloud res Author: Alain Spineux Date: Tue Sep 24 10:17:51 2019 +0200 dedup: move all dedup code into the plugin - remove global variables: dedupengine and bucker_manager - update protocol SD<->DIR - extend dedup_dev and BufferedMsgSD to suuport all function - create "dummy" functions in dev.h Warning : - include a dedupengine use counter - .status storage=XXX dedupengine display all the dedupengine on the SD instead of the one(s) related to the "device" Author: Alain Spineux Date: Thu Jun 6 11:59:15 2019 +0200 Fix #3987 Scrub limit set in the storage{} not taken into account - we were using two different variables Author: Alain Spineux Date: Mon Apr 8 14:56:33 2019 +0200 PSK: Add new "TLS PSK Enable" directive to all resources - add the field to the resources - create default "psk_ctx" CONTEXT for each of them at startup Author: Eric Bollengier Date: Thu Sep 6 17:14:02 2018 +0200 Add SSDDirectory resource to the Storage Daemon Author: Norbert Bizet Date: Thu Jun 21 18:19:05 2018 +0200 cloud: First implementation and scenario test for generic_driver Author: Eric Bollengier Date: Mon May 1 09:39:57 2017 +0200 dedup: Add directive Storage::DedupCheckHash to control the rehydration control. Default is no. Author: Alain Spineux Date: Wed Sep 28 16:44:27 2016 +0200 dedup: add MaxContainerSize directive to SD ( FIX #1728 ) - this directive must limit the size of the container - add max_container_size to bacula-sd.conf and to BucketManager - new method BucketManager::grow0() like grow() but without any lock - new method BucketManager::add_buckets() that add a bucket - new method BucketManager::can_grow_more() that tell if a bucket can grow - allocate the max size (511 entries) for the container table (this is the maximum for standard 64K header) - use BucketManager::alloc_block() instead of Bucket::alloc_block() to alloc chunks BucketManager::alloc_block() will handle the choice of the bucket, extend or create a new one when required. - use Bucket::capacity() instead of ba.size - dedup: link extra container together and keep ba.size < max_container_bidx . container of the same max_blocksize are linked . the head is got by BucketManager::choose_bucket() . the current container is head->current . they can be iterated up to NULL vi head->current->next . ba.size is kept < max_container_bidx by grow0() (I started doing a "runtime" limitation in alloc_block()) . brc32 must use the right size (ba.size,max_container_bidx) at startup . add_buckets() return a bucket and handle "md_fsmonly" - extend clone_fsm() to handle extra containers - extend BucketManager::alloc_block() to keep vacuum_fsm geometry in sync - add a lock parameter to BucketManager::get_bucket(blockaddr addr, bool lck=true) - new method DedupEngine::vacuum_mark0() with a lock parameter - can modifying MaxContainerSize at any time . containers that are already bigger than the new value will no grow anymore . container that are smaller than the new value will continue to grow - add bucketmanager::Check Author: Kern Sibbald Date: Thu Sep 1 20:16:25 2016 +0200 First cut Cloud resource for SD Device Author: Eric Bollengier Date: Mon May 2 09:29:57 2016 +0200 Add an option to configure the FileMedia record interval Author: Kern Sibbald Date: Thu May 8 10:09:13 2014 +0200 Redefine SD dedup directives Author: Alain Spineux Date: Sun Apr 6 20:05:35 2014 +0200 Beginning of dedup code --- diff --git a/bacula/src/stored/stored_conf.h b/bacula/src/stored/stored_conf.h index 5d2a5d0de7..893d7a4134 100644 --- a/bacula/src/stored/stored_conf.h +++ b/bacula/src/stored/stored_conf.h @@ -52,8 +52,9 @@ enum { R_AUTOCHANGER = 3005, R_CLOUD = 3006, R_COLLECTOR = 3007, + R_DEDUP = 3008, R_FIRST = R_DIRECTOR, - R_LAST = R_COLLECTOR /* keep this updated */ + R_LAST = R_DEDUP /* keep this updated */ }; enum { @@ -70,6 +71,12 @@ enum { /* * Cloud drivers */ +enum { + CLOUD_RESTORE_PRIO_HIGH = 0, + CLOUD_RESTORE_PRIO_MEDIUM, + CLOUD_RESTORE_PRIO_LOW +}; + class CLOUD { public: RES hdr; @@ -92,6 +99,27 @@ public: uint32_t max_concurrent_downloads; uint64_t upload_limit; uint64_t download_limit; + char *driver_command; + int32_t transfer_priority; + utime_t transfer_retention; +}; + +/* + * Dedup drivers + */ +class DedupEngine; +class DEDUPRES { +public: + RES hdr; + char *dedup_dir; /* DEDUP directory */ + char *dedup_index_dir; /* Directory for index (db) file */ + int64_t max_container_size; /* Maximum container size then split */ + bool dedup_check_hash; /* Check Hash of each chunk after rehydration */ + int64_t dedup_scrub_max_bandwidth; /* Maximum disk bandwidth usable for scrub */ + uint32_t driver_type; /* dedup driver type */ + DedupEngine *dedupengine; + int dedupengine_use_count; + char *dedup_err_msg; /* is set for any error status */ }; /* @@ -106,6 +134,7 @@ public: bool monitor; /* Have only access to status and .status functions */ bool tls_authenticate; /* Authenticate with TLS */ bool tls_enable; /* Enable TLS */ + bool tls_psk_enable; /* Enable TLS-PSK */ bool tls_require; /* Require TLS */ bool tls_verify_peer; /* TLS Verify Client Certificate */ char *tls_ca_certfile; /* TLS CA Certificate File */ @@ -116,6 +145,7 @@ public: alist *tls_allowed_cns; /* TLS Allowed Clients */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ + TLS_CONTEXT *psk_ctx; /* Shared TLS-PSK Context */ }; @@ -131,14 +161,17 @@ public: char *subsys_directory; char *plugin_directory; /* Plugin directory */ char *scripts_directory; + char *ssd_directory; uint32_t max_concurrent_jobs; /* maximum concurrent jobs to run */ MSGS *messages; /* Daemon message handler */ utime_t ClientConnectTimeout; /* Max time to wait to connect client */ utime_t heartbeat_interval; /* Interval to send hb to FD */ utime_t client_wait; /* Time to wait for FD to connect */ bool comm_compression; /* Set to allow comm line compression */ + bool require_fips; /* Check for FIPS module */ bool tls_authenticate; /* Authenticate with TLS */ bool tls_enable; /* Enable TLS */ + bool tls_psk_enable; /* Enable TLS-PSK */ bool tls_require; /* Require TLS */ bool tls_verify_peer; /* TLS Verify Client Certificate */ char *tls_ca_certfile; /* TLS CA Certificate File */ @@ -149,7 +182,13 @@ public: alist *tls_allowed_cns; /* TLS Allowed Clients */ char *verid; /* Custom Id to print in version command */ TLS_CONTEXT *tls_ctx; /* Shared TLS Context */ + TLS_CONTEXT *psk_ctx; /* Shared TLS-PSK Context */ + char *dedup_dir; /* DEDUP directory */ + char *dedup_index_dir; /* Directory for index (db) file */ + int64_t max_container_size; /* Maximum container size then split */ + bool dedup_check_hash; /* Check Hash of each chunk after rehydration */ + int64_t dedup_scrub_max_bandwidth; /* Maximum disk bandwidth usable for scrub */ }; typedef class s_res_store STORES; @@ -200,6 +239,7 @@ public: int64_t max_volume_files; /* max files to put on one volume */ int64_t max_volume_size; /* max bytes to put on one volume */ int64_t max_file_size; /* max file size in bytes */ + int64_t max_file_index; /* max file size between two FileMedia in bytes */ /* ***BEEF*** */ int64_t volume_capacity; /* advisory capacity */ int64_t min_free_space; /* Minimum disk free space */ int64_t max_spool_size; /* Max spool size for all jobs */ @@ -212,6 +252,7 @@ public: char *write_part_command; /* Write part command */ char *free_space_command; /* Free space command */ CLOUD *cloud; /* pointer to cloud resource */ + DEDUPRES *dedup; /* pointer to dedup resource */ /* The following are set at runtime */ DEVICE *dev; /* Pointer to phyical dev -- set at runtime */ @@ -225,6 +266,10 @@ union URES { MSGS res_msgs; AUTOCHANGER res_changer; CLOUD res_cloud; + DEDUPRES res_dedup; RES hdr; COLLECTOR res_collector; }; + +/* Get the size of a give resource */ +int get_resource_size(int type);