} else if (strcmp(policy, "ListedOrder") == 0) {
Dmsg1(dbglvl_store_mngr, "Setting ListedOrder storage group policy for JobId: %d\n", jcr->JobId);
jcr->store_mngr = New(ListedOrderStore());
+ } else if (strcmp(policy, "FreeSpace") == 0) {
+ Dmsg1(dbglvl_store_mngr, "Setting FreeSpace storage group policy for JobId: %d\n", jcr->JobId);
+ jcr->store_mngr = New(FreeSpaceStore());
+ } else {
+ Dmsg1(dbglvl_store_mngr, "Invalid policy for JobId: %d, setting default (ListedOrder)\n", jcr->JobId);
+ jcr->store_mngr = New(ListedOrderStore());
}
} else {
Dmsg1(dbglvl_store_mngr, "Setting ListedOrder storage group policy for JobId: %d\n", jcr->JobId);
#include "bacula.h"
#include "dird.h"
#include "lib/lockmgr.h"
+#include "lib/store_mngr_cmds.h"
static const int dbglvl = 200;
*v2 = temp;
}
+void QueryStore::apply_policy(bool write_store) {
+ alist *list = write_store ? wstore.get_list() : rstore.get_list();
+ STORE *store = NULL;
+ JCR *tmp_jcr = new_jcr(sizeof(JCR), dird_free_jcr); // temporary jcr needed for connecting to the SD
+ BSOCK *sd = tmp_jcr->store_bsock;
+ sm_ctx *context = NULL;
+ dlist *d_list = New(dlist(context, &context->link));
+
+ /* Init store mngr so that we can assing current storage to the jcr - needed for the connect call below */
+ tmp_jcr->store_mngr = New(ListedOrderStore());
+
+ /* Query Storage daemon for needed information for each storage in the list */
+ foreach_alist(store, list) {
+ if (write_store) {
+ tmp_jcr->store_mngr->set_wstorage(store, "qeuerystore_applypolicy");
+ } else {
+ tmp_jcr->store_mngr->set_rstore(store, "qeuerystore_applypolicy");
+ }
+
+ if (!connect_to_storage_daemon(tmp_jcr, 1, 1, 0)) {
+ Dmsg1(dbglvl, "Failed to connect to the Storage: %s during policy quering\n", store->name());
+ continue;
+ }
+
+ sd = tmp_jcr->store_bsock;
+
+ /* Init storage querying */
+ if (!sd->fsend(store_query)) {
+ Dmsg2(dbglvl, "Failed to send \"%s\" command to the %s Storage\n", store_query, store->name());
+ continue;
+ }
+
+ sm_ctx *context = New(sm_ctx(store));
+
+ /* Execute the policy-specific querying */
+ int ret = query(sd, d_list, context);
+
+ sd->signal(BNET_TERMINATE);
+ sd->close();
+
+ if(!ret) {
+ Dmsg1(dbglvl, "Failed to query '\%s\' Storage\n", store->name());
+ delete context; /* Free context since it was not added to the list */
+
+ /* Continue in case some more processing is needed - we don't want to work on
+ invalid data */
+ continue;
+ }
+ }
+
+ /* Let the policy reorder storage list according to the information gathered */
+ reorder_list(list, d_list);
+
+ if (d_list) {
+ delete d_list;
+ }
+
+ if (tmp_jcr) {
+ free_jcr(tmp_jcr);
+ }
+}
+
+void FreeSpaceStore::reorder_list(alist *list, dlist *d_list) {
+ sm_ctx *ctx;
+ STORE *store;
+
+ list->destroy();
+ list->init(10, not_owned_by_alist);
+
+ Dmsg0(dbglvl, "Storage List after applying FreeSpace policy:\n");
+ foreach_dlist(ctx, d_list) {
+ store = ctx->store;
+ list->append(store);
+ Dmsg2(dbglvl, "*** store: %s size: %llu\n", ctx->store->name(), ctx->number);
+ }
+}
+
+bool FreeSpaceStore::query(BSOCK *sd, dlist *d_list, sm_ctx *context) {
+ STORE *store = context->store;
+ bool ret = false;
+ POOL_MEM device_name;
+ DEVICE *dev;
+ uint64_t fs;
+ uint64_t *size = (uint64_t *) &context->number;
+
+ if (sd->recv() >= 0) {
+ Dmsg1(dbglvl, "<stored: %s", sd->msg);
+ if (strncmp(sd->msg, OK_store_query, strlen(OK_store_query)) != 0) {
+ Dmsg2(dbglvl, "Failed to query storage: %s msg=%s\n", store->name(), sd->msg);
+ goto bail_out;
+ }
+ } else {
+ Dmsg1(dbglvl, "Failed to query storage: %s\n", store->name());
+ goto bail_out;
+ }
+
+ /* Now go through the list of devices for the storage resource and
+ * query each for available storage space */
+ foreach_alist(dev, store->device) {
+ pm_strcpy(device_name, dev->name());
+ bash_spaces(device_name.c_str());
+ sd->fsend(store_query_freespace, device_name.c_str());
+ }
+
+ sd->signal(BNET_EOD);
+
+ if (sd->recv() >= 0) {
+ ret = sscanf(sd->msg, OK_store_size, &fs) == 1;
+ if (!ret) {
+ Dmsg1(dbglvl, "Failed to get size for storage: %s\n", store->name());
+ goto bail_out;
+ }
+
+ *size = fs;
+ }
+
+ // Insert store size to the list
+ d_list->binary_insert_multiple(context, &FreeSpaceStore::cmp);
+
+ ret = true;
+
+bail_out:
+ return ret;
+}
+
void LeastUsedStore::apply_policy(bool write_store) {
alist *store = write_store ? wstore.get_list() : rstore.get_list();
alist tmp_list(10, not_owned_by_alist);
/* Forward delcaration */
class STORE;
-
/*
* Helper class to make managing each storage type (write/read) easier.
* It contains storage resource ('store' member) which is currently used as well as list of all
}
};
+/* Context with per-policy specific data (as of now there's only single uint64_t value available for each policy)*/
+class sm_ctx : public SMARTALLOC {
+ public:
+ STORE *store;
+ dlink link;
+ uint64_t number; /* Per-policy specific number (e.g. free space, cpu usage...) */
+
+ sm_ctx(STORE *s) {
+ store = s;
+ number = 0;
+ }
+};
+
+/*
+ * Abstract class for policies which require DIR<->SD querying.
+ * Each querying policy has to implement query() and reorder() methods - theese act
+ * as a callbacks for QueryStore's generic methods.
+ */
+class QueryStore : public StorageManager {
+ protected:
+ /* Policy-specific query method */
+ virtual bool query(BSOCK *sd, dlist *d_list, sm_ctx *context) = 0;
+
+ /* Reorder storage list after querying all storages in the list for needed information */
+ virtual void reorder_list(alist *list, dlist *d_list) = 0;
+
+ public:
+ void apply_policy(bool write_store);
+
+ QueryStore (const char *policy="VirtualPolicy_QueryStore"): StorageManager(policy) {
+ }
+
+ ~QueryStore() {
+ }
+};
+
+/*
+ * Policy which reorders storage list based on FreeSpace available on each Storage
+ */
+class FreeSpaceStore : public QueryStore {
+ private:
+ /* Private helper struct */
+ struct store_free_size {
+ STORE *store;
+ uint64_t free_size;
+ dlink link;
+ };
+
+ bool query(BSOCK *sd, dlist *d_list, sm_ctx *context);
+
+ /* Comparator for easy list ordering */
+ static int cmp(void *item1, void *item2) {
+ sm_ctx *ctx1 = (sm_ctx *) item1;
+ sm_ctx *ctx2 = (sm_ctx *) item2;
+ uint64_t s1 = ctx1->number;
+ uint64_t s2 = ctx2->number;
+
+ if (s1 < s2) {
+ return 1;
+ } else if (s1 > s2) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
+
+ void reorder_list(alist *list, dlist *d_list);
+
+ public:
+ void apply_write_policy() {
+ return apply_policy(true);
+ }
+ void apply_read_policy() {
+ return apply_policy(false);
+ }
+
+ FreeSpaceStore(): QueryStore("FreeSpace") {
+ }
+
+ ~FreeSpaceStore() {
+ }
+};
+
#endif // STORE_MNGR_H
static char const *storage_mngmt_policy[] = {
"LeastUsed",
"ListedOrder",
+ "FreeSpace",
NULL
};
--- /dev/null
+/*
+ Bacula(R) - The Network Backup Solution
+
+ Copyright (C) 2000-2023 Kern Sibbald
+
+ The original author of Bacula is Kern Sibbald, with contributions
+ from many others, a complete list can be found in the file AUTHORS.
+
+ You may use this file and others of this release according to the
+ license defined in the LICENSE file, which includes the Affero General
+ Public License, v3.0 ("AGPLv3") and some additional permissions and
+ terms pursuant to its AGPLv3 Section 7.
+
+ This notice must be preserved when any source code is
+ conveyed and/or propagated.
+
+ Bacula(R) is a registered trademark of Kern Sibbald.
+*/
+/*
+ * Commands defined for the Storage Manager, for communication between the DIR and SD.
+ */
+
+#ifndef STORE_MNGR_CMDS_H
+#define STORE_MNGR_CMDS_H 1
+
+static const char store_query[] = "store_mngr query\n";
+static const char store_query_freespace[] = "store_mngr freespace device=%s\n";
+static char OK_store_query[] = "3000 OK query store\n";
+static char store_query_unsupp_policy[] = "3200 unsupported policy\n";
+static char OK_store_size[] = "3000 OK size=%llu\n";
+
+#endif // STORE_MNGR_CMDS_H
# bacula-sd
SDOBJS = \
- stored.o append.o authenticate.o dircmd.o fd_cmds.o job.o \
+ stored.o append.o authenticate.o store_mngr.o dircmd.o fd_cmds.o job.o \
hello.o status.o vbackup.o \
sdcollect.o $(SDCORE_OBJS) $(EXTRA_STORED_OBJS)
extern bool status_cmd(JCR *sjcr);
extern bool qstatus_cmd(JCR *jcr);
extern bool collect_cmd(JCR *jcr);
+extern bool query_store(JCR *jcr);
//extern bool query_cmd(JCR *jcr);
/* Forward referenced functions */
#endif
{"run", run_cmd, 0},
{"statistics", collect_cmd, 0},
+ {"store_mngr", query_store, 0},
// {"query", query_cmd, 0},
{NULL, NULL} /* list terminator */
};
--- /dev/null
+/*
+ Bacula(R) - The Network Backup Solution
+
+ Copyright (C) 2000-2023 Kern Sibbald
+
+ The original author of Bacula is Kern Sibbald, with contributions
+ from many others, a complete list can be found in the file AUTHORS.
+
+ You may use this file and others of this release according to the
+ license defined in the LICENSE file, which includes the Affero General
+ Public License, v3.0 ("AGPLv3") and some additional permissions and
+ terms pursuant to its AGPLv3 Section 7.
+
+ This notice must be preserved when any source code is
+ conveyed and/or propagated.
+
+ Bacula(R) is a registered trademark of Kern Sibbald.
+*/
+/*
+ * Storage daemon's part of Storage Manager - responsible for retrieving needed
+ * information in response to Director's query.
+ */
+
+#include "bacula.h"
+#include "stored.h"
+#include "store_mngr.h"
+#include "lib/store_mngr_cmds.h"
+
+static const int dbglvl = 200;
+
+/* Helper Compare method for the FreeSpace policy.
+ * It's purpose is to have list with unique devices. */
+static int query_size_cmp(void *item1, void *item2) {
+ struct query_size *qs1 = (struct query_size *) item1;
+ struct query_size *qs2 = (struct query_size *) item2;
+
+ /* We only want to avoid duplicates, order does not matter at all */
+ if (qs1->devno == qs2->devno) {
+ return 0;
+ }
+
+ return 1;
+}
+
+static bool query_freespace(JCR *jcr, const char *dev_name) {
+ BSOCK *dir = jcr->dir_bsock;
+ DEVRES *device;
+ AUTOCHANGER *changer;
+ uint64_t ts, fs, free_sum = 0;
+ struct query_size *qs;
+ alist dev_list(10, not_owned_by_alist);
+ dlist size_list;
+ bool found;
+
+ /* Device sent can be either single device or an autochanger */
+ foreach_res(changer, R_AUTOCHANGER) {
+ if (strcmp(changer->hdr.name, dev_name) == 0) {
+ /* For now we sum free size of all autochanger's unique devices and report it as a single value
+ * back to the DIR in response to the FreeSpace policy querying.
+ * We may want to change it at some point if we see that some other calculations/reporting make more sense */
+ foreach_alist(device, changer->device) {
+ /* devno == 0 means that it's just an different directory of a device we already have */
+ if (device && device->dev->devno) {
+ device->dev->get_freespace(&fs, &ts);
+ /* Alloc helper struct, set correct size for this dev and add it to list
+ * so that later we can sum size of all unique devices */
+ qs = (struct query_size *) malloc(sizeof(struct query_size));
+ if (!qs) {
+ Dmsg1(dbglvl, "Failed to alloc query_size for dev: %s, policy querying stopped\n", device->device_name);
+ break;
+ } else {
+ qs->size = fs;
+ qs->devno = device->dev->devno;
+ dev_list.append(qs);
+ }
+ }
+ }
+
+ found = true; /* No need to go through the device resource list now */
+ break;
+ }
+ }
+
+ if (!found) {
+ /* Device sent to query is a single device instead of an autochanger */
+ foreach_res(device, R_DEVICE) {
+ if (strcmp(device->hdr.name, dev_name) == 0) {
+ device->dev->get_freespace(&fs, &ts);
+ qs = (struct query_size *) malloc(sizeof(struct query_size));
+ if (!qs) {
+ Dmsg1(dbglvl, "Failed to alloc query_size for dev: %s, policy querying stopped\n", device->device_name);
+ break;
+ } else {
+ qs->size = fs;
+ qs->devno = device->dev->devno;
+ }
+ dev_list.append(qs);
+ found = true;
+ break;
+ }
+ }
+ }
+
+ /* Add free size for each unique device */
+ foreach_alist(qs, &dev_list) {
+ struct query_size *tmp = (struct query_size *) size_list.binary_insert(qs, query_size_cmp);
+ if (tmp) {
+ free_sum+=tmp->size;
+ }
+ }
+
+ dir->fsend(OK_store_size, free_sum);
+
+ return found;
+}
+
+/* Handler for DIR's 'store_mngr query' command */
+bool query_store(JCR *jcr) {
+ BSOCK *dir = jcr->dir_bsock;
+ POOL_MEM dev_name;
+ bool ok = false;
+
+ if (strncmp(dir->msg, store_query, strlen(store_query)) != 0) {
+ return false;
+ }
+
+ dir->fsend(OK_store_query);
+
+ /* Return parameters asked by the DIR */
+ if (dir->recv() >= 0) {
+ if (sscanf(dir->msg, store_query_freespace, dev_name.c_str()) == 1) {
+ unbash_spaces(dev_name.c_str());
+
+ ok = query_freespace(jcr, dev_name.c_str());
+ } else {
+ dir->fsend(store_query_unsupp_policy);
+ }
+ }
+
+ return ok;
+}
--- /dev/null
+/*
+ Bacula(R) - The Network Backup Solution
+
+ Copyright (C) 2000-2023 Kern Sibbald
+
+ The original author of Bacula is Kern Sibbald, with contributions
+ from many others, a complete list can be found in the file AUTHORS.
+
+ You may use this file and others of this release according to the
+ license defined in the LICENSE file, which includes the Affero General
+ Public License, v3.0 ("AGPLv3") and some additional permissions and
+ terms pursuant to its AGPLv3 Section 7.
+
+ This notice must be preserved when any source code is
+ conveyed and/or propagated.
+
+ Bacula(R) is a registered trademark of Kern Sibbald.
+*/
+/*
+ * Storage Daemon's part of Storage Manager - responsible for retrieving needed
+ * information in response to Director's query.
+ */
+
+#ifndef STORE_MNGR_H
+#define STORE_MNGR_H 1
+
+#include "bacula.h"
+
+
+/* FreeSpace policy struct to easily get rid of device duplicates in the list */
+struct query_size {
+ struct dlink link;
+ uint64_t devno; /* Unique device number */
+ uint64_t size; /* Device's freespace in bytes */
+};
+
+/* Main FreeSpace policy handler */
+bool query_store(JCR *jcr);
+
+#endif // STORE_MNGR_H
#include "vtape_dev.h"
#include "cloud_dev.h"
#include "aligned_dev.h"
+#include "store_mngr.h"
#ifdef SD_DEDUP_SUPPORT
#include "dedup_dev.h"
#endif
projects / thirdparty / bacula.git / commitdiff
