From 7196826271e15a4e7083a351eebd0eb0ca48a58b Mon Sep 17 00:00:00 2001 From: Michal Rakowski Date: Tue, 9 Mar 2021 16:28:38 +0100 Subject: [PATCH] Fix for plugin file's metadata backing up flow Fix backing up of file metadata - previously it was not related to each single file as it should be. Now each file's metadata (if exists) is being backed up separately. Some cleaning around that, as well as adding more comments. --- bacula/src/filed/backup.c | 11 +++++- bacula/src/filed/fd_plugins.c | 25 +++++++++--- bacula/src/filed/fd_plugins.h | 49 +++++++++++------------- bacula/src/filetypes.h | 1 - bacula/src/findlib/find.h | 6 +-- bacula/src/plugins/fd/test-plugin-fd.c | 53 +++++++++++--------------- bacula/src/streams.h | 5 +-- 7 files changed, 80 insertions(+), 70 deletions(-) diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index cf2f18d40..1c4973446 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -231,7 +231,16 @@ bool blast_data_to_storage_daemon(JCR *jcr, char *addr) return ok; } -bool metadata_save(JCR *jcr, plugin_metadata *plug_meta) +/* + * Called by plugin_metadata_backup() for each file for which there is + * metadata provided by the plugin. + * Using the plugin_metadata class it iterates over metadata packets, serialize it and sends + * it to the sd one by one. + * + * Returns: true if OK + * false if error (size of the meta packet is too big or communication with the sd failed) + */ +bool metadata_save(JCR *jcr, const plugin_metadata *plug_meta) { bool stat = false; BSOCK *sd = jcr->store_bsock; diff --git a/bacula/src/filed/fd_plugins.c b/bacula/src/filed/fd_plugins.c index b4004de06..4964fe10e 100644 --- a/bacula/src/filed/fd_plugins.c +++ b/bacula/src/filed/fd_plugins.c @@ -40,7 +40,7 @@ const char *plugin_type = "-fd.so"; extern bool save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level); extern bool check_changes(JCR *jcr, FF_PKT *ff_pkt); -extern int metadata_save(JCR *jcr, plugin_metadata *plug_meta); +extern int metadata_save(JCR *jcr, const plugin_metadata *plug_meta); /* Function pointers to be set here */ extern DLL_IMP_EXP int (*plugin_bopen)(BFILE *bfd, const char *fname, uint64_t flags, mode_t mode); @@ -702,8 +702,6 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level) ff_pkt->restore_obj.object = sp.restore_obj.object; ff_pkt->restore_obj.object_len = sp.restore_obj.object_len; } - } else if (sp.type == FT_PLUGIN_METADATA) { - ff_pkt->plug_meta = sp.plug_meta; } else { Dsm_check(999); if (!sp.fname) { @@ -714,6 +712,11 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level) pm_strcpy(fname, sp.fname); pm_strcpy(link, sp.link); + if (sp.plug_meta) { + /* File has some metadata assigned to it */ + ff_pkt->plug_meta = sp.plug_meta; + } + ff_pkt->fname = fname.c_str(); ff_pkt->snap_fname = fname.c_str(); ff_pkt->link = link.c_str(); @@ -1215,15 +1218,25 @@ bool plugin_set_attributes(JCR *jcr, ATTR *attr, BFILE *ofd) return true; } +/* + * The Plugin file's metadata backup. + * Stream of metadata packets is being decoded, serialized and sent to the storage daemon for storing + * on the volume. Each metadata packet type has its own STREAM_* type. + * + * in: + * jcr - Job Control Record + * ff_pkt - file save packet + * out: + * true - backup completed successfully (or there was no metadata to backup at all) + * false - backup of metadata failed + */ bool plugin_backup_metadata(JCR *jcr, FF_PKT *ff_pkt) { - Plugin *plugin = jcr->plugin; - /* Backup metadata if provided */ if (ff_pkt->plug_meta) { if (!metadata_save(jcr, ff_pkt->plug_meta)) { Jmsg2(jcr, M_ERROR, 0, _("Failed to backup metadata for plugin: \"%s\" fname: %s"), - plugin->file, ff_pkt->fname); + jcr->plugin->file, ff_pkt->fname); return false; } } diff --git a/bacula/src/filed/fd_plugins.h b/bacula/src/filed/fd_plugins.h index e97e2f9d8..ce55aa462 100644 --- a/bacula/src/filed/fd_plugins.h +++ b/bacula/src/filed/fd_plugins.h @@ -121,11 +121,12 @@ enum metadata_type { }; /* - * This packet is used for storing plugin's metadata. + * This class is used to store single plugin's metadata packet along with providing some + * helper methods (e.g. for serialization of the data so that it can be send to the sd). */ class meta_pkt: public SMARTALLOC { private: - bool decoded; /* Was metadata packed decoded from serialized stream or not */ + bool decoded; /* Was metadata packed decoded from serialized stream or not */ public: uint32_t total_size; /* Total size of metadata stream (consiting of packets) */ @@ -135,6 +136,7 @@ class meta_pkt: public SMARTALLOC { uint32_t buf_len; /* Length of buffer */ void *buf; /* Can be either passed by the user or allocated for deserialization */ + /* Constructor, allows to assign most of needed fields in place */ meta_pkt(metadata_type type=plugin_meta_invalid, uint32_t len=0, void *buf=NULL, uint16_t idx=0) { this->buf_len = len; this->type = type; @@ -146,7 +148,8 @@ class meta_pkt: public SMARTALLOC { decoded = false; }; - /* Build metadata packet from serialized stream */ + /* Constructor, class object can be either created from stream (which will be deserialized) + * or just initialized with some init values. */ meta_pkt(void *stream) { if (stream) { unser_declare; @@ -173,7 +176,7 @@ class meta_pkt: public SMARTALLOC { ~meta_pkt() { if (decoded) { - bfree(buf); + bfree(buf); /* Metadata packet contains buffer allocated by this class, needs freeing */ } }; @@ -207,31 +210,25 @@ class meta_pkt: public SMARTALLOC { * meta_mgr->add_packet(plugin_meta_blog, buf1, buf1_len); * meta_mgr->add_packet(plugin_meta_catalog_email, buf2, buf2_len); * - * Then just simply return meta_mgr as an 'plug_meta' field in save_packet structure and - * set save_packet`s type to FT_PLUGIN_METADATA. Bacula will then take care of all of the packets - * added to the list and store it onto the volume one by one. + * Then just simply return meta_mgr as an 'plug_meta' field in save_packet structure. + * Bacula will then take care of all of the packets added to the list and store it + * onto the volume one by one. */ class plugin_metadata: public SMARTALLOC { private: uint32_t total_size; /* Total size of metadata stream (consiting of many packets) */ uint16_t total_count; /* Total count of metadata packets in the stream */ - alist *packets; /* List of packets in the stream */ + mutable alist packets; /* List of packets in the stream */ public: plugin_metadata() { - packets = New(alist(5, false)); + packets.init(5, false); total_size = 0; total_count = 0; }; ~plugin_metadata() { - /* Remove packets from list, delete each of them */ - while (!packets->empty()) { - meta_pkt *mp = (meta_pkt *)packets->pop(); - delete mp; - } - - delete packets; + reset(); }; /* Create packet with specified attributes, add it to the list */ @@ -241,32 +238,32 @@ class plugin_metadata: public SMARTALLOC { mp->total_size = total_size; mp->total_count = total_count; - packets->push(mp); + packets.push(mp); /* Update all packets with new total size and count */ - foreach_alist(mp, packets) { + foreach_alist(mp, &packets) { mp->total_size = total_size; mp->total_count = total_count; } }; - uint32_t size() { + uint32_t size() const { return total_size; }; - uint16_t count() { + uint16_t count() const { return total_count; }; - meta_pkt *get(int index) { - return (meta_pkt *)packets->get(index); + meta_pkt *get(int index) const { + return (meta_pkt *)packets.get(index); }; void reset() { - //Free allocated metadata packets - while (!packets->empty()) { - meta_pkt *mp = (meta_pkt *)packets->pop(); // remove from list + /* Remove packets from list, delete each of them */ + while (!packets.empty()) { + meta_pkt *mp = (meta_pkt *)packets.pop(); // remove from list delete mp; } @@ -294,7 +291,7 @@ struct save_pkt { char *cmd; /* command */ struct restore_object restore_obj; /* Info about restore object */ struct plugin_object plugin_obj; /* Plugin Object */ - plugin_metadata *plug_meta; /* Metadata packet provided by plugin */ + const plugin_metadata *plug_meta; /* Metadata packet provided by plugin */ uint32_t delta_seq; /* Delta sequence number */ int32_t LinkFI; /* LinkFI if LINKSAVED */ int32_t pkt_end; /* end packet sentinel */ diff --git a/bacula/src/filetypes.h b/bacula/src/filetypes.h index 239712e1b..80467014d 100644 --- a/bacula/src/filetypes.h +++ b/bacula/src/filetypes.h @@ -71,7 +71,6 @@ #define FT_PLUGIN_CONFIG 27 /* Object for Plugin configuration */ #define FT_PLUGIN_CONFIG_FILLED 28 /* Object for Plugin configuration filled by Director */ #define FT_PLUGIN_OBJECT 29 /* Opaque Plugin Object used for Object Management*/ -#define FT_PLUGIN_METADATA 30 /* Plugin metadata */ /* Definitions for upper part of type word (see above). */ #define AR_DATA_STREAM (1<<16) /* Data stream id present */ diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index 7a771cf42..19424541a 100644 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -173,9 +173,9 @@ struct FF_PKT { int32_t FileIndex; /* FileIndex of this file */ int32_t LinkFI; /* FileIndex of main hard linked file */ int32_t delta_seq; /* Delta Sequence number */ - struct restore_object restore_obj; - struct plugin_object plugin_obj; - plugin_metadata *plug_meta; + struct restore_object restore_obj; /* Info about restore object */ + struct plugin_object plugin_obj; /* Plugin Object */ + const plugin_metadata *plug_meta; /* Metadata packet provided by plugin */ struct f_link *linked; /* Set if this file is hard linked */ int type; /* FT_ type from above */ int ff_errno; /* errno */ diff --git a/bacula/src/plugins/fd/test-plugin-fd.c b/bacula/src/plugins/fd/test-plugin-fd.c index 01e9693bf..87f1dece1 100644 --- a/bacula/src/plugins/fd/test-plugin-fd.c +++ b/bacula/src/plugins/fd/test-plugin-fd.c @@ -135,7 +135,7 @@ struct plugin_ctx { int nb_obj; /* Number of objects created */ int nb; /* used in queryParameter */ char *query_buf; /* buffer used to not loose memory */ - plugin_metadata *meta_mgr; + plugin_metadata *meta_mgr; /* Metadata manager */ int job_level; /* current Job level */ POOLMEM *buf; /* store ConfigFile */ }; @@ -184,7 +184,7 @@ static bRC newPlugin(bpContext *ctx) memset(p_ctx, 0, sizeof(struct plugin_ctx)); ctx->pContext = (void *)p_ctx; /* set our context pointer */ - // Create metadata menager class + /* Create metadata manager class */ p_ctx->meta_mgr = New(plugin_metadata); return bRC_OK; @@ -209,7 +209,7 @@ static bRC freePlugin(bpContext *ctx) free(p_ctx->cmd); /* free any allocated command string */ } if (p_ctx->meta_mgr) { - delete p_ctx->meta_mgr; + delete p_ctx->meta_mgr; /* delete metadata manager object */ } free(p_ctx); /* free our private context */ ctx->pContext = NULL; @@ -377,8 +377,8 @@ static bRC metadataRestore(bpContext *ctx, struct meta_pkt *mp) /*switch (mp->type) {*/ /*case plugin_meta_blob:*/ /*Dmsg0(0, "Restoring metadata of 'blob' type!\n");*/ - /*Dmsg1(0, _("---- [pluginrestore] len: %lld\n"), mp->len);*/ - /*Dmsg1(0, _("---- [pluginrestore] buf: %s\n"), mp->buf);*/ + /*Dmsg1(0, _("---- [pluginrestore] len: %lld\n"), mp->buf_len);*/ + /*Dmsg2(0, _("---- [pluginrestore] buf: %.*s\n"), mp->buf_len, mp->buf);*/ /*break;*/ /*case plugin_meta_catalog_email:*/ /*Dmsg0(0, "Restoring metadata of 'email catalog' type!\n");*/ @@ -690,35 +690,28 @@ static bRC startBackupFile(bpContext *ctx, struct save_pkt *sp) return bRC_OK; } else if (p_ctx->nb_obj == 7) { - // Remove any metadata packets related to previous files - p_ctx->meta_mgr->reset(); - - const char* m1 = - "{\ - \"key1\": \"val1\", \ - \"key2\": \"val2\", \ - \"key3\": \"val3\" \ - }"; - - /*TODO change payload to catalog packet when it's defined*/ - const char *m2 = "meta_type=email,title=msg"; - - p_ctx->meta_mgr->add_packet(plugin_meta_blob, strlen(m1), (void *)m1); - p_ctx->meta_mgr->add_packet(plugin_meta_catalog_email, strlen(m2), (void *)m2); - - sp->plug_meta = p_ctx->meta_mgr; - - p_ctx->nb_obj++; - sp->type = FT_PLUGIN_METADATA; - - return bRC_OK; - - } else if (p_ctx->nb_obj == 8) { p_ctx->nb_obj++; if (p_ctx->job_level == 'F') { sp->type = FT_REG; sp->link = sp->fname = (char *)"/@testplugin/test1.zero"; + /* Assign some metadata for the fake file */ + p_ctx->meta_mgr->reset(); + + const char* m1 = + "{\ + \"key1\": \"val1\", \ + \"key2\": \"val2\", \ + \"key3\": \"val3\" \ + }"; + + /*TODO change payload to catalog packet when it's defined*/ + const char *m2 = "meta_type=email,title=msg"; + p_ctx->meta_mgr->add_packet(plugin_meta_blob, strlen(m1), (void *)m1); + p_ctx->meta_mgr->add_packet(plugin_meta_catalog_email, strlen(m2), (void *)m2); + + sp->plug_meta = p_ctx->meta_mgr; + } else { return bRC_Stop; } @@ -762,7 +755,7 @@ static bRC endBackupFile(bpContext *ctx) * We would return bRC_More if we wanted startBackupFile to be * called again to backup another file */ - if (p_ctx->nb_obj >= 9) { + if (p_ctx->nb_obj >= 8) { return bRC_OK; } else { return bRC_More; diff --git a/bacula/src/streams.h b/bacula/src/streams.h index ade7fcad6..b85a7bda9 100644 --- a/bacula/src/streams.h +++ b/bacula/src/streams.h @@ -102,9 +102,8 @@ #define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA 32 /* Encrypted, compressed data */ #define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 33 /* Encrypted, compressed Win32 BackupRead data */ #define STREAM_PLUGIN_OBJECT 34 /* Plugin object */ -#define STREAM_PLUGIN_META_HEADER 35 /* Plugin metadata header for file being backed up */ -#define STREAM_PLUGIN_META_BLOB 36 /* Plugin metadata (blob) for file being backed up */ -#define STREAM_PLUGIN_META_CATALOG 37 /* Plugin metadata (to be stored in catalog) for file being backed up */ +#define STREAM_PLUGIN_META_BLOB 35 /* Plugin metadata (blob) for file being backed up */ +#define STREAM_PLUGIN_META_CATALOG 36 /* Plugin metadata (to be stored in catalog) for file being backed up */ #define STREAM_ADATA_BLOCK_HEADER 200 /* Adata block header */ #define STREAM_ADATA_RECORD_HEADER 201 /* Adata record header */ -- 2.47.3