From: Michal Rakowski Date: Wed, 9 Sep 2020 15:27:56 +0000 (+0200) Subject: plugins: Add Object Management handling X-Git-Tag: Release-11.3.2~1028 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=8d2c4d4fa4147f739ae881b4eb703bde9473f178;p=thirdparty%2Fbacula.git plugins: Add Object Management handling --- diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 1b60b6acf8..ca40a165f9 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -248,7 +248,7 @@ public: char *Path; char *Filename; char *PluginName; - char *ObjectType; + char ObjectType[MAX_NAME_LENGTH]; char *ObjectName; char *ObjectSource; char *ObjectUUID; diff --git a/bacula/src/cats/protos.h b/bacula/src/cats/protos.h index b54b583dc0..53fdbc0b6e 100644 --- a/bacula/src/cats/protos.h +++ b/bacula/src/cats/protos.h @@ -90,6 +90,7 @@ int db_string_list_handler(void *ctx, int num_fields, char **row); int db_int_handler(void *ctx, int num_fields, char **row); void bdb_debug_print(JCR *jcr, FILE *fp); void db_free_restoreobject_record(JCR *jcr, ROBJECT_DBR *rr); +void db_free_pluginobject_record(JCR *jcr, OBJECT_DBR *obj_r); #define db_open_batch_connexion(jcr, mdb) \ mdb->bdb_open_batch_connexion(jcr) diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index 6ad3bc5f20..c052a0f544 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -739,32 +739,66 @@ void db_free_restoreobject_record(JCR *jcr, ROBJECT_DBR *rr) rr->object = rr->plugin_name = rr->object_name = NULL; } +void db_free_pluginobject_record(JCR *jcr, OBJECT_DBR *obj_r) +{ + bfree_and_null(obj_r->Path); + bfree_and_null(obj_r->Filename); + bfree_and_null(obj_r->PluginName); + bfree_and_null(obj_r->ObjectName); + bfree_and_null(obj_r->ObjectSource); + bfree_and_null(obj_r->ObjectUUID); +} + /* * TODO Update doc * */ bool BDB::bdb_get_plugin_object_record(JCR *jcr, OBJECT_DBR *obj_r) { SQL_ROW row; + POOLMEM *where_str = get_pool_memory(PM_MESSAGE); int stat = false; - /*TODO Probably only specified - ObjectId/JobId records should be retrieved hennce Mmsg without additional args for now - */ + Mmsg(where_str, "JobID=%lu AND ObjectID=%lu", obj_r->JobId, obj_r->ObjectId); Mmsg(cmd, - "SELECT JobId, Path, Filename, PluginName, " + "SELECT ObjectId, JobId, Path, Filename, PluginName, " "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize " - "FROM Object"); + "FROM Object WHERE %s", where_str); bdb_lock(); if (QueryDB(jcr, cmd)) { - //TODO fill that part + if (sql_num_rows() > 1) { + char ed1[30]; + Mmsg1(errmsg, _("Error got %s RestoreObjects but expected only one!\n"), + edit_uint64(sql_num_rows(), ed1)); + sql_data_seek(sql_num_rows()-1); + } + if ((row = sql_fetch_row()) == NULL) { + Mmsg2(errmsg, _("PluginOjbect with JobId=%lu ObjectId=%lu not found.\n"), + obj_r->JobId, obj_r->ObjectId); + } else { + db_free_pluginobject_record(jcr, obj_r); + + obj_r->ObjectId = str_to_uint64(row[0]); + obj_r->JobId = str_to_uint64(row[1]); + obj_r->Path = bstrdup(row[2]); + obj_r->Filename = bstrdup(row[3]); + obj_r->PluginName = bstrdup(row[4]); + bstrncpy(obj_r->ObjectType, row[5], 128); + obj_r->ObjectName = row[6]; + obj_r->ObjectName = bstrdup(row[7]); + obj_r->ObjectSource = bstrdup(row[8]); + obj_r->ObjectUUID = bstrdup(row[8]); + obj_r->ObjectSize = str_to_uint64(row[9]); stat = true; + } } else { - //err - Jmsg(jcr, M_ERROR, 0, _("Query %s failed!\n"), cmd); + Jmsg(jcr, M_ERROR, 0, _("PluginObject query %s failed!\n"), cmd); } bdb_unlock(); + +bail_out: + free_pool_memory(where_str); return stat; } /* diff --git a/bacula/src/cats/sql_list.c b/bacula/src/cats/sql_list.c index 899fcfe63d..e2825d729c 100644 --- a/bacula/src/cats/sql_list.c +++ b/bacula/src/cats/sql_list.c @@ -144,15 +144,39 @@ void BDB::bdb_list_client_records(JCR *jcr, DB_LIST_HANDLER *sendit, void *ctx, */ void BDB::bdb_list_plugin_objects(JCR *jcr, OBJECT_DBR *obj_r, DB_LIST_HANDLER *sendit, void *ctx, e_list_type type) { + char esc[MAX_ESCAPE_NAME_LENGTH]; - //TODO Fixup fields displayed - Mmsg(cmd, - "SELECT ObjectId, JobId, " - "ObjectType, ObjectName " - "FROM Object"); + bdb_lock(); + if (type == HORZ_LIST) { + if (obj_r->ObjectType[0] != 0) { + bdb_escape_string(jcr, esc, obj_r->ObjectType, strlen(obj_r->ObjectType)); + Mmsg(cmd, + "SELECT ObjectId, JobId, " + "ObjectType, ObjectName " + "FROM Object WHERE ObjectType='%s'", + esc); + } else { + Mmsg(cmd, + "SELECT ObjectId, JobId, " + "ObjectType, ObjectName " + "FROM Object ORDER BY ObjectId"); + } + } else { + if (obj_r->ObjectType[0] != 0) { + bdb_escape_string(jcr, esc, obj_r->ObjectType, strlen(obj_r->ObjectType)); + Mmsg(cmd, + "SELECT ObjectId, JobId, Path, Filename, " + "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize " + "FROM Object WHERE ObjectType='%s'", esc); + } else { + Mmsg(cmd, + "SELECT ObjectId, JobId, Path, Filename, " + "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize " + "FROM Object ORDER BY ObjectId"); + } + } - bdb_lock(); if (!QueryDB(jcr, cmd)) { Jmsg(jcr, M_ERROR, 0, _("Query %s failed!\n"), cmd); bdb_unlock(); diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index 8105b40f77..ee028ce781 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -560,6 +560,20 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) * Plugin_name * Object_name * Binary Object data + * + * Plugin Object + * File_index + * File_type + * JobId + * Path + * File_name + * Plugin_name + * Object_name + * Objest_type + * Object_source + * Object_UUID + * Object_size + * */ Dmsg1(400, "UpdCat msg=%s\n", msg); @@ -624,6 +638,82 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) Dmsg2(400, "dirddb, &obj_r)) { + Jmsg1(jcr, M_FATAL, 0, _("Plugin object create error. %s"), db_strerror(jcr->db)); + } + + } else if (Stream == STREAM_RESTORE_OBJECT) { ROBJECT_DBR ro; diff --git a/bacula/src/dird/ua_output.c b/bacula/src/dird/ua_output.c index d75f1c61ae..8a017f19ef 100644 --- a/bacula/src/dird/ua_output.c +++ b/bacula/src/dird/ua_output.c @@ -326,6 +326,7 @@ bail_out: * list nextvol job=xx - list the next vol to be used by job * list nextvolume job=xx - same as above. * list copies jobid=x,y,z + * list objects [type=objecttype] [job_id=n objectid=m] - list plugin objects * list pluginrestoreconf jobid=x,y,z [id=k] * list filemedia jobid=x fileindex=z * @@ -661,6 +662,47 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) db_free_restoreobject_record(ua->jcr, &rr); return 1; + } else if (strcasecmp(ua->argk[i], NT_("object")) == 0 || + strcasecmp(ua->argk[i], NT_("objects")) == 0) { + class OBJECT_DBR obj_r; + j = find_arg_with_value(ua, NT_("type")); + if (j >= 0) { + bstrncpy(obj_r.ObjectType, ua->argv[j], sizeof(obj_r.ObjectType)); + } + + for (j=i+1; jargc; j++) { + //TODO job_id arg should be handled different probably because of the collision with the 'list jobid=nn' cmd' + if (strcasecmp(ua->argk[j], NT_("job_id")) == 0 && ua->argv[j]) { + if (is_a_number(ua->argv[j])) { + obj_r.JobId = str_to_uint64(ua->argv[j]); + } else { + ua->error_msg(_("Invalid jobid argument\n")); + return 1; + } + } else if ((strcasecmp(ua->argk[j], NT_("objectid")) == 0) && + ua->argv[j]) + { + if (is_a_number(ua->argv[j])) { + obj_r.ObjectId = str_to_uint64(ua->argv[j]); + } else { + ua->error_msg(_("Invalid objectid argument\n")); + return 1; + } + } + } + + if (obj_r.JobId != 0 && obj_r.ObjectId != 0) { + db_get_plugin_object_record(ua->jcr, ua->db, &obj_r); + return 1; + } + + int k = find_arg_with_value(ua, NT_("type")); + if (k >= 0) { + bstrncpy(obj_r.ObjectType, ua->argv[k], sizeof(obj_r.ObjectType)); + } + + db_list_plugin_objects(ua->jcr, ua->db, &obj_r, prtit, ua, llist); + /* List MEDIA or VOLUMES */ } else if (strcasecmp(ua->argk[i], NT_("media")) == 0 || strcasecmp(ua->argk[i], NT_("volume")) == 0 || diff --git a/bacula/src/dird/ua_purge.c b/bacula/src/dird/ua_purge.c index 1a0f67cd50..75ac8e56c7 100644 --- a/bacula/src/dird/ua_purge.c +++ b/bacula/src/dird/ua_purge.c @@ -445,6 +445,10 @@ void purge_jobs_from_catalog(UAContext *ua, char *jobs) db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); Dmsg1(050, "Delete Log sql=%s\n", query.c_str()); + Mmsg(query, "DELETE FROM Object WHERE JobId IN (%s)", jobs); + db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); + Dmsg1(050, "Delete Object sql=%s\n", query.c_str()); + Mmsg(query, "DELETE FROM RestoreObject WHERE JobId IN (%s)", jobs); db_sql_query(ua->db, query.c_str(), NULL, (void *)NULL); Dmsg1(050, "Delete RestoreObject sql=%s\n", query.c_str()); diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 061766b7f1..c5b2a4e6ea 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -297,6 +297,8 @@ int save_file(JCR *jcr, FF_PKT *ff_pkt, bool top_level) case FT_PLUGIN_CONFIG: Dmsg1(100, "FT_PLUGIN_CONFIG saving: %s\n", ff_pkt->fname); break; + case FT_PLUGIN_OBJECT: + break; case FT_DIRBEGIN: jcr->num_files_examined--; /* correct file count */ return 1; /* not used */ @@ -948,8 +950,10 @@ bool encode_and_send_attributes(bctx_t &bctx) encode_stat(attribs, &ff_pkt->statp, sizeof(ff_pkt->statp), ff_pkt->LinkFI, bctx.data_stream); /** Now possibly extend the attributes */ - if (IS_FT_OBJECT(ff_pkt->type)) { + if (ff_pkt->type != FT_PLUGIN_OBJECT && IS_FT_OBJECT(ff_pkt->type)) { attr_stream = STREAM_RESTORE_OBJECT; + } else if (ff_pkt->type == FT_PLUGIN_OBJECT) { + attr_stream = STREAM_PLUGIN_OBJECT; } else { attribsEx = attribsExBuf; attr_stream = encode_attribsEx(jcr, attribsEx, ff_pkt); @@ -1066,6 +1070,13 @@ bool encode_and_send_attributes(bctx_t &bctx) } break; case FT_PLUGIN_OBJECT: + sd->msglen = Mmsg(sd->msg, "%d %d %d %s %s %s %s %s %s %s %llu", + jcr->JobFiles, ff_pkt->type, ff_pkt->plugin_obj.JobId, ff_pkt->plugin_obj.path, + ff_pkt->plugin_obj.filename, + ff_pkt->plugin_obj.plugin_name, ff_pkt->plugin_obj.object_type, + ff_pkt->plugin_obj.object_name, ff_pkt->plugin_obj.object_source, + ff_pkt->plugin_obj.object_uuid, ff_pkt->plugin_obj.object_size); + stat = sd->send(); break; case FT_REG: stat = sd->fsend("%ld %d %s%c%s%c%c%s%c%d%c", jcr->JobFiles, diff --git a/bacula/src/filed/fd_plugins.c b/bacula/src/filed/fd_plugins.c index 88a21f45ae..979ee49d4f 100644 --- a/bacula/src/filed/fd_plugins.c +++ b/bacula/src/filed/fd_plugins.c @@ -679,17 +679,29 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level) */ ff_pkt->type = sp.type; if (IS_FT_OBJECT(sp.type)) { - if (!sp.restore_obj.object_name) { + if (sp.type != FT_PLUGIN_OBJECT && !sp.restore_obj.object_name) { Jmsg1(jcr, M_FATAL, 0, _("Command plugin \"%s\": no object_name in startBackupFile packet.\n"), cmd); goto bail_out; } ff_pkt->fname = cmd; /* full plugin string */ - ff_pkt->restore_obj.object_name = sp.restore_obj.object_name; - ff_pkt->restore_obj.index = sp.restore_obj.index; /* restore object index */ - ff_pkt->restore_obj.object_compression = 0; /* no compression for now */ - ff_pkt->restore_obj.object = sp.restore_obj.object; - ff_pkt->restore_obj.object_len = sp.restore_obj.object_len; + if (sp.type == FT_PLUGIN_OBJECT) { + ff_pkt->plugin_obj.JobId = jcr->JobId; + ff_pkt->plugin_obj.path = sp.plugin_obj.path; + ff_pkt->plugin_obj.filename = sp.plugin_obj.filename; + ff_pkt->plugin_obj.plugin_name = sp.plugin_obj.plugin_name; + ff_pkt->plugin_obj.object_type = sp.plugin_obj.object_type; + ff_pkt->plugin_obj.object_name = sp.plugin_obj.object_name; + ff_pkt->plugin_obj.object_source = sp.plugin_obj.object_source; + ff_pkt->plugin_obj.object_uuid = sp.plugin_obj.object_uuid; + ff_pkt->plugin_obj.object_size = sp.plugin_obj.object_size; + } else { + ff_pkt->restore_obj.object_name = sp.restore_obj.object_name; + ff_pkt->restore_obj.index = sp.restore_obj.index; /* restore object index */ + ff_pkt->restore_obj.object_compression = 0; /* no compression for now */ + ff_pkt->restore_obj.object = sp.restore_obj.object; + ff_pkt->restore_obj.object_len = sp.restore_obj.object_len; + } } else { Dsm_check(999); if (!sp.fname) { diff --git a/bacula/src/filed/fd_plugins.h b/bacula/src/filed/fd_plugins.h index 182460a3bd..60b2cf03a7 100644 --- a/bacula/src/filed/fd_plugins.h +++ b/bacula/src/filed/fd_plugins.h @@ -99,6 +99,18 @@ struct restore_object { int32_t object_compression; /* set to compression type */ }; +struct plugin_object { + uint32_t JobId; + char *path; + char *filename; + char *plugin_name; + char *object_type; + char *object_name; + char *object_source; + char *object_uuid; + uint64_t object_size; +}; + /* * This packet is used for file save info transfer. */ @@ -117,6 +129,7 @@ struct save_pkt { bool do_dedup; /* True if we deal with a dedup storage system */ char *cmd; /* command */ struct restore_object restore_obj; /* Info about restore object */ + struct plugin_object plugin_obj; /* Plugin Object */ uint32_t delta_seq; /* Delta sequence number */ int32_t LinkFI; /* LinkFI if LINKSAVED */ int32_t pkt_end; /* end packet sentinel */ diff --git a/bacula/src/filetypes.h b/bacula/src/filetypes.h index c1bc1d9fd3..80467014dd 100644 --- a/bacula/src/filetypes.h +++ b/bacula/src/filetypes.h @@ -76,6 +76,7 @@ #define AR_DATA_STREAM (1<<16) /* Data stream id present */ /* Quick way to know if a Filetype is about a plugin "Object" */ -#define IS_FT_OBJECT(x) (((x) == FT_RESTORE_FIRST) || ((x) == FT_PLUGIN_CONFIG_FILLED) || ((x) == FT_PLUGIN_CONFIG)) +#define IS_FT_OBJECT(x) (((x) == FT_RESTORE_FIRST) || ((x) == FT_PLUGIN_CONFIG_FILLED) || ((x) == FT_PLUGIN_CONFIG) \ + || ((x) == FT_PLUGIN_OBJECT)) #endif /* __BFILETYPES_H */ diff --git a/bacula/src/findlib/find.h b/bacula/src/findlib/find.h index ceeedc86a8..037965e7f1 100644 --- a/bacula/src/findlib/find.h +++ b/bacula/src/findlib/find.h @@ -174,6 +174,7 @@ struct FF_PKT { int32_t LinkFI; /* FileIndex of main hard linked file */ int32_t delta_seq; /* Delta Sequence number */ struct restore_object restore_obj; + struct plugin_object plugin_obj; struct f_link *linked; /* Set if this file is hard linked */ int type; /* FT_ type from above */ int ff_errno; /* errno */ diff --git a/bacula/src/stored/append.c b/bacula/src/stored/append.c index e8d27a44df..3d5cefef1c 100644 --- a/bacula/src/stored/append.c +++ b/bacula/src/stored/append.c @@ -443,6 +443,7 @@ bool send_attrs_to_dir(JCR *jcr, DEV_RECORD *rec) if (rec->maskedStream == STREAM_UNIX_ATTRIBUTES || rec->maskedStream == STREAM_UNIX_ATTRIBUTES_EX || rec->maskedStream == STREAM_RESTORE_OBJECT || + rec->maskedStream == STREAM_PLUGIN_OBJECT || crypto_digest_stream_type(rec->maskedStream) != CRYPTO_DIGEST_NONE) { if (!jcr->no_attributes) { BSOCK *dir = jcr->dir_bsock; diff --git a/bacula/src/stored/bextract.c b/bacula/src/stored/bextract.c index 01b6462ae0..5fa997bc0e 100644 --- a/bacula/src/stored/bextract.c +++ b/bacula/src/stored/bextract.c @@ -472,6 +472,7 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) break; case STREAM_RESTORE_OBJECT: + case STREAM_PLUGIN_OBJECT: /* nothing to do */ break; diff --git a/bacula/src/stored/bls.c b/bacula/src/stored/bls.c index 8a97008fae..5fa792f57e 100644 --- a/bacula/src/stored/bls.c +++ b/bacula/src/stored/bls.c @@ -451,6 +451,8 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) Dmsg1(100, "Plugin data: %s\n", data); } else if (rec->maskedStream == STREAM_RESTORE_OBJECT) { Dmsg0(100, "Restore Object record\n"); + } else if (rec->maskedStream == STREAM_PLUGIN_OBJECT) { + Dmsg0(100, "Plugin Object record\n"); } else if (rec->maskedStream == STREAM_ADATA_BLOCK_HEADER) { Dmsg0(000, "Adata block header\n"); } else if (rec->maskedStream == STREAM_ADATA_RECORD_HEADER) { diff --git a/bacula/src/stored/record_util.c b/bacula/src/stored/record_util.c index 6ed21bd6ff..316889da2b 100644 --- a/bacula/src/stored/record_util.c +++ b/bacula/src/stored/record_util.c @@ -111,6 +111,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "contUNIX-ATTR-EX"; case STREAM_RESTORE_OBJECT: return "contRESTORE-OBJECT"; + case STREAM_PLUGIN_OBJECT: + return "contPLUGIN-OBJECT"; case STREAM_SPARSE_DATA: return "contSPARSE-DATA"; case STREAM_SPARSE_GZIP_DATA: @@ -183,6 +185,8 @@ const char *stream_to_ascii(char *buf, int stream, int fi) return "UNIX-ATTR-EX"; case STREAM_RESTORE_OBJECT: return "RESTORE-OBJECT"; + case STREAM_PLUGIN_OBJECT: + return "PLUGIN-OBJECT"; case STREAM_SPARSE_DATA: return "SPARSE-DATA"; case STREAM_SPARSE_GZIP_DATA: diff --git a/bacula/src/streams.h b/bacula/src/streams.h index 75af69ec09..005c778171 100644 --- a/bacula/src/streams.h +++ b/bacula/src/streams.h @@ -90,17 +90,18 @@ #define STREAM_PLUGIN_NAME 26 /* Plugin "file" string */ #define STREAM_PLUGIN_DATA 27 /* Plugin specific data */ #define STREAM_RESTORE_OBJECT 28 /* Plugin restore object */ +#define STREAM_PLUGIN_OBJECT 29 /* Plugin object */ /* * Non-gzip compressed streams. Those streams can handle arbitrary * compression algorithm data as an additional header is stored * at the beginning of the stream. See comp_stream_header definition * in ch.h for more details. */ -#define STREAM_COMPRESSED_DATA 29 /* Compressed file data */ -#define STREAM_SPARSE_COMPRESSED_DATA 30 /* Sparse compressed data stream */ -#define STREAM_WIN32_COMPRESSED_DATA 31 /* Compressed Win32 BackupRead data */ -#define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA 32 /* Encrypted, compressed data */ -#define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 33 /* Encrypted, compressed Win32 BackupRead data */ +#define STREAM_COMPRESSED_DATA 30 /* Compressed file data */ +#define STREAM_SPARSE_COMPRESSED_DATA 31 /* Sparse compressed data stream */ +#define STREAM_WIN32_COMPRESSED_DATA 32 /* Compressed Win32 BackupRead data */ +#define STREAM_ENCRYPTED_FILE_COMPRESSED_DATA 33 /* Encrypted, compressed data */ +#define STREAM_ENCRYPTED_WIN32_COMPRESSED_DATA 34 /* Encrypted, compressed Win32 BackupRead data */ #define STREAM_ADATA_BLOCK_HEADER 200 /* Adata block header */ #define STREAM_ADATA_RECORD_HEADER 201 /* Adata record header */