From: Michal Rakowski Date: Wed, 31 Mar 2021 17:04:20 +0000 (+0200) Subject: Add ObjectStatus and ObjectCount to Plugin Objects X-Git-Tag: Release-11.3.2~576 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=68b2885c4abb949bb6f94a0448bacd1810e4e6d1;p=thirdparty%2Fbacula.git Add ObjectStatus and ObjectCount to Plugin Objects --- diff --git a/bacula/src/cats/cats.c b/bacula/src/cats/cats.c index 4dbfedf97..975bcba20 100644 --- a/bacula/src/cats/cats.c +++ b/bacula/src/cats/cats.c @@ -31,6 +31,8 @@ #if HAVE_SQLITE3 || HAVE_MYSQL || HAVE_POSTGRESQL #include "cats.h" + +static int dbglvl=100; void append_filter(POOLMEM **buf, char *cond) { @@ -160,89 +162,124 @@ void BDB::print_lock_info(FILE *fp) } } -void OBJECT_DBR::parse_plugin_object_string(char **obj_str) +/* Parse stream of tags, return next one from the stream (it will be null terminated, + * original buffer will be changed) */ +static char *get_next_tag(char **buf) { - /* TODO: Do some sanity checks on the input format */ - char *p = *obj_str; - int fnl, pnl; - char* tmp_path = p; - skip_nonspaces(&p); /* skip Path */ + char *p = *buf; + char *tmp = p; + skip_nonspaces(&p); skip_spaces(&p); - char *c = strpbrk(tmp_path, " "); + char *c = strpbrk(tmp, " "); if (c) { *c = '\0'; + } else { + Dmsg0(dbglvl, "No tag found!\n"); + return NULL; } + *buf = p; + + Dmsg1(dbglvl, "Found tag: %s\n", tmp); + return tmp; +} + + +bool OBJECT_DBR::parse_plugin_object_string(char **obj_str) +{ + bool ret = false; + int fnl, pnl; - if (tmp_path[strlen(tmp_path) - 1] == '/') { - pm_strcpy(Path, tmp_path); + char *tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; + } + + if (tmp[strlen(tmp) - 1] == '/') { + pm_strcpy(Path, tmp); unbash_spaces(Path); } else { - split_path_and_filename(tmp_path, &Path, &pnl, &Filename, &fnl); + split_path_and_filename(tmp, &Path, &pnl, &Filename, &fnl); unbash_spaces(Path); unbash_spaces(Filename); } - char *tmp = p; - skip_nonspaces(&p); /* skip PluginName */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } pm_strcpy(PluginName, tmp); unbash_spaces(PluginName); - tmp = p; - skip_nonspaces(&p); /* skip ObjectType */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } bstrncpy(ObjectCategory, tmp, sizeof(ObjectCategory)); unbash_spaces(ObjectCategory); - tmp = p; - skip_nonspaces(&p); /* skip ObjectType */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } bstrncpy(ObjectType, tmp, sizeof(ObjectType)); unbash_spaces(ObjectType); - tmp = p; - skip_nonspaces(&p); /* skip ObjectName */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } bstrncpy(ObjectName, tmp, sizeof(ObjectName)); unbash_spaces(ObjectName); - tmp = p; - skip_nonspaces(&p); /* skip ObjectSource */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } bstrncpy(ObjectSource, tmp, sizeof(ObjectSource)); unbash_spaces(ObjectSource); - tmp = p; - skip_nonspaces(&p); /* skip ObjectUuid */ - skip_spaces(&p); - c = strpbrk(tmp, " "); - if (c) { - *c = '\0'; + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; } bstrncpy(ObjectUUID, tmp, sizeof(ObjectUUID)); unbash_spaces(ObjectUUID); - ObjectSize = str_to_uint64(p); + tmp = get_next_tag(obj_str); + if (tmp) { + ObjectSize = str_to_uint64(tmp); + } else if (*obj_str) { + /* Object size is the last tag here, we are not expecting to have status in the stream */ + ObjectSize = str_to_uint64(*obj_str); + ret = true; + goto bail_out; + } else { + goto bail_out; + } + + /* We should have status string in the end */ + if (*obj_str) { + tmp = get_next_tag(obj_str); + if (!tmp) { + goto bail_out; + } + ObjectStatus = (int)*tmp; + if (*obj_str) { + ObjectCount = str_to_uint64(*obj_str); + } + } else { + goto bail_out; + } + + ret = true; + +bail_out: + if (!ret) { + /* Reset parsed fields */ + reset(); + } + + return ret; } void OBJECT_DBR::create_db_filter(JCR *jcr, POOLMEM **where) @@ -310,6 +347,11 @@ void OBJECT_DBR::create_db_filter(JCR *jcr, POOLMEM **where) Mmsg(tmp, " Object.ObjectSize=%llu", ObjectSize); append_filter(where, tmp.c_str()); } + + if (ObjectStatus != 0) { + Mmsg(tmp, " Object.ObjectStatus='%c'", ObjectStatus); + append_filter(where, tmp.c_str()); + } } } @@ -344,7 +386,7 @@ void parse_restore_object_string(char **r_obj_str, ROBJECT_DBR *robj_r) len = strlen(robj_r->object_name); robj_r->object = &robj_r->object_name[len+1]; /* point to object */ robj_r->object[robj_r->object_len] = 0; /* add zero for those who attempt printing */ - Dmsg7(100, "oname=%s stream=%d FT=%d FI=%d JobId=%ld, obj_len=%d\nobj=\"%s\"\n", + Dmsg7(dbglvl, "oname=%s stream=%d FT=%d FI=%d JobId=%ld, obj_len=%d\nobj=\"%s\"\n", robj_r->object_name, robj_r->Stream, robj_r->FileType, robj_r->FileIndex, robj_r->JobId, robj_r->object_len, robj_r->object); } diff --git a/bacula/src/cats/cats.h b/bacula/src/cats/cats.h index 31fdf0b0d..6ccd46840 100644 --- a/bacula/src/cats/cats.h +++ b/bacula/src/cats/cats.h @@ -241,17 +241,19 @@ struct ATTR_DBR { class OBJECT_DBR { public: - OBJECT_DBR() { + OBJECT_DBR() { Path = get_pool_memory(PM_FNAME); Filename = get_pool_memory(PM_FNAME); PluginName = get_pool_memory(PM_FNAME); reset(); }; - ~OBJECT_DBR() { + + ~OBJECT_DBR() { free_pool_memory(Path); free_pool_memory(Filename); free_pool_memory(PluginName); }; + /* reset memory */ void reset() { JobId = 0; @@ -261,9 +263,12 @@ public: *ObjectCategory = *ObjectType = *ObjectName = *ObjectSource = *ObjectUUID = *ClientName = 0; limit = 0; order = 0; + ObjectStatus = 'U'; + ObjectCount = 0; }; + /* Parse OBJECT record from stream */ - void parse_plugin_object_string(char **obj_str); + bool parse_plugin_object_string(char **obj_str); /* Helper for creating the 'where' part of jcr's related sql query based on fields from the Object */ void create_db_filter(JCR *jcr, POOLMEM **where); @@ -278,6 +283,8 @@ public: char ObjectSource[MAX_NAME_LENGTH]; char ObjectUUID[MAX_NAME_LENGTH]; uint64_t ObjectSize; + int32_t ObjectStatus; + uint32_t ObjectCount; /* Fields not stored in db directly */ char ClientName[MAX_NAME_LENGTH]; diff --git a/bacula/src/cats/make_mysql_tables.in b/bacula/src/cats/make_mysql_tables.in index 14616f9cb..28c87fb2f 100644 --- a/bacula/src/cats/make_mysql_tables.in +++ b/bacula/src/cats/make_mysql_tables.in @@ -64,7 +64,7 @@ CREATE TABLE Object ( ObjectId BIGINT UNSIGNED NOT NULL AUTO_INCREMENT, - JobId INTEGER UNSIGNED not null, + JobId INTEGER UNSIGNED NOT NULL, Path BLOB NOT NULL, Filename BLOB NOT NULL, PluginName TINYBLOB NOT NULL, @@ -75,6 +75,8 @@ CREATE TABLE Object ObjectSource TINYBLOB NOT NULL, ObjectUUID TINYBLOB NOT NULL, ObjectSize bigint NOT NULL, + ObjectStatus BINARY(1) NOT NULL DEFAULT 'U', + ObjectCount INTEGER UNSIGNED NOT NULL DEFAULT 1, primary key (ObjectId) ); diff --git a/bacula/src/cats/make_postgresql_tables.in b/bacula/src/cats/make_postgresql_tables.in index f1e8ae3cd..f4b1c2b2b 100644 --- a/bacula/src/cats/make_postgresql_tables.in +++ b/bacula/src/cats/make_postgresql_tables.in @@ -58,6 +58,8 @@ CREATE TABLE Object ObjectSource text not null, ObjectUUID text not null, ObjectSize bigint not null, + ObjectStatus char(1) not null default 'U', + ObjectCount integer not null default 1, primary key (ObjectId) ); diff --git a/bacula/src/cats/make_sqlite3_tables.in b/bacula/src/cats/make_sqlite3_tables.in index 9c142bab1..96dec7787 100644 --- a/bacula/src/cats/make_sqlite3_tables.in +++ b/bacula/src/cats/make_sqlite3_tables.in @@ -55,6 +55,8 @@ CREATE TABLE Object ObjectSource text not null, ObjectUUID text not null, ObjectSize integer not null, + ObjectStatus char(1) not null default 'U', + ObjectCount integer not null default 1, primary key (ObjectId) ); diff --git a/bacula/src/cats/sql_create.c b/bacula/src/cats/sql_create.c index 166fb53f8..2a97ad1e8 100644 --- a/bacula/src/cats/sql_create.c +++ b/bacula/src/cats/sql_create.c @@ -1210,11 +1210,11 @@ bool BDB::bdb_create_object_record(JCR *jcr, OBJECT_DBR *obj) bdb_lock(); Mmsg(cmd, "INSERT INTO Object (JobId, Path, Filename, PluginName, ObjectCategory, " - "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize) " - "VALUES (%lu, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %llu)", + "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize, ObjectStatus, ObjectCount) " + "VALUES (%lu, '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', %llu, '%c', %lu)", obj->JobId, esc_path, esc_filename, esc_plugin_name, esc_obj_category, esc_obj_type, esc_obj_name, esc_obj_source, esc_obj_uuid, - obj->ObjectSize); + obj->ObjectSize, (char)obj->ObjectStatus, obj->ObjectCount); obj->ObjectId = sql_insert_autokey_record(cmd, NT_("Object")); if (obj->ObjectId == 0) { diff --git a/bacula/src/cats/sql_get.c b/bacula/src/cats/sql_get.c index 663de7480..53a733754 100644 --- a/bacula/src/cats/sql_get.c +++ b/bacula/src/cats/sql_get.c @@ -779,7 +779,7 @@ bool BDB::bdb_get_plugin_object_record(JCR *jcr, OBJECT_DBR *obj_r) Mmsg(cmd, "SELECT ObjectId, JobId, Path, Filename, PluginName, ObjectCategory, " - "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize " + "ObjectType, ObjectName, ObjectSource, ObjectUUID, ObjectSize, ObjectStatus, ObjectCount " "FROM Object %s", where.c_str()); bdb_lock(); @@ -807,6 +807,8 @@ bool BDB::bdb_get_plugin_object_record(JCR *jcr, OBJECT_DBR *obj_r) bstrncpy(obj_r->ObjectSource, row[8], sizeof(obj_r->ObjectSource)); bstrncpy(obj_r->ObjectUUID, row[9], sizeof(obj_r->ObjectUUID)); obj_r->ObjectSize = str_to_uint64(row[10]); + obj_r->ObjectStatus = row[11] != NULL ? (int)*row[11] : 'U'; + obj_r->ObjectCount = str_to_uint64(row[12]); stat = true; } diff --git a/bacula/src/cats/sql_list.c b/bacula/src/cats/sql_list.c index 65b8ea203..37172d2c5 100644 --- a/bacula/src/cats/sql_list.c +++ b/bacula/src/cats/sql_list.c @@ -184,13 +184,14 @@ void BDB::bdb_list_plugin_objects(JCR *jcr, OBJECT_DBR *obj_r, DB_LIST_HANDLER * case VERT_LIST: Mmsg(cmd, "SELECT Object.ObjectId, Object.JobId, Object.Path, Object.Filename, Object.PluginName, Object.ObjectCategory, " - "Object.ObjectType, Object.ObjectName, Object.ObjectSource, Object.ObjectUUID, Object.ObjectSize " + "Object.ObjectType, Object.ObjectName, Object.ObjectSource, " + "Object.ObjectUUID, Object.ObjectSize, Object.ObjectStatus, Object.ObjectCount " "FROM Object %s %s", join.c_str(), where.c_str()); break; case HORZ_LIST: Mmsg(cmd, "SELECT Object.ObjectId, Object.JobId, Object.ObjectCategory, " - "Object.ObjectType, Object.ObjectName " + "Object.ObjectType, Object.ObjectName, Object.ObjectStatus " "FROM Object %s %s", join.c_str(), where.c_str()); break; default: @@ -221,13 +222,14 @@ void BDB::bdb_list_plugin_objects_ids(JCR *jcr, char* id_list, DB_LIST_HANDLER * case VERT_LIST: Mmsg(cmd, "SELECT Object.ObjectId, Object.JobId, Object.Path, Object.Filename, Object.PluginName, Object.ObjectCategory, " - "Object.ObjectType, Object.ObjectName, Object.ObjectSource, Object.ObjectUUID, Object.ObjectSize " + "Object.ObjectType, Object.ObjectName, Object.ObjectSource, Object.ObjectUUID, " + "Object.ObjectSize, Object.ObjectStatus, Object.ObjectCount " "FROM Object WHERE ObjectId IN (%s) ORDER BY ObjectId ASC", id_list); break; case HORZ_LIST: Mmsg(cmd, "SELECT Object.ObjectId, Object.JobId, Object.ObjectCategory, " - "Object.ObjectType, Object.ObjectName, Object.ObjectUUID " + "Object.ObjectType, Object.ObjectName, Object.ObjectUUID, Object.ObjectStatus " "FROM Object WHERE ObjectId IN (%s) ORDER BY ObjectId ASC", id_list); break; default: diff --git a/bacula/src/dird/catreq.c b/bacula/src/dird/catreq.c index da11056b3..289b465db 100644 --- a/bacula/src/dird/catreq.c +++ b/bacula/src/dird/catreq.c @@ -641,7 +641,10 @@ static void update_attribute(JCR *jcr, char *msg, int32_t msglen) } else if (Stream == STREAM_PLUGIN_OBJECT) { OBJECT_DBR obj_r; - obj_r.parse_plugin_object_string(&p); + if (!obj_r.parse_plugin_object_string(&p)) { + Jmsg0(jcr, M_FATAL, 0, _("Failed to parse plugin object!\n")); + return; + } if (jcr->wjcr) { obj_r.JobId = jcr->wjcr->JobId; diff --git a/bacula/src/dird/ua_cmds.c b/bacula/src/dird/ua_cmds.c index 385d51d1b..083ec405a 100644 --- a/bacula/src/dird/ua_cmds.c +++ b/bacula/src/dird/ua_cmds.c @@ -137,7 +137,7 @@ static struct cmdstruct commands[] = { /* C "\tfilemedia jobid= fileindex= | clients\n" "\tevents [type= | limit= | order= | days= | start= | end= |\n" "\t\t source= | code= | type= ]\n" - "\tobjects [jobid= client= type= | category= | limit= | order= ]\n" + "\tobjects [jobid= client= type= | category= | status= | limit= | order= ]\n" ), false}, { NT_("llist"), llist_cmd, _("Full or long list like list command"), diff --git a/bacula/src/dird/ua_output.c b/bacula/src/dird/ua_output.c index 13e31833f..8305687a0 100644 --- a/bacula/src/dird/ua_output.c +++ b/bacula/src/dird/ua_output.c @@ -326,7 +326,7 @@ bail_out: * list nextvol job=xx - list the next vol to be used by job * list nextvolume job=xx - same as above. * list copies jobid=x,y,z - * list objects [type=objecttype job_id=id clientname=n] - list plugin objects + * list objects [type=objecttype job_id=id clientname=n,status=S] - list plugin objects * list pluginrestoreconf jobid=x,y,z [id=k] * list filemedia jobid=x fileindex=z * @@ -709,6 +709,8 @@ static int do_list_cmd(UAContext *ua, const char *cmd, e_list_type llist) } else if (strcasecmp(ua->argk[j], NT_("category")) == 0) { bstrncpy(obj_r.ObjectCategory, ua->argv[j], sizeof(obj_r.ObjectCategory)); + } else if (strcasecmp(ua->argk[j], NT_("status")) == 0) { + obj_r.ObjectStatus = (int32_t)ua->argv[j][0]; } else if (strcasecmp(ua->argk[j], NT_("limit")) == 0 && ua->argv[j]) { obj_r.limit = atoi(ua->argv[j]); diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index e0fb6c7e1..f558e27df 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -1186,10 +1186,14 @@ bool encode_and_send_attributes(bctx_t &bctx) bash_spaces(object_source.c_str()); pm_strcpy(object_uuid, NPRTB(ff_pkt->plugin_obj.object_uuid)); bash_spaces(object_uuid.c_str()); - sd->msglen = Mmsg(sd->msg, "%s %s %s %s %s %s %s %llu%c", + sd->msglen = Mmsg(sd->msg, "%s %s %s %s %s %s %s %llu %c %lu%c", path.c_str(), plugin_name.c_str(), object_category.c_str(), object_type.c_str(), object_name.c_str(), object_source.c_str(), object_uuid.c_str(), - ff_pkt->plugin_obj.object_size, 0); + ff_pkt->plugin_obj.object_size, + /* Plugin can left status unset, we need to handle that */ + ff_pkt->plugin_obj.status != 0 ? (char)ff_pkt->plugin_obj.status : 'U', + ff_pkt->plugin_obj.count, + 0); stat = sd->send(); break; diff --git a/bacula/src/filed/fd_plugins.c b/bacula/src/filed/fd_plugins.c index ba5281016..f13d37950 100644 --- a/bacula/src/filed/fd_plugins.c +++ b/bacula/src/filed/fd_plugins.c @@ -696,6 +696,8 @@ int plugin_save(JCR *jcr, FF_PKT *ff_pkt, bool top_level) ff_pkt->plugin_obj.object_source = sp.plugin_obj.object_source; ff_pkt->plugin_obj.object_uuid = sp.plugin_obj.object_uuid; ff_pkt->plugin_obj.object_size = sp.plugin_obj.object_size; + ff_pkt->plugin_obj.status = sp.plugin_obj.status; + ff_pkt->plugin_obj.count = sp.plugin_obj.count; } else { ff_pkt->restore_obj.object_name = sp.restore_obj.object_name; ff_pkt->restore_obj.index = sp.restore_obj.index; /* restore object index */ diff --git a/bacula/src/filed/fd_plugins.h b/bacula/src/filed/fd_plugins.h index 232e958b6..2d68e250c 100644 --- a/bacula/src/filed/fd_plugins.h +++ b/bacula/src/filed/fd_plugins.h @@ -103,6 +103,13 @@ struct restore_object { //TODO probably more specific db types needed #define PLUGIN_OBJECT_DATABASE "database" /* For all database plugins */ +/* Possible statuses of plugin objects. Based on Job status */ +#define PLUG_OBJ_STATUS_UNSET 'U' /* Status was not set at all by the plugin */ +#define PLUG_OBJ_STATUS_TERMINATED 'T' /* Plugin object is OK */ +#define PLUG_OBJ_STATUS_WARNING 'W' /* Plugin object OK, with some non-fatal warning */ +#define PLUG_OBJ_STATUS_ERROR 'e' /* Non-fatal error */ +#define PLUG_OBJ_STATUS_FATAL 'f' /* Fatal error occured */ + struct plugin_object { char *path; char *plugin_name; @@ -112,6 +119,8 @@ struct plugin_object { char *object_source; char *object_uuid; uint64_t object_size; + int32_t status; + uint32_t count; }; enum metadata_type { diff --git a/bacula/src/plugins/fd/test-plugin-fd.c b/bacula/src/plugins/fd/test-plugin-fd.c index a5751f55e..77bfb45d7 100644 --- a/bacula/src/plugins/fd/test-plugin-fd.c +++ b/bacula/src/plugins/fd/test-plugin-fd.c @@ -669,6 +669,8 @@ static bRC startBackupFile(bpContext *ctx, struct save_pkt *sp) sp->plugin_obj.object_name = (char *)NT_("test vm"); sp->plugin_obj.object_source = (char *)NT_("test plugin source"); sp->plugin_obj.object_uuid = (char *)NT_("1234-abc-testplugin"); + sp->plugin_obj.status = PLUG_OBJ_STATUS_TERMINATED; + sp->plugin_obj.count = 2; sp->plugin_obj.object_size = obj_uuid++; sp->type = FT_PLUGIN_OBJECT; diff --git a/bacula/src/stored/bscan.c b/bacula/src/stored/bscan.c index e645dcb98..dffa2cc4a 100644 --- a/bacula/src/stored/bscan.c +++ b/bacula/src/stored/bscan.c @@ -823,7 +823,10 @@ static bool record_cb(DCR *dcr, DEV_RECORD *rec) char *buf = rec->data; num_plugin_objects++; - obj_r.parse_plugin_object_string(&buf); + if (!obj_r.parse_plugin_object_string(&buf)) { + Pmsg0(000, _("Failed to parse plugin object!\n")); + break; + } // Need to get new jobId if possible mjcr = get_jcr_by_session(rec->VolSessionId, rec->VolSessionTime);