From: Michal Rakowski Date: Sun, 2 Aug 2020 20:55:28 +0000 (+0200) Subject: mysql: add Primary Key to each table created when sql_require_primary_key flag is... X-Git-Tag: Release-11.3.2~1312 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=e014cd2bc46dad5ae6b31744fa3d79b03bd96717;p=thirdparty%2Fbacula.git mysql: add Primary Key to each table created when sql_require_primary_key flag is turned on --- diff --git a/bacula/src/cats/bdb_mysql.h b/bacula/src/cats/bdb_mysql.h index 56e4bfff21..1aac3ac0da 100644 --- a/bacula/src/cats/bdb_mysql.h +++ b/bacula/src/cats/bdb_mysql.h @@ -33,6 +33,10 @@ private: MYSQL *m_db_handle; MYSQL m_instance; MYSQL_RES *m_result; + POOLMEM *m_pkey_query_buffer; + + bool is_pkey_required(); + const char* enable_pkey(const char *query); public: BDB_MYSQL(); diff --git a/bacula/src/cats/bvfs.c b/bacula/src/cats/bvfs.c index 1b01fc06aa..985e504ddc 100644 --- a/bacula/src/cats/bvfs.c +++ b/bacula/src/cats/bvfs.c @@ -1425,7 +1425,8 @@ bool Bvfs::insert_hardlinks(char *output_table) } Dmsg1(dbglevel, "Inserting %d hardlink records\n", missing_hardlinks->size()); - Mmsg(query, "CREATE TEMPORARY TABLE h%s (JobId integer, FileIndex integer)", output_table); + Mmsg(query, "CREATE TEMPORARY TABLE h%s (JobId INTEGER, FileIndex INTEGER" + "/*PKEY, DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY*/)", output_table); Dmsg1(dbglevel, "q=%s\n", query.c_str()); if (!db->bdb_sql_query(query.c_str(), NULL, NULL)) { Dmsg1(dbglevel, "Can't execute query=%s\n", query.c_str()); @@ -1527,7 +1528,7 @@ bool Bvfs::compute_restore_list(char *fileid, char *dirid, char *output_table) /* Now start the transaction to protect ourself from other commands */ db->bdb_start_transaction(jcr); - Mmsg(query, "CREATE TABLE btemp%s AS ", output_table); + Mmsg(query, "CREATE TABLE btemp%s /*PKEY (DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY)*/ AS ", output_table); if (*fileid) { /* Select files with their direct id */ init=true; diff --git a/bacula/src/cats/mysql.c b/bacula/src/cats/mysql.c index 7bf02fd6e4..67444efcb1 100644 --- a/bacula/src/cats/mysql.c +++ b/bacula/src/cats/mysql.c @@ -187,7 +187,22 @@ get_out: V(mutex); return mdb; } - + +bool BDB_MYSQL::is_pkey_required() +{ + int ret = mysql_query(m_db_handle, "show variables like 'sql_require_primary_key'"); + bool req_pkey = false; + if (!ret) { + if ((m_result = mysql_use_result(m_db_handle)) != NULL) { + SQL_ROW row; + while ((row = mysql_fetch_row(m_result))) { + req_pkey = !strncmp(row[1], "ON", 2); + } + sql_free_result(); + } + } + return req_pkey; +} /* * Now actually open the database. This can generate errors, @@ -302,6 +317,13 @@ bool BDB_MYSQL::bdb_open_database(JCR *jcr) if (!bdb_check_version(jcr)) { goto get_out; } + + /* Check if all tables created must have a Primary Key */ + if (is_pkey_required()) { + m_pkey_query_buffer = get_pool_memory(PM_FNAME); + } else { + m_pkey_query_buffer = NULL; + } Dmsg3(100, "opendb ref=%d connected=%d db=%p\n", mdb->m_ref_count, mdb->m_connected, mdb->m_db_handle); @@ -348,6 +370,9 @@ void BDB_MYSQL::bdb_close_database(JCR *jcr) free_pool_memory(mdb->esc_name); free_pool_memory(mdb->esc_path); free_pool_memory(mdb->esc_obj); + if (mdb->m_pkey_query_buffer) { + free_pool_memory(mdb->m_pkey_query_buffer); + } if (mdb->m_db_driver) { free(mdb->m_db_driver); } @@ -464,7 +489,24 @@ void BDB_MYSQL::bdb_start_transaction(JCR *jcr) void BDB_MYSQL::bdb_end_transaction(JCR *jcr) { } - + +/* + * Helper function to dynamically add Primary Keys to the tables being created. + */ +const char* BDB_MYSQL::enable_pkey(const char *query) +{ + if (!m_pkey_query_buffer) return query; + const char *tag = strstr(query, "/*PKEY"); + if (!tag) return query; + + pm_strcpy(m_pkey_query_buffer, query); + char *pos = strstr(m_pkey_query_buffer, "/*PKEY"); + for (int i=0; i<6; i++) *pos++=' '; + pos = strstr(pos, "*/"); + for (int i=0; i<2; i++) *pos++=' '; + return m_pkey_query_buffer; +} + /* * Submit a general SQL command (cmd), and for each row returned, * the result_handler is called with the ctx. @@ -481,10 +523,12 @@ bool BDB_MYSQL::bdb_sql_query(const char *query, DB_RESULT_HANDLER *result_handl bdb_lock(); errmsg[0] = 0; + + query = enable_pkey(query); ret = mysql_query(m_db_handle, query); if (ret != 0) { Mmsg(mdb->errmsg, _("Query failed: %s: ERR=%s\n"), query, sql_strerror()); - Dmsg0(500, "db_sql_query failed\n"); + Dmsg0(500, "db_sql_query failed\n"); goto get_out; } @@ -519,7 +563,7 @@ get_out: bdb_unlock(); return retval; } - + bool BDB_MYSQL::sql_query(const char *query, int flags) { int ret; @@ -539,6 +583,7 @@ bool BDB_MYSQL::sql_query(const char *query, int flags) mdb->m_result = NULL; } + query = enable_pkey(query); ret = mysql_query(mdb->m_db_handle, query); if (ret == 0) { Dmsg0(500, "we have a result\n"); @@ -687,13 +732,14 @@ bool BDB_MYSQL::sql_batch_start(JCR *jcr) bdb_lock(); retval = sql_query("CREATE TEMPORARY TABLE batch (" - "FileIndex integer," - "JobId integer," + "FileIndex integer not null," + "JobId integer not null," "Path blob," "Name blob," "LStat tinyblob," "MD5 tinyblob," - "DeltaSeq integer)"); + "DeltaSeq integer" + "/*PKEY, DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY*/)"); bdb_unlock(); /* @@ -751,7 +797,7 @@ bool BDB_MYSQL::sql_batch_insert(JCR *jcr, ATTR_DBR *ar) * Try to batch up multiple inserts using multi-row inserts. */ if (mdb->changes == 0) { - Mmsg(cmd, "INSERT INTO batch VALUES " + Mmsg(cmd, "INSERT INTO batch(FileIndex, JobId, Path, Name, LStat, MD5, DeltaSeq) VALUES " "(%d,%s,'%s','%s','%s','%s',%u)", ar->FileIndex, edit_int64(ar->JobId,ed1), mdb->esc_path, mdb->esc_name, ar->attr, digest, ar->DeltaSeq); diff --git a/bacula/src/cats/sql_cmds.c b/bacula/src/cats/sql_cmds.c index 7446f92ebe..f5ea4ae656 100644 --- a/bacula/src/cats/sql_cmds.c +++ b/bacula/src/cats/sql_cmds.c @@ -126,7 +126,7 @@ const char *uar_del_temp = "DROP TABLE IF EXISTS temp"; const char *uar_del_temp1 = "DROP TABLE IF EXISTS temp1"; const char *uar_last_full = - "INSERT INTO temp1 SELECT Job.JobId,JobTdate " + "INSERT INTO temp1(JobId,JobTDate) SELECT Job.JobId,JobTdate " "FROM Client,Job,JobMedia,Media,FileSet WHERE Client.ClientId=%s " "AND Job.ClientId=%s " "AND Job.StartTime < '%s' " @@ -140,7 +140,9 @@ const char *uar_last_full = "ORDER BY Job.JobTDate DESC LIMIT 1"; const char *uar_full = - "INSERT INTO temp SELECT Job.JobId,Job.JobTDate," + "INSERT INTO temp (JobId,JobTDate,ClientId,Level,JobFiles,JobBytes,StartTime,VolumeName,StartFile,VolSessionId,VolSessionTime) " + + "SELECT Job.JobId,Job.JobTDate," "Job.ClientId,Job.Level,Job.JobFiles,Job.JobBytes," "StartTime,VolumeName,JobMedia.StartFile,VolSessionId,VolSessionTime " "FROM temp1,Job,JobMedia,Media WHERE temp1.JobId=Job.JobId " @@ -150,7 +152,8 @@ const char *uar_full = "AND JobMedia.MediaId=Media.MediaId"; const char *uar_dif = - "INSERT INTO temp SELECT Job.JobId,Job.JobTDate,Job.ClientId," + "INSERT INTO temp (JobId,JobTDate,ClientId,Level,JobFiles,JobBytes,StartTime,VolumeName,StartFile,VolSessionId,VolSessionTime) " + "SELECT Job.JobId,Job.JobTDate,Job.ClientId," "Job.Level,Job.JobFiles,Job.JobBytes," "Job.StartTime,Media.VolumeName,JobMedia.StartFile," "Job.VolSessionId,Job.VolSessionTime " @@ -167,7 +170,8 @@ const char *uar_dif = "ORDER BY Job.JobTDate DESC LIMIT 1"; const char *uar_inc = - "INSERT INTO temp SELECT Job.JobId,Job.JobTDate,Job.ClientId," + "INSERT INTO temp (JobId,JobTDate,ClientId,Level,JobFiles,JobBytes,StartTime,VolumeName,StartFile,VolSessionId,VolSessionTime) " + "SELECT Job.JobId,Job.JobTDate,Job.ClientId," "Job.Level,Job.JobFiles,Job.JobBytes," "Job.StartTime,Media.VolumeName,JobMedia.StartFile," "Job.VolSessionId,Job.VolSessionTime " @@ -409,9 +413,9 @@ const char *select_recent_version[] = /* We don't create this table as TEMPORARY because MySQL MyISAM 5.0 and 5.1 are unable to run further queries in this mode */ -static const char *create_temp_accurate_jobids_default = - "CREATE TABLE btemp3%s AS " - "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles " +static const char *create_temp_accurate_jobids_default = + "CREATE TABLE btemp3%s AS " + "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles, FileSetId " "FROM Job JOIN FileSet USING (FileSetId) " "WHERE ClientId = %s " "AND Level='F' AND JobStatus IN ('T','W') AND Type='B' " @@ -419,23 +423,35 @@ static const char *create_temp_accurate_jobids_default = "AND FileSet.FileSet=(SELECT FileSet FROM FileSet WHERE FileSetId = %s) " " %s " /* Any filter */ "ORDER BY Job.JobTDate DESC LIMIT 1"; - + +static const char *create_temp_accurate_jobids_mysql = + "CREATE TABLE btemp3%s /*PKEY (DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY)*/ AS (" + "SELECT JobId, StartTime, EndTime, JobTDate, PurgedFiles, FileSetId " + "FROM Job JOIN FileSet USING (FileSetId) " + "WHERE ClientId = %s " + "AND Level='F' AND JobStatus IN ('T','W') AND Type='B' " + "AND StartTime<'%s' " + "AND FileSet.FileSet=(SELECT FileSet FROM FileSet WHERE FileSetId = %s) " + " %s " /* Any filter */ + "ORDER BY Job.JobTDate DESC LIMIT 1)"; + const char *create_temp_accurate_jobids[] = { /* MySQL */ - create_temp_accurate_jobids_default, + create_temp_accurate_jobids_mysql, /* PostgreSQL */ - create_temp_accurate_jobids_default, + create_temp_accurate_jobids_default, /* SQLite */ create_temp_accurate_jobids_default -}; +}; const char *create_temp_basefile[] = { /* MySQL */ "CREATE TEMPORARY TABLE basefile%lld" - "(Path BLOB NOT NULL, Name BLOB NOT NULL," - " INDEX (Path(255), Name(255)))", + "(Path BLOB NOT NULL, Name BLOB NOT NULL, " + "INDEX (Path(255), Name(255))" + "/*PKEY, DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY*/)", /* PostgreSQL */ "CREATE TEMPORARY TABLE basefile%lld" "(Path TEXT, Name TEXT)", @@ -447,7 +463,7 @@ const char *create_temp_basefile[] = const char *create_temp_new_basefile[] = { /* MySQL */ - "CREATE TEMPORARY TABLE new_basefile%lld AS " + "CREATE TEMPORARY TABLE new_basefile%lld /*PKEY(DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY)*/ AS " "SELECT Path.Path AS Path, Temp.Filename AS Name, Temp.FileIndex AS FileIndex," " Temp.JobId AS JobId, Temp.LStat AS LStat, Temp.FileId AS FileId," " Temp.MD5 AS MD5" @@ -485,7 +501,8 @@ const char *create_deltabs[] = "PurgedFiles TINYINT, " "FileSetId INTEGER UNSIGNED, " "JobFiles INTEGER UNSIGNED, " - "JobStatus BINARY(1))", + "JobStatus BINARY(1)" + "/*PKEY, DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY*/)", /* PostgreSQL */ "CREATE TEMPORARY TABLE DelCandidates ( " @@ -509,7 +526,7 @@ const char *create_deltabs[] = * to a Backup job when the original backup job is expired. */ static const char *uap_upgrade_copies_oldest_job_default = -"CREATE TEMPORARY TABLE cpy_tmp AS " + "CREATE TEMPORARY TABLE cpy_tmp /*PKEY (DummyPkey INTEGER PRIMARY KEY DEFAULT 1)*/ AS " "SELECT MIN(JobId) AS JobId FROM Job " /* Choose the oldest job */ "WHERE Type='%c' " /* JT_JOB_COPY */ "AND ( PriorJobId IN (%s) " /* JobId selection */ @@ -583,7 +600,8 @@ const char *uar_create_temp[] = "VolumeName TEXT," "StartFile INTEGER UNSIGNED," "VolSessionId INTEGER UNSIGNED," - "VolSessionTime INTEGER UNSIGNED)", + "VolSessionTime INTEGER UNSIGNED" + "/*PKEY, DummyPKey INTEGER AUTO_INCREMENT PRIMARY KEY*/)", /* PostgreSQL */ "CREATE TEMPORARY TABLE temp (" @@ -619,7 +637,8 @@ const char *uar_create_temp1[] = /* MySQL */ "CREATE TEMPORARY TABLE temp1 (" "JobId INTEGER UNSIGNED NOT NULL," - "JobTDate BIGINT UNSIGNED)", + "JobTDate BIGINT UNSIGNED" + "/*PKEY, DummyPKey INTEGER AUTO_INCREMENT PRIMARY KEY*/)", /* PostgreSQL */ "CREATE TEMPORARY TABLE temp1 (" "JobId INTEGER NOT NULL," @@ -694,7 +713,21 @@ const char *sql_get_max_connections[] = * version of a file in the same dataset. */ const char *default_sql_bvfs_select = -"CREATE TABLE %s AS " +"CREATE TABLE %s AS" +"SELECT File.JobId, File.FileIndex, File.FileId " +"FROM Job, File, ( " + "SELECT MAX(JobTDate) AS JobTDate, PathId, Filename " + "FROM btemp%s GROUP BY PathId, Filename " + ") AS T1 " +"WHERE T1.JobTDate = Job.JobTDate " + "AND Job.JobId = File.JobId " + "AND T1.PathId = File.PathId " + "AND T1.Filename = File.Filename " + "AND File.FileIndex > 0 " + "AND Job.JobId IN (SELECT DISTINCT JobId FROM btemp%s) "; + +const char *default_sql_bvfs_select_mysql = +"CREATE TABLE %s /*PKEY (DummyPkey INTEGER AUTO_INCREMENT PRIMARY KEY)*/ AS " "SELECT File.JobId, File.FileIndex, File.FileId " "FROM Job, File, ( " "SELECT MAX(JobTDate) AS JobTDate, PathId, Filename " @@ -710,7 +743,7 @@ const char *default_sql_bvfs_select = const char *sql_bvfs_select[] = { /* MySQL */ - default_sql_bvfs_select, + default_sql_bvfs_select_mysql, /* PostgreSQL */ "CREATE TABLE %s AS ( " "SELECT JobId, FileIndex, FileId " diff --git a/bacula/src/dird/ua_prune.c b/bacula/src/dird/ua_prune.c index 066bf366a9..5b05c327fe 100644 --- a/bacula/src/dird/ua_prune.c +++ b/bacula/src/dird/ua_prune.c @@ -631,7 +631,7 @@ int prune_jobs(UAContext *ua, CLIENT *client, POOL *pool, int JobType) * and add them into the "DeletionCandidates" table. */ Mmsg(query, - "INSERT INTO DelCandidates " + "INSERT INTO DelCandidates(JobId, PurgedFiles, FileSetId, JobFiles, JobStatus) " "SELECT JobId,PurgedFiles,FileSetId,JobFiles,JobStatus " "FROM Job %s " /* JOIN Pool/Client */ "WHERE Type IN ('B', 'C', 'M', 'V', 'D', 'R', 'c', 'm', 'g') "