From: Marcin Haba Date: Mon, 17 Apr 2023 07:54:58 +0000 (+0200) Subject: baculum: Rework and improve sources endpoint X-Git-Tag: Release-13.0.3~78 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=bd6139fd5a546efaecf69a630066cc372581c604;p=thirdparty%2Fbacula.git baculum: Rework and improve sources endpoint --- diff --git a/gui/baculum/protected/API/Modules/SourceManager.php b/gui/baculum/protected/API/Modules/SourceManager.php index 052e4dea1..bf3629a77 100644 --- a/gui/baculum/protected/API/Modules/SourceManager.php +++ b/gui/baculum/protected/API/Modules/SourceManager.php @@ -22,6 +22,11 @@ namespace Baculum\API\Modules; +use PDO; +use Prado\Prado; +use Baculum\API\Modules\ConsoleOutputPage; +Prado::using('Baculum.API.Pages.API.JobsShow'); + /** * Source (client + fileset + job) manager module. * @@ -31,38 +36,72 @@ namespace Baculum\API\Modules; */ class SourceManager extends APIModule { - public function getSources($criteria = [], $limit_val = null, $offset_val = 0) { - $limit = ''; - if(is_int($limit_val) && $limit_val > 0) { - $limit = ' LIMIT ' . $limit_val; - } - $offset = ''; - if (is_int($offset_val) && $offset_val > 0) { - $offset = ' OFFSET ' . $offset_val; + /** + * Source result modes. + * Modes: + * - normal - record list without any additional data + * - overview - record list in sections (vSphere, MySQL, PostgreSQL...) with items and summary count + */ + const SOURCE_RESULT_MODE_NORMAL = 'normal'; + const SOURCE_RESULT_MODE_OVERVIEW = 'overview'; + + public function getSources($criteria = [], $props = [], $limit_val = 0, $offset_val = 0, $mode = self::SOURCE_RESULT_MODE_NORMAL) { + $jobs_show = new \JobsShow; + $config = $jobs_show->show( + ConsoleOutputPage::OUTPUT_FORMAT_JSON + ); + + $fs_content = $this->getFileSetAndContent(); + $fs_count = []; + + $sources = $jobs = []; + if ($config->exitcode === 0) { + for ($i = 0; $i < count($config->output); $i++) { + if (!key_exists('name', $config->output[$i])) { + continue; + } + if ($config->output[$i]['jobtype'] != 66) { + // only backup jobs are taken into account in sources + continue; + } + $job = $config->output[$i]['name']; + $fileset = $config->output[$i]['fileset']; + $client = $config->output[$i]['client']; + + // Use filters + if (key_exists('job', $props) && $props['job'] != $job) { + continue; + } elseif (key_exists('fileset', $props) && $props['fileset'] != $fileset) { + continue; + } elseif (key_exists('client', $props) && $props['client'] != $client) { + continue; + } + + $jobs[] = $job; + + $sources[] = [ + 'job' => $job, + 'client' => $client, + 'fileset' => $fileset + ]; + } } + + $where_job = count($jobs) > 0 ? ' Job.Name IN (\'' . implode('\',\'', $jobs) . '\') ' : ''; + $where = Database::getWhere($criteria, true); - $sql = 'SELECT DISTINCT - sres.fileset AS fileset, - sres.client AS client, - sres.job AS job, + $sql = 'SELECT + ores.job AS job, jres.StartTime AS starttime, jres.EndTime AS endtime, ores.jobid AS jobid, - fres.Content AS content, + regexp_split_to_table(fres.Content, \',\') AS content, jres.Type AS type, jres.JobStatus AS jobstatus, jres.JobErrors AS joberrors FROM Job AS jres, FileSet AS fres, - ( - SELECT DISTINCT - FileSet.FileSet AS fileset, - Client.Name AS client, - Job.Name AS job - FROM Job - JOIN FileSet USING (FileSetId) - JOIN Client USING (ClientId) - ) AS sres, ( + ( SELECT MAX(JobId) AS jobid, FileSet.FileSet AS fileset, @@ -71,25 +110,143 @@ class SourceManager extends APIModule { FROM Job JOIN FileSet USING (FileSetId) JOIN Client USING (ClientId) + ' . (!empty($where_job) ? ' WHERE ' . $where_job : '') . ' GROUP BY FileSet.FileSet, Client.Name, Job.Name ) AS ores LEFT JOIN Object USING (JobId) WHERE jres.JobId = ores.jobid AND jres.FileSetId = fres.FileSetId - AND sres.job = ores.job - AND sres.client = ores.client - AND sres.fileset = ores.fileset AND jres.Type = \'B\' ' . (!empty($where['where']) ? ' AND ' . $where['where'] : '') . ' - ORDER BY sres.fileset ASC, sres.client ASC, sres.job ASC, jres.starttime ASC ' . $limit . $offset; - - $connection = SourceRecord::finder()->getDbConnection(); - $connection->setActive(true); - $pdo = $connection->getPdoInstance(); - $sth = $pdo->prepare($sql); - $sth->execute($where['params']); - return $sth->fetchAll(\PDO::FETCH_ASSOC); + ORDER BY jres.starttime ASC'; + + $statement = Database::runQuery($sql, $where['params']); + $result = $statement->fetchAll(PDO::FETCH_GROUP | PDO::FETCH_ASSOC); + + /* + * This query is only to know if source is filtered. + * Otherwise there is not possible to make a difference between + * sources filtered and sources that have not been exected yet. + * It there will be an other way to make this difference, please + * remove this query. + */ + $sql = 'SELECT DISTINCT + Job.Name AS job, + FileSet.FileSet AS fileset, + Client.Name AS client + FROM Job + JOIN FileSet USING (FileSetId) + JOIN Client USING (ClientId) + WHERE Job.Type = \'B\''; + $statement = Database::runQuery($sql); + $jobs_all = $statement->fetchAll(PDO::FETCH_GROUP | PDO::FETCH_ASSOC); + + function is_item($data, $item, $type) { + $found = false; + for ($i = 0; $i < count($data); $i++) { + if ($data[$i][$type] === $item) { + $found = true; + break; + } + } + return $found; + } + + $sources_len = count($sources); + $sources_ft = []; + for ($i = 0; $i < $sources_len; $i++) { + if (key_exists($sources[$i]['job'], $result)) { + // job has jobids in the catalog - job was executed + $sources[$i] = array_merge($sources[$i], $result[$sources[$i]['job']][0]); + + } elseif (key_exists($sources[$i]['job'], $jobs_all) && is_item($jobs_all[$sources[$i]['job']], $sources[$i]['client'], 'client') && is_item($jobs_all[$sources[$i]['job']], $sources[$i]['fileset'], 'fileset')) { + // Source exists but it has been filtered + continue; + } elseif (count($criteria) > 0) { + // If SQL criteria used, all sources that has not been executed anytime are hidden. + continue; + } else { + // No jobs in the catalog - source was not executed and no SQL criteria used. + $sources[$i]['starttime'] = ''; + $sources[$i]['endtime'] = ''; + $sources[$i]['jobid'] = 0; + $sources[$i]['content'] = ''; + $sources[$i]['type'] = ''; + $sources[$i]['jobstatus'] = ''; + $sources[$i]['joberrors'] = ''; + } + + // Set count for each section. It has to be done for all sources here. + $fileset = $sources[$i]['fileset']; + if (key_exists($fileset, $fs_content)) { + for ($j = 0; $j < count($fs_content[$fileset]); $j++) { + $content = $fs_content[$fileset][$j]['content']; + if (!key_exists($content, $fs_count)) { + $fs_count[$content] = 0; + } + $fs_count[$content]++; + } + } + $sources_ft[] = $sources[$i]; + } + + // below work only with filtered jobs + $sources = $sources_ft; + + if ($mode == self::SOURCE_RESULT_MODE_OVERVIEW) { + // Overview mode. + $res = []; + $ofs = []; + $sources_len = count($sources); + for ($i = 0; $i < $sources_len; $i++) { + $content = $fs_content[$sources[$i]['fileset']]; + $content_len = count($content); + for ($j = 0; $j < $content_len; $j++) { + if (!key_exists($content[$j]['content'], $res)) { + $res[$content[$j]['content']] = [ + 'sources' => [], + 'count' => $fs_count[$content[$j]['content']] + ]; + $ofs[$content[$j]['content']] = 0; + } + if ($offset_val > 0 && $ofs[$content[$j]['content']] < $offset_val) { + // offset taken into account + $ofs[$content[$j]['content']]++; + continue; + } + if ($limit_val > 0 && count($res[$content[$j]['content']]['sources']) >= $limit_val) { + // limit reached + break; + } + $sources[$i]['content'] = $content[$j]['content']; + + $res[$content[$j]['content']]['sources'][] = $sources[$i]; + } + } + $sources = $res; + } else { + if ($limit_val > 0) { + $sources = array_slice($sources, $offset_val, $limit_val); + } elseif ($limit_val == 0 && $offset_val > 0) { + $sources = array_slice($sources, $offset_val); + } + } + return $sources; + } + + /** + * Get fileset and content. + * + * @return array content counts + */ + public function getFileSetAndContent() { + $sql = 'SELECT DISTINCT FileSet.FileSet AS fileset, + regexp_split_to_table(FileSet.Content, \',\') AS content + FROM + FileSet'; + $statement = Database::runQuery($sql); + return $statement->fetchAll(PDO::FETCH_GROUP | PDO::FETCH_ASSOC); } } ?> diff --git a/gui/baculum/protected/API/Pages/API/Sources.php b/gui/baculum/protected/API/Pages/API/Sources.php index 9b1c56f05..a2e561000 100644 --- a/gui/baculum/protected/API/Pages/API/Sources.php +++ b/gui/baculum/protected/API/Pages/API/Sources.php @@ -21,6 +21,7 @@ */ use Baculum\API\Modules\BaculumAPIServer; +use Baculum\API\Modules\SourceManager; use Baculum\Common\Modules\Errors\SourceError; /** @@ -36,33 +37,30 @@ class Sources extends BaculumAPIServer { $misc = $this->getModule('misc'); $limit = $this->Request->contains('limit') && $misc->isValidInteger($this->Request['limit']) ? (int)$this->Request['limit'] : 0; $offset = $this->Request->contains('offset') && $misc->isValidInteger($this->Request['offset']) ? (int)$this->Request['offset'] : 0; - $job = $this->Request->contains('job') && $misc->isValidName($this->Request['job']) ? $this->Request['job'] : ''; - $client = $this->Request->contains('client') && $misc->isValidName($this->Request['client']) ? $this->Request['client'] : ''; - $fileset = $this->Request->contains('fileset') && $misc->isValidName($this->Request['fileset']) ? $this->Request['fileset'] : ''; + $job = $this->Request->contains('job') && $misc->isValidName($this->Request['job']) ? $this->Request['job'] : null; + $client = $this->Request->contains('client') && $misc->isValidName($this->Request['client']) ? $this->Request['client'] : null; + $fileset = $this->Request->contains('fileset') && $misc->isValidName($this->Request['fileset']) ? $this->Request['fileset'] : null; $starttime_from = $this->Request->contains('starttime_from') && $misc->isValidInteger($this->Request['starttime_from']) ? (int)$this->Request['starttime_from'] : null; $starttime_to = $this->Request->contains('starttime_to') && $misc->isValidInteger($this->Request['starttime_to']) ? (int)$this->Request['starttime_to'] : null; $endtime_from = $this->Request->contains('endtime_from') && $misc->isValidInteger($this->Request['endtime_from']) ? (int)$this->Request['endtime_from'] : null; $endtime_to = $this->Request->contains('endtime_to') && $misc->isValidInteger($this->Request['endtime_to']) ? (int)$this->Request['endtime_to'] : null; $jobstatus = $this->Request->contains('jobstatus') && $misc->isValidState($this->Request['jobstatus']) ? $this->Request['jobstatus'] : ''; $hasobject = $this->Request->contains('hasobject') && $misc->isValidBoolean($this->Request['hasobject']) ? $this->Request['hasobject'] : null; + $mode = ($this->Request->contains('overview') && $misc->isValidBooleanTrue($this->Request['overview'])) ? SourceManager::SOURCE_RESULT_MODE_OVERVIEW : SourceManager::SOURCE_RESULT_MODE_NORMAL; - // @TODO: Fix using sres and jres in this place. It can lead to a problem when sres or jres will be changed in manager. - $params = []; - if (!empty($job)) { - $params['sres.job'] = [[ - 'vals' => $job - ]]; + $props = []; + if (!is_null($job)) { + $props['job'] = $job; } - if (!empty($client)) { - $params['sres.client'] = [[ - 'vals' => $client - ]]; + if (!is_null($client)) { + $props['client'] = $client; } - if (!empty($fileset)) { - $params['sres.fileset'] = [[ - 'vals' => $fileset - ]]; + if (!is_null($fileset)) { + $props['fileset'] = $fileset; } + + // @TODO: Fix using jres in this place. It can lead to a problem when sres or jres will be changed in manager. + $params = []; if (!empty($jobstatus)) { $params['jres.jobstatus'] = [[ 'vals' => $jobstatus @@ -116,7 +114,13 @@ class Sources extends BaculumAPIServer { } } - $sources = $this->getModule('source')->getSources($params, $limit, $offset); + $sources = $this->getModule('source')->getSources( + $params, + $props, + $limit, + $offset, + $mode + ); $this->output = $sources; $this->error = SourceError::ERROR_NO_ERRORS; } diff --git a/gui/baculum/protected/API/openapi_baculum.json b/gui/baculum/protected/API/openapi_baculum.json index a53e4c454..7119e415a 100644 --- a/gui/baculum/protected/API/openapi_baculum.json +++ b/gui/baculum/protected/API/openapi_baculum.json @@ -8092,7 +8092,7 @@ "name": "starttime_from", "in": "query", "required": false, - "description": "Start time from (UNIX timestamp format, seconds)", + "description": "Start time from (UNIX timestamp format, seconds). It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", "schema": { "type": "integer" } @@ -8101,7 +8101,7 @@ "name": "starttime_to", "in": "query", "required": false, - "description": "Start time to (UNIX timestamp format, seconds)", + "description": "Start time to (UNIX timestamp format, seconds). It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", "schema": { "type": "integer" } @@ -8110,7 +8110,7 @@ "name": "endtime_from", "in": "query", "required": false, - "description": "End time from (UNIX timestamp format, seconds)", + "description": "End time from (UNIX timestamp format, seconds). It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", "schema": { "type": "integer" } @@ -8119,7 +8119,7 @@ "name": "endtime_to", "in": "query", "required": false, - "description": "End time to (UNIX timestamp format, seconds)", + "description": "End time to (UNIX timestamp format, seconds). It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", "schema": { "type": "integer" } @@ -8128,7 +8128,7 @@ "name": "jobstatus", "in": "query", "required": false, - "description": "Job status letter", + "description": "Job status letter. It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", "schema": { "type": "string", "enum": ["C", "R", "B", "T", "W", "E", "e", "f", "D", "A", "I", "F", "S", "m", "M", "s", "j", "c", "d", "t", "p", "i", "a", "l", "L"] @@ -8138,7 +8138,16 @@ "name": "hasobject", "in": "query", "required": false, - "description": "Show sources that have object, 1 - show sources with objects created, 0 - show sources without objects created", + "description": "Show sources that have object, 1 - show sources with objects created, 0 - show sources without objects created. It applies for sources that job has been executed at least one time. For sources that job has not been executed anytime this filter not work.", + "schema": { + "type": "boolean" + } + }, + { + "name": "overview", + "in": "query", + "required": false, + "description": "If set, it puts sources in source categories - content field of FileSet (files, gw, m365, vSphere...etc.). NOTE: Offset and limit parameters do not apply to overview counts.", "schema": { "type": "boolean" }