From: Marcin Haba Date: Mon, 27 Feb 2023 09:22:16 +0000 (+0100) Subject: baculum: Add a new endpoint to list jobs together with objects X-Git-Tag: Release-13.0.3~129 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=aa785f77a2c92b542d730b4fa245b2636be74f82;p=thirdparty%2Fbacula.git baculum: Add a new endpoint to list jobs together with objects Other changes: - add realstarttime_from and realstarttime_to filters to job list and job object list endpoints - general performance improvements and optimizations - add job result modes - refactor job overview part --- diff --git a/gui/baculum/protected/API/Modules/APIDbModule.php b/gui/baculum/protected/API/Modules/APIDbModule.php index 3a683c896..04d7c670a 100644 --- a/gui/baculum/protected/API/Modules/APIDbModule.php +++ b/gui/baculum/protected/API/Modules/APIDbModule.php @@ -37,6 +37,11 @@ use Prado\Data\TDbConnection; */ class APIDbModule extends TActiveRecord { + /** + * API database connection handler. + */ + private static $db_connection; + /** * Get Data Source Name (DSN). * @@ -87,28 +92,28 @@ class APIDbModule extends TActiveRecord { * @throws BCatalogException if cataloga access is not supported */ public static function getAPIDbConnection(array $db_params, $force = false) { - $db_connection = null; if ((array_key_exists('enabled', $db_params) && $db_params['enabled'] === '1') || $force === true) { - $dsn = self::getDsn($db_params); - $db_connection = null; - if (array_key_exists('login', $db_params) && array_key_exists('password', $db_params)) { - $db_connection = new TDbConnection($dsn, $db_params['login'], $db_params['password']); - } else { - $db_connection = new TDbConnection($dsn); + if (is_null(self::$db_connection)) { + $dsn = self::getDsn($db_params); + if (array_key_exists('login', $db_params) && array_key_exists('password', $db_params)) { + self::$db_connection = new TDbConnection($dsn, $db_params['login'], $db_params['password']); + } else { + self::$db_connection = new TDbConnection($dsn); + } + self::$db_connection->setActive(true); + if ($db_params['type'] === Database::MYSQL_TYPE) { + self::$db_connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); + } + self::$db_connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); + self::$db_connection->setAttribute(PDO::ATTR_CASE, PDO::CASE_LOWER); } - $db_connection->setActive(true); - if ($db_params['type'] === Database::MYSQL_TYPE) { - $db_connection->setAttribute(PDO::ATTR_EMULATE_PREPARES, false); - } - $db_connection->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION); - $db_connection->setAttribute(PDO::ATTR_CASE, PDO::CASE_LOWER); } else { throw new BCatalogException( DatabaseError::MSG_ERROR_DATABASE_ACCESS_NOT_SUPPORTED, DatabaseError::ERROR_DATABASE_ACCESS_NOT_SUPPORTED ); } - return $db_connection; + return self::$db_connection; } public function getColumnValue($column_name) { diff --git a/gui/baculum/protected/API/Modules/JobManager.php b/gui/baculum/protected/API/Modules/JobManager.php index ec6803ded..c100e4b61 100644 --- a/gui/baculum/protected/API/Modules/JobManager.php +++ b/gui/baculum/protected/API/Modules/JobManager.php @@ -34,7 +34,126 @@ use Prado\Data\ActiveRecord\TActiveRecordCriteria; */ class JobManager extends APIModule { - public function getJobs($criteria = array(), $limit_val = null, $offset_val = 0, $sort_col = 'JobId', $sort_order = 'ASC', $overview = false) { + /** + * SQL query builder. + * + * @var TDbCommandBuilder command builder + */ + private static $query_builder; + + /** + * Job statuses in some parts are not compatible with rest of the API. + * NOTE: Used here are also internal job statuses that are not used in the Catalog + * but they are used internally by Bacula. + */ + private $js_successful = ['T']; + private $js_unsuccessful = ['A', 'E', 'f']; + private $js_warning = ['I', 'e']; + private $js_running = ['C', 'B', 'D', 'F', 'L', 'M', 'R', 'S', 'a', 'c', 'd', 'i', 'j', 'l', 'm', 'p', 'q', 's', 't']; + + /** + * Job result in job and object endpoint can be displayed in on of the two views: + * - basic - display only base job and object properties + * - full - display all properties + * Here are job properties for basic view. + */ + private $basic_mode_job_props = [ + 'Job.JobId', + 'Job.Job', + 'Job.Name', + 'Job.Type', + 'Job.Level', + 'Job.JobStatus', + 'Job.SchedTime', + 'Job.RealEndTime', + 'Job.JobFiles', + 'Job.JobBytes', + 'Job.JobErrors', + 'Job.Reviewed', + 'Job.Comment', + 'Job.RealStartTime', + 'Job.IsVirtualFull', + 'Job.CompressRatio', + 'Job.Rate', + 'Job.StatusInfo', + 'Job.Encrypted', + 'Fileset.Content' + ]; + + /** + * Job statuses. + * @see JobManager::getJobsObjectsOverview() + */ + const JS_GROUP_SUCCESSFUL = 'successful'; + const JS_GROUP_UNSUCCESSFUL = 'unsuccessful'; + const JS_GROUP_WARNING = 'warning'; + const JS_GROUP_RUNNING = 'running'; + const JS_GROUP_ALL_TERMINATED = 'all_terminated'; + + /** + * Job result modes. + * Modes: + * - normal - job record list without any additional data + * - overview - job record list with some summary (successful, unsuccessful, warning...) + * - group - job record list with grouped by jobid (jobids as keys) + */ + const JOB_RESULT_MODE_NORMAL = 'normal'; + const JOB_RESULT_MODE_OVERVIEW = 'overview'; + const JOB_RESULT_MODE_GROUP = 'group'; + + /** + * Job result record view. + * Views: + * - basic - list only limited record properties + * - full - list all record properties + */ + const JOB_RESULT_VIEW_BASIC = 'basic'; + const JOB_RESULT_VIEW_FULL = 'full'; + + /** + * Get the SQL query builder instance. + * Note: Singleton + * + * @return TDbCommandBuilder command builder + */ + private function getQueryBuilder() { + if (is_null(self::$query_builder)) { + $record = JobRecord::finder(); + $connection = $record->getDbConnection(); + $tableInfo = $record->getRecordGateway()->getRecordTableInfo($record); + self::$query_builder = $tableInfo->createCommandBuilder($connection); + } + return self::$query_builder; + } + + /** + * Get job status groups. + * + * @return array job status groups + */ + private function getJSGroups() { + return [ + self::JS_GROUP_SUCCESSFUL, + self::JS_GROUP_UNSUCCESSFUL, + self::JS_GROUP_WARNING, + self::JS_GROUP_RUNNING, + self::JS_GROUP_ALL_TERMINATED + ]; + } + + /** + * Get job list. + * + * @param array $criteria SQL criteria to get job list + * @param mixed $limit_val result limit value + * @param int $offset_val result offset value + * @param string $sort_col sort by selected SQL column (default: JobId) + * @param string $sort_order sort order:'ASC' or 'DESC' (default: ASC, ascending) + * @param string $mode job result mode (normal, overview, group) + * @param string $view job records view (basic, full) + * @return array job list records or empty list if no job found + */ + public function getJobs($criteria = array(), $limit_val = null, $offset_val = 0, $sort_col = 'JobId', $sort_order = 'ASC', $mode = self::JOB_RESULT_MODE_NORMAL, $view = self::JOB_RESULT_VIEW_FULL) { $db_params = $this->getModule('api_config')->getConfig('db'); if ($db_params['type'] === Database::PGSQL_TYPE) { $sort_col = strtolower($sort_col); @@ -51,7 +170,12 @@ class JobManager extends APIModule { $where = Database::getWhere($criteria); - $sql = 'SELECT Job.*, + $job_record = 'Job.*,'; + if ($view == self::JOB_RESULT_VIEW_BASIC) { + $job_record = implode(',', $this->basic_mode_job_props) . ','; + } + + $sql = 'SELECT ' . $job_record . ' Client.Name as client, Pool.Name as pool, FileSet.FileSet as fileset @@ -61,40 +185,230 @@ LEFT JOIN Pool USING (PoolId) LEFT JOIN FileSet USING (FilesetId)' . $where['where'] . $order . $limit . $offset; - $result = JobRecord::finder()->findAllBySql($sql, $where['params']); + $builder = $this->getQueryBuilder(); + $command = $builder->applyCriterias($sql, $where['params']); + $statement = $command->getPdoStatement(); + $command->query(); + $result = []; + if ($mode == self::JOB_RESULT_MODE_OVERVIEW) { + // Overview mode. + $result = $statement->fetchAll(\PDO::FETCH_OBJ); + $result = [ + 'jobs' => $result, + 'overview' => $this->getJobCountByJSGroup($criteria) + ]; + } elseif ($mode == self::JOB_RESULT_MODE_GROUP) { + // Group mode. + $result = $statement->fetchAll(\PDO::FETCH_GROUP | \PDO::FETCH_OBJ); + } else { + // Normal mode. + $result = $statement->fetchAll(\PDO::FETCH_OBJ); + } + return $result; + } + + /** + * Get job records with objects in one of the two flavours: normal or overview. + * + * @param array $criteria SQL criteria to get job list + * @param mixed $limit_val result limit value + * @param int $offset_val result offset value + * @param string $sort_col sort by selected SQL column (default: JobId) + * @param string $sort_order sort order:'ASC' or 'DESC' (default: ASC, ascending) + * @param mixed $object_limit limit for object results + * @param bool $overview if true, results are displayed in overview mode, otherwise normal mode + * @param string $view job records view (basic, full) + * @return array job record list with objects or empty list if no job found + */ + public function getJobsObjectsOverview($criteria = array(), $limit_val = null, $offset_val = 0, $sort_col = 'Job.JobId', $sort_order = 'ASC', $object_limit = null, $overview = false, $view = self::JOB_RESULT_VIEW_FULL) { + + // First get total job count by job status group + $job_count_by_js_group_criteria = []; + if (key_exists('Job.Name', $criteria)) { + $job_count_by_js_group_criteria = [ + 'Job.Name' => $criteria['Job.Name'] + ]; + } + $job_count_by_js_group = $this->getJobCountByJSGroup( + $job_count_by_js_group_criteria + ); + + + // Then get job list + $job_list_result = $this->getJobs( + $criteria, + $limit_val, + $offset_val, + $sort_col, + $sort_order, + self::JOB_RESULT_MODE_GROUP, + $view + ); + + // Prepare job identifiers and job records + $jobids = array_keys($job_list_result); + $job_list = array_values($job_list_result); + + $obj_criteria = $criteria; // we use the same criteria as for jobs plus limit jobids + if (count($jobids) > 0) { + // Prepare object criteria + $obj_criteria['Job.JobId'] = []; + $obj_criteria['Job.JobId'][] = [ + 'operator' => 'IN', + 'vals' => $jobids + ]; + } + + // Get objects + $obj = $this->getModule('object'); + $object_list = $obj->getObjects( + $obj_criteria, + null, + 0, + 'ObjectId', + 'ASC', + 'jobid', + false, + $view + ); + + // Get object categories for jobs + $ocs = $obj->getObjectCategories( + $criteria + ); + $ocs_count = count($ocs); + + $out = []; + $ovw = []; + $js_groups = []; + if ($overview) { + $js_groups = $this->getJSGroups(); + + // Init overview results + for ($i = 0; $i < count($js_groups); $i++) { + $ovw[$js_groups[$i]] = ['count' => 0, 'jobs' => []]; + } + } + + $job_list_count = count($job_list); + for ($i = 0; $i < $job_list_count; $i++) { + $jobid = $jobids[$i]; + $job_list[$i][0]->jobid = $jobid; + $job_obj_list = key_exists($jobid, $object_list) ? $object_list[$jobid] : []; + $job = [ + 'job' => $job_list[$i][0], + 'objects' => [ + 'overview' => [], + 'totalcount' => count($job_obj_list) + ] + ]; + for ($j = 0; $j < $ocs_count; $j++) { + // current object category + $objectcategory = $ocs[$j]['objectcategory']; + + // Take only objects from current category + $job_obj_cat_list = array_filter($job_obj_list, function ($item) use ($objectcategory) { + return ($item->objectcategory == $objectcategory); + }); + $job_obj_cat_count = count($job_obj_cat_list); + if ($job_obj_cat_count == 0) { + // empty categories are not listed + continue; + } + + // Prepare object slice if limit used, otherwise take all objects + $job_obj_cat_list_f = is_int($object_limit) && $object_limit > 0 ? array_slice($job_obj_cat_list, 0, $object_limit) : $job_obj_cat_list; + + $job['objects']['overview'][$objectcategory] = [ + 'count' => $job_obj_cat_count, + 'objects' => $job_obj_cat_list_f + ]; + } + if ($overview) { + // Overview mode. + // Put jobs to specific categories + if (in_array($job['job']->jobstatus, $this->js_successful) && $job['job']->joberrors == 0) { + $ovw[self::JS_GROUP_SUCCESSFUL]['jobs'][] = $job; + } elseif (in_array($job['job']->jobstatus, $this->js_unsuccessful)) { + $ovw[self::JS_GROUP_UNSUCCESSFUL]['jobs'][] = $job; + } elseif (in_array($job['job']->jobstatus, $this->js_warning) || (in_array($job['job']->jobstatus, $this->js_successful) && $job['job']->joberrors > 0)) { + $ovw[self::JS_GROUP_WARNING]['jobs'][] = $job; + } elseif (in_array($job['job']->jobstatus, $this->js_running)) { + $ovw[self::JS_GROUP_RUNNING]['jobs'][] = $job; + } + if (!in_array($job['job']->jobstatus, $this->js_running)) { + $ovw[self::JS_GROUP_ALL_TERMINATED]['jobs'][] = $job; + } + + } else { + // Normal mode. + $out[$i] = $job; + } + } + + if ($overview) { + // Overview mode. + for ($i = 0; $i < count($js_groups); $i++) { + // Set all job count + $ovw[$js_groups[$i]]['count'] = $job_count_by_js_group[$js_groups[$i]]; + + if (is_int($limit_val) && $limit_val > 0) { + // If limit used, prepare a slice of jobs + $ovw[$js_groups[$i]]['jobs'] = array_slice($ovw[$js_groups[$i]]['jobs'], 0, $limit_val); + } + } + } + return ($overview ? $ovw : $out); + } + + /** + * Get job count by job status group. + * + * @param array $criteria SQL criteria + * @return array job count by job status group + */ + public function getJobCountByJSGroup($criteria = []) { + $where = Database::getWhere($criteria, true); + $cond = ''; + if (!empty($where['where'])) { + $cond = $where['where'] . ' AND '; + } + $sql = 'SELECT +(SELECT COUNT(1) FROM Job WHERE ' . $cond . ' Job.JobStatus IN (\'' . implode('\',\'', $this->js_successful) . '\') AND Job.JobErrors = 0) AS successful, +(SELECT COUNT(1) FROM Job WHERE ' . $cond . ' Job.JobStatus IN (\'' . implode('\',\'', $this->js_unsuccessful) . '\')) AS unsuccessful, +(SELECT COUNT(1) FROM Job WHERE ' . $cond . ' (Job.JobStatus IN (\'' . implode('\',\'', $this->js_warning) . '\') OR (Job.JobStatus IN (\'' . implode('\',\'', $this->js_successful) . '\') AND Job.JobErrors > 0))) AS warning, +(SELECT COUNT(1) FROM Job WHERE ' . $cond . ' Job.JobStatus IN (\'' . implode('\',\'', $this->js_running) . '\')) AS running, +(SELECT COUNT(1) FROM Job WHERE ' . $cond . ' Job.JobStatus NOT IN (\'' . implode('\',\'', $this->js_running) . '\')) AS all_terminated, +(SELECT COUNT(1) FROM Job ' . (!empty($where['where']) ? ' WHERE ' . $where['where'] : '') . ') AS all + '; + + $builder = $this->getQueryBuilder(); + if (count($where['params']) == 0) { /** - * Job statuses in some parts are not compatible with rest of the API. - * NOTE: Used here are also internal job statuses that are not used in the Catalog - * but they are used internally by Bacula. + * Please note that in case no params the TDbCommandBuilder::applyCriterias() + * returns empty the PDO statement handler. From this reason here + * the query is called directly by PDO. */ - $successful = ['T']; - $unsuccessful = ['A', 'E', 'f']; - $warning = ['I', 'e']; - $running = ['C', 'B', 'D', 'F', 'L', 'M', 'R', 'S', 'a', 'c', 'd', 'i', 'j', 'l', 'm', 'p', 'q', 's', 't']; - $sql = 'SELECT - (SELECT COUNT(1) FROM Job ' . $where['where'] . ' AND Job.JobStatus IN (\'' . implode('\',\'', $successful) . '\') AND Job.JobErrors = 0) AS successful, - (SELECT COUNT(1) FROM Job ' . $where['where'] . ' AND Job.JobStatus IN (\'' . implode('\',\'', $unsuccessful) . '\')) AS unsuccessful, - (SELECT COUNT(1) FROM Job ' . $where['where'] . ' AND (Job.JobStatus IN (\'' . implode('\',\'', $warning) . '\') OR (Job.JobStatus IN (\'' . implode('\',\'', $successful) . '\') AND JobErrors > 0))) AS warning, - (SELECT COUNT(1) FROM Job ' . $where['where'] . ' AND Job.JobStatus IN (\'' . implode('\',\'', $running) . '\')) AS running, - (SELECT COUNT(1) FROM Job ' . $where['where'] . ') AS all - '; + $connection = JobRecord::finder()->getDbConnection(); + $connection->setActive(true); + $pdo = $connection->getPdoInstance(); + $statement = $pdo->query($sql); - $record = JobRecord::finder(); - $connection = $record->getDbConnection(); - $tableInfo = $record->getRecordGateway()->getRecordTableInfo($record); - $builder = $tableInfo->createCommandBuilder($connection); + } else { $command = $builder->applyCriterias($sql, $where['params']); - $res = $command->query(); - $ov = $res->read(); - $result = [ - 'jobs' => $result, - 'overview' => $ov - ]; + $statement = $command->getPdoStatement(); + $command->query(); } - return $result; + return $statement->fetch(\PDO::FETCH_ASSOC); } + /** + * Get job record by job identifier. + * + * @param integer job identifier + * @return JobRecord|false job record or false is no job record found + */ public function getJobById($jobid) { $job = $this->getJobs(array( 'Job.JobId' => [[ @@ -104,6 +418,8 @@ LEFT JOIN FileSet USING (FilesetId)' ), 1); if (is_array($job) && count($job) > 0) { $job = array_shift($job); + } else { + $job = false; } return $job; } diff --git a/gui/baculum/protected/API/Modules/ObjectManager.php b/gui/baculum/protected/API/Modules/ObjectManager.php index fecc01ce4..2f0367ccd 100644 --- a/gui/baculum/protected/API/Modules/ObjectManager.php +++ b/gui/baculum/protected/API/Modules/ObjectManager.php @@ -31,6 +31,56 @@ namespace Baculum\API\Modules; */ class ObjectManager extends APIModule { + /** + * Object result in job and object endpoint can be displayed in on of the two views: + * - basic - display only base job and object properties + * - full - display all properties + * Here are object properties for basic view. + */ + private $basic_mode_obj_props = [ + 'ObjectId', + 'JobId', + 'ObjectCategory', + 'ObjectType', + 'ObjectName', + 'ObjectSource', + 'ObjectSize', + 'ObjectStatus', + 'ObjectCount' + ]; + + /** + * SQL query builder. + * + * @var TDbCommandBuilder command builder + */ + private static $query_builder; + + /** + * Object result record view. + * Views: + * - basic - list only limited record properties + * - full - list all record properties + */ + const OBJ_RESULT_VIEW_BASIC = 'basic'; + const OBJ_RESULT_VIEW_FULL = 'full'; + + + /** + * Get the SQL query builder instance. + * Note: Singleton + * + * @return TDbCommandBuilder command builder + */ + private static function getQueryBuilder() { + if (is_null(self::$query_builder)) { + $record = ObjectRecord::finder(); + $connection = $record->getDbConnection(); + $tableInfo = $record->getRecordGateway()->getRecordTableInfo($record); + self::$query_builder = $tableInfo->createCommandBuilder($connection); + } + return self::$query_builder; + } /** * Get objects. @@ -42,9 +92,10 @@ class ObjectManager extends APIModule * @param string $sort_order sort order (asc - ascending, desc - descending) * @param string $group_by column to group * @param integer $group_limit maximum number of elements in one group + * @param string $view job records view (basic, full) * @return array object list */ - public function getObjects($criteria = array(), $limit_val = null, $offset_val = 0, $sort_col = 'ObjectId', $sort_order = 'DESC', $group_by = null, $group_limit = 0) { + public function getObjects($criteria = array(), $limit_val = null, $offset_val = 0, $sort_col = 'ObjectId', $sort_order = 'DESC', $group_by = null, $group_limit = 0, $view = self::OBJ_RESULT_VIEW_FULL) { $db_params = $this->getModule('api_config')->getConfig('db'); if ($db_params['type'] === Database::PGSQL_TYPE) { $sort_col = strtolower($sort_col); @@ -71,17 +122,44 @@ class ObjectManager extends APIModule $where = Database::getWhere($criteria); - $sql = 'SELECT Object.*, -Job.Name as jobname + $obj_record = 'Object.*, Job.Name as jobname '; + if ($view == self::OBJ_RESULT_VIEW_BASIC) { + $obj_record = implode(',', $this->basic_mode_obj_props); + } + $sql = 'SELECT ' . $obj_record . ' FROM Object LEFT JOIN Job USING (JobId) ' . $where['where'] . $order . $limit . $offset; - $result = ObjectRecord::finder()->findAllBySql($sql, $where['params']); + $builder = $this->getQueryBuilder(); + $command = $builder->applyCriterias($sql, $where['params']); + $statement = $command->getPdoStatement(); + $command->query(); + $result = $statement->fetchAll(\PDO::FETCH_OBJ); Database::groupBy($group_by, $result, $group_limit); return $result; } + /** + * Get object categories based on criterias. + * + * @param array $criteria SQL criteria to get job list + * @return array category list or empty list if no category found + */ + public function getObjectCategories($criteria = []) { + $where = Database::getWhere($criteria); + + $sql = 'SELECT DISTINCT ObjectCategory as objectcategory +FROM Object +JOIN Job USING (JobId) ' +. $where['where']; + $builder = $this->getQueryBuilder(); + $command = $builder->applyCriterias($sql, $where['params']); + $statement = $command->getPdoStatement(); + $command->query(); + return $statement->fetchAll(\PDO::FETCH_ASSOC); + } + public function getObjectById($objectid) { $params = [ 'Object.ObjectId' => [[ diff --git a/gui/baculum/protected/API/Pages/API/Jobs.php b/gui/baculum/protected/API/Pages/API/Jobs.php index c166eb1de..2845e1be7 100644 --- a/gui/baculum/protected/API/Pages/API/Jobs.php +++ b/gui/baculum/protected/API/Pages/API/Jobs.php @@ -22,6 +22,7 @@ use Baculum\API\Modules\BaculumAPIServer; use Baculum\API\Modules\JobRecord; +use Baculum\API\Modules\JobManager; use Baculum\Common\Modules\Errors\JobError; /** @@ -50,12 +51,14 @@ class Jobs extends BaculumAPIServer { $starttime_to = $this->Request->contains('starttime_to') && $misc->isValidInteger($this->Request['starttime_to']) ? (int)$this->Request['starttime_to'] : null; $endtime_from = $this->Request->contains('endtime_from') && $misc->isValidInteger($this->Request['endtime_from']) ? (int)$this->Request['endtime_from'] : null; $endtime_to = $this->Request->contains('endtime_to') && $misc->isValidInteger($this->Request['endtime_to']) ? (int)$this->Request['endtime_to'] : null; + $realstarttime_from = $this->Request->contains('realstarttime_from') && $misc->isValidInteger($this->Request['realstarttime_from']) ? (int)$this->Request['realstarttime_from'] : null; + $realstarttime_to = $this->Request->contains('realstarttime_to') && $misc->isValidInteger($this->Request['realstarttime_to']) ? (int)$this->Request['realstarttime_to'] : null; $realendtime_from = $this->Request->contains('realendtime_from') && $misc->isValidInteger($this->Request['realendtime_from']) ? (int)$this->Request['realendtime_from'] : null; $realendtime_to = $this->Request->contains('realendtime_to') && $misc->isValidInteger($this->Request['realendtime_to']) ? (int)$this->Request['realendtime_to'] : null; $age = $this->Request->contains('age') && $misc->isValidInteger($this->Request['age']) ? (int)$this->Request['age'] : null; $order_by = $this->Request->contains('order_by') && $misc->isValidColumn($this->Request['order_by']) ? $this->Request['order_by']: 'JobId'; $order_direction = $this->Request->contains('order_direction') && $misc->isValidOrderDirection($this->Request['order_direction']) ? $this->Request['order_direction']: 'DESC'; - $overview = ($this->Request->contains('overview') && $misc->isValidBooleanTrue($this->Request['overview'])); + $mode = ($this->Request->contains('overview') && $misc->isValidBooleanTrue($this->Request['overview'])) ? JobManager::JOB_RESULT_MODE_OVERVIEW : JobManager::JOB_RESULT_MODE_NORMAL; if (!empty($jobids)) { /** @@ -205,6 +208,23 @@ class Jobs extends BaculumAPIServer { } } + // Real start time range + if (!empty($realstarttime_from) || !empty($realstarttime_to)) { + $params['Job.RealStartTime'] = []; + if (!empty($realstarttime_from)) { + $params['Job.RealStartTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $realstarttime_from) + ]; + } + if (!empty($realstarttime_to)) { + $params['Job.RealStartTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $realstarttime_to) + ]; + } + } + // Real end time range if (!empty($realendtime_from) || !empty($realendtime_to)) { $params['Job.RealEndTime'] = []; @@ -285,7 +305,7 @@ class Jobs extends BaculumAPIServer { $offset, $order_by, $order_direction, - $overview + $mode ); $this->output = $result; $this->error = JobError::ERROR_NO_ERRORS; diff --git a/gui/baculum/protected/API/Pages/API/JobsObjects.php b/gui/baculum/protected/API/Pages/API/JobsObjects.php new file mode 100644 index 000000000..8224372ea --- /dev/null +++ b/gui/baculum/protected/API/Pages/API/JobsObjects.php @@ -0,0 +1,331 @@ + + * @category API + * @package Baculum API + */ +class JobsObjects extends BaculumAPIServer { + + public function get() { + $misc = $this->getModule('misc'); + $jobids = $this->Request->contains('jobids') && $misc->isValidIdsList($this->Request['jobids']) ? $this->Request['jobids'] : ''; + $afterjobid = $this->Request->contains('afterjobid') && $misc->isValidInteger($this->Request['afterjobid']) ? $this->Request['afterjobid'] : 0; + $limit = $this->Request->contains('limit') && $misc->isValidInteger($this->Request['limit']) ? (int)$this->Request['limit'] : 0; + $object_limit = $this->Request->contains('object_limit') && $misc->isValidInteger($this->Request['object_limit']) ? (int)$this->Request['object_limit'] : 0; + $offset = $this->Request->contains('offset') && $misc->isValidInteger($this->Request['offset']) ? (int)$this->Request['offset'] : 0; + $jobstatus = $this->Request->contains('jobstatus') ? $this->Request['jobstatus'] : ''; + $level = $this->Request->contains('level') && $misc->isValidJobLevel($this->Request['level']) ? $this->Request['level'] : ''; + $type = $this->Request->contains('type') && $misc->isValidJobType($this->Request['type']) ? $this->Request['type'] : ''; + $jobname = $this->Request->contains('name') && $misc->isValidName($this->Request['name']) ? $this->Request['name'] : ''; + $clientid = $this->Request->contains('clientid') ? $this->Request['clientid'] : ''; + $schedtime_from = $this->Request->contains('schedtime_from') && $misc->isValidInteger($this->Request['schedtime_from']) ? (int)$this->Request['schedtime_from'] : null; + $schedtime_to = $this->Request->contains('schedtime_to') && $misc->isValidInteger($this->Request['schedtime_to']) ? (int)$this->Request['schedtime_to'] : null; + $starttime_from = $this->Request->contains('starttime_from') && $misc->isValidInteger($this->Request['starttime_from']) ? (int)$this->Request['starttime_from'] : null; + $starttime_to = $this->Request->contains('starttime_to') && $misc->isValidInteger($this->Request['starttime_to']) ? (int)$this->Request['starttime_to'] : null; + $endtime_from = $this->Request->contains('endtime_from') && $misc->isValidInteger($this->Request['endtime_from']) ? (int)$this->Request['endtime_from'] : null; + $endtime_to = $this->Request->contains('endtime_to') && $misc->isValidInteger($this->Request['endtime_to']) ? (int)$this->Request['endtime_to'] : null; + $realstarttime_from = $this->Request->contains('realstarttime_from') && $misc->isValidInteger($this->Request['realstarttime_from']) ? (int)$this->Request['realstarttime_from'] : null; + $realstarttime_to = $this->Request->contains('realstarttime_to') && $misc->isValidInteger($this->Request['realstarttime_to']) ? (int)$this->Request['realstarttime_to'] : null; + $realendtime_from = $this->Request->contains('realendtime_from') && $misc->isValidInteger($this->Request['realendtime_from']) ? (int)$this->Request['realendtime_from'] : null; + $realendtime_to = $this->Request->contains('realendtime_to') && $misc->isValidInteger($this->Request['realendtime_to']) ? (int)$this->Request['realendtime_to'] : null; + $age = $this->Request->contains('age') && $misc->isValidInteger($this->Request['age']) ? (int)$this->Request['age'] : null; + $order_by = $this->Request->contains('order_by') && $misc->isValidColumn($this->Request['order_by']) ? $this->Request['order_by']: 'JobId'; + $order_direction = $this->Request->contains('order_direction') && $misc->isValidOrderDirection($this->Request['order_direction']) ? $this->Request['order_direction']: 'DESC'; + $overview = ($this->Request->contains('overview') && $misc->isValidBooleanTrue($this->Request['overview'])); + $view = ($this->Request->contains('view') && $misc->isValidResultView($this->Request['view'])) ? $this->Request['view'] : JobManager::JOB_RESULT_VIEW_FULL; + + if (!empty($jobids)) { + /** + * If jobids parameter provided, all other parameters are not used. + */ + $params['Job.JobId'] = []; + $params['Job.JobId'][] = [ + 'operator' => 'IN', + 'vals' => explode(',', $jobids) + ]; + $result = $this->getModule('job')->getJobsObjectsOverview( + $params, + null, + 0, + $order_by, + $order_direction, + $object_limit, + $overview, + $view + ); + $this->output = $result; + $this->error = JobError::ERROR_NO_ERRORS; + return; + } + + if (!empty($clientid) && !$misc->isValidId($clientid)) { + $this->output = JobError::MSG_ERROR_CLIENT_DOES_NOT_EXISTS; + $this->error = JobError::ERROR_CLIENT_DOES_NOT_EXISTS; + return; + } + + $client = $this->Request->contains('client') ? $this->Request['client'] : ''; + if (!empty($client) && !$misc->isValidName($client)) { + $this->output = JobError::MSG_ERROR_CLIENT_DOES_NOT_EXISTS; + $this->error = JobError::ERROR_CLIENT_DOES_NOT_EXISTS; + return; + } + $jr = new \ReflectionClass('Baculum\API\Modules\JobRecord'); + $sort_cols = $jr->getProperties(); + if (strpos($order_by, '.') !== false) { + $order_by_ex = explode('.', $order_by); + $order_by = array_shift($order_by_ex); + + } + $order_by_lc = strtolower($order_by); + $cols_excl = ['client', 'fileset', 'pool']; + $columns = []; + foreach ($sort_cols as $cols) { + $name = $cols->getName(); + // skip columns not existing in the catalog + if (in_array($name, $cols_excl)) { + continue; + } + $columns[] = $name; + } + if (!in_array($order_by_lc, $columns)) { + $this->output = JobError::MSG_ERROR_INVALID_PROPERTY; + $this->error = JobError::ERROR_INVALID_PROPERTY; + return; + } + + + $params = []; + + if ($afterjobid > 0) { + $params['Job.JobId'] = []; + $params['Job.JobId'][] = [ + 'operator' => '>', + 'vals' => $afterjobid + ]; + } + + $jobstatuses = array_keys($misc->getJobState()); + $sts = str_split($jobstatus); + $js_counter = 0; + for ($i = 0; $i < count($sts); $i++) { + if (in_array($sts[$i], $jobstatuses)) { + if (!key_exists('Job.JobStatus', $params)) { + $params['Job.JobStatus'] = []; + $params['Job.JobStatus'][$js_counter] = [ + 'operator' => 'OR', + 'vals' => [] + ]; + } + $params['Job.JobStatus'][$js_counter]['vals'][] = $sts[$i]; + } + } + if (!empty($level)) { + $params['Job.Level'] = []; + $params['Job.Level'][] = [ + 'vals' => $level + ]; + } + if (!empty($type)) { + $params['Job.Type'] = []; + $params['Job.Type'][] = [ + 'vals' => $type + ]; + } + + // Scheduled time range + if (!empty($schedtime_from) || !empty($schedtime_to)) { + $params['Job.SchedTime'] = []; + if (!empty($schedtime_from)) { + $params['Job.SchedTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $schedtime_from) + ]; + } + if (!empty($schedtime_to)) { + $params['Job.SchedTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $schedtime_to) + ]; + } + } + + // Start time range + if (!empty($starttime_from) || !empty($starttime_to)) { + $params['Job.StartTime'] = []; + if (!empty($starttime_from)) { + $params['Job.StartTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $starttime_from) + ]; + } + if (!empty($starttime_to)) { + $params['Job.StartTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $starttime_to) + ]; + } + } elseif (!empty($age)) { // Job age (now() - age) + $params['Job.StartTime'] = []; + $params['Job.StartTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', (time() - $age)) + ]; + } + + // End time range + if (!empty($endtime_from) || !empty($endtime_to)) { + $params['Job.EndTime'] = []; + if (!empty($endtime_from)) { + $params['Job.EndTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $endtime_from) + ]; + } + if (!empty($endtime_to)) { + $params['Job.EndTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $endtime_to) + ]; + } + } + + // Real start time range + if (!empty($realstarttime_from) || !empty($realstarttime_to)) { + $params['Job.RealStartTime'] = []; + if (!empty($realstarttime_from)) { + $params['Job.RealStartTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $realstarttime_from) + ]; + } + if (!empty($realstarttime_to)) { + $params['Job.RealStartTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $realstarttime_to) + ]; + } + } + + // Real end time range + if (!empty($realendtime_from) || !empty($realendtime_to)) { + $params['Job.RealEndTime'] = []; + if (!empty($realendtime_from)) { + $params['Job.RealEndTime'][] = [ + 'operator' => '>=', + 'vals' => date('Y-m-d H:i:s', $realendtime_from) + ]; + } + if (!empty($realendtime_to)) { + $params['Job.RealEndTime'][] = [ + 'operator' => '<=', + 'vals' => date('Y-m-d H:i:s', $realendtime_to) + ]; + } + } + + $result = $this->getModule('bconsole')->bconsoleCommand( + $this->director, + ['.jobs'], + null, + true + ); + if ($result->exitcode === 0) { + $vals = []; + if (!empty($jobname) && in_array($jobname, $result->output)) { + $vals = [$jobname]; + } else { + $vals = $result->output; + } + if (count($vals) == 0) { + // no $vals criteria means that user has no job resources assigned. + $this->output = []; + $this->error = JobError::ERROR_NO_ERRORS; + return; + } + + $params['Job.Name'] = []; + $params['Job.Name'][] = [ + 'operator' => 'IN', + 'vals' => $vals + ]; + + $error = false; + // Client name and clientid filter + if (!empty($client) || !empty($clientid)) { + $result = $this->getModule('bconsole')->bconsoleCommand($this->director, array('.client')); + if ($result->exitcode === 0) { + array_shift($result->output); + $cli = null; + if (!empty($client)) { + $cli = $this->getModule('client')->getClientByName($client); + } elseif (!empty($clientid)) { + $cli = $this->getModule('client')->getClientById($clientid); + } + if (is_object($cli) && in_array($cli->name, $result->output)) { + $params['Job.ClientId'] = []; + $params['Job.ClientId'][] = [ + 'operator' => 'AND', + 'vals' => [$cli->clientid] + ]; + } else { + $error = true; + $this->output = JobError::MSG_ERROR_CLIENT_DOES_NOT_EXISTS; + $this->error = JobError::ERROR_CLIENT_DOES_NOT_EXISTS; + } + } else { + $error = true; + $this->output = $result->output; + $this->error = $result->exitcode; + } + } + + if ($error === false) { + $result = $this->getModule('job')->getJobsObjectsOverview( + $params, + $limit, + $offset, + $order_by, + $order_direction, + $object_limit, + $overview, + $view + ); + $this->output = $result; + $this->error = JobError::ERROR_NO_ERRORS; + } + } else { + $this->output = $result->output; + $this->error = $result->exitcode; + } + } +} diff --git a/gui/baculum/protected/API/Pages/API/endpoints.xml b/gui/baculum/protected/API/Pages/API/endpoints.xml index 29136cc8a..9e50080d5 100644 --- a/gui/baculum/protected/API/Pages/API/endpoints.xml +++ b/gui/baculum/protected/API/Pages/API/endpoints.xml @@ -85,6 +85,7 @@ + diff --git a/gui/baculum/protected/API/openapi_baculum.json b/gui/baculum/protected/API/openapi_baculum.json index de36d1ca1..63e1d5de2 100644 --- a/gui/baculum/protected/API/openapi_baculum.json +++ b/gui/baculum/protected/API/openapi_baculum.json @@ -958,6 +958,24 @@ "type": "integer" } }, + { + "name": "realstarttime_from", + "in": "query", + "required": false, + "description": "Real start time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "realstarttime_to", + "in": "query", + "required": false, + "description": "Real start time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, { "name": "endtime_from", "in": "query", @@ -1813,6 +1831,297 @@ ] } }, + "/api/v2/jobs/objects": { + "get": { + "tags": ["jobs"], + "summary": "Get jobs together with objects", + "description": "Get jobs together with objects.", + "responses": { + "200": { + "description": "Jobs with objects", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "output": { + "type": "array", + "items": { + "type": "object", + "properties": { + "job": { + "$ref": "#/components/schemas/Jobs" + }, + "objects": { + "type": "object", + "properties": { + "overview": { + "type": "object", + "properties": { + "objectcategory": { + "type": "object", + "properties": { + "count": { + "type": "integer", + "description": "Total number of objects given category" + }, + "objects": { + "$ref": "#/components/schemas/Objects" + } + } + } + } + }, + "totalcount": { + "type": "integer", + "description": "Total number of objects" + } + } + } + } + } + }, + "error": { + "type": "integer", + "description": "Error code", + "enum": [0, 1, 2, 3, 4, 5, 6, 7, 10, 11, 1000] + } + } + } + } + } + } + }, + "parameters": [ + { + "$ref": "#/components/parameters/Limit" + }, + { + "$ref": "#/components/parameters/Offset" + }, + { + "name": "object_limit", + "in": "query", + "description": "Object item limit", + "required": false, + "schema": { + "type": "integer" + } + }, + { + "name": "name", + "in": "query", + "required": false, + "description": "Job name", + "schema": { + "type": "string", + "pattern": "[a-zA-Z0-9:.-_ ]+" + } + }, + { + "name": "jobids", + "in": "query", + "required": false, + "description": "Comma separated job identifiers. Using this filter causes that all other filters are ignored. Sorting is possible.", + "schema": { + "type": "string" + } + }, + { + "name": "afterjobid", + "in": "query", + "required": false, + "description": "Displays jobs after given job identifier with excluding the given jobid.", + "schema": { + "type": "integer" + } + }, + { + "name": "jobstatus", + "in": "query", + "required": false, + "description": "Job status letter(s). Possible multiple values like 'Ef' or 'Tef'", + "schema": { + "type": "string" + } + }, + { + "name": "clientid", + "in": "query", + "required": false, + "description": "Client identifier", + "schema": { + "type": "integer" + } + }, + { + "name": "client", + "in": "query", + "required": false, + "description": "Client name", + "schema": { + "type": "string" + } + }, + { + "name": "type", + "in": "query", + "required": false, + "description": "Job type letter", + "schema": { + "type": "string" + } + }, + { + "name": "level", + "in": "query", + "required": false, + "description": "Job level letter", + "schema": { + "type": "string" + } + }, + { + "name": "schedtime_from", + "in": "query", + "required": false, + "description": "Scheduled time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "schedtime_to", + "in": "query", + "required": false, + "description": "Scheduled time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "starttime_from", + "in": "query", + "required": false, + "description": "Start time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "starttime_to", + "in": "query", + "required": false, + "description": "Start time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "realstarttime_from", + "in": "query", + "required": false, + "description": "Real start time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "realstarttime_to", + "in": "query", + "required": false, + "description": "Real start time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "endtime_from", + "in": "query", + "required": false, + "description": "End time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "endtime_to", + "in": "query", + "required": false, + "description": "End time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "realendtime_from", + "in": "query", + "required": false, + "description": "Real end time from (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + "name": "realendtime_to", + "in": "query", + "required": false, + "description": "Real end time to (UNIX timestamp format, seconds)", + "schema": { + "type": "integer" + } + }, + { + + "name": "age", + "in": "query", + "required": false, + "description": "Job age in seconds (used is start time). starttime_from and starttime_to take precedence over this parameter.", + "schema": { + "type": "integer" + } + }, + { + "name": "order_by", + "in": "query", + "required": false, + "description": "Sort by selected job property (default jobid). There can be any job property (jobid, job, clientid ...etc.) except client, fileset and pool.", + "schema": { + "type": "string" + } + }, + { + "name": "order_direction", + "in": "query", + "required": false, + "description": "Order direction. It can be 'asc' (ascending order) or 'desc' (descending order - default)", + "schema": { + "type": "string", + "enum": ["asc", "desc"] + } + }, + { + "name": "overview", + "in": "query", + "required": false, + "description": "If set, it puts jobs (and objects) in job status groups (successful, unsuccessful, running, all). NOTE: Offset and limit parameters do not apply to overview counts.", + "schema": { + "type": "boolean" + } + }, + { + "name": "view", + "in": "query", + "required": false, + "description": "Set result view type: full (default) - display all job and object properties, basic - display only base properties", + "schema": { + "type": "string", + "enum": ["basic", "full"] + } + } + ] + } + }, "/api/v2/jobs/run": { "post": { "tags": ["jobs"], diff --git a/gui/baculum/protected/Common/Modules/Miscellaneous.php b/gui/baculum/protected/Common/Modules/Miscellaneous.php index 8d4d1e954..c4403c8d3 100644 --- a/gui/baculum/protected/Common/Modules/Miscellaneous.php +++ b/gui/baculum/protected/Common/Modules/Miscellaneous.php @@ -311,6 +311,11 @@ class Miscellaneous extends TModule { return (preg_match('/^(asc|desc)$/i', $order) === 1); } + public function isValidResultView($view) { + return (preg_match('/^(basic|full)$/', $view) === 1); + } + + public function escapeCharsToConsole($path) { return preg_replace('/([$])/', '\\\${1}', $path); }