From: Marcin Haba Date: Sat, 15 Jul 2023 17:38:38 +0000 (+0200) Subject: baculum: Speed up dashboard page loading X-Git-Tag: Release-13.0.4~37 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=2b2837eea904263c96d7cf64b451c03147577ee5;p=thirdparty%2Fbacula.git baculum: Speed up dashboard page loading This change is ported from Bacularis. This slow dashboard problem was reported by Yakup Kaya on bacula-users mailing list: https://sourceforge.net/p/bacula/mailman/bacula-users/thread/cb8a6ea35db8404f964bf32f67baa7c5%40indra-avitech.aero/#msg37870165 --- diff --git a/gui/baculum/protected/API/Modules/JobManager.php b/gui/baculum/protected/API/Modules/JobManager.php index 27f56a048..000377dc0 100644 --- a/gui/baculum/protected/API/Modules/JobManager.php +++ b/gui/baculum/protected/API/Modules/JobManager.php @@ -552,23 +552,92 @@ LEFT JOIN FileSet USING (FilesetId)' return $jobids; } - public function getJobTotals($allowed_jobs = array()) { - $jobtotals = array('bytes' => 0, 'files' => 0); - $connection = JobRecord::finder()->getDbConnection(); - $connection->setActive(true); + public function getJobTotals($criteria = []) + { + $jobtotals = [ + 'job_count' => 0, + 'most_occupied_client' => 0, + 'most_occupied_client_count' => 0, + 'most_occupied_job' => 0, + 'most_occupied_job_count' => 0, + 'most_occupied_pool' => 0, + 'most_occupied_pool_count' => 0, + 'bytes' => 0, + 'files' => 0 + ]; + $where = Database::getWhere($criteria); - $where = ''; - if (count($allowed_jobs) > 0) { - $where = " WHERE name='" . implode("' OR name='", $allowed_jobs) . "'"; + /** + * NOTE: All SQL queries could be provided in one query. + * It could speed up the loading but there is risk that + * the query can be too long (many jobs in where clause). + * Safely the queries are provided separately. + */ + + // Job count and total bytes + $sql = "SELECT + COUNT(1) AS job_count, + SUM(Job.JobFiles) AS files, + SUM(Job.JobBytes) AS bytes + FROM + Job + {$where['where']}"; + + $statement = Database::runQuery($sql, $where['params']); + $ret = $statement->fetch(\PDO::FETCH_ASSOC); + if ($ret) { + $jobtotals = array_merge($jobtotals, $ret); } - $sql = "SELECT sum(JobFiles) AS files, sum(JobBytes) AS bytes FROM Job $where"; - $pdo = $connection->getPdoInstance(); - $result = $pdo->query($sql); - $ret = $result->fetch(); - $jobtotals['bytes'] = $ret['bytes']; - $jobtotals['files'] = $ret['files']; - $pdo = null; + // The most occupied client job stats + $sql = "SELECT + Client.Name AS most_occupied_client, + COUNT(1) AS most_occupied_client_count + FROM + Job + JOIN Client USING (ClientId) + {$where['where']} + GROUP BY Client.Name + ORDER BY most_occupied_client_count DESC + LIMIT 1"; + $statement = Database::runQuery($sql, $where['params']); + $ret = $statement->fetch(\PDO::FETCH_ASSOC); + if ($ret) { + $jobtotals = array_merge($jobtotals, $ret); + } + + // The most occupied job stats + $sql = "SELECT + Name AS most_occupied_job, + COUNT(1) AS most_occupied_job_count + FROM + Job + {$where['where']} + GROUP BY Name + ORDER BY most_occupied_job_count DESC + LIMIT 1"; + $statement = Database::runQuery($sql, $where['params']); + $ret = $statement->fetch(\PDO::FETCH_ASSOC); + if ($ret) { + $jobtotals = array_merge($jobtotals, $ret); + } + + // The most occupied pool job stats + $sql = "SELECT + Pool.Name AS most_occupied_pool, + COUNT(1) AS most_occupied_pool_count + FROM + Job + JOIN Pool USING (PoolId) + {$where['where']} + GROUP BY Pool.Name + ORDER BY most_occupied_pool_count DESC + LIMIT 1"; + $statement = Database::runQuery($sql, $where['params']); + $ret = $statement->fetch(\PDO::FETCH_ASSOC); + if ($ret) { + $jobtotals = array_merge($jobtotals, $ret); + } return $jobtotals; } diff --git a/gui/baculum/protected/API/Pages/API/JobTotals.php b/gui/baculum/protected/API/Pages/API/JobTotals.php index d824ddb6d..5d7d6decb 100644 --- a/gui/baculum/protected/API/Pages/API/JobTotals.php +++ b/gui/baculum/protected/API/Pages/API/JobTotals.php @@ -55,7 +55,11 @@ class JobTotals extends BaculumAPIServer { } if ($error === false) { - $jobtotals = $this->getModule('job')->getJobTotals($allowed); + $params = ['Job.Name' => [[ + 'operator' => 'IN', + 'vals' => $allowed + ]]]; + $jobtotals = $this->getModule('job')->getJobTotals($params); $this->output = $jobtotals; $this->error = JobError::ERROR_NO_ERRORS; } diff --git a/gui/baculum/protected/API/openapi_baculum.json b/gui/baculum/protected/API/openapi_baculum.json index 564061924..870406ad9 100644 --- a/gui/baculum/protected/API/openapi_baculum.json +++ b/gui/baculum/protected/API/openapi_baculum.json @@ -2682,6 +2682,34 @@ "output": { "type": "object", "properties": { + "job_count": { + "type": "integer", + "description": "Total number of jobs" + }, + "most_occupied_client": { + "type": "string", + "description": "The most occupied client by jobs" + }, + "most_occupied_client_count": { + "type": "integer", + "description": "Job count ran on the most occupied client" + }, + "most_occupied_job": { + "type": "string", + "description": "The most often running job" + }, + "most_occupied_job_count": { + "type": "integer", + "description": "Job count the most often running job" + }, + "most_occupied_pool": { + "type": "string", + "description": "The most occupied pool by jobs" + }, + "most_occupied_pool_count": { + "type": "integer", + "description": "Job count ran on the most occupied pool" + }, "bytes": { "type": "integer", "description": "Total number backed up bytes" diff --git a/gui/baculum/protected/Web/JavaScript/misc.js b/gui/baculum/protected/Web/JavaScript/misc.js index 464916b4c..251f8f169 100644 --- a/gui/baculum/protected/Web/JavaScript/misc.js +++ b/gui/baculum/protected/Web/JavaScript/misc.js @@ -868,23 +868,12 @@ var Dashboard = { this.update_database(); }, update_clients: function() { - var clients = this.stats.clients_occupancy; - var most_occuped_client = this.noval; - var occupancy = -1; - for (client in clients) { - if (occupancy < clients[client]) { - most_occuped_client = client; - occupancy = clients[client]; - } - } - - if (occupancy === -1) { - occupancy = 0; - } - - document.getElementById(this.ids.clients.no).textContent = Object.keys(this.stats.clients).length; - document.getElementById(this.ids.clients.most).setAttribute('title', most_occuped_client); - document.getElementById(this.ids.clients.most).textContent = Strings.get_short_label(most_occuped_client); + const client_no = Object.keys(this.stats.clients).length; + const most_occupied_client = this.stats.jobtotals.most_occupied_client || this.noval; + const occupancy = this.stats.jobtotals.most_occupied_client_count || this.noval; + document.getElementById(this.ids.clients.no).textContent = client_no; + document.getElementById(this.ids.clients.most).setAttribute('title', most_occupied_client); + document.getElementById(this.ids.clients.most).textContent = Strings.get_short_label(most_occupied_client); document.getElementById(this.ids.clients.jobs).textContent = occupancy; }, update_job_access: function() { @@ -895,23 +884,13 @@ var Dashboard = { }); }, update_jobs: function() { - var jobs = this.stats.jobs_occupancy; - var most_occuped_job = this.noval; - var occupancy = -1; - for (job in jobs) { - if (occupancy < jobs[job]) { - most_occuped_job = job; - occupancy = jobs[job]; - } - } + const job_no = this.stats.jobtotals.job_count || Object.keys(this.stats.jobs).length || this.noval; + const most_occupied_job = this.stats.jobtotals.most_occupied_job || this.noval; + const occupancy = this.stats.jobtotals.most_occupied_job_count || this.noval; - if (occupancy === -1) { - occupancy = 0; - } - - document.getElementById(this.ids.jobs.no).textContent = Object.keys(this.stats.jobs).length; - document.getElementById(this.ids.jobs.most).setAttribute('title',most_occuped_job); - document.getElementById(this.ids.jobs.most).textContent = Strings.get_short_label(most_occuped_job); + document.getElementById(this.ids.jobs.no).textContent = job_no; + document.getElementById(this.ids.jobs.most).setAttribute('title', most_occupied_job); + document.getElementById(this.ids.jobs.most).textContent = Strings.get_short_label(most_occupied_job); document.getElementById(this.ids.jobs.most_count).textContent = occupancy; }, update_jobtotals: function() { @@ -925,23 +904,12 @@ var Dashboard = { } }, update_pools: function() { - var pools = this.stats.pools_occupancy; - var most_occuped_pool = this.noval; - var occupancy = -1; - for (pool in pools) { - if (occupancy < pools[pool]) { - most_occuped_pool = pool; - occupancy = pools[pool]; - } - } - - if (occupancy === -1) { - occupancy = 0; - } - - document.getElementById(this.ids.pools.no).textContent = Object.keys(this.stats.pools).length; - document.getElementById(this.ids.pools.most).setAttribute('title', most_occuped_pool); - document.getElementById(this.ids.pools.most).textContent = Strings.get_short_label(most_occuped_pool); + const pool_no = Object.keys(this.stats.pools).length; + const most_occupied_pool = this.stats.jobtotals.most_occupied_pool || this.noval; + const occupancy = this.stats.jobtotals.most_occupied_pool_count || this.noval; + document.getElementById(this.ids.pools.no).textContent = pool_no; + document.getElementById(this.ids.pools.most).setAttribute('title', most_occupied_pool); + document.getElementById(this.ids.pools.most).textContent = Strings.get_short_label(most_occupied_pool); document.getElementById(this.ids.pools.jobs).textContent = occupancy; }, update_pie_jobstatus: function() { diff --git a/gui/baculum/protected/Web/JavaScript/statistics.js b/gui/baculum/protected/Web/JavaScript/statistics.js index 3861c40ae..b96e77ee7 100644 --- a/gui/baculum/protected/Web/JavaScript/statistics.js +++ b/gui/baculum/protected/Web/JavaScript/statistics.js @@ -4,8 +4,6 @@ var Statistics = { pools: null, jobtotals: null, dbsize: null, - clients_occupancy: {}, - pools_occupancy: {}, jobs_summary: [], grab_statistics: function(data, opts) { this.jobs = data.jobs; @@ -14,9 +12,6 @@ var Statistics = { this.jobtotals = data.jobtotals; this.dbsize = data.dbsize; var jobs_count = this.jobs.length; - var clients_occupancy = {}; - var pools_occupancy = {}; - var jobs_occupancy = {}; var jobs_summary = { ok: [], error: [], @@ -33,23 +28,6 @@ var Statistics = { if (opts.job_age > 0 && job_time_ts < start_time_ts) { continue; } - if (typeof(clients_occupancy[this.jobs[i].clientid]) === 'undefined') { - clients_occupancy[this.jobs[i].clientid] = 1; - } else { - clients_occupancy[this.jobs[i].clientid] += 1; - } - - if (typeof(pools_occupancy[this.jobs[i].poolid]) === 'undefined') { - pools_occupancy[this.jobs[i].poolid] = 1; - } else { - pools_occupancy[this.jobs[i].poolid] += 1; - } - - if (typeof(jobs_occupancy[this.jobs[i].name]) === 'undefined') { - jobs_occupancy[this.jobs[i].name] = 1; - } else { - jobs_occupancy[this.jobs[i].name] += 1; - } if (opts.job_states.hasOwnProperty(this.jobs[i].jobstatus)) { status_type = opts.job_states[this.jobs[i].jobstatus].type; if (status_type == 'ok' && this.jobs[i].joberrors > 0) { @@ -61,25 +39,7 @@ var Statistics = { jobs_summary[status_type].push(this.jobs[i]); } } - var clients_ids = Object.keys(clients_occupancy); - for (var i = 0; i < clients_ids.length; i++) { - for (var j = 0; j < this.clients.length; j++) { - if (clients_ids[i] == this.clients[j].clientid) { - this.clients_occupancy[this.clients[j].name] = clients_occupancy[clients_ids[i]]; - } - } - } - - var pools_ids = Object.keys(pools_occupancy); - for (var i = 0; i < pools_ids.length; i++) { - for (var j = 0; j < this.pools.length; j++) { - if (pools_ids[i] == this.pools[j].poolid) { - this.pools_occupancy[this.pools[j].name] = pools_occupancy[pools_ids[i]]; - } - } - } - this.jobs_occupancy = jobs_occupancy; this.jobs_summary = jobs_summary; } } diff --git a/gui/baculum/protected/Web/Pages/Monitor.php b/gui/baculum/protected/Web/Pages/Monitor.php index 27362f1f1..8ccf949c7 100644 --- a/gui/baculum/protected/Web/Pages/Monitor.php +++ b/gui/baculum/protected/Web/Pages/Monitor.php @@ -56,10 +56,15 @@ class Monitor extends BaculumPage { $web_config = $this->getModule('web_config')->getConfig(); $job_limit = WebConfig::DEF_MAX_JOBS; - if (count($web_config) > 0 && key_exists('max_jobs', $web_config['baculum'])) { - $job_limit = $web_config['baculum']['max_jobs']; - } - + $job_age = 0; + if (count($web_config) > 0) { + if (key_exists('max_jobs', $web_config['baculum'])) { + $job_limit = $web_config['baculum']['max_jobs']; + } + if (key_exists('job_age_on_job_status_graph', $web_config['baculum'])) { + $job_age = $web_config['baculum']['job_age_on_job_status_graph']; + } + } $error = null; $params = $this->Request->contains('params') ? $this->Request['params'] : []; if (!is_array($params)) { @@ -74,11 +79,13 @@ class Monitor extends BaculumPage { if (is_array($params['jobs'])) { if (key_exists('name', $params['jobs']) && is_array($params['jobs']['name'])) { for ($i = 0; $i < count($params['jobs']['name']); $i++) { + // @TODO: Add support for multiple names in query $job_query['name'] = $params['jobs']['name'][$i]; } } if (key_exists('client', $params['jobs']) && is_array($params['jobs']['client'])) { for ($i = 0; $i < count($params['jobs']['client']); $i++) { + // @TODO: Add support for multiple clients in query $job_query['client'] = $params['jobs']['client'][$i]; } } @@ -86,6 +93,10 @@ class Monitor extends BaculumPage { if ($this->Request->contains('use_limit') && $this->Request['use_limit'] == 1) { $job_query['limit'] = $job_limit; } + if ($this->Request->contains('use_age') && $this->Request['use_age'] == 1) { + $job_query['age'] = $job_age; + } + if (count($job_query) > 0) { $job_params[] = '?' . http_build_query($job_query); } diff --git a/gui/baculum/protected/Web/Portlets/ResourceMonitor.tpl b/gui/baculum/protected/Web/Portlets/ResourceMonitor.tpl index b06ec80d3..a57e2346b 100644 --- a/gui/baculum/protected/Web/Portlets/ResourceMonitor.tpl +++ b/gui/baculum/protected/Web/Portlets/ResourceMonitor.tpl @@ -16,6 +16,7 @@ $(function() { data: { 'params': (typeof(MonitorParams) == 'object' ? MonitorParams : []), 'use_limit' : <%=$this->Service->getRequestedPagePath() == "Dashboard" ? '0' : '1'%>, + 'use_age' : <%=$this->Service->getRequestedPagePath() == "Dashboard" ? '1' : '0'%> }, beforeSend: function() { last_callback_time = new Date().getTime();