]> git.ipfire.org Git - thirdparty/bacula.git/commitdiff
baculum: Speed up dashboard page loading
authorMarcin Haba <marcin.haba@bacula.pl>
Sat, 15 Jul 2023 17:38:38 +0000 (19:38 +0200)
committerMarcin Haba <marcin.haba@bacula.pl>
Sat, 15 Jul 2023 17:38:38 +0000 (19:38 +0200)
This change is ported from Bacularis.

This slow dashboard problem was reported by Yakup Kaya on bacula-users mailing list:
https://sourceforge.net/p/bacula/mailman/bacula-users/thread/cb8a6ea35db8404f964bf32f67baa7c5%40indra-avitech.aero/#msg37870165

gui/baculum/protected/API/Modules/JobManager.php
gui/baculum/protected/API/Pages/API/JobTotals.php
gui/baculum/protected/API/openapi_baculum.json
gui/baculum/protected/Web/JavaScript/misc.js
gui/baculum/protected/Web/JavaScript/statistics.js
gui/baculum/protected/Web/Pages/Monitor.php
gui/baculum/protected/Web/Portlets/ResourceMonitor.tpl

index 27f56a0488f53bbea60a41b6ec678e67a16434ef..000377dc0cd5602bd243322e9f019b3e412ac9de 100644 (file)
@@ -552,23 +552,92 @@ LEFT JOIN FileSet USING (FilesetId)'
                return $jobids;
        }
 
-       public function getJobTotals($allowed_jobs = array()) {
-               $jobtotals = array('bytes' => 0, 'files' => 0);
-               $connection = JobRecord::finder()->getDbConnection();
-               $connection->setActive(true);
+       public function getJobTotals($criteria = [])
+       {
+               $jobtotals = [
+                       'job_count' => 0,
+                       'most_occupied_client' => 0,
+                       'most_occupied_client_count' => 0,
+                       'most_occupied_job' => 0,
+                       'most_occupied_job_count' => 0,
+                       'most_occupied_pool' => 0,
+                       'most_occupied_pool_count' => 0,
+                       'bytes' => 0,
+                       'files' => 0
+               ];
+               $where = Database::getWhere($criteria);
 
-               $where = '';
-               if (count($allowed_jobs) > 0) {
-                       $where = " WHERE name='" . implode("' OR name='", $allowed_jobs) . "'";
+               /**
+                * NOTE: All SQL queries could be provided in one query.
+                * It could speed up the loading but there is risk that
+                * the query can be too long (many jobs in where clause).
+                * Safely the queries are provided separately.
+                */
+
+               // Job count and total bytes
+               $sql = "SELECT
+                               COUNT(1)          AS job_count,
+                               SUM(Job.JobFiles) AS files,
+                               SUM(Job.JobBytes) AS bytes
+                       FROM
+                               Job
+                       {$where['where']}";
+
+               $statement = Database::runQuery($sql, $where['params']);
+               $ret = $statement->fetch(\PDO::FETCH_ASSOC);
+               if ($ret) {
+                       $jobtotals = array_merge($jobtotals, $ret);
                }
 
-               $sql = "SELECT sum(JobFiles) AS files, sum(JobBytes) AS bytes FROM Job $where";
-               $pdo = $connection->getPdoInstance();
-               $result = $pdo->query($sql);
-               $ret = $result->fetch();
-               $jobtotals['bytes'] = $ret['bytes'];
-               $jobtotals['files'] = $ret['files'];
-               $pdo = null;
+               // The most occupied client job stats
+               $sql = "SELECT
+                               Client.Name       AS most_occupied_client,
+                               COUNT(1)          AS most_occupied_client_count
+                       FROM
+                               Job
+                               JOIN Client USING (ClientId)
+                       {$where['where']}
+                       GROUP BY Client.Name
+                       ORDER BY most_occupied_client_count DESC
+                       LIMIT 1";
+               $statement = Database::runQuery($sql, $where['params']);
+               $ret = $statement->fetch(\PDO::FETCH_ASSOC);
+               if ($ret) {
+                       $jobtotals = array_merge($jobtotals, $ret);
+               }
+
+               // The most occupied job stats
+               $sql = "SELECT
+                               Name     AS most_occupied_job,
+                               COUNT(1) AS most_occupied_job_count
+                       FROM
+                               Job
+                       {$where['where']}
+                       GROUP BY Name
+                       ORDER BY most_occupied_job_count DESC
+                       LIMIT 1";
+               $statement = Database::runQuery($sql, $where['params']);
+               $ret = $statement->fetch(\PDO::FETCH_ASSOC);
+               if ($ret) {
+                       $jobtotals = array_merge($jobtotals, $ret);
+               }
+
+               // The most occupied pool job stats
+               $sql = "SELECT
+                               Pool.Name AS most_occupied_pool,
+                               COUNT(1)  AS most_occupied_pool_count
+                       FROM
+                               Job
+                               JOIN Pool USING (PoolId)
+                       {$where['where']}
+                       GROUP BY Pool.Name
+                       ORDER BY most_occupied_pool_count DESC
+                       LIMIT 1";
+               $statement = Database::runQuery($sql, $where['params']);
+               $ret = $statement->fetch(\PDO::FETCH_ASSOC);
+               if ($ret) {
+                       $jobtotals = array_merge($jobtotals, $ret);
+               }
                return $jobtotals;
        }
 
index d824ddb6d5920978c750faa5bf2e0b4d86d1fe53..5d7d6decbad4d9cedb175b17d4a43d207d13074e 100644 (file)
@@ -55,7 +55,11 @@ class JobTotals extends BaculumAPIServer {
                }
 
                if ($error === false) {
-                       $jobtotals = $this->getModule('job')->getJobTotals($allowed);
+                       $params = ['Job.Name' => [[
+                               'operator' => 'IN',
+                               'vals' => $allowed
+                       ]]];
+                       $jobtotals = $this->getModule('job')->getJobTotals($params);
                        $this->output = $jobtotals;
                        $this->error = JobError::ERROR_NO_ERRORS;
                }
index 5640619240392118ce183c91ef2ddf7ada3bfaeb..870406ad907580cebdb9c535f6fc5630d172e086 100644 (file)
                                                                                "output": {
                                                                                        "type": "object",
                                                                                        "properties": {
+                                                                                               "job_count": {
+                                                                                                       "type": "integer",
+                                                                                                       "description": "Total number of jobs"
+                                                                                               },
+                                                                                               "most_occupied_client": {
+                                                                                                       "type": "string",
+                                                                                                       "description": "The most occupied client by jobs"
+                                                                                               },
+                                                                                               "most_occupied_client_count": {
+                                                                                                       "type": "integer",
+                                                                                                       "description": "Job count ran on the most occupied client"
+                                                                                               },
+                                                                                               "most_occupied_job": {
+                                                                                                       "type": "string",
+                                                                                                       "description": "The most often running job"
+                                                                                               },
+                                                                                               "most_occupied_job_count": {
+                                                                                                       "type": "integer",
+                                                                                                       "description": "Job count the most often running job"
+                                                                                               },
+                                                                                               "most_occupied_pool": {
+                                                                                                       "type": "string",
+                                                                                                       "description": "The most occupied pool by jobs"
+                                                                                               },
+                                                                                               "most_occupied_pool_count": {
+                                                                                                       "type": "integer",
+                                                                                                       "description": "Job count ran on the most occupied pool"
+                                                                                               },
                                                                                                "bytes": {
                                                                                                        "type": "integer",
                                                                                                        "description": "Total number backed up bytes"
index 464916b4c419a6d8fe408a3599abde5911f29105..251f8f169f276e1926f7e5aaf7d7817d5301b8d4 100644 (file)
@@ -868,23 +868,12 @@ var Dashboard = {
                this.update_database();
        },
        update_clients: function() {
-               var clients = this.stats.clients_occupancy;
-               var most_occuped_client = this.noval;
-               var occupancy = -1;
-               for (client in clients) {
-                       if (occupancy < clients[client]) {
-                               most_occuped_client = client;
-                               occupancy = clients[client];
-                       }
-               }
-
-               if (occupancy === -1) {
-                       occupancy = 0;
-               }
-
-               document.getElementById(this.ids.clients.no).textContent = Object.keys(this.stats.clients).length;
-               document.getElementById(this.ids.clients.most).setAttribute('title', most_occuped_client);
-               document.getElementById(this.ids.clients.most).textContent = Strings.get_short_label(most_occuped_client);
+               const client_no = Object.keys(this.stats.clients).length;
+               const most_occupied_client = this.stats.jobtotals.most_occupied_client || this.noval;
+               const occupancy = this.stats.jobtotals.most_occupied_client_count || this.noval;
+               document.getElementById(this.ids.clients.no).textContent = client_no;
+               document.getElementById(this.ids.clients.most).setAttribute('title', most_occupied_client);
+               document.getElementById(this.ids.clients.most).textContent = Strings.get_short_label(most_occupied_client);
                document.getElementById(this.ids.clients.jobs).textContent = occupancy;
        },
        update_job_access: function() {
@@ -895,23 +884,13 @@ var Dashboard = {
                });
        },
        update_jobs: function() {
-               var jobs = this.stats.jobs_occupancy;
-               var most_occuped_job = this.noval;
-               var occupancy = -1;
-               for (job in jobs) {
-                       if (occupancy < jobs[job]) {
-                               most_occuped_job = job;
-                               occupancy = jobs[job];
-                       }
-               }
+               const job_no = this.stats.jobtotals.job_count || Object.keys(this.stats.jobs).length || this.noval;
+               const most_occupied_job = this.stats.jobtotals.most_occupied_job || this.noval;
+               const occupancy = this.stats.jobtotals.most_occupied_job_count || this.noval;
 
-               if (occupancy === -1) {
-                       occupancy = 0;
-               }
-
-               document.getElementById(this.ids.jobs.no).textContent = Object.keys(this.stats.jobs).length;
-               document.getElementById(this.ids.jobs.most).setAttribute('title',most_occuped_job);
-               document.getElementById(this.ids.jobs.most).textContent = Strings.get_short_label(most_occuped_job);
+               document.getElementById(this.ids.jobs.no).textContent = job_no;
+               document.getElementById(this.ids.jobs.most).setAttribute('title', most_occupied_job);
+               document.getElementById(this.ids.jobs.most).textContent = Strings.get_short_label(most_occupied_job);
                document.getElementById(this.ids.jobs.most_count).textContent = occupancy;
        },
        update_jobtotals: function() {
@@ -925,23 +904,12 @@ var Dashboard = {
                }
        },
        update_pools: function() {
-               var pools = this.stats.pools_occupancy;
-               var most_occuped_pool = this.noval;
-               var occupancy = -1;
-               for (pool in pools) {
-                       if (occupancy < pools[pool]) {
-                               most_occuped_pool = pool;
-                               occupancy = pools[pool];
-                       }
-               }
-
-               if (occupancy === -1) {
-                       occupancy = 0;
-               }
-
-               document.getElementById(this.ids.pools.no).textContent = Object.keys(this.stats.pools).length;
-               document.getElementById(this.ids.pools.most).setAttribute('title', most_occuped_pool);
-               document.getElementById(this.ids.pools.most).textContent = Strings.get_short_label(most_occuped_pool);
+               const pool_no = Object.keys(this.stats.pools).length;
+               const most_occupied_pool = this.stats.jobtotals.most_occupied_pool || this.noval;
+               const occupancy = this.stats.jobtotals.most_occupied_pool_count || this.noval;
+               document.getElementById(this.ids.pools.no).textContent = pool_no;
+               document.getElementById(this.ids.pools.most).setAttribute('title', most_occupied_pool);
+               document.getElementById(this.ids.pools.most).textContent = Strings.get_short_label(most_occupied_pool);
                document.getElementById(this.ids.pools.jobs).textContent = occupancy;
        },
        update_pie_jobstatus: function() {
index 3861c40ae650e363ec2a34b6639af12ced5e45d6..b96e77ee7df4f0814b8ba41c38de2cae9cf7e711 100644 (file)
@@ -4,8 +4,6 @@ var Statistics = {
        pools: null,
        jobtotals: null,
        dbsize: null,
-       clients_occupancy: {},
-       pools_occupancy: {},
        jobs_summary: [],
        grab_statistics: function(data, opts) {
                this.jobs = data.jobs;
@@ -14,9 +12,6 @@ var Statistics = {
                this.jobtotals = data.jobtotals;
                this.dbsize = data.dbsize;
                var jobs_count = this.jobs.length;
-               var clients_occupancy = {};
-               var pools_occupancy = {};
-               var jobs_occupancy = {};
                var jobs_summary = {
                        ok: [],
                        error: [],
@@ -33,23 +28,6 @@ var Statistics = {
                        if (opts.job_age > 0 && job_time_ts < start_time_ts) {
                                continue;
                        }
-                       if (typeof(clients_occupancy[this.jobs[i].clientid]) === 'undefined') {
-                               clients_occupancy[this.jobs[i].clientid] = 1;
-                       } else {
-                               clients_occupancy[this.jobs[i].clientid] += 1;
-                       }
-
-                       if (typeof(pools_occupancy[this.jobs[i].poolid]) === 'undefined') {
-                               pools_occupancy[this.jobs[i].poolid] = 1;
-                       } else {
-                               pools_occupancy[this.jobs[i].poolid] += 1;
-                       }
-
-                       if (typeof(jobs_occupancy[this.jobs[i].name]) === 'undefined') {
-                               jobs_occupancy[this.jobs[i].name] = 1;
-                       } else {
-                               jobs_occupancy[this.jobs[i].name] += 1;
-                       }
                        if (opts.job_states.hasOwnProperty(this.jobs[i].jobstatus)) {
                                status_type = opts.job_states[this.jobs[i].jobstatus].type;
                                if (status_type == 'ok' && this.jobs[i].joberrors > 0) {
@@ -61,25 +39,7 @@ var Statistics = {
                                jobs_summary[status_type].push(this.jobs[i]);
                        }
                }
-               var clients_ids = Object.keys(clients_occupancy);
-               for (var i = 0; i < clients_ids.length; i++) {
-                       for (var j = 0; j < this.clients.length; j++) {
-                               if (clients_ids[i] == this.clients[j].clientid) {
-                                       this.clients_occupancy[this.clients[j].name] = clients_occupancy[clients_ids[i]];
-                               }
-                       }
-               }
-
-               var pools_ids = Object.keys(pools_occupancy);
-               for (var i = 0; i < pools_ids.length; i++) {
-                       for (var j = 0; j < this.pools.length; j++) {
-                               if (pools_ids[i] == this.pools[j].poolid) {
-                                       this.pools_occupancy[this.pools[j].name] = pools_occupancy[pools_ids[i]];
-                               }
-                       }
-               }
 
-               this.jobs_occupancy = jobs_occupancy;
                this.jobs_summary = jobs_summary;
        }
 }
index 27362f1f1189f31b0cf7bb42d6d910051788c36b..8ccf949c74c74cb6660cb3610855cbe2cb27a697 100644 (file)
@@ -56,10 +56,15 @@ class Monitor extends BaculumPage {
 
                $web_config = $this->getModule('web_config')->getConfig();
                $job_limit = WebConfig::DEF_MAX_JOBS;
-               if (count($web_config) > 0 && key_exists('max_jobs', $web_config['baculum'])) {
-                       $job_limit = $web_config['baculum']['max_jobs'];
-               }
-
+               $job_age = 0;
+               if (count($web_config) > 0) {
+                       if (key_exists('max_jobs', $web_config['baculum'])) {
+                               $job_limit = $web_config['baculum']['max_jobs'];
+                       }
+                       if (key_exists('job_age_on_job_status_graph', $web_config['baculum'])) {
+                               $job_age = $web_config['baculum']['job_age_on_job_status_graph'];
+                       }
+               }
                $error = null;
                $params = $this->Request->contains('params') ? $this->Request['params'] : [];
                if (!is_array($params)) {
@@ -74,11 +79,13 @@ class Monitor extends BaculumPage {
                        if (is_array($params['jobs'])) {
                                if (key_exists('name', $params['jobs']) && is_array($params['jobs']['name'])) {
                                        for ($i = 0; $i < count($params['jobs']['name']); $i++) {
+                                               // @TODO: Add support for multiple names in query
                                                $job_query['name'] = $params['jobs']['name'][$i];
                                        }
                                }
                                if (key_exists('client', $params['jobs']) && is_array($params['jobs']['client'])) {
                                        for ($i = 0; $i < count($params['jobs']['client']); $i++) {
+                                               // @TODO: Add support for multiple clients in query
                                                $job_query['client'] = $params['jobs']['client'][$i];
                                        }
                                }
@@ -86,6 +93,10 @@ class Monitor extends BaculumPage {
                        if ($this->Request->contains('use_limit') && $this->Request['use_limit'] == 1) {
                                $job_query['limit'] = $job_limit;
                        }
+                       if ($this->Request->contains('use_age') && $this->Request['use_age'] == 1) {
+                               $job_query['age'] = $job_age;
+                       }
+
                        if (count($job_query) > 0) {
                                $job_params[] = '?' . http_build_query($job_query);
                        }
index b06ec80d38dc159aad65bd98b188c6f337b4f61b..a57e2346b18fb2c9dc31d970f03c1ab41ebfe88f 100644 (file)
@@ -16,6 +16,7 @@ $(function() {
                        data: {
                                'params': (typeof(MonitorParams) == 'object' ? MonitorParams : []),
                                'use_limit' : <%=$this->Service->getRequestedPagePath() == "Dashboard" ? '0' : '1'%>,
+                               'use_age' : <%=$this->Service->getRequestedPagePath() == "Dashboard" ? '1' : '0'%>
                        },
                        beforeSend: function() {
                                last_callback_time = new Date().getTime();