builders: Fix logging stats

Signed-off-by: Michael Tremer <michael.tremer@ipfire.org>

diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py
index d66a99ad2dd3b46b9a78c149e46ac6f991d3ca60..f3fc80c7b0c1a010f22b75afdb57a68c34e638a6 100644 (file)
--- a/src/buildservice/builders.py
+++ b/src/buildservice/builders.py
@@ -7,7 +7,7 @@ import functools
 import logging
 
 import sqlalchemy
-from sqlalchemy import BigInteger, Boolean, Column, DateTime, ForeignKey, Integer, Text
+from sqlalchemy import BigInteger, Boolean, Column, DateTime, Double, ForeignKey, Integer, Text
 
 from . import base
 from . import database
@@ -20,6 +20,149 @@ from .errors import *
 # Setup logging
 log = logging.getLogger("pbs.builders")
 
+class BuilderStat(database.Base):
+       __tablename__ = "builder_stats"
+
+       # Builder ID
+
+       builder_id = Column(Integer, ForeignKey("builders.id"), primary_key=True, nullable=False)
+
+       # Builder
+
+       builder = sqlalchemy.orm.relationship(
+               "Builder", foreign_keys=[builder_id], lazy="selectin",
+       )
+
+       # Created At
+
+       created_at = Column(DateTime(timezone=False), primary_key=True, nullable=False,
+               server_default=sqlalchemy.func.current_timestamp())
+
+       # CPU User
+
+       cpu_user = Column(Double, nullable=False)
+
+       # CPU Nice
+
+       cpu_nice = Column(Double, nullable=False)
+
+       # CPU System
+
+       cpu_system = Column(Double, nullable=False)
+
+       # CPU Idle
+
+       cpu_idle = Column(Double, nullable=False)
+
+       # CPU IO Wait
+
+       cpu_iowait = Column(Double, nullable=False)
+
+       # CPU IRQ
+
+       cpu_irq = Column(Double, nullable=False)
+
+       # CPU Soft IRQ
+
+       cpu_softirq = Column(Double, nullable=False)
+
+       # CPU Steal
+
+       cpu_steal = Column(Double, nullable=False)
+
+       # CPU Guest
+
+       cpu_guest = Column(Double, nullable=False)
+
+       # CPU Guest Nice
+
+       cpu_guest_nice = Column(Double, nullable=False)
+
+       @property
+       def cpu_usage(self):
+               """
+                       Returns the CPU usage in percent
+               """
+               return 1 - self.cpu_idle
+
+       # Load Average 1
+
+       loadavg1 = Column(Double, nullable=False)
+
+       # Load Average 5
+
+       loadavg5 = Column(Double, nullable=False)
+
+       # Load Average 15
+
+       loadavg15 = Column(Double, nullable=False)
+
+       # Memory Total
+
+       mem_total = Column(BigInteger, nullable=False)
+
+       # Memory Available
+
+       mem_available = Column(BigInteger, nullable=False)
+
+       # Memory Used
+
+       mem_used = Column(BigInteger, nullable=False)
+
+       # Memory Free
+
+       mem_free = Column(BigInteger, nullable=False)
+
+       # Memory Active
+
+       mem_active = Column(BigInteger, nullable=False)
+
+       # Memory Inactive
+
+       mem_inactive = Column(BigInteger, nullable=False)
+
+       # Memory Buffers
+
+       mem_buffers = Column(BigInteger, nullable=False)
+
+       # Memory Cached
+
+       mem_cached = Column(BigInteger, nullable=False)
+
+       # Memory Shared
+
+       mem_shared = Column(BigInteger, nullable=False)
+
+       @property
+       def mem_usage(self):
+               """
+                       Returns the amount of used memory in percent
+               """
+               return (self.mem_total - self.mem_available) / self.mem_total * 100
+
+       # Swap Total
+
+       swap_total = Column(BigInteger, nullable=False)
+
+       # Swap Used
+
+       swap_used = Column(BigInteger, nullable=False)
+
+       # Swap Free
+
+       swap_free = Column(BigInteger, nullable=False)
+
+       @property
+       def swap_usage(self):
+               """
+                       Returns the amount of used swap in percent
+               """
+               if not self.swap_total:
+                       return 0
+
+               return self.swap_used / self.swap_total * 100
+
+
 class Builders(base.Object):
        # Stores any control connections to builders
        connections = {}
@@ -326,135 +469,6 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        maintenance = Column(Boolean, nullable=False, default=False)
 
-       # Stats
-
-       def _get_stats(self, query, *args, **kwargs):
-               res = self.db.get(query, *args, **kwargs)
-
-               if res:
-                       return BuilderStats(self.backend, self, res)
-
-       async def log_stats(self, cpu_model=None, cpu_count=None, cpu_arch=None, pakfire_version=None,
-                       os_name=None, cpu_user=None, cpu_nice=None, cpu_system=None, cpu_idle=None,
-                       cpu_iowait=None, cpu_irq=None, cpu_softirq=None, cpu_steal=None, cpu_guest=None,
-                       cpu_guest_nice=None, loadavg1=None, loadavg5=None, loadavg15=None, mem_total=None,
-                       mem_available=None, mem_used=None, mem_free=None, mem_active=None, mem_inactive=None,
-                       mem_buffers=None, mem_cached=None, mem_shared=None, swap_total=None, swap_used=None,
-                       swap_free=None):
-               """
-                       Logs some stats about this builder
-               """
-               # Update information
-               await self.db.execute("""
-                       UPDATE
-                               builders
-                       SET
-                               updated_at = CURRENT_TIMESTAMP,
-                               cpu_model = %s,
-                               cpu_count = %s,
-                               cpu_arch = %s,
-                               mem_total = %s,
-                               pakfire_version = %s,
-                               os_name = %s
-                       WHERE
-                               id = %s""",
-                       cpu_model,
-                       cpu_count,
-                       cpu_arch,
-                       mem_total,
-                       pakfire_version,
-                       os_name,
-                       self.id,
-               )
-
-               # Log Stats
-               stats = await self._get_stats("""
-                       INSERT INTO
-                               builder_stats
-                       (
-                               builder_id,
-                               cpu_user,
-                               cpu_nice,
-                               cpu_system,
-                               cpu_idle,
-                               cpu_iowait,
-                               cpu_irq,
-                               cpu_softirq,
-                               cpu_steal,
-                               cpu_guest,
-                               cpu_guest_nice,
-                               loadavg1,
-                               loadavg5,
-                               loadavg15,
-                               mem_total,
-                               mem_available,
-                               mem_used,
-                               mem_free,
-                               mem_active,
-                               mem_inactive,
-                               mem_buffers,
-                               mem_cached,
-                               mem_shared,
-                               swap_total,
-                               swap_used,
-                               swap_free
-                       )
-                       VALUES (
-                               %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
-                               %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s
-                       )
-                       RETURNING *""",
-                       self.id,
-                       cpu_user,
-                       cpu_nice,
-                       cpu_system,
-                       cpu_idle,
-                       cpu_iowait,
-                       cpu_irq,
-                       cpu_softirq,
-                       cpu_steal,
-                       cpu_guest,
-                       cpu_guest_nice,
-                       loadavg1,
-                       loadavg5,
-                       loadavg15,
-                       mem_total,
-                       mem_available,
-                       mem_used,
-                       mem_free,
-                       mem_active,
-                       mem_inactive,
-                       mem_buffers,
-                       mem_cached,
-                       mem_shared,
-                       swap_total,
-                       swap_used,
-                       swap_free,
-               )
-
-               # Send out the stats
-               await self.backend.builders.stats.submit_stats(self, stats)
-
-       @lazy_property
-       def stats(self):
-               """
-                       Returns the latest stats data (if any)
-               """
-               return self._get_stats("""
-                       SELECT
-                               *
-                       FROM
-                               builder_stats
-                       WHERE
-                               builder_id = %s
-                       AND
-                               created_at >= CURRENT_TIMESTAMP - INTERVAL '10 minutes'
-                       ORDER BY
-                               created_at DESC
-                       LIMIT 1""",
-                       self.id,
-               )
-
        # Enabled
 
        enabled = Column(Boolean, nullable=False, default=False)
@@ -548,10 +562,6 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        cpu_arch = Column(Text)
 
-       # Mem Total
-
-       mem_total = Column(BigInteger, nullable=False, default=0)
-
        # AWS - Instance ID
 
        instance_id = Column(Text)
@@ -725,6 +735,48 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
 
        # Stats
 
+       async def get_stats(self):
+               """
+                       Fetch the latest stats
+               """
+               stmt = (
+                       sqlalchemy
+                       .select(
+                               BuilderStat,
+                       )
+                       .where(
+                               BuilderStat.builder == self,
+                       )
+                       .order_by(
+                               BuilderStat.created_at.desc(),
+                       )
+                       .limit(1)
+               )
+
+               return await self.db.fetch_one(stmt)
+
+       async def log_stats(self, cpu_model=None, cpu_count=None, cpu_arch=None,
+                       pakfire_version=None, os_name=None, **kwargs):
+               """
+                       Logs some stats about this builder
+               """
+               # Update CPU information
+               self.cpu_model = cpu_model
+               self.cpu_count = cpu_count
+               self.cpu_arch  = cpu_arch
+
+               # Update Pakfire & OS information
+               self.pakfire_version = pakfire_version
+               self.os_name         = os_name
+
+               # Log Stats
+               stats = await self.db.insert(
+                       BuilderStat, builder=self, **kwargs,
+               )
+
+               # Send out the stats
+               await self.backend.builders.stats.submit_stats(self, stats)
+
        async def get_total_build_time(self):
                """
                        Returns the total build time
@@ -843,33 +895,3 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin):
                )
 
                return self.db.fetch(stmt)
-
-
-class BuilderStats(base.Object):
-       def init(self, builder, data):
-               self.builder = builder
-               self.data = data
-
-       @property
-       def cpu_usage(self):
-               """
-                       Returns the CPU usage in percent
-               """
-               return 1 - self.data.cpu_idle
-
-       @property
-       def mem_usage(self):
-               """
-                       Returns the amount of used memory in percent
-               """
-               return (self.data.mem_total - self.data.mem_available) / self.data.mem_total * 100
-
-       @property
-       def swap_usage(self):
-               """
-                       Returns the amount of used swap in percent
-               """
-               if not self.data.swap_total:
-                       return 0
-
-               return self.data.swap_used / self.data.swap_total * 100

diff --git a/src/database.sql b/src/database.sql
index b7ca0d610775f35eac88d7dd981858c54570234d..aff5276456812add19677fe374c2564380ed2895 100644 (file)
--- a/src/database.sql
+++ b/src/database.sql
@@ -201,11 +201,9 @@ CREATE TABLE public.builders (
     cpu_model text,
     cpu_count integer DEFAULT 1 NOT NULL,
     host_key_id text,
-    updated_at timestamp without time zone,
     cpu_arch text,
     instance_id text,
     instance_type text,
-    mem_total bigint,
     created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL,
     created_by_id integer,
     deleted_at timestamp without time zone,

diff --git a/src/templates/builders/show.html b/src/templates/builders/show.html
index 2e4b4effe52bf68552db724ec5e606b58dcf6b22..d02a65231d5509285000fe538b485215013c0937 100644 (file)
--- a/src/templates/builders/show.html
+++ b/src/templates/builders/show.html
@@ -45,6 +45,9 @@
                                        {% endif %}
                                </div>
 
+                               {# Fetch stats #}
+                               {% set stats = builder.get_stats() %}
+
                                <div class="level">
                                        {% if builder.cpu_model %}
                                                <div class="level-item has-text-centered">
@@ -60,12 +63,12 @@
                                                </div>
                                        {% endif %}
 
-                                       {% if builder.mem_total %}
+                                       {% if stats and stats.mem_total %}
                                                <div class="level-item has-text-centered">
                                                        <div>
                                                                <p class="heading">{{ _("Memory") }}</p>
                                                                <p>
-                                                                       {{ builder.mem_total | filesizeformat(binary=True) }}
+                                                                       {{ stats.mem_total | filesizeformat(binary=True) }}
                                                                </p>
                                                        </div>
                                                </div>

diff --git a/src/web/builders.py b/src/web/builders.py
index 31c47e1c53b9d3c8b83c66b88172045672d87e9f..5cbb149d290c370dfa0c63b4e55453f47484bb1a 100644 (file)
--- a/src/web/builders.py
+++ b/src/web/builders.py
@@ -42,19 +42,13 @@ class APIv1ControlHandler(base.APIMixin, base.BackendMixin, tornado.websocket.We
 
                # Handle stats
                if type == "stats":
-                       await self._handle_stats(data)
+                       async with await self.db.transaction():
+                               await builder.log_stats(**data)
 
                # Log an error and ignore any other messages
                else:
                        log.error("Received message of type '%s' which we cannot handle here" % type)
 
-       async def _handle_stats(self, data):
-               """
-                       Handles stats messages
-               """
-               async with await self.db.transaction():
-                       await self.builder.log_stats(**data)
-
 
 class StatsHandler(base.BaseHandler, tornado.websocket.WebSocketHandler):
        # No authentication required