From: Michael Tremer Date: Wed, 22 Jan 2025 12:47:33 +0000 (+0000) Subject: builders: Fix logging stats X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=97dccfcbabb06ea993dacaf0c9a0375ef05ba02f;p=pbs.git builders: Fix logging stats Signed-off-by: Michael Tremer --- diff --git a/src/buildservice/builders.py b/src/buildservice/builders.py index d66a99ad..f3fc80c7 100644 --- a/src/buildservice/builders.py +++ b/src/buildservice/builders.py @@ -7,7 +7,7 @@ import functools import logging import sqlalchemy -from sqlalchemy import BigInteger, Boolean, Column, DateTime, ForeignKey, Integer, Text +from sqlalchemy import BigInteger, Boolean, Column, DateTime, Double, ForeignKey, Integer, Text from . import base from . import database @@ -20,6 +20,149 @@ from .errors import * # Setup logging log = logging.getLogger("pbs.builders") +class BuilderStat(database.Base): + __tablename__ = "builder_stats" + + # Builder ID + + builder_id = Column(Integer, ForeignKey("builders.id"), primary_key=True, nullable=False) + + # Builder + + builder = sqlalchemy.orm.relationship( + "Builder", foreign_keys=[builder_id], lazy="selectin", + ) + + # Created At + + created_at = Column(DateTime(timezone=False), primary_key=True, nullable=False, + server_default=sqlalchemy.func.current_timestamp()) + + # CPU User + + cpu_user = Column(Double, nullable=False) + + # CPU Nice + + cpu_nice = Column(Double, nullable=False) + + # CPU System + + cpu_system = Column(Double, nullable=False) + + # CPU Idle + + cpu_idle = Column(Double, nullable=False) + + # CPU IO Wait + + cpu_iowait = Column(Double, nullable=False) + + # CPU IRQ + + cpu_irq = Column(Double, nullable=False) + + # CPU Soft IRQ + + cpu_softirq = Column(Double, nullable=False) + + # CPU Steal + + cpu_steal = Column(Double, nullable=False) + + # CPU Guest + + cpu_guest = Column(Double, nullable=False) + + # CPU Guest Nice + + cpu_guest_nice = Column(Double, nullable=False) + + @property + def cpu_usage(self): + """ + Returns the CPU usage in percent + """ + return 1 - self.cpu_idle + + # Load Average 1 + + loadavg1 = Column(Double, nullable=False) + + # Load Average 5 + + loadavg5 = Column(Double, nullable=False) + + # Load Average 15 + + loadavg15 = Column(Double, nullable=False) + + # Memory Total + + mem_total = Column(BigInteger, nullable=False) + + # Memory Available + + mem_available = Column(BigInteger, nullable=False) + + # Memory Used + + mem_used = Column(BigInteger, nullable=False) + + # Memory Free + + mem_free = Column(BigInteger, nullable=False) + + # Memory Active + + mem_active = Column(BigInteger, nullable=False) + + # Memory Inactive + + mem_inactive = Column(BigInteger, nullable=False) + + # Memory Buffers + + mem_buffers = Column(BigInteger, nullable=False) + + # Memory Cached + + mem_cached = Column(BigInteger, nullable=False) + + # Memory Shared + + mem_shared = Column(BigInteger, nullable=False) + + @property + def mem_usage(self): + """ + Returns the amount of used memory in percent + """ + return (self.mem_total - self.mem_available) / self.mem_total * 100 + + # Swap Total + + swap_total = Column(BigInteger, nullable=False) + + # Swap Used + + swap_used = Column(BigInteger, nullable=False) + + # Swap Free + + swap_free = Column(BigInteger, nullable=False) + + @property + def swap_usage(self): + """ + Returns the amount of used swap in percent + """ + if not self.swap_total: + return 0 + + return self.swap_used / self.swap_total * 100 + + class Builders(base.Object): # Stores any control connections to builders connections = {} @@ -326,135 +469,6 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): maintenance = Column(Boolean, nullable=False, default=False) - # Stats - - def _get_stats(self, query, *args, **kwargs): - res = self.db.get(query, *args, **kwargs) - - if res: - return BuilderStats(self.backend, self, res) - - async def log_stats(self, cpu_model=None, cpu_count=None, cpu_arch=None, pakfire_version=None, - os_name=None, cpu_user=None, cpu_nice=None, cpu_system=None, cpu_idle=None, - cpu_iowait=None, cpu_irq=None, cpu_softirq=None, cpu_steal=None, cpu_guest=None, - cpu_guest_nice=None, loadavg1=None, loadavg5=None, loadavg15=None, mem_total=None, - mem_available=None, mem_used=None, mem_free=None, mem_active=None, mem_inactive=None, - mem_buffers=None, mem_cached=None, mem_shared=None, swap_total=None, swap_used=None, - swap_free=None): - """ - Logs some stats about this builder - """ - # Update information - await self.db.execute(""" - UPDATE - builders - SET - updated_at = CURRENT_TIMESTAMP, - cpu_model = %s, - cpu_count = %s, - cpu_arch = %s, - mem_total = %s, - pakfire_version = %s, - os_name = %s - WHERE - id = %s""", - cpu_model, - cpu_count, - cpu_arch, - mem_total, - pakfire_version, - os_name, - self.id, - ) - - # Log Stats - stats = await self._get_stats(""" - INSERT INTO - builder_stats - ( - builder_id, - cpu_user, - cpu_nice, - cpu_system, - cpu_idle, - cpu_iowait, - cpu_irq, - cpu_softirq, - cpu_steal, - cpu_guest, - cpu_guest_nice, - loadavg1, - loadavg5, - loadavg15, - mem_total, - mem_available, - mem_used, - mem_free, - mem_active, - mem_inactive, - mem_buffers, - mem_cached, - mem_shared, - swap_total, - swap_used, - swap_free - ) - VALUES ( - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, - %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s - ) - RETURNING *""", - self.id, - cpu_user, - cpu_nice, - cpu_system, - cpu_idle, - cpu_iowait, - cpu_irq, - cpu_softirq, - cpu_steal, - cpu_guest, - cpu_guest_nice, - loadavg1, - loadavg5, - loadavg15, - mem_total, - mem_available, - mem_used, - mem_free, - mem_active, - mem_inactive, - mem_buffers, - mem_cached, - mem_shared, - swap_total, - swap_used, - swap_free, - ) - - # Send out the stats - await self.backend.builders.stats.submit_stats(self, stats) - - @lazy_property - def stats(self): - """ - Returns the latest stats data (if any) - """ - return self._get_stats(""" - SELECT - * - FROM - builder_stats - WHERE - builder_id = %s - AND - created_at >= CURRENT_TIMESTAMP - INTERVAL '10 minutes' - ORDER BY - created_at DESC - LIMIT 1""", - self.id, - ) - # Enabled enabled = Column(Boolean, nullable=False, default=False) @@ -548,10 +562,6 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): cpu_arch = Column(Text) - # Mem Total - - mem_total = Column(BigInteger, nullable=False, default=0) - # AWS - Instance ID instance_id = Column(Text) @@ -725,6 +735,48 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): # Stats + async def get_stats(self): + """ + Fetch the latest stats + """ + stmt = ( + sqlalchemy + .select( + BuilderStat, + ) + .where( + BuilderStat.builder == self, + ) + .order_by( + BuilderStat.created_at.desc(), + ) + .limit(1) + ) + + return await self.db.fetch_one(stmt) + + async def log_stats(self, cpu_model=None, cpu_count=None, cpu_arch=None, + pakfire_version=None, os_name=None, **kwargs): + """ + Logs some stats about this builder + """ + # Update CPU information + self.cpu_model = cpu_model + self.cpu_count = cpu_count + self.cpu_arch = cpu_arch + + # Update Pakfire & OS information + self.pakfire_version = pakfire_version + self.os_name = os_name + + # Log Stats + stats = await self.db.insert( + BuilderStat, builder=self, **kwargs, + ) + + # Send out the stats + await self.backend.builders.stats.submit_stats(self, stats) + async def get_total_build_time(self): """ Returns the total build time @@ -843,33 +895,3 @@ class Builder(database.Base, database.BackendMixin, database.SoftDeleteMixin): ) return self.db.fetch(stmt) - - -class BuilderStats(base.Object): - def init(self, builder, data): - self.builder = builder - self.data = data - - @property - def cpu_usage(self): - """ - Returns the CPU usage in percent - """ - return 1 - self.data.cpu_idle - - @property - def mem_usage(self): - """ - Returns the amount of used memory in percent - """ - return (self.data.mem_total - self.data.mem_available) / self.data.mem_total * 100 - - @property - def swap_usage(self): - """ - Returns the amount of used swap in percent - """ - if not self.data.swap_total: - return 0 - - return self.data.swap_used / self.data.swap_total * 100 diff --git a/src/database.sql b/src/database.sql index b7ca0d61..aff52764 100644 --- a/src/database.sql +++ b/src/database.sql @@ -201,11 +201,9 @@ CREATE TABLE public.builders ( cpu_model text, cpu_count integer DEFAULT 1 NOT NULL, host_key_id text, - updated_at timestamp without time zone, cpu_arch text, instance_id text, instance_type text, - mem_total bigint, created_at timestamp without time zone DEFAULT CURRENT_TIMESTAMP NOT NULL, created_by_id integer, deleted_at timestamp without time zone, diff --git a/src/templates/builders/show.html b/src/templates/builders/show.html index 2e4b4eff..d02a6523 100644 --- a/src/templates/builders/show.html +++ b/src/templates/builders/show.html @@ -45,6 +45,9 @@ {% endif %} + {# Fetch stats #} + {% set stats = builder.get_stats() %} +
{% if builder.cpu_model %}
@@ -60,12 +63,12 @@
{% endif %} - {% if builder.mem_total %} + {% if stats and stats.mem_total %}

{{ _("Memory") }}

- {{ builder.mem_total | filesizeformat(binary=True) }} + {{ stats.mem_total | filesizeformat(binary=True) }}

diff --git a/src/web/builders.py b/src/web/builders.py index 31c47e1c..5cbb149d 100644 --- a/src/web/builders.py +++ b/src/web/builders.py @@ -42,19 +42,13 @@ class APIv1ControlHandler(base.APIMixin, base.BackendMixin, tornado.websocket.We # Handle stats if type == "stats": - await self._handle_stats(data) + async with await self.db.transaction(): + await builder.log_stats(**data) # Log an error and ignore any other messages else: log.error("Received message of type '%s' which we cannot handle here" % type) - async def _handle_stats(self, data): - """ - Handles stats messages - """ - async with await self.db.transaction(): - await self.builder.log_stats(**data) - class StatsHandler(base.BaseHandler, tornado.websocket.WebSocketHandler): # No authentication required