From: Michael Tremer Date: Tue, 31 Oct 2023 10:53:39 +0000 (+0000) Subject: buildservice: Move submitting stats into C X-Git-Tag: 0.9.30~1376 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=d39f3da5a64ee17110da73919a1ac8003b2a57c7;p=pakfire.git buildservice: Move submitting stats into C Signed-off-by: Michael Tremer --- diff --git a/configure.ac b/configure.ac index f417f8af2..8bf8dc96a 100644 --- a/configure.ac +++ b/configure.ac @@ -274,9 +274,7 @@ AC_CHECK_FUNCS([ \ # Python Stuff AM_PATH_PYTHON([3.6]) -AX_PYTHON_MODULE([cpuinfo], [fatal]) AX_PYTHON_MODULE([kerberos], [fatal]) -AX_PYTHON_MODULE([psutil], [fatal]) AX_PYTHON_MODULE([setproctitle], [fatal]) AX_PYTHON_MODULE([systemd], [fatal]) AX_PYTHON_MODULE([tornado], [fatal]) diff --git a/src/_pakfire/buildservice.c b/src/_pakfire/buildservice.c index 2db33330b..9a1263206 100644 --- a/src/_pakfire/buildservice.c +++ b/src/_pakfire/buildservice.c @@ -69,6 +69,19 @@ static PyObject* BuildService_get_url(BuildServiceObject* self) { return PyUnicode_FromString(url); } +static PyObject* BuildService_submit_stats(BuildServiceObject* self) { + int r; + + // Do what it says on the tin... + r = pakfire_buildservice_submit_stats(self->service); + if (r < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + Py_RETURN_NONE; +} + static PyObject* BuildService_upload(BuildServiceObject* self, PyObject* args, PyObject* kwds) { const char* kwlist[] = { "path", "filename", NULL }; const char* filename = NULL; @@ -103,6 +116,12 @@ ERROR: } static struct PyMethodDef BuildService_methods[] = { + { + "submit_stats", + (PyCFunction)BuildService_submit_stats, + METH_NOARGS, + NULL, + }, { "upload", (PyCFunction)BuildService_upload, diff --git a/src/libpakfire/buildservice.c b/src/libpakfire/buildservice.c index 5b469d9bd..3043e0e51 100644 --- a/src/libpakfire/buildservice.c +++ b/src/libpakfire/buildservice.c @@ -24,12 +24,14 @@ #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -974,3 +976,240 @@ ERROR: return r; } + +PAKFIRE_EXPORT int pakfire_buildservice_submit_stats(struct pakfire_buildservice* service) { + struct pakfire_cpuinfo cpuinfo = {}; + struct pakfire_cpustat cpustat = {}; + struct pakfire_loadavg loadavg = {}; + struct pakfire_meminfo meminfo = {}; + const char* arch = NULL; + int r; + + struct pakfire_xfer* xfer = NULL; + json_object* response = NULL; + char* buffer = NULL; + size_t length = 0; + + // Fetch CPU info + r = pakfire_cpuinfo(&cpuinfo); + if (r) { + CTX_ERROR(service->ctx, "Failed to fetch CPU info: %s\n", strerror(-r)); + goto ERROR; + } + + // Fetch CPU stats + r = pakfire_cpustat(&cpustat); + if (r) { + CTX_ERROR(service->ctx, "Failed to fetch CPU stats: %s\n", strerror(-r)); + goto ERROR; + } + + // Fetch load average + r = pakfire_loadavg(&loadavg); + if (r) { + CTX_ERROR(service->ctx, "Failed to fetch load average: %s\n", strerror(-r)); + goto ERROR; + } + + // Fetch meminfo + r = pakfire_meminfo(&meminfo); + if (r) { + CTX_ERROR(service->ctx, "Failed to fetch meminfo: %s\n", strerror(-r)); + goto ERROR; + } + + // Fetch native arch + arch = pakfire_arch_native(); + if (!arch) { + CTX_ERROR(service->ctx, "Failed to fetch native arch: %s\n", strerror(errno)); + r = -errno; + goto ERROR; + } + + // Create a new xfer + r = pakfire_buildservice_create_xfer(&xfer, service, "/api/v1/builders/stats"); + if (r) + goto ERROR; + + // Enable authentication + r = pakfire_xfer_auth(xfer); + if (r) + goto ERROR; + + // CPU Model + r = pakfire_xfer_add_param(xfer, "cpu_model", "%s", cpuinfo.model); + if (r) + goto ERROR; + + // CPU Count + r = pakfire_xfer_add_param(xfer, "cpu_count", "%u", cpuinfo.count); + if (r) + goto ERROR; + + // CPU Arch + r = pakfire_xfer_add_param(xfer, "cpu_arch", "%s", arch); + if (r) + goto ERROR; + + // Pakfire Version + r = pakfire_xfer_add_param(xfer, "pakfire_version", "%s", PACKAGE_VERSION); + if (r) + goto ERROR; + + // OS + r = pakfire_xfer_add_param(xfer, "os_name", "%s", "???"); + if (r) + goto ERROR; + + // CPU Stats: User + r = pakfire_xfer_add_param(xfer, "cpu_user", "%.6f", cpustat.user); + if (r) + goto ERROR; + + // CPU Stats: Nice + r = pakfire_xfer_add_param(xfer, "cpu_nice", "%.6f", cpustat.nice); + if (r) + goto ERROR; + + // CPU Stats: System + r = pakfire_xfer_add_param(xfer, "cpu_system", "%.6f", cpustat.system); + if (r) + goto ERROR; + + // CPU Stats: Idle + r = pakfire_xfer_add_param(xfer, "cpu_idle", "%.6f", cpustat.idle); + if (r) + goto ERROR; + + // CPU Stats: IO Wait + r = pakfire_xfer_add_param(xfer, "cpu_iowait", "%.6f", cpustat.iowait); + if (r) + goto ERROR; + + // CPU Stats: IRQ + r = pakfire_xfer_add_param(xfer, "cpu_irq", "%.6f", cpustat.irq); + if (r) + goto ERROR; + + // CPU Stats: Soft IRQ + r = pakfire_xfer_add_param(xfer, "cpu_softirq", "%.6f", cpustat.softirq); + if (r) + goto ERROR; + + // CPU Stats: Steal + r = pakfire_xfer_add_param(xfer, "cpu_steal", "%.6f", cpustat.steal); + if (r) + goto ERROR; + + // CPU Stats: Guest + r = pakfire_xfer_add_param(xfer, "cpu_guest", "%.6f", cpustat.guest); + if (r) + goto ERROR; + + // CPU Stats: Guest Nice + r = pakfire_xfer_add_param(xfer, "cpu_guest_nice", "%.6f", cpustat.guest_nice); + if (r) + goto ERROR; + + // Load Average: 1 + r = pakfire_xfer_add_param(xfer, "loadavg1", "%.2f", loadavg.load1); + if (r) + goto ERROR; + + // Load Average: 5 + r = pakfire_xfer_add_param(xfer, "loadavg5", "%.2f", loadavg.load5); + if (r) + goto ERROR; + + // Load Average: 15 + r = pakfire_xfer_add_param(xfer, "loadavg15", "%.2f", loadavg.load15); + if (r) + goto ERROR; + + // Memory: Total + r = pakfire_xfer_add_param(xfer, "mem_total", "%lu", meminfo.total); + if (r) + goto ERROR; + + // Memory: Available + r = pakfire_xfer_add_param(xfer, "mem_available", "%lu", meminfo.available); + if (r) + goto ERROR; + + // Memory: Used + r = pakfire_xfer_add_param(xfer, "mem_used", "%lu", meminfo.used); + if (r) + goto ERROR; + + // Memory: Free + r = pakfire_xfer_add_param(xfer, "mem_free", "%lu", meminfo.free); + if (r) + goto ERROR; + + // Memory: Active + r = pakfire_xfer_add_param(xfer, "mem_active", "%lu", meminfo.active); + if (r) + goto ERROR; + + // Memory: Inactive + r = pakfire_xfer_add_param(xfer, "mem_inactive", "%lu", meminfo.inactive); + if (r) + goto ERROR; + + // Memory: Buffers + r = pakfire_xfer_add_param(xfer, "mem_buffers", "%lu", meminfo.buffers); + if (r) + goto ERROR; + + // Memory: Cached + r = pakfire_xfer_add_param(xfer, "mem_cached", "%lu", meminfo.cached); + if (r) + goto ERROR; + + // Memory: Shared + r = pakfire_xfer_add_param(xfer, "mem_shared", "%lu", meminfo.shared); + if (r) + goto ERROR; + + // Swap: Total + r = pakfire_xfer_add_param(xfer, "swap_total", "%lu", meminfo.swap_total); + if (r) + goto ERROR; + + // Swap: Used + r = pakfire_xfer_add_param(xfer, "swap_used", "%lu", meminfo.swap_used); + if (r) + goto ERROR; + + // Swap: Free + r = pakfire_xfer_add_param(xfer, "swap_free", "%lu", meminfo.swap_free); + if (r) + goto ERROR; + + // Write the response to the buffer + r = pakfire_xfer_set_output_buffer(xfer, &buffer, &length); + if (r) + goto ERROR; + + // Run the xfer + r = pakfire_xfer_run(xfer, PAKFIRE_XFER_NO_PROGRESS); + if (r) + goto ERROR; + + // Parse the response + r = pakfire_buildservice_parse_response(service, xfer, buffer, length, &response); + if (r) { + CTX_ERROR(service->ctx, "Could not parse the response: %s\n", strerror(-r)); + goto ERROR; + } + +ERROR: + if (xfer) + pakfire_xfer_unref(xfer); + if (response) + json_object_put(response); + if (buffer) + free(buffer); + + return r; +} diff --git a/src/libpakfire/include/pakfire/buildservice.h b/src/libpakfire/include/pakfire/buildservice.h index b711c0324..d705aad18 100644 --- a/src/libpakfire/include/pakfire/buildservice.h +++ b/src/libpakfire/include/pakfire/buildservice.h @@ -63,4 +63,8 @@ int pakfire_buildservice_create_repo(struct pakfire_buildservice* service, int pakfire_buildservice_delete_repo(struct pakfire_buildservice* service, const char* distro, const char* name); +// Stats + +int pakfire_buildservice_submit_stats(struct pakfire_buildservice* service); + #endif /* PAKFIRE_BUILDSERVICE_H */ diff --git a/src/libpakfire/libpakfire.sym b/src/libpakfire/libpakfire.sym index 872d1a9e0..80493ffad 100644 --- a/src/libpakfire/libpakfire.sym +++ b/src/libpakfire/libpakfire.sym @@ -94,6 +94,7 @@ global: pakfire_buildservice_create_repo; pakfire_buildservice_delete_repo; pakfire_buildservice_get_url; + pakfire_buildservice_submit_stats; # dependencies pakfire_static_version_compare; diff --git a/src/pakfire/buildservice.py b/src/pakfire/buildservice.py index 5df32f9c7..f05acfd6b 100644 --- a/src/pakfire/buildservice.py +++ b/src/pakfire/buildservice.py @@ -19,12 +19,10 @@ ############################################################################### import asyncio -import cpuinfo import json import kerberos import logging import os -import psutil import subprocess import tempfile import tornado.httpclient @@ -401,75 +399,6 @@ class ControlConnection(Connection): "job" : self.daemon.job_received, } - # Fetch processor information - self.cpu = cpuinfo.get_cpu_info() - - # Fetch the native architecture - self.native_arch = _pakfire.native_arch() - - async def submit_stats(self): - """ - Sends stats about this builder - """ - log.debug("Sending stats...") - - # Fetch processor information - cpu_times = psutil.cpu_times_percent() - - # Fetch memory/swap information - mem = psutil.virtual_memory() - swap = psutil.swap_memory() - - # Fetch load average - loadavg = psutil.getloadavg() - - await self.write_message({ - "type" : "stats", - "data" : { - # CPU info - "cpu_model" : self.cpu.get("brand"), - "cpu_count" : self.cpu.get("count"), - "cpu_arch" : self.native_arch, - - # Pakfire + OS - "pakfire_version" : PAKFIRE_VERSION, - "os_name" : util.get_distro_name(), - - # CPU Times - "cpu_user" : cpu_times.user, - "cpu_nice" : cpu_times.nice, - "cpu_system" : cpu_times.system, - "cpu_idle" : cpu_times.idle, - "cpu_iowait" : cpu_times.iowait, - "cpu_irq" : cpu_times.irq, - "cpu_softirq" : cpu_times.softirq, - "cpu_steal" : cpu_times.steal, - "cpu_guest" : cpu_times.guest, - "cpu_guest_nice" : cpu_times.guest_nice, - - # Load average - "loadavg1" : loadavg[0], - "loadavg5" : loadavg[1], - "loadavg15" : loadavg[2], - - # Memory - "mem_total" : mem.total, - "mem_available" : mem.available, - "mem_used" : mem.used, - "mem_free" : mem.free, - "mem_active" : mem.active, - "mem_inactive" : mem.inactive, - "mem_buffers" : mem.buffers, - "mem_cached" : mem.cached, - "mem_shared" : mem.shared, - - # Swap - "swap_total" : swap.total, - "swap_used" : swap.used, - "swap_free" : swap.free, - }, - }) - class JobControlConnection(Connection): """ diff --git a/src/pakfire/daemon.py b/src/pakfire/daemon.py index f1e25db38..ec794ee6f 100644 --- a/src/pakfire/daemon.py +++ b/src/pakfire/daemon.py @@ -74,7 +74,7 @@ class Daemon(object): # Run main loop while True: # Submit stats - await self.control.submit_stats() + self.service.submit_stats() # Check if we are running by awaiting the shutdown signal try: