From: Michael Tremer Date: Sun, 10 May 2015 18:30:48 +0000 (+0000) Subject: Add dbus interface X-Git-Tag: 002~10 X-Git-Url: http://git.ipfire.org/?p=collecty.git;a=commitdiff_plain;h=c968f6d9744a12474be417bfca1056c44c1eadc9 Add dbus interface Collecty will be able to generate graph images and send them to the collecty-client via dbus. --- diff --git a/Makefile.am b/Makefile.am index bccf86d..72c06ad 100644 --- a/Makefile.am +++ b/Makefile.am @@ -32,6 +32,10 @@ SUBDIRS = . po pythondir = $(pyexecdir) +# Dirs of external packages +dbuspolicydir=@dbuspolicydir@ +dbussystemservicedir=@dbussystemservicedir@ + CLEANFILES = DISTCLEANFILES = EXTRA_DIST = @@ -45,6 +49,8 @@ update-po: $(MAKE) -C po update-po DISTCHECK_CONFIGURE_FLAGS = \ + --with-dbuspolicydir=$$dc_install_base/$(dbuspolicydir) \ + --with-dbussystemservicedir=$$dc_install_base/$(dbussystemservicedir) \ --with-systemdsystemunitdir=$$dc_install_base/$(systemdsystemunitdir) # ------------------------------------------------------------------------------ @@ -66,6 +72,7 @@ dist_bin_SCRIPTS = \ collecty_PYTHON = \ src/collecty/__init__.py \ + src/collecty/bus.py \ src/collecty/client.py \ src/collecty/constants.py \ src/collecty/daemon.py \ @@ -89,6 +96,24 @@ collectyplugins_PYTHON = \ collectypluginsdir = $(collectydir)/plugins +dist_dbuspolicy_DATA = \ + src/dbus/org.ipfire.collecty1.conf + +dist_dbussystemservice_DATA = \ + src/dbus/org.ipfire.collecty1.service + +systemdsystemunit_DATA = \ + src/systemd/collecty.service + +dist_systemdsystemunit_DATA = \ + src/systemd/org.ipfire.collecty1.busname + +EXTRA_DIST += \ + src/systemd/collecty.service.in + +CLEANFILES += \ + src/systemd/collecty.service + # ------------------------------------------------------------------------------ if ENABLE_MANPAGES @@ -142,22 +167,6 @@ endif # ------------------------------------------------------------------------------ -if HAVE_SYSTEMD -systemdsystemunit_DATA = \ - src/systemd/collecty.service - -CLEANFILES += \ - $(systemdsystemunit_DATA) - -INSTALL_DIRS += \ - $(systemdsystemunitdir) -endif - -EXTRA_DIST += \ - src/systemd/collecty.service.in - -# ------------------------------------------------------------------------------ - substitutions = \ '|PACKAGE_NAME=$(PACKAGE_NAME)|' \ '|PACKAGE_VERSION=$(PACKAGE_VERSION)|' \ diff --git a/configure.ac b/configure.ac index 899fc2d..cdfcb66 100644 --- a/configure.ac +++ b/configure.ac @@ -76,6 +76,18 @@ AC_ARG_WITH([systemd], AS_HELP_STRING([--with-systemd], [Enable systemd support.]) ) +AC_ARG_WITH([dbuspolicydir], + AS_HELP_STRING([--with-dbuspolicydir=DIR], [D-Bus policy directory]), + [], + [with_dbuspolicydir=${sysconfdir}/dbus-1/system.d] +) + +AC_ARG_WITH([dbussystemservicedir], + AS_HELP_STRING([--with-dbussystemservicedir=DIR], [D-Bus system service directory]), + [], + [with_dbussystemservicedir=${datadir}/dbus-1/system-services] +) + AS_IF([test "x$with_systemd" != "xno"], [PKG_CHECK_MODULES(systemd, [libsystemd-daemon], [have_systemd=yes], [have_systemd=no])], @@ -104,6 +116,9 @@ AS_IF([test "x$have_systemd" = "xyes"], AM_CONDITIONAL(HAVE_SYSTEMD, [test "x$have_systemd" = "xyes"]) +AC_SUBST([dbuspolicydir], [$with_dbuspolicydir]) +AC_SUBST([dbussystemservicedir], [$with_dbussystemservicedir]) + # ------------------------------------------------------------------------------ AC_CONFIG_FILES([ @@ -116,6 +131,8 @@ AC_MSG_RESULT([ ${PACKAGE_NAME} ${VERSION} prefix: ${prefix} + D-Bus policy dir: ${with_dbuspolicydir} + D-Bus system dir: ${with_dbussystemservicedir} Systemd support ${have_systemd} Generate man-pages: ${have_manpages} diff --git a/po/POTFILES.in b/po/POTFILES.in index 193d1d9..77fbb19 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -1,3 +1,4 @@ +src/collecty/bus.py src/collecty/client.py src/collecty/constants.py src/collecty/daemon.py diff --git a/src/collecty-client b/src/collecty-client index a52be35..e6cdeed 100755 --- a/src/collecty-client +++ b/src/collecty-client @@ -19,13 +19,7 @@ # # ############################################################################### -import collecty +import collecty.client -client = collecty.CollectyClient() - -for ds in client.data_sources: - for template in ds.templates: - t = template(ds) - - for interval in ("-3h", "day", "week", "year"): - t.graph("graphs/%s-%s-%s.png" % (t.name, ds.id, interval), interval) +client = collecty.client.CollectyClient() +client.run_cli() diff --git a/src/collecty/bus.py b/src/collecty/bus.py new file mode 100644 index 0000000..6a3f0bd --- /dev/null +++ b/src/collecty/bus.py @@ -0,0 +1,90 @@ +#!/usr/bin/python +############################################################################### +# # +# collecty - A system statistics collection daemon for IPFire # +# Copyright (C) 2015 IPFire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +import dbus +import dbus.mainloop.glib +import dbus.service +import gobject +import threading + +from constants import * +from i18n import _ + +import logging +log = logging.getLogger("collecty.bus") +log.propagate = 1 + +# Initialise the glib main loop +dbus.mainloop.glib.DBusGMainLoop(set_as_default=True) +dbus.mainloop.glib.threads_init() + +class Bus(threading.Thread): + def __init__(self, collecty): + threading.Thread.__init__(self) + self.daemon = True + + self.collecty = collecty + + # Initialise the main loop + gobject.threads_init() + self.loop = gobject.MainLoop() + + # Register the GraphGenerator interface + self.generator = GraphGenerator(self.collecty) + + def run(self): + log.debug(_("Bus thread has started")) + + # Run the main loop + self.loop.run() + + def shutdown(self): + log.debug(_("Stopping bus thread")) + + # End the main loop + self.loop.quit() + + # Return when this thread has finished + return self.join() + + +class GraphGenerator(dbus.service.Object): + def __init__(self, collecty): + bus_name = dbus.service.BusName(BUS_DOMAIN, bus=dbus.SystemBus()) + dbus.service.Object.__init__(self, bus_name, "/%s" % self.__class__.__name__) + + self.collecty = collecty + + @dbus.service.method(BUS_DOMAIN, in_signature="sa{sv}", out_signature="ay") + def GenerateGraph(self, template_name, kwargs): + """ + Returns a graph generated from the given template and object. + """ + graph = self.collecty.generate_graph(template_name, **kwargs) + + return dbus.ByteArray(graph or []) + + @dbus.service.method(BUS_DOMAIN, in_signature="", out_signature="as") + def ListTemplates(self): + """ + Returns a list of all available templates + """ + return [t.name for t in self.collecty.templates] diff --git a/src/collecty/client.py b/src/collecty/client.py index c805870..4570c69 100644 --- a/src/collecty/client.py +++ b/src/collecty/client.py @@ -19,28 +19,92 @@ # # ############################################################################### -import daemon +import argparse +import dbus +import sys + +from constants import * +from i18n import _ import logging log = logging.getLogger("collectly.client") class CollectyClient(object): - def __init__(self, **settings): - self.collecty = daemon.Collecty(**settings) + def __init__(self): + self.bus = dbus.SystemBus() + + self.proxy = self.bus.get_object(BUS_DOMAIN, "/GraphGenerator") + + def list_templates(self): + templates = self.proxy.ListTemplates() + + return ["%s" % t for t in templates] + + def list_templates_cli(self, ns): + templates = self.list_templates() + + for t in sorted(templates): + print t + + def generate_graph(self, template_name, **kwargs): + byte_array = self.proxy.GenerateGraph(template_name, kwargs, + signature="sa{sv}") + + # Convert the byte array into a byte string again + if byte_array: + return "".join((chr(b) for b in byte_array)) + + def generate_graph_cli(self, ns): + kwargs = { + "object_id" : ns.object, + } + + if ns.height or ns.width: + kwargs.update({ + "height" : ns.height or 0, + "width" : ns.width or 0, + }) + + if ns.interval: + kwargs["interval"] = ns.interval + + # Generate the graph image + graph = self.generate_graph(ns.template, **kwargs) + + # Write file to disk + with open(ns.filename, "wb") as f: + f.write(graph) + + def parse_cli(self, args): + parser = argparse.ArgumentParser(prog="collecty-client") + subparsers = parser.add_subparsers(help="sub-command help") + + # generate-graph + parser_generate_graph = subparsers.add_parser("generate-graph", + help=_("Generate a graph image")) + parser_generate_graph.set_defaults(func=self.generate_graph_cli) + parser_generate_graph.add_argument("--filename", + help=_("filename"), required=True) + parser_generate_graph.add_argument("--interval", help=_("interval")) + parser_generate_graph.add_argument("--object", + help=_("Object identifier"), default="default") + parser_generate_graph.add_argument("--template", + help=_("The graph template identifier"), required=True) - @property - def data_sources(self): - return self.collecty.data_sources + # Dimensions + parser_generate_graph.add_argument("--height", type=int, default=0, + help=_("Height of the generated image")) + parser_generate_graph.add_argument("--width", type=int, default=0, + help=_("Width of the generated image")) - def get_data_source_by_id(self, id): - for ds in self.data_sources: - if not ds.id == id: - continue + # list-templates + parser_list_templates = subparsers.add_parser("list-templates", + help=_("Lists all graph templates")) + parser_list_templates.set_defaults(func=self.list_templates_cli) - return ds + return parser.parse_args(args) - def graph(self, id, filename, interval=None, **kwargs): - ds = self.get_data_source_by_id(id) - assert ds, "Could not find data source: %s" % id + def run_cli(self, args=None): + args = self.parse_cli(args or sys.argv[1:]) - ds.graph(filename, interval=interval, **kwargs) + return args.func(args) diff --git a/src/collecty/constants.py b/src/collecty/constants.py index 20bdc35..6094808 100644 --- a/src/collecty/constants.py +++ b/src/collecty/constants.py @@ -23,7 +23,9 @@ from i18n import _ DATABASE_DIR = "/var/lib/collecty" -GRAPH_DEFAULT_ARGUMENTS = [ +BUS_DOMAIN = "org.ipfire.collecty1" + +GRAPH_DEFAULT_ARGUMENTS = ( # Always generate graphs in PNG format. "--imgformat", "PNG", @@ -42,7 +44,7 @@ GRAPH_DEFAULT_ARGUMENTS = [ # Brand all generated graphs. "--watermark", _("Created by collecty"), -] +) GRAPH_DEFAULT_WIDTH = 768 GRAPH_DEFAULT_HEIGHT = 480 diff --git a/src/collecty/daemon.py b/src/collecty/daemon.py index e678899..2fa000f 100644 --- a/src/collecty/daemon.py +++ b/src/collecty/daemon.py @@ -25,6 +25,7 @@ import signal import threading import time +import bus import plugins from constants import * @@ -55,6 +56,10 @@ class Collecty(object): # will be written to disk later. self.write_queue = WriteQueue(self, self.SUBMIT_INTERVAL) + # Create a thread that connects to dbus and processes requests we + # get from there. + self.bus = bus.Bus(self) + # Add all plugins for plugin in plugins.get(): self.add_plugin(plugin) @@ -73,10 +78,19 @@ class Collecty(object): self.plugins.append(plugin) + @property + def templates(self): + for plugin in self.plugins: + for template in plugin.templates: + yield template + def run(self): # Register signal handlers. self.register_signal_handler() + # Start the bus + self.bus.start() + # Start all data source threads. for p in self.plugins: p.start() @@ -96,6 +110,9 @@ class Collecty(object): for p in self.plugins: p.join() + # Stop the bus thread + self.bus.shutdown() + # Write all collected data to disk before ending the main thread self.write_queue.shutdown() @@ -129,9 +146,19 @@ class Collecty(object): # Commit all data. self.write_queue.commit() - @property - def graph_default_arguments(self): - return GRAPH_DEFAULT_ARGUMENTS + def get_plugin_from_template(self, template_name): + for plugin in self.plugins: + if not template_name in [t.name for t in plugin.templates]: + continue + + return plugin + + def generate_graph(self, template_name, *args, **kwargs): + plugin = self.get_plugin_from_template(template_name) + if not plugin: + raise RuntimeError("Could not find template %s" % template_name) + + return plugin.generate_graph(template_name, *args, **kwargs) class WriteQueue(threading.Thread): diff --git a/src/collecty/ping.py b/src/collecty/ping.py index 7982b6a..60b5537 100644 --- a/src/collecty/ping.py +++ b/src/collecty/ping.py @@ -211,8 +211,11 @@ class Ping(object): try: return socket.gethostbyname(host) - except PingResolvError: - raise PingResolveError + except socket.gaierror as e: + if e.errno == -3: + raise PingResolveError + + raise def _is_valid_ipv4_address(self, addr): """ diff --git a/src/collecty/plugins/base.py b/src/collecty/plugins/base.py index 94b0bc0..8d38cad 100644 --- a/src/collecty/plugins/base.py +++ b/src/collecty/plugins/base.py @@ -26,6 +26,7 @@ import logging import math import os import rrdtool +import tempfile import threading import time @@ -196,6 +197,35 @@ class Plugin(threading.Thread): if self.timer: self.timer.cancel() + def get_object(self, id): + for object in self.objects: + if not object.id == id: + continue + + return object + + def get_template(self, template_name): + for template in self.templates: + if not template.name == template_name: + continue + + return template(self) + + def generate_graph(self, template_name, object_id="default", **kwargs): + template = self.get_template(template_name) + if not template: + raise RuntimeError("Could not find template %s" % template_name) + + time_start = time.time() + + graph = template.generate_graph(object_id=object_id, **kwargs) + + duration = time.time() - time_start + self.log.info(_("Generated graph %s in %.1fms") \ + % (template, duration * 1000)) + + return graph + class Object(object): # The schema of the RRD database. @@ -354,41 +384,89 @@ class GraphTemplate(object): # Extra arguments passed to rrdgraph. rrd_graph_args = [] - def __init__(self, ds): - self.ds = ds + intervals = { + None : "-3h", + "hour" : "-1h", + "day" : "-25h", + "week" : "-360h", + "year" : "-365d", + } + + # Default dimensions for this graph + height = GRAPH_DEFAULT_HEIGHT + width = GRAPH_DEFAULT_WIDTH + + def __init__(self, plugin): + self.plugin = plugin + + def __repr__(self): + return "<%s>" % self.__class__.__name__ @property def collecty(self): - return self.ds.collecty + return self.plugin.collecty - def graph(self, file, interval=None, - width=GRAPH_DEFAULT_WIDTH, height=GRAPH_DEFAULT_HEIGHT): - args = [ - "--width", "%d" % width, - "--height", "%d" % height, + @property + def log(self): + return self.plugin.log + + def _make_command_line(self, interval, width=None, height=None): + args = [] + + args += GRAPH_DEFAULT_ARGUMENTS + + args += [ + "--height", "%s" % (height or self.height), + "--width", "%s" % (width or self.width), ] - args += self.collecty.graph_default_arguments - args += self.rrd_graph_args - intervals = { - None : "-3h", - "hour" : "-1h", - "day" : "-25h", - "week" : "-360h", - "year" : "-365d", - } + args += self.rrd_graph_args + # Add interval args.append("--start") + try: - args.append(intervals[interval]) + args.append(self.intervals[interval]) except KeyError: - args.append(interval) + args.append(str(interval)) + + return args + + def get_object_table(self, object_id): + return { + "file" : self.plugin.get_object(object_id), + } + + def get_object_files(self, object_id): + files = {} + + for id, obj in self.get_object_table(object_id).items(): + files[id] = obj.file + + return files + + def generate_graph(self, object_id, interval=None, **kwargs): + args = self._make_command_line(interval, **kwargs) + + self.log.info(_("Generating graph %s") % self) + self.log.debug(" args: %s" % args) + + object_files = self.get_object_files(object_id) - info = { "file" : self.ds.file } for item in self.rrd_graph: try: - args.append(item % info) + args.append(item % object_files) except TypeError: args.append(item) - rrdtool.graph(file, *args) + return self.write_graph(*args) + + def write_graph(self, *args): + with tempfile.NamedTemporaryFile() as f: + rrdtool.graph(f.name, *args) + + # Get back to the beginning of the file + f.seek(0) + + # Return all the content + return f.read() diff --git a/src/collecty/plugins/cpu.py b/src/collecty/plugins/cpu.py index c9825b7..c2f0183 100644 --- a/src/collecty/plugins/cpu.py +++ b/src/collecty/plugins/cpu.py @@ -25,8 +25,8 @@ import base from ..i18n import _ -class GraphTemplateCPU(base.GraphTemplate): - name = "cpu" +class GraphTemplateProcessor(base.GraphTemplate): + name = "processor" rrd_graph = [ "DEF:user=%(file)s:user:AVERAGE", @@ -154,7 +154,7 @@ class ProcessorPlugin(base.Plugin): name = "processor" description = "Processor Usage Plugin" - templates = [GraphTemplateCPU,] + templates = [GraphTemplateProcessor] interval = 30 diff --git a/src/collecty/plugins/entropy.py b/src/collecty/plugins/entropy.py index f800f9a..4296c77 100644 --- a/src/collecty/plugins/entropy.py +++ b/src/collecty/plugins/entropy.py @@ -71,7 +71,7 @@ class EntropyPlugin(base.Plugin): name = "entropy" description = "Entropy Plugin" - templates = [GraphTemplateEntropy,] + templates = [GraphTemplateEntropy] @property def objects(self): diff --git a/src/collecty/plugins/latency.py b/src/collecty/plugins/latency.py index 98e6936..589afa8 100644 --- a/src/collecty/plugins/latency.py +++ b/src/collecty/plugins/latency.py @@ -105,7 +105,7 @@ class LatencyObject(base.Object): try: ping = collecty.ping.Ping(destination=self.hostname, timeout=20000) ping.run(count=5, deadline=self.deadline) - + except collecty.ping.PingError, e: self.log.warning(_("Could not run latency check for %(host)s: %(msg)s") \ % { "host" : self.hostname, "msg" : e.msg }) @@ -122,7 +122,7 @@ class LatencyPlugin(base.Plugin): name = "latency" description = "Latency (ICMP ping) Plugin" - templates = [GraphTemplateLatency,] + templates = [GraphTemplateLatency] interval = 60 diff --git a/src/collecty/plugins/loadavg.py b/src/collecty/plugins/loadavg.py index b5550eb..b8cf07a 100644 --- a/src/collecty/plugins/loadavg.py +++ b/src/collecty/plugins/loadavg.py @@ -88,7 +88,7 @@ class LoadAvgPlugin(base.Plugin): name = "loadavg" description = "Load Average Plugin" - templates = [GraphTemplateLoadAvg,] + templates = [GraphTemplateLoadAvg] interval = 30 diff --git a/src/collecty/plugins/memory.py b/src/collecty/plugins/memory.py index b58cbca..d640077 100644 --- a/src/collecty/plugins/memory.py +++ b/src/collecty/plugins/memory.py @@ -141,7 +141,7 @@ class MemoryPlugin(base.Plugin): name = "memory" description = "Memory Usage Plugin" - templates = [GraphTemplateMemory,] + templates = [GraphTemplateMemory] @property def objects(self): diff --git a/src/dbus/org.ipfire.collecty1.conf b/src/dbus/org.ipfire.collecty1.conf new file mode 100644 index 0000000..39fef28 --- /dev/null +++ b/src/dbus/org.ipfire.collecty1.conf @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + diff --git a/src/dbus/org.ipfire.collecty1.service b/src/dbus/org.ipfire.collecty1.service new file mode 100644 index 0000000..f61a417 --- /dev/null +++ b/src/dbus/org.ipfire.collecty1.service @@ -0,0 +1,5 @@ +[D-BUS Service] +Name=org.ipfire.collecty1 +Exec=usr/bin/collectyd +User=root +SystemdService=collecty.service diff --git a/src/systemd/collecty.service.in b/src/systemd/collecty.service.in index 8a8bbca..a515b8e 100644 --- a/src/systemd/collecty.service.in +++ b/src/systemd/collecty.service.in @@ -2,8 +2,10 @@ Description=collecty - A system data collecting daemon [Service] +Type=dbus ExecStart=@bindir@/collectyd ExecReload=/bin/kill -HUP $MAINPID +BusName=org.ipfire.collecty1 [Install] WantedBy=multi-user.target diff --git a/src/systemd/org.ipfire.collecty1.busname b/src/systemd/org.ipfire.collecty1.busname new file mode 100644 index 0000000..c4de87c --- /dev/null +++ b/src/systemd/org.ipfire.collecty1.busname @@ -0,0 +1,6 @@ +[Unit] +Description=Collecty Service Bus Name + +[BusName] +Service=collecty.service +AllowWorld=talk