-#!/usr/bin/python
+#!/usr/bin/python3
###############################################################################
# #
# collecty - A system statistics collection daemon for IPFire #
# #
###############################################################################
-from __future__ import division
-
+import datetime
import logging
import math
import os
import rrdtool
+import tempfile
import threading
import time
+import unicodedata
+from .. import locales
from ..constants import *
from ..i18n import _
return self.elapsed > self.timeout
-class DataSource(threading.Thread):
+class Environment(object):
+ """
+ Sets the correct environment for rrdtool to create
+ localised graphs and graphs in the correct timezone.
+ """
+ def __init__(self, timezone, locale):
+ # Build the new environment
+ self.new_environment = {
+ "TZ" : timezone or DEFAULT_TIMEZONE,
+ }
+
+ for k in ("LANG", "LC_ALL"):
+ self.new_environment[k] = locale or DEFAULT_LOCALE
+
+ def __enter__(self):
+ # Save the current environment
+ self.old_environment = {}
+ for k in self.new_environment:
+ self.old_environment[k] = os.environ.get(k, None)
+
+ # Apply the new one
+ os.environ.update(self.new_environment)
+
+ def __exit__(self, type, value, traceback):
+ # Roll back to the previous environment
+ for k, v in self.old_environment.items():
+ if v is None:
+ try:
+ del os.environ[k]
+ except KeyError:
+ pass
+ else:
+ os.environ[k] = v
+
+
+class PluginRegistration(type):
+ plugins = {}
+
+ def __init__(plugin, name, bases, dict):
+ type.__init__(plugin, name, bases, dict)
+
+ # The main class from which is inherited is not registered
+ # as a plugin.
+ if name == "Plugin":
+ return
+
+ if not all((plugin.name, plugin.description)):
+ raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
+
+ PluginRegistration.plugins[plugin.name] = plugin
+
+
+def get():
+ """
+ Returns a list with all automatically registered plugins.
+ """
+ return PluginRegistration.plugins.values()
+
+class Plugin(object, metaclass=PluginRegistration):
# The name of this plugin.
name = None
# the data from this data source.
templates = []
- # The schema of the RRD database.
- rrd_schema = None
-
- # RRA properties.
- rra_types = ["AVERAGE", "MIN", "MAX"]
- rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
- rra_rows = 2880
-
- # The default interval of this plugin.
- default_interval = 60
+ # The default interval for all plugins
+ interval = 60
def __init__(self, collecty, **kwargs):
- threading.Thread.__init__(self, name=self.description)
- self.daemon = True
-
self.collecty = collecty
# Check if this plugin was configured correctly.
assert self.name, "Name of the plugin is not set: %s" % self.name
assert self.description, "Description of the plugin is not set: %s" % self.description
- assert self.rrd_schema
# Initialize the logger.
self.log = logging.getLogger("collecty.plugins.%s" % self.name)
# Run some custom initialization.
self.init(**kwargs)
- # Create the database file.
- self.create()
-
- # Keepalive options
- self.running = True
- self.timer = Timer(self.interval)
-
- self.log.info(_("Successfully initialized (%s).") % self.id)
-
- def __repr__(self):
- return "<%s %s>" % (self.__class__.__name__, self.id)
+ self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
@property
- def id(self):
+ def path(self):
"""
- A unique ID of the plugin instance.
+ Returns the name of the sub directory in which all RRD files
+ for this plugin should be stored in.
"""
return self.name
- @property
- def interval(self):
+ ### Basic methods
+
+ def init(self, **kwargs):
+ """
+ Do some custom initialization stuff here.
"""
- Returns the interval in milliseconds, when the read method
- should be called again.
+ pass
+
+ def collect(self):
"""
- # XXX read this from the settings
+ Gathers the statistical data, this plugin collects.
+ """
+ time_start = time.time()
+
+ # Run through all objects of this plugin and call the collect method.
+ for o in self.objects:
+ now = datetime.datetime.utcnow()
+ try:
+ result = o.collect()
+
+ result = self._format_result(result)
+ except:
+ self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
+ continue
+
+ if not result:
+ self.log.warning(_("Received empty result: %s") % o)
+ continue
- # Otherwise return the default.
- return self.default_interval
+ self.log.debug(_("Collected %s: %s") % (o, result))
+
+ # Add the object to the write queue so that the data is written
+ # to the databases later.
+ self.collecty.write_queue.add(o, now, result)
+
+ # Returns the time this function took to complete.
+ delay = time.time() - time_start
+
+ # Log some warning when a collect method takes too long to return some data
+ if delay >= 60:
+ self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
+
+ @staticmethod
+ def _format_result(result):
+ if not isinstance(result, tuple) and not isinstance(result, list):
+ return result
+
+ # Replace all Nones by NaN
+ s = []
+
+ for e in result:
+ if e is None:
+ e = "NaN"
+
+ # Format as string
+ e = "%s" % e
+
+ s.append(e)
+
+ return ":".join(s)
+
+ def get_object(self, id):
+ for object in self.objects:
+ if not object.id == id:
+ continue
+
+ return object
+
+ def get_template(self, template_name, object_id, locale=None, timezone=None):
+ for template in self.templates:
+ if not template.name == template_name:
+ continue
+
+ return template(self, object_id, locale=locale, timezone=timezone)
+
+ def generate_graph(self, template_name, object_id="default",
+ timezone=None, locale=None, **kwargs):
+ template = self.get_template(template_name, object_id=object_id,
+ timezone=timezone, locale=locale)
+ if not template:
+ raise RuntimeError("Could not find template %s" % template_name)
+
+ time_start = time.time()
+
+ graph = template.generate_graph(**kwargs)
+
+ duration = time.time() - time_start
+ self.log.debug(_("Generated graph %s in %.1fms") \
+ % (template, duration * 1000))
+
+ return graph
+
+
+class Object(object):
+ # The schema of the RRD database.
+ rrd_schema = None
+
+ # RRA properties.
+ rra_types = ("AVERAGE", "MIN", "MAX")
+ rra_timespans = (
+ ("1m", "10d"),
+ ("1h", "18M"),
+ ("1d", "5y"),
+ )
+
+ def __init__(self, plugin, *args, **kwargs):
+ self.plugin = plugin
+
+ # Indicates if this object has collected its data
+ self.collected = False
+
+ # Initialise this object
+ self.init(*args, **kwargs)
+
+ # Create the database file.
+ self.create()
+
+ def __repr__(self):
+ return "<%s>" % self.__class__.__name__
@property
- def stepsize(self):
- return self.interval
+ def collecty(self):
+ return self.plugin.collecty
@property
- def heartbeat(self):
- return self.stepsize * 2
+ def log(self):
+ return self.plugin.log
+
+ @property
+ def id(self):
+ """
+ Returns a UNIQUE identifier for this object. As this is incorporated
+ into the path of RRD file, it must only contain ASCII characters.
+ """
+ raise NotImplementedError
@property
def file(self):
"""
The absolute path to the RRD file of this plugin.
"""
- return os.path.join(DATABASE_DIR, "%s.rrd" % self.id)
+ filename = self._normalise_filename("%s.rrd" % self.id)
+
+ return os.path.join(DATABASE_DIR, self.plugin.path, filename)
+
+ @staticmethod
+ def _normalise_filename(filename):
+ # Convert the filename into ASCII characters only
+ filename = unicodedata.normalize("NFKC", filename)
+
+ # Replace any spaces by dashes
+ filename = filename.replace(" ", "-")
+
+ return filename
+
+ ### Basic methods
+
+ def init(self, *args, **kwargs):
+ """
+ Do some custom initialization stuff here.
+ """
+ pass
def create(self):
"""
for arg in args:
self.log.debug(" %s" % arg)
+ def info(self):
+ return rrdtool.info(self.file)
+
+ @property
+ def stepsize(self):
+ return self.plugin.interval
+
+ @property
+ def heartbeat(self):
+ return self.stepsize * 2
+
def get_rrd_schema(self):
schema = [
"--step", "%s" % self.stepsize,
xff = 0.1
- cdp_length = 0
- for rra_timespan in self.rra_timespans:
- if (rra_timespan / self.stepsize) < self.rra_rows:
- rra_timespan = self.stepsize * self.rra_rows
-
- if cdp_length == 0:
- cdp_length = 1
- else:
- cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
-
- cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
-
- for rra_type in self.rra_types:
- schema.append("RRA:%s:%.10f:%d:%d" % \
- (rra_type, xff, cdp_length, cdp_number))
+ for steps, rows in self.rra_timespans:
+ for type in self.rra_types:
+ schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
return schema
- def info(self):
- return rrdtool.info(self.file)
+ def execute(self):
+ if self.collected:
+ raise RuntimeError("This object has already collected its data")
- ### Basic methods
+ self.collected = True
+ self.now = datetime.datetime.utcnow()
- def init(self, **kwargs):
- """
- Do some custom initialization stuff here.
- """
- pass
+ # Call the collect
+ result = self.collect()
- def read(self):
+ def commit(self):
"""
- Gathers the statistical data, this plugin collects.
+ Will commit the collected data to the database.
"""
- raise NotImplementedError
+ # Make sure that the RRD database has been created
+ self.create()
- def submit(self):
- """
- Flushes the read data to disk.
- """
- # Do nothing in case there is no data to submit.
- if not self.data:
- return
- self.log.debug(_("Submitting data to database. %d entries.") % len(self.data))
- for data in self.data:
- self.log.debug(" %s" % data)
+class GraphTemplate(object):
+ # A unique name to identify this graph template.
+ name = None
- # Create the RRD files (if they don't exist yet or
- # have vanished for some reason).
- self.create()
+ # Headline of the graph image
+ graph_title = None
- rrdtool.update(self.file, *self.data)
- self.data = []
+ # Vertical label of the graph
+ graph_vertical_label = None
- def _read(self, *args, **kwargs):
- """
- This method catches errors from the read() method and logs them.
- """
- start_time = time.time()
+ # Limits
+ lower_limit = None
+ upper_limit = None
- try:
- data = self.read(*args, **kwargs)
- if data is None:
- self.log.warning(_("Received empty data."))
- else:
- self.data.append("%d:%s" % (start_time, data))
+ # Instructions how to create the graph.
+ rrd_graph = None
- # Catch any exceptions, so collecty does not crash.
- except Exception, e:
- self.log.critical(_("Unhandled exception in read()!"), exc_info=True)
+ # Extra arguments passed to rrdgraph.
+ rrd_graph_args = []
- # Return the elapsed time since _read() has been called.
- return (time.time() - start_time)
+ intervals = {
+ None : "-3h",
+ "hour" : "-1h",
+ "day" : "-25h",
+ "month": "-30d",
+ "week" : "-360h",
+ "year" : "-365d",
+ }
- def _submit(self, *args, **kwargs):
- """
- This method catches errors from the submit() method and logs them.
- """
- try:
- return self.submit(*args, **kwargs)
+ # Default dimensions for this graph
+ height = GRAPH_DEFAULT_HEIGHT
+ width = GRAPH_DEFAULT_WIDTH
- # Catch any exceptions, so collecty does not crash.
- except Exception, e:
- self.log.critical(_("Unhandled exception in submit()!"), exc_info=True)
+ def __init__(self, plugin, object_id, locale=None, timezone=None):
+ self.plugin = plugin
- def run(self):
- self.log.debug(_("Started."))
+ # Save localisation parameters
+ self.locale = locales.get(locale)
+ self.timezone = timezone
- while self.running:
- # Reset the timer.
- self.timer.reset()
+ # Get all required RRD objects
+ self.object_id = object_id
- # Wait until the timer has successfully elapsed.
- if self.timer.wait():
- self.log.debug(_("Collecting..."))
- delay = self._read()
+ # Get the main object
+ self.object = self.get_object(self.object_id)
- self.timer.reset(delay)
+ def __repr__(self):
+ return "<%s>" % self.__class__.__name__
- self._submit()
- self.log.debug(_("Stopped."))
+ @property
+ def collecty(self):
+ return self.plugin.collecty
- def shutdown(self):
- self.log.debug(_("Received shutdown signal."))
- self.running = False
+ @property
+ def log(self):
+ return self.plugin.log
- # Kill any running timers.
- if self.timer:
- self.timer.cancel()
+ def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
+ width=None, height=None):
+ args = []
+ args += GRAPH_DEFAULT_ARGUMENTS
-class GraphTemplate(object):
- # A unique name to identify this graph template.
- name = None
+ args += [
+ "--imgformat", format,
+ "--height", "%s" % (height or self.height),
+ "--width", "%s" % (width or self.width),
+ ]
- # Instructions how to create the graph.
- rrd_graph = None
+ args += self.rrd_graph_args
- # Extra arguments passed to rrdgraph.
- rrd_graph_args = []
+ # Graph title
+ if self.graph_title:
+ args += ["--title", self.graph_title]
- def __init__(self, ds):
- self.ds = ds
+ # Vertical label
+ if self.graph_vertical_label:
+ args += ["--vertical-label", self.graph_vertical_label]
- @property
- def collecty(self):
- return self.ds.collecty
+ if self.lower_limit is not None or self.upper_limit is not None:
+ # Force to honour the set limits
+ args.append("--rigid")
- def graph(self, file, interval=None,
- width=GRAPH_DEFAULT_WIDTH, height=GRAPH_DEFAULT_HEIGHT):
- args = [
- "--width", "%d" % width,
- "--height", "%d" % height,
- ]
- args += self.collecty.graph_default_arguments
- args += self.rrd_graph_args
+ if self.lower_limit is not None:
+ args += ["--lower-limit", self.lower_limit]
- intervals = {
- None : "-3h",
- "hour" : "-1h",
- "day" : "-25h",
- "week" : "-360h",
- "year" : "-365d",
- }
+ if self.upper_limit is not None:
+ args += ["--upper-limit", self.upper_limit]
- args.append("--start")
try:
- args.append(intervals[interval])
+ interval = self.intervals[interval]
except KeyError:
- args.append(interval)
+ interval = "end-%s" % interval
+
+ # Add interval
+ args += ["--start", interval]
+
+ return args
+
+ def get_object(self, *args, **kwargs):
+ return self.plugin.get_object(*args, **kwargs)
+
+ def get_object_table(self):
+ return {
+ "file" : self.object,
+ }
+
+ @property
+ def object_table(self):
+ if not hasattr(self, "_object_table"):
+ self._object_table = self.get_object_table()
+
+ return self._object_table
+
+ def get_object_files(self):
+ files = {}
+
+ for id, obj in self.object_table.items():
+ files[id] = obj.file
+
+ return files
+
+ def generate_graph(self, interval=None, **kwargs):
+ args = self._make_command_line(interval, **kwargs)
+
+ self.log.info(_("Generating graph %s") % self)
+ self.log.debug(" args: %s" % args)
+
+ object_files = self.get_object_files()
- info = { "file" : self.ds.file }
for item in self.rrd_graph:
try:
- args.append(item % info)
+ args.append(item % object_files)
except TypeError:
args.append(item)
- rrdtool.graph(file, *args)
+ self.log.debug(" %s" % args[-1])
+
+ # Convert arguments to string
+ args = [str(e) for e in args]
+
+ with Environment(self.timezone, self.locale.lang):
+ graph = rrdtool.graphv("-", *args)
+
+ return graph.get("image")
projects / collecty.git / blobdiff