]>
git.ipfire.org Git - collecty.git/blob - src/collecty/plugins/base.py
2 ###############################################################################
4 # collecty - A system statistics collection daemon for IPFire #
5 # Copyright (C) 2012 IPFire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
33 from .. import locales
35 from ..constants
import *
38 DEF_MATCH
= r
"C?DEF:([A-Za-z0-9_]+)="
40 class Environment(object):
42 Sets the correct environment for rrdtool to create
43 localised graphs and graphs in the correct timezone.
45 def __init__(self
, timezone
, locale
):
46 # Build the new environment
47 self
.new_environment
= {
48 "TZ" : timezone
or DEFAULT_TIMEZONE
,
51 for k
in ("LANG", "LC_ALL"):
52 self
.new_environment
[k
] = locale
or DEFAULT_LOCALE
55 # Save the current environment
56 self
.old_environment
= {}
57 for k
in self
.new_environment
:
58 self
.old_environment
[k
] = os
.environ
.get(k
, None)
61 os
.environ
.update(self
.new_environment
)
63 def __exit__(self
, type, value
, traceback
):
64 # Roll back to the previous environment
65 for k
, v
in self
.old_environment
.items():
75 class PluginRegistration(type):
78 def __init__(plugin
, name
, bases
, dict):
79 type.__init
__(plugin
, name
, bases
, dict)
81 # The main class from which is inherited is not registered
86 if not all((plugin
.name
, plugin
.description
)):
87 raise RuntimeError(_("Plugin is not properly configured: %s") % plugin
)
89 PluginRegistration
.plugins
[plugin
.name
] = plugin
94 Returns a list with all automatically registered plugins.
96 return PluginRegistration
.plugins
.values()
98 class Plugin(object, metaclass
=PluginRegistration
):
99 # The name of this plugin.
102 # A description for this plugin.
105 # Templates which can be used to generate a graph out of
106 # the data from this data source.
109 # The default interval for all plugins
115 def __init__(self
, collecty
, **kwargs
):
116 self
.collecty
= collecty
118 # Check if this plugin was configured correctly.
119 assert self
.name
, "Name of the plugin is not set: %s" % self
.name
120 assert self
.description
, "Description of the plugin is not set: %s" % self
.description
122 # Initialize the logger.
123 self
.log
= logging
.getLogger("collecty.plugins.%s" % self
.name
)
125 # Run some custom initialization.
128 self
.log
.debug(_("Successfully initialized %s") % self
.__class
__.__name
__)
133 Returns the name of the sub directory in which all RRD files
134 for this plugin should be stored in.
140 def init(self
, **kwargs
):
142 Do some custom initialization stuff here.
148 Gathers the statistical data, this plugin collects.
150 time_start
= time
.time()
152 # Run through all objects of this plugin and call the collect method.
153 for o
in self
.objects
:
154 now
= datetime
.datetime
.utcnow()
158 result
= self
._format
_result
(result
)
160 self
.log
.warning(_("Unhandled exception in %s.collect()") % o
, exc_info
=True)
164 self
.log
.warning(_("Received empty result: %s") % o
)
167 self
.log
.debug(_("Collected %s: %s") % (o
, result
))
169 # Add the object to the write queue so that the data is written
170 # to the databases later.
171 self
.collecty
.write_queue
.add(o
, now
, result
)
173 # Returns the time this function took to complete.
174 delay
= time
.time() - time_start
176 # Log some warning when a collect method takes too long to return some data
178 self
.log
.warning(_("A worker thread was stalled for %.4fs") % delay
)
181 def _format_result(result
):
182 if not isinstance(result
, tuple) and not isinstance(result
, list):
185 # Replace all Nones by UNKNOWN
196 def get_object(self
, id):
197 for object in self
.objects
:
198 if not object.id == id:
203 def get_template(self
, template_name
, object_id
, locale
=None, timezone
=None):
204 for template
in self
.templates
:
205 if not template
.name
== template_name
:
208 return template(self
, object_id
, locale
=locale
, timezone
=timezone
)
210 def generate_graph(self
, template_name
, object_id
="default",
211 timezone
=None, locale
=None, **kwargs
):
212 template
= self
.get_template(template_name
, object_id
=object_id
,
213 timezone
=timezone
, locale
=locale
)
215 raise RuntimeError("Could not find template %s" % template_name
)
217 time_start
= time
.time()
219 graph
= template
.generate_graph(**kwargs
)
221 duration
= time
.time() - time_start
222 self
.log
.debug(_("Generated graph %s in %.1fms") \
223 % (template
, duration
* 1000))
227 def graph_info(self
, template_name
, object_id
="default",
228 timezone
=None, locale
=None, **kwargs
):
229 template
= self
.get_template(template_name
, object_id
=object_id
,
230 timezone
=timezone
, locale
=locale
)
232 raise RuntimeError("Could not find template %s" % template_name
)
234 return template
.graph_info()
236 def last_update(self
, object_id
="default"):
237 object = self
.get_object(object_id
)
239 raise RuntimeError("Could not find object %s" % object_id
)
241 return object.last_update()
244 class Object(object):
245 # The schema of the RRD database.
249 rra_types
= ("AVERAGE", "MIN", "MAX")
256 def __init__(self
, plugin
, *args
, **kwargs
):
259 # Indicates if this object has collected its data
260 self
.collected
= False
262 # Initialise this object
263 self
.init(*args
, **kwargs
)
265 # Create the database file.
269 return "<%s>" % self
.__class
__.__name
__
271 def __lt__(self
, other
):
272 return self
.id < other
.id
276 return self
.plugin
.collecty
280 return self
.plugin
.log
285 Returns a UNIQUE identifier for this object. As this is incorporated
286 into the path of RRD file, it must only contain ASCII characters.
288 raise NotImplementedError
293 The absolute path to the RRD file of this plugin.
295 filename
= self
._normalise
_filename
("%s.rrd" % self
.id)
297 return os
.path
.join(DATABASE_DIR
, self
.plugin
.path
, filename
)
300 def _normalise_filename(filename
):
301 # Convert the filename into ASCII characters only
302 filename
= unicodedata
.normalize("NFKC", filename
)
304 # Replace any spaces by dashes
305 filename
= filename
.replace(" ", "-")
311 def init(self
, *args
, **kwargs
):
313 Do some custom initialization stuff here.
319 Creates an empty RRD file with the desired data structures.
321 # Skip if the file does already exist.
322 if os
.path
.exists(self
.file):
325 dirname
= os
.path
.dirname(self
.file)
326 if not os
.path
.exists(dirname
):
329 # Create argument list.
330 args
= self
.get_rrd_schema()
332 rrdtool
.create(self
.file, *args
)
334 self
.log
.debug(_("Created RRD file %s.") % self
.file)
336 self
.log
.debug(" %s" % arg
)
339 return rrdtool
.info(self
.file)
341 def last_update(self
):
343 Returns a dictionary with the timestamp and
344 data set of the last database update.
347 "dataset" : self
.last_dataset
,
348 "timestamp" : self
.last_updated
,
351 def _last_update(self
):
352 return rrdtool
.lastupdate(self
.file)
355 def last_updated(self
):
357 Returns the timestamp when this database was last updated
359 lu
= self
._last
_update
()
362 return lu
.get("date")
365 def last_dataset(self
):
367 Returns the latest dataset in the database
369 lu
= self
._last
_update
()
376 return self
.plugin
.interval
380 return self
.stepsize
* 2
382 def get_rrd_schema(self
):
384 "--step", "%s" % self
.stepsize
,
386 for line
in self
.rrd_schema
:
387 if line
.startswith("DS:"):
389 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
395 "%s" % self
.heartbeat
,
406 for steps
, rows
in self
.rra_timespans
:
407 for type in self
.rra_types
:
408 schema
.append("RRA:%s:%s:%s:%s" % (type, xff
, steps
, rows
))
413 def rrd_schema_names(self
):
416 for line
in self
.rrd_schema
:
417 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
422 def make_rrd_defs(self
, prefix
=None):
425 for name
in self
.rrd_schema_names
:
427 p
= "%s_%s" % (prefix
, name
)
432 "DEF:%s=%s:%s:AVERAGE" % (p
, self
.file, name
),
437 def get_stddev(self
, interval
=None):
438 args
= self
.make_rrd_defs()
440 # Add the correct interval
441 args
+= ["--start", util
.make_interval(interval
)]
443 for name
in self
.rrd_schema_names
:
445 "VDEF:%s_stddev=%s,STDEV" % (name
, name
),
446 "PRINT:%s_stddev:%%lf" % name
,
449 x
, y
, vals
= rrdtool
.graph("/dev/null", *args
)
450 return dict(zip(self
.rrd_schema_names
, vals
))
454 raise RuntimeError("This object has already collected its data")
456 self
.collected
= True
457 self
.now
= datetime
.datetime
.utcnow()
460 result
= self
.collect()
464 Will commit the collected data to the database.
466 # Make sure that the RRD database has been created
469 # Write everything to disk that is in the write queue
470 self
.collecty
.write_queue
.commit_file(self
.file)
472 # Convenience functions for plugin authors
474 def read_file(self
, *args
, strip
=True):
476 Reads the content of the given file
478 filename
= os
.path
.join(*args
)
480 with
open(filename
) as f
:
483 # Strip any excess whitespace
485 value
= value
.strip()
489 def read_file_integer(self
, filename
):
491 Reads the content from a file and returns it as an integer
493 value
= self
.read_file(filename
)
500 def read_proc_stat(self
):
502 Reads /proc/stat and returns it as a dictionary
506 with
open("/proc/stat") as f
:
508 # Split the key from the rest of the line
509 key
, line
= line
.split(" ", 1)
511 # Remove any line breaks
512 ret
[key
] = line
.rstrip()
517 class GraphTemplate(object):
518 # A unique name to identify this graph template.
521 # Headline of the graph image
524 # Vertical label of the graph
525 graph_vertical_label
= None
531 # Instructions how to create the graph.
534 # Extra arguments passed to rrdgraph.
537 # Default dimensions for this graph
538 height
= GRAPH_DEFAULT_HEIGHT
539 width
= GRAPH_DEFAULT_WIDTH
541 def __init__(self
, plugin
, object_id
, locale
=None, timezone
=None):
544 # Save localisation parameters
545 self
.locale
= locales
.get(locale
)
546 self
.timezone
= timezone
548 # Get all required RRD objects
549 self
.object_id
= object_id
551 # Get the main object
552 self
.objects
= self
.get_objects(self
.object_id
)
556 return "<%s>" % self
.__class
__.__name
__
560 return self
.plugin
.collecty
564 return self
.plugin
.log
569 Shortcut to the main object
571 if len(self
.objects
) == 1:
572 return self
.objects
[0]
574 def _make_command_line(self
, interval
, format
=DEFAULT_IMAGE_FORMAT
,
575 width
=None, height
=None, with_title
=True, thumbnail
=False):
576 args
= [e
for e
in GRAPH_DEFAULT_ARGUMENTS
]
578 # Set the default dimensions
579 default_height
, default_width
= GRAPH_DEFAULT_HEIGHT
, GRAPH_DEFAULT_WIDTH
581 # A thumbnail doesn't have a legend and other labels
583 args
.append("--only-graph")
585 default_height
= THUMBNAIL_DEFAULT_HEIGHT
586 default_width
= THUMBNAIL_DEFAULT_WIDTH
589 "--imgformat", format
,
590 "--height", "%s" % (height
or default_height
),
591 "--width", "%s" % (width
or default_width
),
594 args
+= self
.rrd_graph_args
597 if with_title
and self
.graph_title
:
598 args
+= ["--title", self
.graph_title
]
601 if self
.graph_vertical_label
:
602 args
+= ["--vertical-label", self
.graph_vertical_label
]
604 if self
.lower_limit
is not None or self
.upper_limit
is not None:
605 # Force to honour the set limits
606 args
.append("--rigid")
608 if self
.lower_limit
is not None:
609 args
+= ["--lower-limit", self
.lower_limit
]
611 if self
.upper_limit
is not None:
612 args
+= ["--upper-limit", self
.upper_limit
]
615 args
+= ["--start", util
.make_interval(interval
)]
620 use_prefix
= len(self
.objects
) >= 2
623 for object in self
.objects
:
625 args
+= object.make_rrd_defs(object.id)
627 args
+= object.make_rrd_defs()
631 def _add_vdefs(self
, args
):
637 # Search for all DEFs and CDEFs
638 m
= re
.match(DEF_MATCH
, "%s" % arg
)
642 # Add the VDEFs for minimum, maximum, etc. values
644 "VDEF:%s_cur=%s,LAST" % (name
, name
),
645 "VDEF:%s_avg=%s,AVERAGE" % (name
, name
),
646 "VDEF:%s_max=%s,MAXIMUM" % (name
, name
),
647 "VDEF:%s_min=%s,MINIMUM" % (name
, name
),
652 def get_objects(self
, *args
, **kwargs
):
653 object = self
.plugin
.get_object(*args
, **kwargs
)
660 def generate_graph(self
, interval
=None, **kwargs
):
661 assert self
.objects
, "Cannot render graph without any objects"
663 # Make sure that all collected data is in the database
664 # to get a recent graph image
665 for object in self
.objects
:
668 args
= self
._make
_command
_line
(interval
, **kwargs
)
670 self
.log
.info(_("Generating graph %s") % self
)
672 rrd_graph
= self
.rrd_graph
674 # Add DEFs for all objects
675 if not any((e
.startswith("DEF:") for e
in rrd_graph
)):
676 args
+= self
._add
_defs
()
679 args
= self
._add
_vdefs
(args
)
681 # Convert arguments to string
682 args
= [str(e
) for e
in args
]
685 self
.log
.debug(" %s" % arg
)
687 with
Environment(self
.timezone
, self
.locale
.lang
):
688 graph
= rrdtool
.graphv("-", *args
)
691 "image" : graph
.get("image"),
692 "image_height" : graph
.get("image_height"),
693 "image_width" : graph
.get("image_width"),
696 def graph_info(self
):
698 Returns a dictionary with useful information
702 "title" : self
.graph_title
,
703 "object_id" : self
.object_id
or "",
704 "template" : self
.name
,