]>
git.ipfire.org Git - collecty.git/blob - src/collecty/plugins/base.py
2 ###############################################################################
4 # collecty - A system statistics collection daemon for IPFire #
5 # Copyright (C) 2012 IPFire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
30 from .. import locales
32 from ..constants
import *
35 DEF_MATCH
= r
"C?DEF:([A-Za-z0-9_]+)="
37 class Environment(object):
39 Sets the correct environment for rrdtool to create
40 localised graphs and graphs in the correct timezone.
42 def __init__(self
, timezone
, locale
):
43 # Build the new environment
44 self
.new_environment
= {
45 "TZ" : timezone
or DEFAULT_TIMEZONE
,
48 for k
in ("LANG", "LC_ALL"):
49 self
.new_environment
[k
] = locale
or DEFAULT_LOCALE
52 # Save the current environment
53 self
.old_environment
= {}
54 for k
in self
.new_environment
:
55 self
.old_environment
[k
] = os
.environ
.get(k
, None)
58 os
.environ
.update(self
.new_environment
)
60 def __exit__(self
, type, value
, traceback
):
61 # Roll back to the previous environment
62 for k
, v
in self
.old_environment
.items():
72 class PluginRegistration(type):
75 def __init__(plugin
, name
, bases
, dict):
76 type.__init
__(plugin
, name
, bases
, dict)
78 # The main class from which is inherited is not registered
83 if not all((plugin
.name
, plugin
.description
)):
84 raise RuntimeError(_("Plugin is not properly configured: %s") % plugin
)
86 PluginRegistration
.plugins
[plugin
.name
] = plugin
91 Returns a list with all automatically registered plugins.
93 return PluginRegistration
.plugins
.values()
95 class Plugin(object, metaclass
=PluginRegistration
):
96 # The name of this plugin.
99 # A description for this plugin.
102 # Templates which can be used to generate a graph out of
103 # the data from this data source.
106 # The default interval for all plugins
112 def __init__(self
, collecty
, **kwargs
):
113 self
.collecty
= collecty
115 # Check if this plugin was configured correctly.
116 assert self
.name
, "Name of the plugin is not set: %s" % self
.name
117 assert self
.description
, "Description of the plugin is not set: %s" % self
.description
119 # Initialize the logger.
120 self
.log
= logging
.getLogger("collecty.plugins.%s" % self
.name
)
122 # Run some custom initialization.
125 self
.log
.debug(_("Successfully initialized %s") % self
.__class
__.__name
__)
130 Returns the name of the sub directory in which all RRD files
131 for this plugin should be stored in.
137 def init(self
, **kwargs
):
139 Do some custom initialization stuff here.
145 Gathers the statistical data, this plugin collects.
147 time_start
= time
.time()
149 # Run through all objects of this plugin and call the collect method.
150 for object in self
.objects
:
151 now
= datetime
.datetime
.utcnow()
155 result
= object.collect()
157 # Catch any unhandled exceptions
158 except Exception as e
:
159 self
.log
.warning(_("Unhandled exception in %s.collect()") % object, exc_info
=True)
163 self
.log
.warning(_("Received empty result: %s") % object)
166 # Format the result for RRDtool
167 result
= self
._format
_result
(result
)
169 self
.log
.debug(_("Collected %s: %s") % (object, result
))
171 # Add the object to the write queue so that the data is written
172 # to the databases later.
173 self
.collecty
.write_queue
.add(object, now
, result
)
175 # Returns the time this function took to complete.
176 delay
= time
.time() - time_start
178 # Log some warning when a collect method takes too long to return some data
180 self
.log
.warning(_("A worker thread was stalled for %.4fs") % delay
)
182 self
.log
.debug(_("Collection finished in %.2fms") % (delay
* 1000))
185 def _format_result(result
):
186 if not isinstance(result
, tuple) and not isinstance(result
, list):
189 # Replace all Nones by UNKNOWN
200 def get_object(self
, id):
201 for object in self
.objects
:
202 if not object.id == id:
207 def get_template(self
, template_name
, object_id
, locale
=None, timezone
=None):
208 for template
in self
.templates
:
209 if not template
.name
== template_name
:
212 return template(self
, object_id
, locale
=locale
, timezone
=timezone
)
214 def generate_graph(self
, template_name
, object_id
="default",
215 timezone
=None, locale
=None, **kwargs
):
216 template
= self
.get_template(template_name
, object_id
=object_id
,
217 timezone
=timezone
, locale
=locale
)
219 raise RuntimeError("Could not find template %s" % template_name
)
221 time_start
= time
.time()
223 graph
= template
.generate_graph(**kwargs
)
225 duration
= time
.time() - time_start
226 self
.log
.debug(_("Generated graph %s in %.1fms") \
227 % (template
, duration
* 1000))
231 def graph_info(self
, template_name
, object_id
="default",
232 timezone
=None, locale
=None, **kwargs
):
233 template
= self
.get_template(template_name
, object_id
=object_id
,
234 timezone
=timezone
, locale
=locale
)
236 raise RuntimeError("Could not find template %s" % template_name
)
238 return template
.graph_info()
240 def last_update(self
, object_id
="default"):
241 object = self
.get_object(object_id
)
243 raise RuntimeError("Could not find object %s" % object_id
)
245 return object.last_update()
248 class Object(object):
249 # The schema of the RRD database.
253 rra_types
= ("AVERAGE", "MIN", "MAX")
260 def __init__(self
, plugin
, *args
, **kwargs
):
263 # Indicates if this object has collected its data
264 self
.collected
= False
266 # Initialise this object
267 self
.init(*args
, **kwargs
)
269 # Create the database file.
273 return "<%s %s>" % (self
.__class
__.__name
__, self
.id)
275 def __lt__(self
, other
):
276 return self
.id < other
.id
280 return self
.plugin
.collecty
284 return self
.plugin
.log
289 Returns a UNIQUE identifier for this object. As this is incorporated
290 into the path of RRD file, it must only contain ASCII characters.
292 raise NotImplementedError
297 The absolute path to the RRD file of this plugin.
299 filename
= self
._normalise
_filename
("%s.rrd" % self
.id)
301 return os
.path
.join(DATABASE_DIR
, self
.plugin
.path
, filename
)
304 def _normalise_filename(filename
):
305 # Convert the filename into ASCII characters only
306 filename
= unicodedata
.normalize("NFKC", filename
)
308 # Replace any spaces by dashes
309 filename
= filename
.replace(" ", "-")
315 def init(self
, *args
, **kwargs
):
317 Do some custom initialization stuff here.
323 Creates an empty RRD file with the desired data structures.
325 # Skip if the file does already exist.
326 if os
.path
.exists(self
.file):
329 dirname
= os
.path
.dirname(self
.file)
330 if not os
.path
.exists(dirname
):
333 # Create argument list.
334 args
= self
.get_rrd_schema()
336 rrdtool
.create(self
.file, *args
)
338 self
.log
.debug(_("Created RRD file %s.") % self
.file)
340 self
.log
.debug(" %s" % arg
)
343 return rrdtool
.info(self
.file)
345 def last_update(self
):
347 Returns a dictionary with the timestamp and
348 data set of the last database update.
351 "dataset" : self
.last_dataset
,
352 "timestamp" : self
.last_updated
,
355 def _last_update(self
):
356 return rrdtool
.lastupdate(self
.file)
359 def last_updated(self
):
361 Returns the timestamp when this database was last updated
363 lu
= self
._last
_update
()
366 return lu
.get("date")
369 def last_dataset(self
):
371 Returns the latest dataset in the database
373 lu
= self
._last
_update
()
380 return self
.plugin
.interval
384 return self
.stepsize
* 2
386 def get_rrd_schema(self
):
388 "--step", "%s" % self
.stepsize
,
390 for line
in self
.rrd_schema
:
391 if line
.startswith("DS:"):
393 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
399 "%s" % self
.heartbeat
,
410 for steps
, rows
in self
.rra_timespans
:
411 for type in self
.rra_types
:
412 schema
.append("RRA:%s:%s:%s:%s" % (type, xff
, steps
, rows
))
417 def rrd_schema_names(self
):
420 for line
in self
.rrd_schema
:
421 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
426 def make_rrd_defs(self
, prefix
=None):
429 for name
in self
.rrd_schema_names
:
431 p
= "%s_%s" % (prefix
, name
)
436 "DEF:%s=%s:%s:AVERAGE" % (p
, self
.file, name
),
441 def get_stddev(self
, interval
=None):
442 args
= self
.make_rrd_defs()
444 # Add the correct interval
445 args
+= ["--start", util
.make_interval(interval
)]
447 for name
in self
.rrd_schema_names
:
449 "VDEF:%s_stddev=%s,STDEV" % (name
, name
),
450 "PRINT:%s_stddev:%%lf" % name
,
453 x
, y
, vals
= rrdtool
.graph("/dev/null", *args
)
454 return dict(zip(self
.rrd_schema_names
, vals
))
458 raise RuntimeError("This object has already collected its data")
460 self
.collected
= True
461 self
.now
= datetime
.datetime
.utcnow()
464 result
= self
.collect()
468 Will commit the collected data to the database.
470 # Make sure that the RRD database has been created
473 # Write everything to disk that is in the write queue
474 self
.collecty
.write_queue
.commit_file(self
.file)
476 # Convenience functions for plugin authors
478 def read_file(self
, *args
, strip
=True):
480 Reads the content of the given file
482 filename
= os
.path
.join(*args
)
484 with
open(filename
) as f
:
487 # Strip any excess whitespace
489 value
= value
.strip()
493 def read_file_integer(self
, filename
):
495 Reads the content from a file and returns it as an integer
497 value
= self
.read_file(filename
)
504 def read_proc_stat(self
):
506 Reads /proc/stat and returns it as a dictionary
510 with
open("/proc/stat") as f
:
512 # Split the key from the rest of the line
513 key
, line
= line
.split(" ", 1)
515 # Remove any line breaks
516 ret
[key
] = line
.rstrip()
520 def read_proc_meminfo(self
):
523 with
open("/proc/meminfo") as f
:
525 # Split the key from the rest of the line
526 key
, line
= line
.split(":", 1)
528 # Remove any whitespace
531 # Remove any trailing kB
532 if line
.endswith(" kB"):
535 # Try to convert to integer
538 except (TypeError, ValueError):
546 class GraphTemplate(object):
547 # A unique name to identify this graph template.
550 # Headline of the graph image
553 # Vertical label of the graph
554 graph_vertical_label
= None
560 # Instructions how to create the graph.
563 # Extra arguments passed to rrdgraph.
566 # Default dimensions for this graph
567 height
= GRAPH_DEFAULT_HEIGHT
568 width
= GRAPH_DEFAULT_WIDTH
570 def __init__(self
, plugin
, object_id
, locale
=None, timezone
=None):
573 # Save localisation parameters
574 self
.locale
= locales
.get(locale
)
575 self
.timezone
= timezone
577 # Get all required RRD objects
578 self
.object_id
= object_id
580 # Get the main object
581 self
.objects
= self
.get_objects(self
.object_id
)
585 return "<%s>" % self
.__class
__.__name
__
589 return self
.plugin
.collecty
593 return self
.plugin
.log
598 Shortcut to the main object
600 if len(self
.objects
) == 1:
601 return self
.objects
[0]
603 def _make_command_line(self
, interval
, format
=DEFAULT_IMAGE_FORMAT
,
604 width
=None, height
=None, with_title
=True, thumbnail
=False):
605 args
= [e
for e
in GRAPH_DEFAULT_ARGUMENTS
]
607 # Set the default dimensions
608 default_height
, default_width
= GRAPH_DEFAULT_HEIGHT
, GRAPH_DEFAULT_WIDTH
610 # A thumbnail doesn't have a legend and other labels
612 args
.append("--only-graph")
614 default_height
= THUMBNAIL_DEFAULT_HEIGHT
615 default_width
= THUMBNAIL_DEFAULT_WIDTH
618 "--imgformat", format
,
619 "--height", "%s" % (height
or default_height
),
620 "--width", "%s" % (width
or default_width
),
623 args
+= self
.rrd_graph_args
626 if with_title
and self
.graph_title
:
627 args
+= ["--title", self
.graph_title
]
630 if self
.graph_vertical_label
:
631 args
+= ["--vertical-label", self
.graph_vertical_label
]
633 if self
.lower_limit
is not None or self
.upper_limit
is not None:
634 # Force to honour the set limits
635 args
.append("--rigid")
637 if self
.lower_limit
is not None:
638 args
+= ["--lower-limit", self
.lower_limit
]
640 if self
.upper_limit
is not None:
641 args
+= ["--upper-limit", self
.upper_limit
]
644 args
+= ["--start", util
.make_interval(interval
)]
649 use_prefix
= len(self
.objects
) >= 2
652 for object in self
.objects
:
654 args
+= object.make_rrd_defs(object.id)
656 args
+= object.make_rrd_defs()
660 def _add_vdefs(self
, args
):
666 # Search for all DEFs and CDEFs
667 m
= re
.match(DEF_MATCH
, "%s" % arg
)
671 # Add the VDEFs for minimum, maximum, etc. values
673 "VDEF:%s_cur=%s,LAST" % (name
, name
),
674 "VDEF:%s_avg=%s,AVERAGE" % (name
, name
),
675 "VDEF:%s_max=%s,MAXIMUM" % (name
, name
),
676 "VDEF:%s_min=%s,MINIMUM" % (name
, name
),
681 def get_objects(self
, *args
, **kwargs
):
682 object = self
.plugin
.get_object(*args
, **kwargs
)
689 def generate_graph(self
, interval
=None, **kwargs
):
690 assert self
.objects
, "Cannot render graph without any objects"
692 # Make sure that all collected data is in the database
693 # to get a recent graph image
694 for object in self
.objects
:
697 args
= self
._make
_command
_line
(interval
, **kwargs
)
699 self
.log
.info(_("Generating graph %s") % self
)
701 rrd_graph
= self
.rrd_graph
703 # Add DEFs for all objects
704 if not any((e
.startswith("DEF:") for e
in rrd_graph
)):
705 args
+= self
._add
_defs
()
708 args
= self
._add
_vdefs
(args
)
710 # Convert arguments to string
711 args
= [str(e
) for e
in args
]
714 self
.log
.debug(" %s" % arg
)
716 with
Environment(self
.timezone
, self
.locale
.lang
):
717 graph
= rrdtool
.graphv("-", *args
)
720 "image" : graph
.get("image"),
721 "image_height" : graph
.get("image_height"),
722 "image_width" : graph
.get("image_width"),
725 def graph_info(self
):
727 Returns a dictionary with useful information
731 "title" : self
.graph_title
,
732 "object_id" : self
.object_id
or "",
733 "template" : self
.name
,