]>
git.ipfire.org Git - collecty.git/blob - src/collecty/plugins/base.py
2 ###############################################################################
4 # collecty - A system statistics collection daemon for IPFire #
5 # Copyright (C) 2012 IPFire development team #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
20 ###############################################################################
30 from ..constants
import *
33 DEF_MATCH
= r
"C?DEF:([A-Za-z0-9_]+)="
35 class Environment(object):
37 Sets the correct environment for rrdtool to create
38 localised graphs and graphs in the correct timezone.
40 def __init__(self
, timezone
="UTC", locale
="en_US.utf-8"):
41 # Build the new environment
42 self
.new_environment
= {
49 # Save the current environment
50 self
.old_environment
= {}
52 for k
in self
.new_environment
:
54 self
.old_environment
[k
] = os
.environ
.get(k
, None)
57 if self
.new_environment
[k
]:
58 os
.environ
[k
] = self
.new_environment
[k
]
60 def __exit__(self
, type, value
, traceback
):
61 # Roll back to the previous environment
62 for k
, v
in self
.old_environment
.items():
72 class PluginRegistration(type):
75 def __init__(plugin
, name
, bases
, dict):
76 type.__init
__(plugin
, name
, bases
, dict)
78 # The main class from which is inherited is not registered
83 if not all((plugin
.name
, plugin
.description
)):
84 raise RuntimeError(_("Plugin is not properly configured: %s") % plugin
)
86 PluginRegistration
.plugins
[plugin
.name
] = plugin
91 Returns a list with all automatically registered plugins.
93 return PluginRegistration
.plugins
.values()
95 class Plugin(object, metaclass
=PluginRegistration
):
96 # The name of this plugin.
99 # A description for this plugin.
102 # Templates which can be used to generate a graph out of
103 # the data from this data source.
106 # The default interval for all plugins
112 def __init__(self
, collecty
, **kwargs
):
113 self
.collecty
= collecty
115 # Check if this plugin was configured correctly.
116 assert self
.name
, "Name of the plugin is not set: %s" % self
.name
117 assert self
.description
, "Description of the plugin is not set: %s" % self
.description
119 # Initialize the logger.
120 self
.log
= logging
.getLogger("collecty.plugins.%s" % self
.name
)
122 # Run some custom initialization.
125 self
.log
.debug(_("Successfully initialized %s") % self
.__class
__.__name
__)
130 Returns the name of the sub directory in which all RRD files
131 for this plugin should be stored in.
137 def init(self
, **kwargs
):
139 Do some custom initialization stuff here.
145 Gathers the statistical data, this plugin collects.
147 time_start
= time
.time()
149 # Run through all objects of this plugin and call the collect method.
150 for object in self
.objects
:
153 result
= object.collect()
155 # Catch any unhandled exceptions
156 except Exception as e
:
157 self
.log
.warning(_("Unhandled exception in %s.collect()") % object, exc_info
=True)
161 self
.log
.warning(_("Received empty result: %s") % object)
164 # Add the object to the write queue so that the data is written
165 # to the databases later.
166 result
= self
.collecty
.write_queue
.submit(object, result
)
168 self
.log
.debug(_("Collected %s: %s") % (object, result
))
170 # Returns the time this function took to complete.
171 delay
= time
.time() - time_start
173 # Log some warning when a collect method takes too long to return some data
175 self
.log
.warning(_("A worker thread was stalled for %.4fs") % delay
)
177 self
.log
.debug(_("Collection finished in %.2fms") % (delay
* 1000))
179 def get_object(self
, id):
180 for object in self
.objects
:
181 if not object.id == id:
186 def get_template(self
, template_name
, object_id
, locale
=None, timezone
=None):
187 for template
in self
.templates
:
188 if not template
.name
== template_name
:
191 return template(self
, object_id
, locale
=locale
, timezone
=timezone
)
193 def generate_graph(self
, template_name
, object_id
="default",
194 timezone
=None, locale
=None, **kwargs
):
195 template
= self
.get_template(template_name
, object_id
=object_id
,
196 timezone
=timezone
, locale
=locale
)
198 raise RuntimeError("Could not find template %s" % template_name
)
200 time_start
= time
.time()
202 with
Environment(timezone
=timezone
, locale
=locale
):
203 graph
= template
.generate_graph(**kwargs
)
205 duration
= time
.time() - time_start
206 self
.log
.debug(_("Generated graph %s in %.1fms") \
207 % (template
, duration
* 1000))
211 def graph_info(self
, template_name
, object_id
="default",
212 timezone
=None, locale
=None, **kwargs
):
213 template
= self
.get_template(template_name
, object_id
=object_id
,
214 timezone
=timezone
, locale
=locale
)
216 raise RuntimeError("Could not find template %s" % template_name
)
218 return template
.graph_info()
220 def last_update(self
, object_id
="default"):
221 object = self
.get_object(object_id
)
223 raise RuntimeError("Could not find object %s" % object_id
)
225 return object.last_update()
228 class Object(object):
229 # The schema of the RRD database.
233 rra_types
= ("AVERAGE", "MIN", "MAX")
240 def __init__(self
, plugin
, *args
, **kwargs
):
243 # Initialise this object
244 self
.init(*args
, **kwargs
)
246 # Create the database file.
250 return "<%s %s>" % (self
.__class
__.__name
__, self
.id)
252 def __lt__(self
, other
):
253 return self
.id < other
.id
257 return self
.plugin
.collecty
261 return self
.plugin
.log
266 Returns a UNIQUE identifier for this object. As this is incorporated
267 into the path of RRD file, it must only contain ASCII characters.
269 raise NotImplementedError
274 The absolute path to the RRD file of this plugin.
276 filename
= self
._normalise
_filename
("%s.rrd" % self
.id)
278 return os
.path
.join(DATABASE_DIR
, self
.plugin
.path
, filename
)
281 def _normalise_filename(filename
):
282 # Convert the filename into ASCII characters only
283 filename
= unicodedata
.normalize("NFKC", filename
)
285 # Replace any spaces by dashes
286 filename
= filename
.replace(" ", "-")
292 def init(self
, *args
, **kwargs
):
294 Do some custom initialization stuff here.
300 Creates an empty RRD file with the desired data structures.
302 # Skip if the file does already exist.
303 if os
.path
.exists(self
.file):
306 dirname
= os
.path
.dirname(self
.file)
307 if not os
.path
.exists(dirname
):
310 # Create argument list.
311 args
= self
.get_rrd_schema()
313 rrdtool
.create(self
.file, *args
)
315 self
.log
.debug(_("Created RRD file %s.") % self
.file)
317 self
.log
.debug(" %s" % arg
)
320 return rrdtool
.info(self
.file)
322 def last_update(self
):
324 Returns a dictionary with the timestamp and
325 data set of the last database update.
328 "dataset" : self
.last_dataset
,
329 "timestamp" : self
.last_updated
,
332 def _last_update(self
):
333 return rrdtool
.lastupdate(self
.file)
336 def last_updated(self
):
338 Returns the timestamp when this database was last updated
340 lu
= self
._last
_update
()
343 return lu
.get("date")
346 def last_dataset(self
):
348 Returns the latest dataset in the database
350 lu
= self
._last
_update
()
357 return self
.plugin
.interval
361 return self
.stepsize
* 2
363 def get_rrd_schema(self
):
365 "--step", "%s" % self
.stepsize
,
367 for line
in self
.rrd_schema
:
368 if line
.startswith("DS:"):
370 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
376 "%s" % self
.heartbeat
,
387 for steps
, rows
in self
.rra_timespans
:
388 for type in self
.rra_types
:
389 schema
.append("RRA:%s:%s:%s:%s" % (type, xff
, steps
, rows
))
394 def rrd_schema_names(self
):
397 for line
in self
.rrd_schema
:
398 (prefix
, name
, type, lower_limit
, upper_limit
) = line
.split(":")
403 def make_rrd_defs(self
, prefix
=None):
406 for name
in self
.rrd_schema_names
:
408 p
= "%s_%s" % (prefix
, name
)
413 "DEF:%s=%s:%s:AVERAGE" % (p
, self
.file, name
),
418 def get_stddev(self
, interval
=None):
419 args
= self
.make_rrd_defs()
421 # Add the correct interval
422 args
+= ["--start", util
.make_interval(interval
)]
424 for name
in self
.rrd_schema_names
:
426 "VDEF:%s_stddev=%s,STDEV" % (name
, name
),
427 "PRINT:%s_stddev:%%lf" % name
,
430 x
, y
, vals
= rrdtool
.graph("/dev/null", *args
)
431 return dict(zip(self
.rrd_schema_names
, vals
))
435 Will commit the collected data to the database.
437 # Make sure that the RRD database has been created
440 # Write everything to disk that is in the write queue
441 self
.collecty
.write_queue
.commit_file(self
.file)
443 # Convenience functions for plugin authors
445 def read_file(self
, *args
, strip
=True):
447 Reads the content of the given file
449 filename
= os
.path
.join(*args
)
452 with
open(filename
) as f
:
454 except FileNotFoundError
as e
:
457 # Strip any excess whitespace
459 value
= value
.strip()
463 def read_file_integer(self
, filename
):
465 Reads the content from a file and returns it as an integer
467 value
= self
.read_file(filename
)
471 except (TypeError, ValueError):
474 def read_proc_stat(self
):
476 Reads /proc/stat and returns it as a dictionary
480 with
open("/proc/stat") as f
:
482 # Split the key from the rest of the line
483 key
, line
= line
.split(" ", 1)
485 # Remove any line breaks
486 ret
[key
] = line
.rstrip()
490 def read_proc_meminfo(self
):
493 with
open("/proc/meminfo") as f
:
495 # Split the key from the rest of the line
496 key
, line
= line
.split(":", 1)
498 # Remove any whitespace
501 # Remove any trailing kB
502 if line
.endswith(" kB"):
505 # Try to convert to integer
508 except (TypeError, ValueError):
516 class GraphTemplate(object):
517 # A unique name to identify this graph template.
520 # Headline of the graph image
523 # Vertical label of the graph
524 graph_vertical_label
= None
530 # Instructions how to create the graph.
533 # Extra arguments passed to rrdgraph.
536 def __init__(self
, plugin
, object_id
, locale
=None, timezone
=None):
539 # Save localisation parameters
541 self
.timezone
= timezone
543 # Get all required RRD objects
544 self
.object_id
= object_id
546 # Get the main object
547 self
.objects
= self
.get_objects(self
.object_id
)
551 return "<%s>" % self
.__class
__.__name
__
555 return self
.plugin
.collecty
559 return self
.plugin
.log
564 Shortcut to the main object
566 if len(self
.objects
) == 1:
567 return self
.objects
[0]
569 def _make_command_line(self
, interval
, format
=DEFAULT_IMAGE_FORMAT
,
570 width
=None, height
=None, with_title
=True, thumbnail
=False):
572 # Change the background colour
573 "--color", "BACK#FFFFFFFF",
575 # Disable the border around the image
578 # Let's width and height define the size of the entire image
581 # Gives the curves a more organic look
587 # Brand all generated graphs
588 "--watermark", _("Created by collecty"),
591 # Set the default dimensions
592 default_width
, default_height
= 960, 480
594 # A thumbnail doesn't have a legend and other labels
596 args
.append("--only-graph")
598 default_width
, default_height
= 80, 20
601 "--imgformat", format
,
602 "--height", "%s" % (height
or default_height
),
603 "--width", "%s" % (width
or default_width
),
606 args
+= self
.rrd_graph_args
609 if with_title
and self
.graph_title
:
610 args
+= ["--title", self
.graph_title
]
613 if self
.graph_vertical_label
:
614 args
+= ["--vertical-label", self
.graph_vertical_label
]
616 if self
.lower_limit
is not None or self
.upper_limit
is not None:
617 # Force to honour the set limits
618 args
.append("--rigid")
620 if self
.lower_limit
is not None:
621 args
+= ["--lower-limit", self
.lower_limit
]
623 if self
.upper_limit
is not None:
624 args
+= ["--upper-limit", self
.upper_limit
]
627 args
+= ["--start", util
.make_interval(interval
)]
632 use_prefix
= len(self
.objects
) >= 2
635 for object in self
.objects
:
637 args
+= object.make_rrd_defs(object.id)
639 args
+= object.make_rrd_defs()
643 def _add_vdefs(self
, args
):
649 # Search for all DEFs and CDEFs
650 m
= re
.match(DEF_MATCH
, "%s" % arg
)
654 # Add the VDEFs for minimum, maximum, etc. values
656 "VDEF:%s_cur=%s,LAST" % (name
, name
),
657 "VDEF:%s_avg=%s,AVERAGE" % (name
, name
),
658 "VDEF:%s_max=%s,MAXIMUM" % (name
, name
),
659 "VDEF:%s_min=%s,MINIMUM" % (name
, name
),
664 def get_objects(self
, *args
, **kwargs
):
665 object = self
.plugin
.get_object(*args
, **kwargs
)
672 def generate_graph(self
, interval
=None, **kwargs
):
673 assert self
.objects
, "Cannot render graph without any objects"
675 # Make sure that all collected data is in the database
676 # to get a recent graph image
677 for object in self
.objects
:
680 args
= self
._make
_command
_line
(interval
, **kwargs
)
682 self
.log
.info(_("Generating graph %s") % self
)
684 rrd_graph
= self
.rrd_graph
686 # Add DEFs for all objects
687 if not any((e
.startswith("DEF:") for e
in rrd_graph
)):
688 args
+= self
._add
_defs
()
691 args
= self
._add
_vdefs
(args
)
693 # Convert arguments to string
694 args
= [str(e
) for e
in args
]
697 self
.log
.debug(" %s" % arg
)
699 graph
= rrdtool
.graphv("-", *args
)
702 "image" : graph
.get("image"),
703 "image_height" : graph
.get("image_height"),
704 "image_width" : graph
.get("image_width"),
707 def graph_info(self
):
709 Returns a dictionary with useful information
713 "title" : self
.graph_title
,
714 "object_id" : self
.object_id
or "",
715 "template" : self
.name
,