graph templates: Make some atttibutes easier to set
[collecty.git] / src / collecty / plugins / base.py
CommitLineData
eed405de
MT
1#!/usr/bin/python
2###############################################################################
3# #
4# collecty - A system statistics collection daemon for IPFire #
5# Copyright (C) 2012 IPFire development team #
6# #
7# This program is free software: you can redistribute it and/or modify #
8# it under the terms of the GNU General Public License as published by #
9# the Free Software Foundation, either version 3 of the License, or #
10# (at your option) any later version. #
11# #
12# This program is distributed in the hope that it will be useful, #
13# but WITHOUT ANY WARRANTY; without even the implied warranty of #
14# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15# GNU General Public License for more details. #
16# #
17# You should have received a copy of the GNU General Public License #
18# along with this program. If not, see <http://www.gnu.org/licenses/>. #
19# #
20###############################################################################
21
965a9c51
MT
22from __future__ import division
23
72364063 24import datetime
4ac0cdf0 25import logging
965a9c51 26import math
4ac0cdf0
MT
27import os
28import rrdtool
c968f6d9 29import tempfile
49ce926e 30import threading
4ac0cdf0
MT
31import time
32
4ac0cdf0 33from ..constants import *
eed405de
MT
34from ..i18n import _
35
5d140577
MT
36_plugins = {}
37
38def get():
39 """
40 Returns a list with all automatically registered plugins.
41 """
42 return _plugins.values()
43
4be39bf9
MT
44class Timer(object):
45 def __init__(self, timeout, heartbeat=1):
46 self.timeout = timeout
47 self.heartbeat = heartbeat
48
e746a56e
MT
49 self.delay = 0
50
4be39bf9
MT
51 self.reset()
52
e746a56e 53 def reset(self, delay=0):
4be39bf9
MT
54 # Save start time.
55 self.start = time.time()
56
e746a56e
MT
57 self.delay = delay
58
4be39bf9
MT
59 # Has this timer been killed?
60 self.killed = False
61
62 @property
63 def elapsed(self):
e746a56e 64 return time.time() - self.start - self.delay
4be39bf9
MT
65
66 def cancel(self):
67 self.killed = True
68
69 def wait(self):
70 while self.elapsed < self.timeout and not self.killed:
71 time.sleep(self.heartbeat)
72
73 return self.elapsed > self.timeout
74
75
5d140577 76class Plugin(threading.Thread):
4ac0cdf0
MT
77 # The name of this plugin.
78 name = None
79
80 # A description for this plugin.
81 description = None
82
b1ea4956
MT
83 # Templates which can be used to generate a graph out of
84 # the data from this data source.
85 templates = []
86
72364063
MT
87 # The default interval for all plugins
88 interval = 60
4ac0cdf0 89
5d140577
MT
90 # Automatically register all providers.
91 class __metaclass__(type):
92 def __init__(plugin, name, bases, dict):
93 type.__init__(plugin, name, bases, dict)
94
95 # The main class from which is inherited is not registered
96 # as a plugin.
97 if name == "Plugin":
98 return
99
100 if not all((plugin.name, plugin.description)):
101 raise RuntimeError(_("Plugin is not properly configured: %s") \
102 % plugin)
103
104 _plugins[plugin.name] = plugin
105
eed405de 106 def __init__(self, collecty, **kwargs):
49ce926e
MT
107 threading.Thread.__init__(self, name=self.description)
108 self.daemon = True
109
eed405de
MT
110 self.collecty = collecty
111
4ac0cdf0
MT
112 # Check if this plugin was configured correctly.
113 assert self.name, "Name of the plugin is not set: %s" % self.name
114 assert self.description, "Description of the plugin is not set: %s" % self.description
4ac0cdf0
MT
115
116 # Initialize the logger.
117 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
118 self.log.propagate = 1
eed405de 119
eed405de
MT
120 self.data = []
121
269f74cd
MT
122 # Run some custom initialization.
123 self.init(**kwargs)
124
4be39bf9
MT
125 # Keepalive options
126 self.running = True
127 self.timer = Timer(self.interval)
128
0ee0c42d 129 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
4ac0cdf0 130
73db5226 131 @property
72364063 132 def path(self):
73db5226 133 """
72364063
MT
134 Returns the name of the sub directory in which all RRD files
135 for this plugin should be stored in.
73db5226
MT
136 """
137 return self.name
138
72364063
MT
139 ### Basic methods
140
141 def init(self, **kwargs):
4ac0cdf0 142 """
72364063 143 Do some custom initialization stuff here.
4ac0cdf0 144 """
72364063
MT
145 pass
146
147 def collect(self):
148 """
149 Gathers the statistical data, this plugin collects.
150 """
151 time_start = time.time()
152
153 # Run through all objects of this plugin and call the collect method.
154 for o in self.objects:
155 now = datetime.datetime.utcnow()
156 try:
157 result = o.collect()
158 except:
159 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
160 continue
161
162 if not result:
163 self.log.warning(_("Received empty result: %s") % o)
164 continue
165
166 self.log.debug(_("Collected %s: %s") % (o, result))
167
168 # Add the object to the write queue so that the data is written
169 # to the databases later.
170 self.collecty.write_queue.add(o, now, result)
171
172 # Returns the time this function took to complete.
173 return (time.time() - time_start)
174
175 def run(self):
176 self.log.debug(_("%s plugin has started") % self.name)
177
178 # Initially collect everything
179 self.collect()
180
181 while self.running:
182 # Reset the timer.
183 self.timer.reset()
184
185 # Wait until the timer has successfully elapsed.
186 if self.timer.wait():
187 delay = self.collect()
188 self.timer.reset(delay)
189
190 self.log.debug(_("%s plugin has stopped") % self.name)
191
192 def shutdown(self):
193 self.log.debug(_("Received shutdown signal."))
194 self.running = False
195
196 # Kill any running timers.
197 if self.timer:
198 self.timer.cancel()
4ac0cdf0 199
c968f6d9
MT
200 def get_object(self, id):
201 for object in self.objects:
202 if not object.id == id:
203 continue
204
205 return object
206
207 def get_template(self, template_name):
208 for template in self.templates:
209 if not template.name == template_name:
210 continue
211
212 return template(self)
213
214 def generate_graph(self, template_name, object_id="default", **kwargs):
215 template = self.get_template(template_name)
216 if not template:
217 raise RuntimeError("Could not find template %s" % template_name)
218
219 time_start = time.time()
220
221 graph = template.generate_graph(object_id=object_id, **kwargs)
222
223 duration = time.time() - time_start
0ee0c42d 224 self.log.debug(_("Generated graph %s in %.1fms") \
c968f6d9
MT
225 % (template, duration * 1000))
226
227 return graph
228
72364063
MT
229
230class Object(object):
231 # The schema of the RRD database.
232 rrd_schema = None
233
234 # RRA properties.
235 rra_types = ["AVERAGE", "MIN", "MAX"]
236 rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
237 rra_rows = 2880
238
239 def __init__(self, plugin, *args, **kwargs):
240 self.plugin = plugin
241
242 # Indicates if this object has collected its data
243 self.collected = False
244
245 # Initialise this object
246 self.init(*args, **kwargs)
247
248 # Create the database file.
249 self.create()
250
251 def __repr__(self):
252 return "<%s>" % self.__class__.__name__
4ac0cdf0 253
965a9c51 254 @property
72364063
MT
255 def collecty(self):
256 return self.plugin.collecty
965a9c51 257
881751ed 258 @property
72364063
MT
259 def log(self):
260 return self.plugin.log
261
262 @property
263 def id(self):
264 """
265 Returns a UNIQUE identifier for this object. As this is incorporated
266 into the path of RRD file, it must only contain ASCII characters.
267 """
268 raise NotImplementedError
881751ed 269
4ac0cdf0
MT
270 @property
271 def file(self):
272 """
273 The absolute path to the RRD file of this plugin.
274 """
72364063
MT
275 return os.path.join(DATABASE_DIR, self.plugin.path, "%s.rrd" % self.id)
276
277 ### Basic methods
278
279 def init(self, *args, **kwargs):
280 """
281 Do some custom initialization stuff here.
282 """
283 pass
eed405de 284
4ac0cdf0
MT
285 def create(self):
286 """
287 Creates an empty RRD file with the desired data structures.
288 """
289 # Skip if the file does already exist.
290 if os.path.exists(self.file):
291 return
eed405de 292
4ac0cdf0
MT
293 dirname = os.path.dirname(self.file)
294 if not os.path.exists(dirname):
295 os.makedirs(dirname)
eed405de 296
965a9c51 297 # Create argument list.
ff0bbd88 298 args = self.get_rrd_schema()
965a9c51
MT
299
300 rrdtool.create(self.file, *args)
eed405de 301
4ac0cdf0 302 self.log.debug(_("Created RRD file %s.") % self.file)
dadb8fb0
MT
303 for arg in args:
304 self.log.debug(" %s" % arg)
eed405de 305
72364063
MT
306 def info(self):
307 return rrdtool.info(self.file)
308
309 @property
310 def stepsize(self):
311 return self.plugin.interval
312
313 @property
314 def heartbeat(self):
315 return self.stepsize * 2
316
965a9c51
MT
317 def get_rrd_schema(self):
318 schema = [
319 "--step", "%s" % self.stepsize,
320 ]
321 for line in self.rrd_schema:
322 if line.startswith("DS:"):
323 try:
324 (prefix, name, type, lower_limit, upper_limit) = line.split(":")
325
326 line = ":".join((
327 prefix,
328 name,
329 type,
881751ed 330 "%s" % self.heartbeat,
965a9c51
MT
331 lower_limit,
332 upper_limit
333 ))
334 except ValueError:
335 pass
336
337 schema.append(line)
338
339 xff = 0.1
340
341 cdp_length = 0
342 for rra_timespan in self.rra_timespans:
343 if (rra_timespan / self.stepsize) < self.rra_rows:
344 rra_timespan = self.stepsize * self.rra_rows
345
346 if cdp_length == 0:
347 cdp_length = 1
348 else:
349 cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
350
351 cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
352
353 for rra_type in self.rra_types:
354 schema.append("RRA:%s:%.10f:%d:%d" % \
355 (rra_type, xff, cdp_length, cdp_number))
356
357 return schema
358
72364063
MT
359 def execute(self):
360 if self.collected:
361 raise RuntimeError("This object has already collected its data")
eed405de 362
72364063
MT
363 self.collected = True
364 self.now = datetime.datetime.utcnow()
4ac0cdf0 365
72364063
MT
366 # Call the collect
367 result = self.collect()
4ac0cdf0 368
72364063 369 def commit(self):
4ac0cdf0 370 """
72364063 371 Will commit the collected data to the database.
4ac0cdf0 372 """
72364063 373 # Make sure that the RRD database has been created
dadb8fb0
MT
374 self.create()
375
b1ea4956
MT
376
377class GraphTemplate(object):
378 # A unique name to identify this graph template.
379 name = None
380
f181246a
MT
381 # Headline of the graph image
382 graph_title = None
383
384 # Vertical label of the graph
385 graph_vertical_label = None
386
387 # Limits
388 lower_limit = None
389 upper_limit = None
390
b1ea4956
MT
391 # Instructions how to create the graph.
392 rrd_graph = None
393
394 # Extra arguments passed to rrdgraph.
395 rrd_graph_args = []
396
c968f6d9
MT
397 intervals = {
398 None : "-3h",
399 "hour" : "-1h",
400 "day" : "-25h",
401 "week" : "-360h",
402 "year" : "-365d",
403 }
404
405 # Default dimensions for this graph
406 height = GRAPH_DEFAULT_HEIGHT
407 width = GRAPH_DEFAULT_WIDTH
408
409 def __init__(self, plugin):
410 self.plugin = plugin
411
412 def __repr__(self):
413 return "<%s>" % self.__class__.__name__
b1ea4956
MT
414
415 @property
416 def collecty(self):
c968f6d9 417 return self.plugin.collecty
b1ea4956 418
c968f6d9
MT
419 @property
420 def log(self):
421 return self.plugin.log
422
5913a52c
MT
423 def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
424 width=None, height=None):
c968f6d9
MT
425 args = []
426
427 args += GRAPH_DEFAULT_ARGUMENTS
428
429 args += [
5913a52c 430 "--imgformat", format,
c968f6d9
MT
431 "--height", "%s" % (height or self.height),
432 "--width", "%s" % (width or self.width),
73db5226 433 ]
eed405de 434
c968f6d9 435 args += self.rrd_graph_args
eed405de 436
f181246a
MT
437 # Graph title
438 if self.graph_title:
439 args += ["--title", self.graph_title]
440
441 # Vertical label
442 if self.graph_vertical_label:
443 args += ["--vertical-label", self.graph_vertical_label]
444
445 if self.lower_limit is not None or self.upper_limit is not None:
446 # Force to honour the set limits
447 args.append("--rigid")
448
449 if self.lower_limit is not None:
450 args += ["--lower-limit", self.lower_limit]
451
452 if self.upper_limit is not None:
453 args += ["--upper-limit", self.upper_limit]
454
c968f6d9 455 # Add interval
eed405de 456 args.append("--start")
c968f6d9 457
b1ea4956 458 try:
c968f6d9 459 args.append(self.intervals[interval])
b1ea4956 460 except KeyError:
c968f6d9
MT
461 args.append(str(interval))
462
463 return args
464
465 def get_object_table(self, object_id):
466 return {
467 "file" : self.plugin.get_object(object_id),
468 }
469
470 def get_object_files(self, object_id):
471 files = {}
472
473 for id, obj in self.get_object_table(object_id).items():
474 files[id] = obj.file
475
476 return files
477
478 def generate_graph(self, object_id, interval=None, **kwargs):
479 args = self._make_command_line(interval, **kwargs)
480
481 self.log.info(_("Generating graph %s") % self)
482 self.log.debug(" args: %s" % args)
483
484 object_files = self.get_object_files(object_id)
eed405de 485
73db5226 486 for item in self.rrd_graph:
eed405de 487 try:
c968f6d9 488 args.append(item % object_files)
eed405de
MT
489 except TypeError:
490 args.append(item)
491
c968f6d9
MT
492 return self.write_graph(*args)
493
494 def write_graph(self, *args):
5913a52c
MT
495 # Convert all arguments to string
496 args = [str(e) for e in args]
497
c968f6d9
MT
498 with tempfile.NamedTemporaryFile() as f:
499 rrdtool.graph(f.name, *args)
500
501 # Get back to the beginning of the file
502 f.seek(0)
503
504 # Return all the content
505 return f.read()