Silence non-debugging output a bit
[collecty.git] / src / collecty / plugins / base.py
1 #!/usr/bin/python
2 ###############################################################################
3 #                                                                             #
4 # collecty - A system statistics collection daemon for IPFire                 #
5 # Copyright (C) 2012 IPFire development team                                  #
6 #                                                                             #
7 # This program is free software: you can redistribute it and/or modify        #
8 # it under the terms of the GNU General Public License as published by        #
9 # the Free Software Foundation, either version 3 of the License, or           #
10 # (at your option) any later version.                                         #
11 #                                                                             #
12 # This program is distributed in the hope that it will be useful,             #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
15 # GNU General Public License for more details.                                #
16 #                                                                             #
17 # You should have received a copy of the GNU General Public License           #
18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
19 #                                                                             #
20 ###############################################################################
21
22 from __future__ import division
23
24 import datetime
25 import logging
26 import math
27 import os
28 import rrdtool
29 import tempfile
30 import threading
31 import time
32
33 from ..constants import *
34 from ..i18n import _
35
36 _plugins = {}
37
38 def get():
39         """
40                 Returns a list with all automatically registered plugins.
41         """
42         return _plugins.values()
43
44 class Timer(object):
45         def __init__(self, timeout, heartbeat=1):
46                 self.timeout = timeout
47                 self.heartbeat = heartbeat
48
49                 self.delay = 0
50
51                 self.reset()
52
53         def reset(self, delay=0):
54                 # Save start time.
55                 self.start = time.time()
56
57                 self.delay = delay
58
59                 # Has this timer been killed?
60                 self.killed = False
61
62         @property
63         def elapsed(self):
64                 return time.time() - self.start - self.delay
65
66         def cancel(self):
67                 self.killed = True
68
69         def wait(self):
70                 while self.elapsed < self.timeout and not self.killed:
71                         time.sleep(self.heartbeat)
72
73                 return self.elapsed > self.timeout
74
75
76 class Plugin(threading.Thread):
77         # The name of this plugin.
78         name = None
79
80         # A description for this plugin.
81         description = None
82
83         # Templates which can be used to generate a graph out of
84         # the data from this data source.
85         templates = []
86
87         # The default interval for all plugins
88         interval = 60
89
90         # Automatically register all providers.
91         class __metaclass__(type):
92                 def __init__(plugin, name, bases, dict):
93                         type.__init__(plugin, name, bases, dict)
94
95                         # The main class from which is inherited is not registered
96                         # as a plugin.
97                         if name == "Plugin":
98                                 return
99
100                         if not all((plugin.name, plugin.description)):
101                                 raise RuntimeError(_("Plugin is not properly configured: %s") \
102                                         % plugin)
103
104                         _plugins[plugin.name] = plugin
105
106         def __init__(self, collecty, **kwargs):
107                 threading.Thread.__init__(self, name=self.description)
108                 self.daemon = True
109
110                 self.collecty = collecty
111
112                 # Check if this plugin was configured correctly.
113                 assert self.name, "Name of the plugin is not set: %s" % self.name
114                 assert self.description, "Description of the plugin is not set: %s" % self.description
115
116                 # Initialize the logger.
117                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
118                 self.log.propagate = 1
119
120                 self.data = []
121
122                 # Run some custom initialization.
123                 self.init(**kwargs)
124
125                 # Keepalive options
126                 self.running = True
127                 self.timer = Timer(self.interval)
128
129                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
130
131         @property
132         def path(self):
133                 """
134                         Returns the name of the sub directory in which all RRD files
135                         for this plugin should be stored in.
136                 """
137                 return self.name
138
139         ### Basic methods
140
141         def init(self, **kwargs):
142                 """
143                         Do some custom initialization stuff here.
144                 """
145                 pass
146
147         def collect(self):
148                 """
149                         Gathers the statistical data, this plugin collects.
150                 """
151                 time_start = time.time()
152
153                 # Run through all objects of this plugin and call the collect method.
154                 for o in self.objects:
155                         now = datetime.datetime.utcnow()
156                         try:
157                                 result = o.collect()
158                         except:
159                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
160                                 continue
161
162                         if not result:
163                                 self.log.warning(_("Received empty result: %s") % o)
164                                 continue
165
166                         self.log.debug(_("Collected %s: %s") % (o, result))
167
168                         # Add the object to the write queue so that the data is written
169                         # to the databases later.
170                         self.collecty.write_queue.add(o, now, result)
171
172                 # Returns the time this function took to complete.
173                 return (time.time() - time_start)
174
175         def run(self):
176                 self.log.debug(_("%s plugin has started") % self.name)
177
178                 # Initially collect everything
179                 self.collect()
180
181                 while self.running:
182                         # Reset the timer.
183                         self.timer.reset()
184
185                         # Wait until the timer has successfully elapsed.
186                         if self.timer.wait():
187                                 delay = self.collect()
188                                 self.timer.reset(delay)
189
190                 self.log.debug(_("%s plugin has stopped") % self.name)
191
192         def shutdown(self):
193                 self.log.debug(_("Received shutdown signal."))
194                 self.running = False
195
196                 # Kill any running timers.
197                 if self.timer:
198                         self.timer.cancel()
199
200         def get_object(self, id):
201                 for object in self.objects:
202                         if not object.id == id:
203                                 continue
204
205                         return object
206
207         def get_template(self, template_name):
208                 for template in self.templates:
209                         if not template.name == template_name:
210                                 continue
211
212                         return template(self)
213
214         def generate_graph(self, template_name, object_id="default", **kwargs):
215                 template = self.get_template(template_name)
216                 if not template:
217                         raise RuntimeError("Could not find template %s" % template_name)
218
219                 time_start = time.time()
220
221                 graph = template.generate_graph(object_id=object_id, **kwargs)
222
223                 duration = time.time() - time_start
224                 self.log.debug(_("Generated graph %s in %.1fms") \
225                         % (template, duration * 1000))
226
227                 return graph
228
229
230 class Object(object):
231         # The schema of the RRD database.
232         rrd_schema = None
233
234         # RRA properties.
235         rra_types     = ["AVERAGE", "MIN", "MAX"]
236         rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
237         rra_rows      = 2880
238
239         def __init__(self, plugin, *args, **kwargs):
240                 self.plugin = plugin
241
242                 # Indicates if this object has collected its data
243                 self.collected = False
244
245                 # Initialise this object
246                 self.init(*args, **kwargs)
247
248                 # Create the database file.
249                 self.create()
250
251         def __repr__(self):
252                 return "<%s>" % self.__class__.__name__
253
254         @property
255         def collecty(self):
256                 return self.plugin.collecty
257
258         @property
259         def log(self):
260                 return self.plugin.log
261
262         @property
263         def id(self):
264                 """
265                         Returns a UNIQUE identifier for this object. As this is incorporated
266                         into the path of RRD file, it must only contain ASCII characters.
267                 """
268                 raise NotImplementedError
269
270         @property
271         def file(self):
272                 """
273                         The absolute path to the RRD file of this plugin.
274                 """
275                 return os.path.join(DATABASE_DIR, self.plugin.path, "%s.rrd" % self.id)
276
277         ### Basic methods
278
279         def init(self, *args, **kwargs):
280                 """
281                         Do some custom initialization stuff here.
282                 """
283                 pass
284
285         def create(self):
286                 """
287                         Creates an empty RRD file with the desired data structures.
288                 """
289                 # Skip if the file does already exist.
290                 if os.path.exists(self.file):
291                         return
292
293                 dirname = os.path.dirname(self.file)
294                 if not os.path.exists(dirname):
295                         os.makedirs(dirname)
296
297                 # Create argument list.
298                 args = self.get_rrd_schema()
299
300                 rrdtool.create(self.file, *args)
301
302                 self.log.debug(_("Created RRD file %s.") % self.file)
303                 for arg in args:
304                         self.log.debug("  %s" % arg)
305
306         def info(self):
307                 return rrdtool.info(self.file)
308
309         @property
310         def stepsize(self):
311                 return self.plugin.interval
312
313         @property
314         def heartbeat(self):
315                 return self.stepsize * 2
316
317         def get_rrd_schema(self):
318                 schema = [
319                         "--step", "%s" % self.stepsize,
320                 ]
321                 for line in self.rrd_schema:
322                         if line.startswith("DS:"):
323                                 try:
324                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
325
326                                         line = ":".join((
327                                                 prefix,
328                                                 name,
329                                                 type,
330                                                 "%s" % self.heartbeat,
331                                                 lower_limit,
332                                                 upper_limit
333                                         ))
334                                 except ValueError:
335                                         pass
336
337                         schema.append(line)
338
339                 xff = 0.1
340
341                 cdp_length = 0
342                 for rra_timespan in self.rra_timespans:
343                         if (rra_timespan / self.stepsize) < self.rra_rows:
344                                 rra_timespan = self.stepsize * self.rra_rows
345
346                         if cdp_length == 0:
347                                 cdp_length = 1
348                         else:
349                                 cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
350
351                         cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
352
353                         for rra_type in self.rra_types:
354                                 schema.append("RRA:%s:%.10f:%d:%d" % \
355                                         (rra_type, xff, cdp_length, cdp_number))
356
357                 return schema
358
359         def execute(self):
360                 if self.collected:
361                         raise RuntimeError("This object has already collected its data")
362
363                 self.collected = True
364                 self.now = datetime.datetime.utcnow()
365
366                 # Call the collect
367                 result = self.collect()
368
369         def commit(self):
370                 """
371                         Will commit the collected data to the database.
372                 """
373                 # Make sure that the RRD database has been created
374                 self.create()
375
376
377 class GraphTemplate(object):
378         # A unique name to identify this graph template.
379         name = None
380
381         # Instructions how to create the graph.
382         rrd_graph = None
383
384         # Extra arguments passed to rrdgraph.
385         rrd_graph_args = []
386
387         intervals = {
388                 None   : "-3h",
389                 "hour" : "-1h",
390                 "day"  : "-25h",
391                 "week" : "-360h",
392                 "year" : "-365d",
393         }
394
395         # Default dimensions for this graph
396         height = GRAPH_DEFAULT_HEIGHT
397         width  = GRAPH_DEFAULT_WIDTH
398
399         def __init__(self, plugin):
400                 self.plugin = plugin
401
402         def __repr__(self):
403                 return "<%s>" % self.__class__.__name__
404
405         @property
406         def collecty(self):
407                 return self.plugin.collecty
408
409         @property
410         def log(self):
411                 return self.plugin.log
412
413         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
414                         width=None, height=None):
415                 args = []
416
417                 args += GRAPH_DEFAULT_ARGUMENTS
418
419                 args += [
420                         "--imgformat", format,
421                         "--height", "%s" % (height or self.height),
422                         "--width", "%s" % (width or self.width),
423                 ]
424
425                 args += self.rrd_graph_args
426
427                 # Add interval
428                 args.append("--start")
429
430                 try:
431                         args.append(self.intervals[interval])
432                 except KeyError:
433                         args.append(str(interval))
434
435                 return args
436
437         def get_object_table(self, object_id):
438                 return {
439                         "file" : self.plugin.get_object(object_id),
440                 }
441
442         def get_object_files(self, object_id):
443                 files = {}
444
445                 for id, obj in self.get_object_table(object_id).items():
446                         files[id] = obj.file
447
448                 return files
449
450         def generate_graph(self, object_id, interval=None, **kwargs):
451                 args = self._make_command_line(interval, **kwargs)
452
453                 self.log.info(_("Generating graph %s") % self)
454                 self.log.debug("  args: %s" % args)
455
456                 object_files = self.get_object_files(object_id)
457
458                 for item in self.rrd_graph:
459                         try:
460                                 args.append(item % object_files)
461                         except TypeError:
462                                 args.append(item)
463
464                 return self.write_graph(*args)
465
466         def write_graph(self, *args):
467                 # Convert all arguments to string
468                 args = [str(e) for e in args]
469
470                 with tempfile.NamedTemporaryFile() as f:
471                         rrdtool.graph(f.name, *args)
472
473                         # Get back to the beginning of the file
474                         f.seek(0)
475
476                         # Return all the content
477                         return f.read()