82b0982b521a9de371aae4dd53155306d36c976a
[collecty.git] / src / collecty / plugins / base.py
1 #!/usr/bin/python
2 ###############################################################################
3 # #
4 # collecty - A system statistics collection daemon for IPFire #
5 # Copyright (C) 2012 IPFire development team #
6 # #
7 # This program is free software: you can redistribute it and/or modify #
8 # it under the terms of the GNU General Public License as published by #
9 # the Free Software Foundation, either version 3 of the License, or #
10 # (at your option) any later version. #
11 # #
12 # This program is distributed in the hope that it will be useful, #
13 # but WITHOUT ANY WARRANTY; without even the implied warranty of #
14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
15 # GNU General Public License for more details. #
16 # #
17 # You should have received a copy of the GNU General Public License #
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. #
19 # #
20 ###############################################################################
21
22 from __future__ import division
23
24 import logging
25 import math
26 import os
27 import rrdtool
28 import threading
29 import time
30
31 from ..constants import *
32 from ..i18n import _
33
34 _plugins = {}
35
36 def get():
37 """
38 Returns a list with all automatically registered plugins.
39 """
40 return _plugins.values()
41
42 class Timer(object):
43 def __init__(self, timeout, heartbeat=1):
44 self.timeout = timeout
45 self.heartbeat = heartbeat
46
47 self.delay = 0
48
49 self.reset()
50
51 def reset(self, delay=0):
52 # Save start time.
53 self.start = time.time()
54
55 self.delay = delay
56
57 # Has this timer been killed?
58 self.killed = False
59
60 @property
61 def elapsed(self):
62 return time.time() - self.start - self.delay
63
64 def cancel(self):
65 self.killed = True
66
67 def wait(self):
68 while self.elapsed < self.timeout and not self.killed:
69 time.sleep(self.heartbeat)
70
71 return self.elapsed > self.timeout
72
73
74 class Plugin(threading.Thread):
75 # The name of this plugin.
76 name = None
77
78 # A description for this plugin.
79 description = None
80
81 # Templates which can be used to generate a graph out of
82 # the data from this data source.
83 templates = []
84
85 # The schema of the RRD database.
86 rrd_schema = None
87
88 # RRA properties.
89 rra_types = ["AVERAGE", "MIN", "MAX"]
90 rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
91 rra_rows = 2880
92
93 # The default interval of this plugin.
94 default_interval = 60
95
96 # Automatically register all providers.
97 class __metaclass__(type):
98 def __init__(plugin, name, bases, dict):
99 type.__init__(plugin, name, bases, dict)
100
101 # The main class from which is inherited is not registered
102 # as a plugin.
103 if name == "Plugin":
104 return
105
106 if not all((plugin.name, plugin.description)):
107 raise RuntimeError(_("Plugin is not properly configured: %s") \
108 % plugin)
109
110 _plugins[plugin.name] = plugin
111
112 def __init__(self, collecty, **kwargs):
113 threading.Thread.__init__(self, name=self.description)
114 self.daemon = True
115
116 self.collecty = collecty
117
118 # Check if this plugin was configured correctly.
119 assert self.name, "Name of the plugin is not set: %s" % self.name
120 assert self.description, "Description of the plugin is not set: %s" % self.description
121 assert self.rrd_schema
122
123 # Initialize the logger.
124 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
125 self.log.propagate = 1
126
127 self.data = []
128
129 # Run some custom initialization.
130 self.init(**kwargs)
131
132 # Create the database file.
133 self.create()
134
135 # Keepalive options
136 self.running = True
137 self.timer = Timer(self.interval)
138
139 self.log.info(_("Successfully initialized (%s).") % self.id)
140
141 def __repr__(self):
142 return "<%s %s>" % (self.__class__.__name__, self.id)
143
144 @property
145 def id(self):
146 """
147 A unique ID of the plugin instance.
148 """
149 return self.name
150
151 @property
152 def interval(self):
153 """
154 Returns the interval in milliseconds, when the read method
155 should be called again.
156 """
157 # XXX read this from the settings
158
159 # Otherwise return the default.
160 return self.default_interval
161
162 @property
163 def stepsize(self):
164 return self.interval
165
166 @property
167 def heartbeat(self):
168 return self.stepsize * 2
169
170 @property
171 def file(self):
172 """
173 The absolute path to the RRD file of this plugin.
174 """
175 return os.path.join(DATABASE_DIR, "%s.rrd" % self.id)
176
177 def create(self):
178 """
179 Creates an empty RRD file with the desired data structures.
180 """
181 # Skip if the file does already exist.
182 if os.path.exists(self.file):
183 return
184
185 dirname = os.path.dirname(self.file)
186 if not os.path.exists(dirname):
187 os.makedirs(dirname)
188
189 # Create argument list.
190 args = self.get_rrd_schema()
191
192 rrdtool.create(self.file, *args)
193
194 self.log.debug(_("Created RRD file %s.") % self.file)
195 for arg in args:
196 self.log.debug(" %s" % arg)
197
198 def get_rrd_schema(self):
199 schema = [
200 "--step", "%s" % self.stepsize,
201 ]
202 for line in self.rrd_schema:
203 if line.startswith("DS:"):
204 try:
205 (prefix, name, type, lower_limit, upper_limit) = line.split(":")
206
207 line = ":".join((
208 prefix,
209 name,
210 type,
211 "%s" % self.heartbeat,
212 lower_limit,
213 upper_limit
214 ))
215 except ValueError:
216 pass
217
218 schema.append(line)
219
220 xff = 0.1
221
222 cdp_length = 0
223 for rra_timespan in self.rra_timespans:
224 if (rra_timespan / self.stepsize) < self.rra_rows:
225 rra_timespan = self.stepsize * self.rra_rows
226
227 if cdp_length == 0:
228 cdp_length = 1
229 else:
230 cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
231
232 cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
233
234 for rra_type in self.rra_types:
235 schema.append("RRA:%s:%.10f:%d:%d" % \
236 (rra_type, xff, cdp_length, cdp_number))
237
238 return schema
239
240 def info(self):
241 return rrdtool.info(self.file)
242
243 ### Basic methods
244
245 def init(self, **kwargs):
246 """
247 Do some custom initialization stuff here.
248 """
249 pass
250
251 def read(self):
252 """
253 Gathers the statistical data, this plugin collects.
254 """
255 raise NotImplementedError
256
257 def submit(self):
258 """
259 Flushes the read data to disk.
260 """
261 # Do nothing in case there is no data to submit.
262 if not self.data:
263 return
264
265 self.log.debug(_("Submitting data to database. %d entries.") % len(self.data))
266 for data in self.data:
267 self.log.debug(" %s" % data)
268
269 # Create the RRD files (if they don't exist yet or
270 # have vanished for some reason).
271 self.create()
272
273 rrdtool.update(self.file, *self.data)
274 self.data = []
275
276 def _read(self, *args, **kwargs):
277 """
278 This method catches errors from the read() method and logs them.
279 """
280 start_time = time.time()
281
282 try:
283 data = self.read(*args, **kwargs)
284 if data is None:
285 self.log.warning(_("Received empty data."))
286 else:
287 self.data.append("%d:%s" % (start_time, data))
288
289 # Catch any exceptions, so collecty does not crash.
290 except Exception, e:
291 self.log.critical(_("Unhandled exception in read()!"), exc_info=True)
292
293 # Return the elapsed time since _read() has been called.
294 return (time.time() - start_time)
295
296 def _submit(self, *args, **kwargs):
297 """
298 This method catches errors from the submit() method and logs them.
299 """
300 try:
301 return self.submit(*args, **kwargs)
302
303 # Catch any exceptions, so collecty does not crash.
304 except Exception, e:
305 self.log.critical(_("Unhandled exception in submit()!"), exc_info=True)
306
307 def run(self):
308 self.log.debug(_("Started."))
309
310 while self.running:
311 # Reset the timer.
312 self.timer.reset()
313
314 # Wait until the timer has successfully elapsed.
315 if self.timer.wait():
316 self.log.debug(_("Collecting..."))
317 delay = self._read()
318
319 self.timer.reset(delay)
320
321 self._submit()
322 self.log.debug(_("Stopped."))
323
324 def shutdown(self):
325 self.log.debug(_("Received shutdown signal."))
326 self.running = False
327
328 # Kill any running timers.
329 if self.timer:
330 self.timer.cancel()
331
332
333 class GraphTemplate(object):
334 # A unique name to identify this graph template.
335 name = None
336
337 # Instructions how to create the graph.
338 rrd_graph = None
339
340 # Extra arguments passed to rrdgraph.
341 rrd_graph_args = []
342
343 def __init__(self, ds):
344 self.ds = ds
345
346 @property
347 def collecty(self):
348 return self.ds.collecty
349
350 def graph(self, file, interval=None,
351 width=GRAPH_DEFAULT_WIDTH, height=GRAPH_DEFAULT_HEIGHT):
352 args = [
353 "--width", "%d" % width,
354 "--height", "%d" % height,
355 ]
356 args += self.collecty.graph_default_arguments
357 args += self.rrd_graph_args
358
359 intervals = {
360 None : "-3h",
361 "hour" : "-1h",
362 "day" : "-25h",
363 "week" : "-360h",
364 "year" : "-365d",
365 }
366
367 args.append("--start")
368 try:
369 args.append(intervals[interval])
370 except KeyError:
371 args.append(interval)
372
373 info = { "file" : self.ds.file }
374 for item in self.rrd_graph:
375 try:
376 args.append(item % info)
377 except TypeError:
378 args.append(item)
379
380 rrdtool.graph(file, *args)