src/collecty/plugins/base.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 from __future__ import division
  23
  24 import datetime
  25 import logging
  26 import math
  27 import os
  28 import rrdtool
  29 import tempfile
  30 import threading
  31 import time
  32
  33 from ..constants import *
  34 from ..i18n import _
  35
  36 _plugins = {}
  37
  38 def get():
  39         """
  40                 Returns a list with all automatically registered plugins.
  41         """
  42         return _plugins.values()
  43
  44 class Timer(object):
  45         def __init__(self, timeout, heartbeat=1):
  46                 self.timeout = timeout
  47                 self.heartbeat = heartbeat
  48
  49                 self.delay = 0
  50
  51                 self.reset()
  52
  53         def reset(self, delay=0):
  54                 # Save start time.
  55                 self.start = time.time()
  56
  57                 self.delay = delay
  58
  59                 # Has this timer been killed?
  60                 self.killed = False
  61
  62         @property
  63         def elapsed(self):
  64                 return time.time() - self.start - self.delay
  65
  66         def cancel(self):
  67                 self.killed = True
  68
  69         def wait(self):
  70                 while self.elapsed < self.timeout and not self.killed:
  71                         time.sleep(self.heartbeat)
  72
  73                 return self.elapsed > self.timeout
  74
  75
  76 class Plugin(threading.Thread):
  77         # The name of this plugin.
  78         name = None
  79
  80         # A description for this plugin.
  81         description = None
  82
  83         # Templates which can be used to generate a graph out of
  84         # the data from this data source.
  85         templates = []
  86
  87         # The default interval for all plugins
  88         interval = 60
  89
  90         # Automatically register all providers.
  91         class __metaclass__(type):
  92                 def __init__(plugin, name, bases, dict):
  93                         type.__init__(plugin, name, bases, dict)
  94
  95                         # The main class from which is inherited is not registered
  96                         # as a plugin.
  97                         if name == "Plugin":
  98                                 return
  99
 100                         if not all((plugin.name, plugin.description)):
 101                                 raise RuntimeError(_("Plugin is not properly configured: %s") \
 102                                         % plugin)
 103
 104                         _plugins[plugin.name] = plugin
 105
 106         def __init__(self, collecty, **kwargs):
 107                 threading.Thread.__init__(self, name=self.description)
 108                 self.daemon = True
 109
 110                 self.collecty = collecty
 111
 112                 # Check if this plugin was configured correctly.
 113                 assert self.name, "Name of the plugin is not set: %s" % self.name
 114                 assert self.description, "Description of the plugin is not set: %s" % self.description
 115
 116                 # Initialize the logger.
 117                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 118                 self.log.propagate = 1
 119
 120                 self.data = []
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 # Keepalive options
 126                 self.running = True
 127                 self.timer = Timer(self.interval)
 128
 129                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 130
 131         @property
 132         def path(self):
 133                 """
 134                         Returns the name of the sub directory in which all RRD files
 135                         for this plugin should be stored in.
 136                 """
 137                 return self.name
 138
 139         ### Basic methods
 140
 141         def init(self, **kwargs):
 142                 """
 143                         Do some custom initialization stuff here.
 144                 """
 145                 pass
 146
 147         def collect(self):
 148                 """
 149                         Gathers the statistical data, this plugin collects.
 150                 """
 151                 time_start = time.time()
 152
 153                 # Run through all objects of this plugin and call the collect method.
 154                 for o in self.objects:
 155                         now = datetime.datetime.utcnow()
 156                         try:
 157                                 result = o.collect()
 158                         except:
 159                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 160                                 continue
 161
 162                         if not result:
 163                                 self.log.warning(_("Received empty result: %s") % o)
 164                                 continue
 165
 166                         self.log.debug(_("Collected %s: %s") % (o, result))
 167
 168                         # Add the object to the write queue so that the data is written
 169                         # to the databases later.
 170                         self.collecty.write_queue.add(o, now, result)
 171
 172                 # Returns the time this function took to complete.
 173                 return (time.time() - time_start)
 174
 175         def run(self):
 176                 self.log.debug(_("%s plugin has started") % self.name)
 177
 178                 # Initially collect everything
 179                 self.collect()
 180
 181                 while self.running:
 182                         # Reset the timer.
 183                         self.timer.reset()
 184
 185                         # Wait until the timer has successfully elapsed.
 186                         if self.timer.wait():
 187                                 delay = self.collect()
 188                                 self.timer.reset(delay)
 189
 190                 self.log.debug(_("%s plugin has stopped") % self.name)
 191
 192         def shutdown(self):
 193                 self.log.debug(_("Received shutdown signal."))
 194                 self.running = False
 195
 196                 # Kill any running timers.
 197                 if self.timer:
 198                         self.timer.cancel()
 199
 200         def get_object(self, id):
 201                 for object in self.objects:
 202                         if not object.id == id:
 203                                 continue
 204
 205                         return object
 206
 207         def get_template(self, template_name, object_id):
 208                 for template in self.templates:
 209                         if not template.name == template_name:
 210                                 continue
 211
 212                         return template(self, object_id)
 213
 214         def generate_graph(self, template_name, object_id="default", **kwargs):
 215                 template = self.get_template(template_name, object_id=object_id)
 216                 if not template:
 217                         raise RuntimeError("Could not find template %s" % template_name)
 218
 219                 time_start = time.time()
 220
 221                 graph = template.generate_graph(**kwargs)
 222
 223                 duration = time.time() - time_start
 224                 self.log.debug(_("Generated graph %s in %.1fms") \
 225                         % (template, duration * 1000))
 226
 227                 return graph
 228
 229
 230 class Object(object):
 231         # The schema of the RRD database.
 232         rrd_schema = None
 233
 234         # RRA properties.
 235         rra_types     = ["AVERAGE", "MIN", "MAX"]
 236         rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
 237         rra_rows      = 2880
 238
 239         def __init__(self, plugin, *args, **kwargs):
 240                 self.plugin = plugin
 241
 242                 # Indicates if this object has collected its data
 243                 self.collected = False
 244
 245                 # Initialise this object
 246                 self.init(*args, **kwargs)
 247
 248                 # Create the database file.
 249                 self.create()
 250
 251         def __repr__(self):
 252                 return "<%s>" % self.__class__.__name__
 253
 254         @property
 255         def collecty(self):
 256                 return self.plugin.collecty
 257
 258         @property
 259         def log(self):
 260                 return self.plugin.log
 261
 262         @property
 263         def id(self):
 264                 """
 265                         Returns a UNIQUE identifier for this object. As this is incorporated
 266                         into the path of RRD file, it must only contain ASCII characters.
 267                 """
 268                 raise NotImplementedError
 269
 270         @property
 271         def file(self):
 272                 """
 273                         The absolute path to the RRD file of this plugin.
 274                 """
 275                 return os.path.join(DATABASE_DIR, self.plugin.path, "%s.rrd" % self.id)
 276
 277         ### Basic methods
 278
 279         def init(self, *args, **kwargs):
 280                 """
 281                         Do some custom initialization stuff here.
 282                 """
 283                 pass
 284
 285         def create(self):
 286                 """
 287                         Creates an empty RRD file with the desired data structures.
 288                 """
 289                 # Skip if the file does already exist.
 290                 if os.path.exists(self.file):
 291                         return
 292
 293                 dirname = os.path.dirname(self.file)
 294                 if not os.path.exists(dirname):
 295                         os.makedirs(dirname)
 296
 297                 # Create argument list.
 298                 args = self.get_rrd_schema()
 299
 300                 rrdtool.create(self.file, *args)
 301
 302                 self.log.debug(_("Created RRD file %s.") % self.file)
 303                 for arg in args:
 304                         self.log.debug("  %s" % arg)
 305
 306         def info(self):
 307                 return rrdtool.info(self.file)
 308
 309         @property
 310         def stepsize(self):
 311                 return self.plugin.interval
 312
 313         @property
 314         def heartbeat(self):
 315                 return self.stepsize * 2
 316
 317         def get_rrd_schema(self):
 318                 schema = [
 319                         "--step", "%s" % self.stepsize,
 320                 ]
 321                 for line in self.rrd_schema:
 322                         if line.startswith("DS:"):
 323                                 try:
 324                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 325
 326                                         line = ":".join((
 327                                                 prefix,
 328                                                 name,
 329                                                 type,
 330                                                 "%s" % self.heartbeat,
 331                                                 lower_limit,
 332                                                 upper_limit
 333                                         ))
 334                                 except ValueError:
 335                                         pass
 336
 337                         schema.append(line)
 338
 339                 xff = 0.1
 340
 341                 cdp_length = 0
 342                 for rra_timespan in self.rra_timespans:
 343                         if (rra_timespan / self.stepsize) < self.rra_rows:
 344                                 rra_timespan = self.stepsize * self.rra_rows
 345
 346                         if cdp_length == 0:
 347                                 cdp_length = 1
 348                         else:
 349                                 cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
 350
 351                         cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
 352
 353                         for rra_type in self.rra_types:
 354                                 schema.append("RRA:%s:%.10f:%d:%d" % \
 355                                         (rra_type, xff, cdp_length, cdp_number))
 356
 357                 return schema
 358
 359         def execute(self):
 360                 if self.collected:
 361                         raise RuntimeError("This object has already collected its data")
 362
 363                 self.collected = True
 364                 self.now = datetime.datetime.utcnow()
 365
 366                 # Call the collect
 367                 result = self.collect()
 368
 369         def commit(self):
 370                 """
 371                         Will commit the collected data to the database.
 372                 """
 373                 # Make sure that the RRD database has been created
 374                 self.create()
 375
 376
 377 class GraphTemplate(object):
 378         # A unique name to identify this graph template.
 379         name = None
 380
 381         # Headline of the graph image
 382         graph_title = None
 383
 384         # Vertical label of the graph
 385         graph_vertical_label = None
 386
 387         # Limits
 388         lower_limit = None
 389         upper_limit = None
 390
 391         # Instructions how to create the graph.
 392         rrd_graph = None
 393
 394         # Extra arguments passed to rrdgraph.
 395         rrd_graph_args = []
 396
 397         intervals = {
 398                 None   : "-3h",
 399                 "hour" : "-1h",
 400                 "day"  : "-25h",
 401                 "week" : "-360h",
 402                 "year" : "-365d",
 403         }
 404
 405         # Default dimensions for this graph
 406         height = GRAPH_DEFAULT_HEIGHT
 407         width  = GRAPH_DEFAULT_WIDTH
 408
 409         def __init__(self, plugin, object_id):
 410                 self.plugin = plugin
 411
 412                 # Get all required RRD objects
 413                 self.object_id = object_id
 414
 415                 # Get the main object
 416                 self.object = self.get_object(self.object_id)
 417
 418         def __repr__(self):
 419                 return "<%s>" % self.__class__.__name__
 420
 421         @property
 422         def collecty(self):
 423                 return self.plugin.collecty
 424
 425         @property
 426         def log(self):
 427                 return self.plugin.log
 428
 429         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 430                         width=None, height=None):
 431                 args = []
 432
 433                 args += GRAPH_DEFAULT_ARGUMENTS
 434
 435                 args += [
 436                         "--imgformat", format,
 437                         "--height", "%s" % (height or self.height),
 438                         "--width", "%s" % (width or self.width),
 439                 ]
 440
 441                 args += self.rrd_graph_args
 442
 443                 # Graph title
 444                 if self.graph_title:
 445                         args += ["--title", self.graph_title]
 446
 447                 # Vertical label
 448                 if self.graph_vertical_label:
 449                         args += ["--vertical-label", self.graph_vertical_label]
 450
 451                 if self.lower_limit is not None or self.upper_limit is not None:
 452                         # Force to honour the set limits
 453                         args.append("--rigid")
 454
 455                         if self.lower_limit is not None:
 456                                 args += ["--lower-limit", self.lower_limit]
 457
 458                         if self.upper_limit is not None:
 459                                 args += ["--upper-limit", self.upper_limit]
 460
 461                 # Add interval
 462                 args.append("--start")
 463
 464                 try:
 465                         args.append(self.intervals[interval])
 466                 except KeyError:
 467                         args.append(str(interval))
 468
 469                 return args
 470
 471         def get_object(self, *args, **kwargs):
 472                 return self.plugin.get_object(*args, **kwargs)
 473
 474         def get_object_table(self):
 475                 return {
 476                         "file" : self.object,
 477                 }
 478
 479         def get_object_files(self):
 480                 files = {}
 481
 482                 for id, obj in self.get_object_table().items():
 483                         files[id] = obj.file
 484
 485                 return files
 486
 487         def generate_graph(self, interval=None, **kwargs):
 488                 args = self._make_command_line(interval, **kwargs)
 489
 490                 self.log.info(_("Generating graph %s") % self)
 491                 self.log.debug("  args: %s" % args)
 492
 493                 object_files = self.get_object_files()
 494
 495                 for item in self.rrd_graph:
 496                         try:
 497                                 args.append(item % object_files)
 498                         except TypeError:
 499                                 args.append(item)
 500
 501                         self.log.debug("  %s" % args[-1])
 502
 503                 return self.write_graph(*args)
 504
 505         def write_graph(self, *args):
 506                 # Convert all arguments to string
 507                 args = [str(e) for e in args]
 508
 509                 with tempfile.NamedTemporaryFile() as f:
 510                         rrdtool.graph(f.name, *args)
 511
 512                         # Get back to the beginning of the file
 513                         f.seek(0)
 514
 515                         # Return all the content
 516                         return f.read()