src/collecty/plugins/base.py

   1 #!/usr/bin/python
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 from __future__ import division
  23
  24 import datetime
  25 import logging
  26 import math
  27 import os
  28 import rrdtool
  29 import tempfile
  30 import threading
  31 import time
  32
  33 from ..constants import *
  34 from ..i18n import _
  35
  36 _plugins = {}
  37
  38 def get():
  39         """
  40                 Returns a list with all automatically registered plugins.
  41         """
  42         return _plugins.values()
  43
  44 class Timer(object):
  45         def __init__(self, timeout, heartbeat=1):
  46                 self.timeout = timeout
  47                 self.heartbeat = heartbeat
  48
  49                 self.delay = 0
  50
  51                 self.reset()
  52
  53         def reset(self, delay=0):
  54                 # Save start time.
  55                 self.start = time.time()
  56
  57                 self.delay = delay
  58
  59                 # Has this timer been killed?
  60                 self.killed = False
  61
  62         @property
  63         def elapsed(self):
  64                 return time.time() - self.start - self.delay
  65
  66         def cancel(self):
  67                 self.killed = True
  68
  69         def wait(self):
  70                 while self.elapsed < self.timeout and not self.killed:
  71                         time.sleep(self.heartbeat)
  72
  73                 return self.elapsed > self.timeout
  74
  75
  76 class Plugin(threading.Thread):
  77         # The name of this plugin.
  78         name = None
  79
  80         # A description for this plugin.
  81         description = None
  82
  83         # Templates which can be used to generate a graph out of
  84         # the data from this data source.
  85         templates = []
  86
  87         # The default interval for all plugins
  88         interval = 60
  89
  90         # Automatically register all providers.
  91         class __metaclass__(type):
  92                 def __init__(plugin, name, bases, dict):
  93                         type.__init__(plugin, name, bases, dict)
  94
  95                         # The main class from which is inherited is not registered
  96                         # as a plugin.
  97                         if name == "Plugin":
  98                                 return
  99
 100                         if not all((plugin.name, plugin.description)):
 101                                 raise RuntimeError(_("Plugin is not properly configured: %s") \
 102                                         % plugin)
 103
 104                         _plugins[plugin.name] = plugin
 105
 106         def __init__(self, collecty, **kwargs):
 107                 threading.Thread.__init__(self, name=self.description)
 108                 self.daemon = True
 109
 110                 self.collecty = collecty
 111
 112                 # Check if this plugin was configured correctly.
 113                 assert self.name, "Name of the plugin is not set: %s" % self.name
 114                 assert self.description, "Description of the plugin is not set: %s" % self.description
 115
 116                 # Initialize the logger.
 117                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 118                 self.log.propagate = 1
 119
 120                 self.data = []
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 # Keepalive options
 126                 self.running = True
 127                 self.timer = Timer(self.interval)
 128
 129                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 130
 131         @property
 132         def path(self):
 133                 """
 134                         Returns the name of the sub directory in which all RRD files
 135                         for this plugin should be stored in.
 136                 """
 137                 return self.name
 138
 139         ### Basic methods
 140
 141         def init(self, **kwargs):
 142                 """
 143                         Do some custom initialization stuff here.
 144                 """
 145                 pass
 146
 147         def collect(self):
 148                 """
 149                         Gathers the statistical data, this plugin collects.
 150                 """
 151                 time_start = time.time()
 152
 153                 # Run through all objects of this plugin and call the collect method.
 154                 for o in self.objects:
 155                         now = datetime.datetime.utcnow()
 156                         try:
 157                                 result = o.collect()
 158
 159                                 if isinstance(result, tuple) or isinstance(result, list):
 160                                         result = ":".join(("%s" % e for e in result))
 161                         except:
 162                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 163                                 continue
 164
 165                         if not result:
 166                                 self.log.warning(_("Received empty result: %s") % o)
 167                                 continue
 168
 169                         self.log.debug(_("Collected %s: %s") % (o, result))
 170
 171                         # Add the object to the write queue so that the data is written
 172                         # to the databases later.
 173                         self.collecty.write_queue.add(o, now, result)
 174
 175                 # Returns the time this function took to complete.
 176                 return (time.time() - time_start)
 177
 178         def run(self):
 179                 self.log.debug(_("%s plugin has started") % self.name)
 180
 181                 # Initially collect everything
 182                 self.collect()
 183
 184                 while self.running:
 185                         # Reset the timer.
 186                         self.timer.reset()
 187
 188                         # Wait until the timer has successfully elapsed.
 189                         if self.timer.wait():
 190                                 delay = self.collect()
 191                                 self.timer.reset(delay)
 192
 193                 self.log.debug(_("%s plugin has stopped") % self.name)
 194
 195         def shutdown(self):
 196                 self.log.debug(_("Received shutdown signal."))
 197                 self.running = False
 198
 199                 # Kill any running timers.
 200                 if self.timer:
 201                         self.timer.cancel()
 202
 203         def get_object(self, id):
 204                 for object in self.objects:
 205                         if not object.id == id:
 206                                 continue
 207
 208                         return object
 209
 210         def get_template(self, template_name, object_id):
 211                 for template in self.templates:
 212                         if not template.name == template_name:
 213                                 continue
 214
 215                         return template(self, object_id)
 216
 217         def generate_graph(self, template_name, object_id="default", **kwargs):
 218                 template = self.get_template(template_name, object_id=object_id)
 219                 if not template:
 220                         raise RuntimeError("Could not find template %s" % template_name)
 221
 222                 time_start = time.time()
 223
 224                 graph = template.generate_graph(**kwargs)
 225
 226                 duration = time.time() - time_start
 227                 self.log.debug(_("Generated graph %s in %.1fms") \
 228                         % (template, duration * 1000))
 229
 230                 return graph
 231
 232
 233 class Object(object):
 234         # The schema of the RRD database.
 235         rrd_schema = None
 236
 237         # RRA properties.
 238         rra_types     = ["AVERAGE", "MIN", "MAX"]
 239         rra_timespans = [3600, 86400, 604800, 2678400, 31622400]
 240         rra_rows      = 2880
 241
 242         def __init__(self, plugin, *args, **kwargs):
 243                 self.plugin = plugin
 244
 245                 # Indicates if this object has collected its data
 246                 self.collected = False
 247
 248                 # Initialise this object
 249                 self.init(*args, **kwargs)
 250
 251                 # Create the database file.
 252                 self.create()
 253
 254         def __repr__(self):
 255                 return "<%s>" % self.__class__.__name__
 256
 257         @property
 258         def collecty(self):
 259                 return self.plugin.collecty
 260
 261         @property
 262         def log(self):
 263                 return self.plugin.log
 264
 265         @property
 266         def id(self):
 267                 """
 268                         Returns a UNIQUE identifier for this object. As this is incorporated
 269                         into the path of RRD file, it must only contain ASCII characters.
 270                 """
 271                 raise NotImplementedError
 272
 273         @property
 274         def file(self):
 275                 """
 276                         The absolute path to the RRD file of this plugin.
 277                 """
 278                 return os.path.join(DATABASE_DIR, self.plugin.path, "%s.rrd" % self.id)
 279
 280         ### Basic methods
 281
 282         def init(self, *args, **kwargs):
 283                 """
 284                         Do some custom initialization stuff here.
 285                 """
 286                 pass
 287
 288         def create(self):
 289                 """
 290                         Creates an empty RRD file with the desired data structures.
 291                 """
 292                 # Skip if the file does already exist.
 293                 if os.path.exists(self.file):
 294                         return
 295
 296                 dirname = os.path.dirname(self.file)
 297                 if not os.path.exists(dirname):
 298                         os.makedirs(dirname)
 299
 300                 # Create argument list.
 301                 args = self.get_rrd_schema()
 302
 303                 rrdtool.create(self.file, *args)
 304
 305                 self.log.debug(_("Created RRD file %s.") % self.file)
 306                 for arg in args:
 307                         self.log.debug("  %s" % arg)
 308
 309         def info(self):
 310                 return rrdtool.info(self.file)
 311
 312         @property
 313         def stepsize(self):
 314                 return self.plugin.interval
 315
 316         @property
 317         def heartbeat(self):
 318                 return self.stepsize * 2
 319
 320         def get_rrd_schema(self):
 321                 schema = [
 322                         "--step", "%s" % self.stepsize,
 323                 ]
 324                 for line in self.rrd_schema:
 325                         if line.startswith("DS:"):
 326                                 try:
 327                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 328
 329                                         line = ":".join((
 330                                                 prefix,
 331                                                 name,
 332                                                 type,
 333                                                 "%s" % self.heartbeat,
 334                                                 lower_limit,
 335                                                 upper_limit
 336                                         ))
 337                                 except ValueError:
 338                                         pass
 339
 340                         schema.append(line)
 341
 342                 xff = 0.1
 343
 344                 cdp_length = 0
 345                 for rra_timespan in self.rra_timespans:
 346                         if (rra_timespan / self.stepsize) < self.rra_rows:
 347                                 rra_timespan = self.stepsize * self.rra_rows
 348
 349                         if cdp_length == 0:
 350                                 cdp_length = 1
 351                         else:
 352                                 cdp_length = rra_timespan // (self.rra_rows * self.stepsize)
 353
 354                         cdp_number = math.ceil(rra_timespan / (cdp_length * self.stepsize))
 355
 356                         for rra_type in self.rra_types:
 357                                 schema.append("RRA:%s:%.10f:%d:%d" % \
 358                                         (rra_type, xff, cdp_length, cdp_number))
 359
 360                 return schema
 361
 362         def execute(self):
 363                 if self.collected:
 364                         raise RuntimeError("This object has already collected its data")
 365
 366                 self.collected = True
 367                 self.now = datetime.datetime.utcnow()
 368
 369                 # Call the collect
 370                 result = self.collect()
 371
 372         def commit(self):
 373                 """
 374                         Will commit the collected data to the database.
 375                 """
 376                 # Make sure that the RRD database has been created
 377                 self.create()
 378
 379
 380 class GraphTemplate(object):
 381         # A unique name to identify this graph template.
 382         name = None
 383
 384         # Headline of the graph image
 385         graph_title = None
 386
 387         # Vertical label of the graph
 388         graph_vertical_label = None
 389
 390         # Limits
 391         lower_limit = None
 392         upper_limit = None
 393
 394         # Instructions how to create the graph.
 395         rrd_graph = None
 396
 397         # Extra arguments passed to rrdgraph.
 398         rrd_graph_args = []
 399
 400         intervals = {
 401                 None   : "-3h",
 402                 "hour" : "-1h",
 403                 "day"  : "-25h",
 404                 "week" : "-360h",
 405                 "year" : "-365d",
 406         }
 407
 408         # Default dimensions for this graph
 409         height = GRAPH_DEFAULT_HEIGHT
 410         width  = GRAPH_DEFAULT_WIDTH
 411
 412         def __init__(self, plugin, object_id):
 413                 self.plugin = plugin
 414
 415                 # Get all required RRD objects
 416                 self.object_id = object_id
 417
 418                 # Get the main object
 419                 self.object = self.get_object(self.object_id)
 420
 421         def __repr__(self):
 422                 return "<%s>" % self.__class__.__name__
 423
 424         @property
 425         def collecty(self):
 426                 return self.plugin.collecty
 427
 428         @property
 429         def log(self):
 430                 return self.plugin.log
 431
 432         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 433                         width=None, height=None):
 434                 args = []
 435
 436                 args += GRAPH_DEFAULT_ARGUMENTS
 437
 438                 args += [
 439                         "--imgformat", format,
 440                         "--height", "%s" % (height or self.height),
 441                         "--width", "%s" % (width or self.width),
 442                 ]
 443
 444                 args += self.rrd_graph_args
 445
 446                 # Graph title
 447                 if self.graph_title:
 448                         args += ["--title", self.graph_title]
 449
 450                 # Vertical label
 451                 if self.graph_vertical_label:
 452                         args += ["--vertical-label", self.graph_vertical_label]
 453
 454                 if self.lower_limit is not None or self.upper_limit is not None:
 455                         # Force to honour the set limits
 456                         args.append("--rigid")
 457
 458                         if self.lower_limit is not None:
 459                                 args += ["--lower-limit", self.lower_limit]
 460
 461                         if self.upper_limit is not None:
 462                                 args += ["--upper-limit", self.upper_limit]
 463
 464                 # Add interval
 465                 args.append("--start")
 466
 467                 try:
 468                         args.append(self.intervals[interval])
 469                 except KeyError:
 470                         args.append(str(interval))
 471
 472                 return args
 473
 474         def get_object(self, *args, **kwargs):
 475                 return self.plugin.get_object(*args, **kwargs)
 476
 477         def get_object_table(self):
 478                 return {
 479                         "file" : self.object,
 480                 }
 481
 482         def get_object_files(self):
 483                 files = {}
 484
 485                 for id, obj in self.get_object_table().items():
 486                         files[id] = obj.file
 487
 488                 return files
 489
 490         def generate_graph(self, interval=None, **kwargs):
 491                 args = self._make_command_line(interval, **kwargs)
 492
 493                 self.log.info(_("Generating graph %s") % self)
 494                 self.log.debug("  args: %s" % args)
 495
 496                 object_files = self.get_object_files()
 497
 498                 for item in self.rrd_graph:
 499                         try:
 500                                 args.append(item % object_files)
 501                         except TypeError:
 502                                 args.append(item)
 503
 504                         self.log.debug("  %s" % args[-1])
 505
 506                 return self.write_graph(*args)
 507
 508         def write_graph(self, *args):
 509                 # Convert all arguments to string
 510                 args = [str(e) for e in args]
 511
 512                 with tempfile.NamedTemporaryFile() as f:
 513                         rrdtool.graph(f.name, *args)
 514
 515                         # Get back to the beginning of the file
 516                         f.seek(0)
 517
 518                         # Return all the content
 519                         return f.read()