src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import rrdtool
  27 import tempfile
  28 import threading
  29 import time
  30 import unicodedata
  31
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 class Timer(object):
  36         def __init__(self, timeout, heartbeat=1):
  37                 self.timeout = timeout
  38                 self.heartbeat = heartbeat
  39
  40                 self.delay = 0
  41
  42                 self.reset()
  43
  44         def reset(self, delay=0):
  45                 # Save start time.
  46                 self.start = time.time()
  47
  48                 self.delay = delay
  49
  50                 # Has this timer been killed?
  51                 self.killed = False
  52
  53         @property
  54         def elapsed(self):
  55                 return time.time() - self.start - self.delay
  56
  57         def cancel(self):
  58                 self.killed = True
  59
  60         def wait(self):
  61                 while self.elapsed < self.timeout and not self.killed:
  62                         time.sleep(self.heartbeat)
  63
  64                 return self.elapsed > self.timeout
  65
  66
  67 class PluginRegistration(type):
  68         plugins = {}
  69
  70         def __init__(plugin, name, bases, dict):
  71                 type.__init__(plugin, name, bases, dict)
  72
  73                 # The main class from which is inherited is not registered
  74                 # as a plugin.
  75                 if name == "Plugin":
  76                         return
  77
  78                 if not all((plugin.name, plugin.description)):
  79                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  80
  81                 PluginRegistration.plugins[plugin.name] = plugin
  82
  83
  84 def get():
  85         """
  86                 Returns a list with all automatically registered plugins.
  87         """
  88         return PluginRegistration.plugins.values()
  89
  90 class Plugin(object, metaclass=PluginRegistration):
  91         # The name of this plugin.
  92         name = None
  93
  94         # A description for this plugin.
  95         description = None
  96
  97         # Templates which can be used to generate a graph out of
  98         # the data from this data source.
  99         templates = []
 100
 101         # The default interval for all plugins
 102         interval = 60
 103
 104         def __init__(self, collecty, **kwargs):
 105                 self.collecty = collecty
 106
 107                 # Check if this plugin was configured correctly.
 108                 assert self.name, "Name of the plugin is not set: %s" % self.name
 109                 assert self.description, "Description of the plugin is not set: %s" % self.description
 110
 111                 # Initialize the logger.
 112                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 113                 self.log.propagate = 1
 114
 115                 self.data = []
 116
 117                 # Run some custom initialization.
 118                 self.init(**kwargs)
 119
 120                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 121
 122         @property
 123         def path(self):
 124                 """
 125                         Returns the name of the sub directory in which all RRD files
 126                         for this plugin should be stored in.
 127                 """
 128                 return self.name
 129
 130         ### Basic methods
 131
 132         def init(self, **kwargs):
 133                 """
 134                         Do some custom initialization stuff here.
 135                 """
 136                 pass
 137
 138         def collect(self):
 139                 """
 140                         Gathers the statistical data, this plugin collects.
 141                 """
 142                 time_start = time.time()
 143
 144                 # Run through all objects of this plugin and call the collect method.
 145                 for o in self.objects:
 146                         now = datetime.datetime.utcnow()
 147                         try:
 148                                 result = o.collect()
 149
 150                                 if isinstance(result, tuple) or isinstance(result, list):
 151                                         result = ":".join(("%s" % e for e in result))
 152                         except:
 153                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 154                                 continue
 155
 156                         if not result:
 157                                 self.log.warning(_("Received empty result: %s") % o)
 158                                 continue
 159
 160                         self.log.debug(_("Collected %s: %s") % (o, result))
 161
 162                         # Add the object to the write queue so that the data is written
 163                         # to the databases later.
 164                         self.collecty.write_queue.add(o, now, result)
 165
 166                 # Returns the time this function took to complete.
 167                 delay = time.time() - time_start
 168
 169                 # Log some warning when a collect method takes too long to return some data
 170                 if delay >= 60:
 171                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 172
 173         def get_object(self, id):
 174                 for object in self.objects:
 175                         if not object.id == id:
 176                                 continue
 177
 178                         return object
 179
 180         def get_template(self, template_name, object_id):
 181                 for template in self.templates:
 182                         if not template.name == template_name:
 183                                 continue
 184
 185                         return template(self, object_id)
 186
 187         def generate_graph(self, template_name, object_id="default", **kwargs):
 188                 template = self.get_template(template_name, object_id=object_id)
 189                 if not template:
 190                         raise RuntimeError("Could not find template %s" % template_name)
 191
 192                 time_start = time.time()
 193
 194                 graph = template.generate_graph(**kwargs)
 195
 196                 duration = time.time() - time_start
 197                 self.log.debug(_("Generated graph %s in %.1fms") \
 198                         % (template, duration * 1000))
 199
 200                 return graph
 201
 202
 203 class Object(object):
 204         # The schema of the RRD database.
 205         rrd_schema = None
 206
 207         # RRA properties.
 208         rra_types     = ("AVERAGE", "MIN", "MAX")
 209         rra_timespans = (
 210                 ("1m", "10d"),
 211                 ("1h", "18M"),
 212                 ("1d",  "5y"),
 213         )
 214
 215         def __init__(self, plugin, *args, **kwargs):
 216                 self.plugin = plugin
 217
 218                 # Indicates if this object has collected its data
 219                 self.collected = False
 220
 221                 # Initialise this object
 222                 self.init(*args, **kwargs)
 223
 224                 # Create the database file.
 225                 self.create()
 226
 227         def __repr__(self):
 228                 return "<%s>" % self.__class__.__name__
 229
 230         @property
 231         def collecty(self):
 232                 return self.plugin.collecty
 233
 234         @property
 235         def log(self):
 236                 return self.plugin.log
 237
 238         @property
 239         def id(self):
 240                 """
 241                         Returns a UNIQUE identifier for this object. As this is incorporated
 242                         into the path of RRD file, it must only contain ASCII characters.
 243                 """
 244                 raise NotImplementedError
 245
 246         @property
 247         def file(self):
 248                 """
 249                         The absolute path to the RRD file of this plugin.
 250                 """
 251                 filename = self._normalise_filename("%s.rrd" % self.id)
 252
 253                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 254
 255         @staticmethod
 256         def _normalise_filename(filename):
 257                 # Convert the filename into ASCII characters only
 258                 filename = unicodedata.normalize("NFKC", filename)
 259
 260                 # Replace any spaces by dashes
 261                 filename = filename.replace(" ", "-")
 262
 263                 return filename
 264
 265         ### Basic methods
 266
 267         def init(self, *args, **kwargs):
 268                 """
 269                         Do some custom initialization stuff here.
 270                 """
 271                 pass
 272
 273         def create(self):
 274                 """
 275                         Creates an empty RRD file with the desired data structures.
 276                 """
 277                 # Skip if the file does already exist.
 278                 if os.path.exists(self.file):
 279                         return
 280
 281                 dirname = os.path.dirname(self.file)
 282                 if not os.path.exists(dirname):
 283                         os.makedirs(dirname)
 284
 285                 # Create argument list.
 286                 args = self.get_rrd_schema()
 287
 288                 rrdtool.create(self.file, *args)
 289
 290                 self.log.debug(_("Created RRD file %s.") % self.file)
 291                 for arg in args:
 292                         self.log.debug("  %s" % arg)
 293
 294         def info(self):
 295                 return rrdtool.info(self.file)
 296
 297         @property
 298         def stepsize(self):
 299                 return self.plugin.interval
 300
 301         @property
 302         def heartbeat(self):
 303                 return self.stepsize * 2
 304
 305         def get_rrd_schema(self):
 306                 schema = [
 307                         "--step", "%s" % self.stepsize,
 308                 ]
 309                 for line in self.rrd_schema:
 310                         if line.startswith("DS:"):
 311                                 try:
 312                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 313
 314                                         line = ":".join((
 315                                                 prefix,
 316                                                 name,
 317                                                 type,
 318                                                 "%s" % self.heartbeat,
 319                                                 lower_limit,
 320                                                 upper_limit
 321                                         ))
 322                                 except ValueError:
 323                                         pass
 324
 325                         schema.append(line)
 326
 327                 xff = 0.1
 328
 329                 for steps, rows in self.rra_timespans:
 330                         for type in self.rra_types:
 331                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 332
 333                 return schema
 334
 335         def execute(self):
 336                 if self.collected:
 337                         raise RuntimeError("This object has already collected its data")
 338
 339                 self.collected = True
 340                 self.now = datetime.datetime.utcnow()
 341
 342                 # Call the collect
 343                 result = self.collect()
 344
 345         def commit(self):
 346                 """
 347                         Will commit the collected data to the database.
 348                 """
 349                 # Make sure that the RRD database has been created
 350                 self.create()
 351
 352
 353 class GraphTemplate(object):
 354         # A unique name to identify this graph template.
 355         name = None
 356
 357         # Headline of the graph image
 358         graph_title = None
 359
 360         # Vertical label of the graph
 361         graph_vertical_label = None
 362
 363         # Limits
 364         lower_limit = None
 365         upper_limit = None
 366
 367         # Instructions how to create the graph.
 368         rrd_graph = None
 369
 370         # Extra arguments passed to rrdgraph.
 371         rrd_graph_args = []
 372
 373         intervals = {
 374                 None   : "-3h",
 375                 "hour" : "-1h",
 376                 "day"  : "-25h",
 377                 "week" : "-360h",
 378                 "year" : "-365d",
 379         }
 380
 381         # Default dimensions for this graph
 382         height = GRAPH_DEFAULT_HEIGHT
 383         width  = GRAPH_DEFAULT_WIDTH
 384
 385         def __init__(self, plugin, object_id):
 386                 self.plugin = plugin
 387
 388                 # Get all required RRD objects
 389                 self.object_id = object_id
 390
 391                 # Get the main object
 392                 self.object = self.get_object(self.object_id)
 393
 394         def __repr__(self):
 395                 return "<%s>" % self.__class__.__name__
 396
 397         @property
 398         def collecty(self):
 399                 return self.plugin.collecty
 400
 401         @property
 402         def log(self):
 403                 return self.plugin.log
 404
 405         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 406                         width=None, height=None):
 407                 args = []
 408
 409                 args += GRAPH_DEFAULT_ARGUMENTS
 410
 411                 args += [
 412                         "--imgformat", format,
 413                         "--height", "%s" % (height or self.height),
 414                         "--width", "%s" % (width or self.width),
 415                 ]
 416
 417                 args += self.rrd_graph_args
 418
 419                 # Graph title
 420                 if self.graph_title:
 421                         args += ["--title", self.graph_title]
 422
 423                 # Vertical label
 424                 if self.graph_vertical_label:
 425                         args += ["--vertical-label", self.graph_vertical_label]
 426
 427                 if self.lower_limit is not None or self.upper_limit is not None:
 428                         # Force to honour the set limits
 429                         args.append("--rigid")
 430
 431                         if self.lower_limit is not None:
 432                                 args += ["--lower-limit", self.lower_limit]
 433
 434                         if self.upper_limit is not None:
 435                                 args += ["--upper-limit", self.upper_limit]
 436
 437                 # Add interval
 438                 args.append("--start")
 439
 440                 try:
 441                         args.append(self.intervals[interval])
 442                 except KeyError:
 443                         args.append(str(interval))
 444
 445                 return args
 446
 447         def get_object(self, *args, **kwargs):
 448                 return self.plugin.get_object(*args, **kwargs)
 449
 450         def get_object_table(self):
 451                 return {
 452                         "file" : self.object,
 453                 }
 454
 455         @property
 456         def object_table(self):
 457                 if not hasattr(self, "_object_table"):
 458                         self._object_table = self.get_object_table()
 459
 460                 return self._object_table
 461
 462         def get_object_files(self):
 463                 files = {}
 464
 465                 for id, obj in self.object_table.items():
 466                         files[id] = obj.file
 467
 468                 return files
 469
 470         def generate_graph(self, interval=None, **kwargs):
 471                 args = self._make_command_line(interval, **kwargs)
 472
 473                 self.log.info(_("Generating graph %s") % self)
 474                 self.log.debug("  args: %s" % args)
 475
 476                 object_files = self.get_object_files()
 477
 478                 for item in self.rrd_graph:
 479                         try:
 480                                 args.append(item % object_files)
 481                         except TypeError:
 482                                 args.append(item)
 483
 484                         self.log.debug("  %s" % args[-1])
 485
 486                 # Convert arguments to string
 487                 args = [str(e) for e in args]
 488
 489                 graph = rrdtool.graphv("-", *args)
 490
 491                 return graph.get("image")