src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import rrdtool
  27 import tempfile
  28 import threading
  29 import time
  30 import unicodedata
  31
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 class Timer(object):
  36         def __init__(self, timeout, heartbeat=1):
  37                 self.timeout = timeout
  38                 self.heartbeat = heartbeat
  39
  40                 self.delay = 0
  41
  42                 self.reset()
  43
  44         def reset(self, delay=0):
  45                 # Save start time.
  46                 self.start = time.time()
  47
  48                 self.delay = delay
  49
  50                 # Has this timer been killed?
  51                 self.killed = False
  52
  53         @property
  54         def elapsed(self):
  55                 return time.time() - self.start - self.delay
  56
  57         def cancel(self):
  58                 self.killed = True
  59
  60         def wait(self):
  61                 while self.elapsed < self.timeout and not self.killed:
  62                         time.sleep(self.heartbeat)
  63
  64                 return self.elapsed > self.timeout
  65
  66
  67 class PluginRegistration(type):
  68         plugins = {}
  69
  70         def __init__(plugin, name, bases, dict):
  71                 type.__init__(plugin, name, bases, dict)
  72
  73                 # The main class from which is inherited is not registered
  74                 # as a plugin.
  75                 if name == "Plugin":
  76                         return
  77
  78                 if not all((plugin.name, plugin.description)):
  79                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  80
  81                 PluginRegistration.plugins[plugin.name] = plugin
  82
  83
  84 def get():
  85         """
  86                 Returns a list with all automatically registered plugins.
  87         """
  88         return PluginRegistration.plugins.values()
  89
  90 class Plugin(object, metaclass=PluginRegistration):
  91         # The name of this plugin.
  92         name = None
  93
  94         # A description for this plugin.
  95         description = None
  96
  97         # Templates which can be used to generate a graph out of
  98         # the data from this data source.
  99         templates = []
 100
 101         # The default interval for all plugins
 102         interval = 60
 103
 104         def __init__(self, collecty, **kwargs):
 105                 self.collecty = collecty
 106
 107                 # Check if this plugin was configured correctly.
 108                 assert self.name, "Name of the plugin is not set: %s" % self.name
 109                 assert self.description, "Description of the plugin is not set: %s" % self.description
 110
 111                 # Initialize the logger.
 112                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 113                 self.log.propagate = 1
 114
 115                 self.data = []
 116
 117                 # Run some custom initialization.
 118                 self.init(**kwargs)
 119
 120                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 121
 122         @property
 123         def path(self):
 124                 """
 125                         Returns the name of the sub directory in which all RRD files
 126                         for this plugin should be stored in.
 127                 """
 128                 return self.name
 129
 130         ### Basic methods
 131
 132         def init(self, **kwargs):
 133                 """
 134                         Do some custom initialization stuff here.
 135                 """
 136                 pass
 137
 138         def collect(self):
 139                 """
 140                         Gathers the statistical data, this plugin collects.
 141                 """
 142                 time_start = time.time()
 143
 144                 # Run through all objects of this plugin and call the collect method.
 145                 for o in self.objects:
 146                         now = datetime.datetime.utcnow()
 147                         try:
 148                                 result = o.collect()
 149
 150                                 if isinstance(result, tuple) or isinstance(result, list):
 151                                         result = ":".join(("%s" % e for e in result))
 152                         except:
 153                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 154                                 continue
 155
 156                         if not result:
 157                                 self.log.warning(_("Received empty result: %s") % o)
 158                                 continue
 159
 160                         self.log.debug(_("Collected %s: %s") % (o, result))
 161
 162                         # Add the object to the write queue so that the data is written
 163                         # to the databases later.
 164                         self.collecty.write_queue.add(o, now, result)
 165
 166                 # Returns the time this function took to complete.
 167                 delay = time.time() - time_start
 168
 169                 # Log some warning when a collect method takes too long to return some data
 170                 if delay >= 60:
 171                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 172
 173         def get_object(self, id):
 174                 for object in self.objects:
 175                         if not object.id == id:
 176                                 continue
 177
 178                         return object
 179
 180         def get_template(self, template_name, object_id):
 181                 for template in self.templates:
 182                         if not template.name == template_name:
 183                                 continue
 184
 185                         return template(self, object_id)
 186
 187         def generate_graph(self, template_name, object_id="default", **kwargs):
 188                 template = self.get_template(template_name, object_id=object_id)
 189                 if not template:
 190                         raise RuntimeError("Could not find template %s" % template_name)
 191
 192                 time_start = time.time()
 193
 194                 graph = template.generate_graph(**kwargs)
 195
 196                 duration = time.time() - time_start
 197                 self.log.debug(_("Generated graph %s in %.1fms") \
 198                         % (template, duration * 1000))
 199
 200                 return graph
 201
 202
 203 class Object(object):
 204         # The schema of the RRD database.
 205         rrd_schema = None
 206
 207         # RRA properties.
 208         rra_types     = ("AVERAGE", "MIN", "MAX")
 209         rra_timespans = (
 210                 ("1m", "10d"),
 211                 ("1h", "18M"),
 212                 ("1d",  "5y"),
 213         )
 214
 215         def __init__(self, plugin, *args, **kwargs):
 216                 self.plugin = plugin
 217
 218                 # Indicates if this object has collected its data
 219                 self.collected = False
 220
 221                 # Initialise this object
 222                 self.init(*args, **kwargs)
 223
 224                 # Create the database file.
 225                 self.create()
 226
 227         def __repr__(self):
 228                 return "<%s>" % self.__class__.__name__
 229
 230         @property
 231         def collecty(self):
 232                 return self.plugin.collecty
 233
 234         @property
 235         def log(self):
 236                 return self.plugin.log
 237
 238         @property
 239         def id(self):
 240                 """
 241                         Returns a UNIQUE identifier for this object. As this is incorporated
 242                         into the path of RRD file, it must only contain ASCII characters.
 243                 """
 244                 raise NotImplementedError
 245
 246         @property
 247         def file(self):
 248                 """
 249                         The absolute path to the RRD file of this plugin.
 250                 """
 251                 filename = self._normalise_filename("%s.rrd" % self.id)
 252
 253                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 254
 255         @staticmethod
 256         def _normalise_filename(filename):
 257                 # Convert the filename into ASCII characters only
 258                 filename = unicodedata.normalize("NFKC", filename)
 259
 260                 # Replace any spaces by dashes
 261                 filename = filename.replace(" ", "-")
 262
 263                 return filename
 264
 265         ### Basic methods
 266
 267         def init(self, *args, **kwargs):
 268                 """
 269                         Do some custom initialization stuff here.
 270                 """
 271                 pass
 272
 273         def create(self):
 274                 """
 275                         Creates an empty RRD file with the desired data structures.
 276                 """
 277                 # Skip if the file does already exist.
 278                 if os.path.exists(self.file):
 279                         return
 280
 281                 dirname = os.path.dirname(self.file)
 282                 if not os.path.exists(dirname):
 283                         os.makedirs(dirname)
 284
 285                 # Create argument list.
 286                 args = self.get_rrd_schema()
 287
 288                 rrdtool.create(self.file, *args)
 289
 290                 self.log.debug(_("Created RRD file %s.") % self.file)
 291                 for arg in args:
 292                         self.log.debug("  %s" % arg)
 293
 294         def info(self):
 295                 return rrdtool.info(self.file)
 296
 297         @property
 298         def stepsize(self):
 299                 return self.plugin.interval
 300
 301         @property
 302         def heartbeat(self):
 303                 return self.stepsize * 2
 304
 305         def get_rrd_schema(self):
 306                 schema = [
 307                         "--step", "%s" % self.stepsize,
 308                 ]
 309                 for line in self.rrd_schema:
 310                         if line.startswith("DS:"):
 311                                 try:
 312                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 313
 314                                         line = ":".join((
 315                                                 prefix,
 316                                                 name,
 317                                                 type,
 318                                                 "%s" % self.heartbeat,
 319                                                 lower_limit,
 320                                                 upper_limit
 321                                         ))
 322                                 except ValueError:
 323                                         pass
 324
 325                         schema.append(line)
 326
 327                 xff = 0.1
 328
 329                 for steps, rows in self.rra_timespans:
 330                         for type in self.rra_types:
 331                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 332
 333                 return schema
 334
 335         def execute(self):
 336                 if self.collected:
 337                         raise RuntimeError("This object has already collected its data")
 338
 339                 self.collected = True
 340                 self.now = datetime.datetime.utcnow()
 341
 342                 # Call the collect
 343                 result = self.collect()
 344
 345         def commit(self):
 346                 """
 347                         Will commit the collected data to the database.
 348                 """
 349                 # Make sure that the RRD database has been created
 350                 self.create()
 351
 352
 353 class GraphTemplate(object):
 354         # A unique name to identify this graph template.
 355         name = None
 356
 357         # Headline of the graph image
 358         graph_title = None
 359
 360         # Vertical label of the graph
 361         graph_vertical_label = None
 362
 363         # Limits
 364         lower_limit = None
 365         upper_limit = None
 366
 367         # Instructions how to create the graph.
 368         rrd_graph = None
 369
 370         # Extra arguments passed to rrdgraph.
 371         rrd_graph_args = []
 372
 373         intervals = {
 374                 None   : "-3h",
 375                 "hour" : "-1h",
 376                 "day"  : "-25h",
 377                 "month": "-30d",
 378                 "week" : "-360h",
 379                 "year" : "-365d",
 380         }
 381
 382         # Default dimensions for this graph
 383         height = GRAPH_DEFAULT_HEIGHT
 384         width  = GRAPH_DEFAULT_WIDTH
 385
 386         def __init__(self, plugin, object_id):
 387                 self.plugin = plugin
 388
 389                 # Get all required RRD objects
 390                 self.object_id = object_id
 391
 392                 # Get the main object
 393                 self.object = self.get_object(self.object_id)
 394
 395         def __repr__(self):
 396                 return "<%s>" % self.__class__.__name__
 397
 398         @property
 399         def collecty(self):
 400                 return self.plugin.collecty
 401
 402         @property
 403         def log(self):
 404                 return self.plugin.log
 405
 406         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 407                         width=None, height=None):
 408                 args = []
 409
 410                 args += GRAPH_DEFAULT_ARGUMENTS
 411
 412                 args += [
 413                         "--imgformat", format,
 414                         "--height", "%s" % (height or self.height),
 415                         "--width", "%s" % (width or self.width),
 416                 ]
 417
 418                 args += self.rrd_graph_args
 419
 420                 # Graph title
 421                 if self.graph_title:
 422                         args += ["--title", self.graph_title]
 423
 424                 # Vertical label
 425                 if self.graph_vertical_label:
 426                         args += ["--vertical-label", self.graph_vertical_label]
 427
 428                 if self.lower_limit is not None or self.upper_limit is not None:
 429                         # Force to honour the set limits
 430                         args.append("--rigid")
 431
 432                         if self.lower_limit is not None:
 433                                 args += ["--lower-limit", self.lower_limit]
 434
 435                         if self.upper_limit is not None:
 436                                 args += ["--upper-limit", self.upper_limit]
 437
 438                 try:
 439                         interval = self.intervals[interval]
 440                 except KeyError:
 441                         interval = "end-%s" % interval
 442
 443                 # Add interval
 444                 args += ["--start", interval]
 445
 446                 return args
 447
 448         def get_object(self, *args, **kwargs):
 449                 return self.plugin.get_object(*args, **kwargs)
 450
 451         def get_object_table(self):
 452                 return {
 453                         "file" : self.object,
 454                 }
 455
 456         @property
 457         def object_table(self):
 458                 if not hasattr(self, "_object_table"):
 459                         self._object_table = self.get_object_table()
 460
 461                 return self._object_table
 462
 463         def get_object_files(self):
 464                 files = {}
 465
 466                 for id, obj in self.object_table.items():
 467                         files[id] = obj.file
 468
 469                 return files
 470
 471         def generate_graph(self, interval=None, **kwargs):
 472                 args = self._make_command_line(interval, **kwargs)
 473
 474                 self.log.info(_("Generating graph %s") % self)
 475                 self.log.debug("  args: %s" % args)
 476
 477                 object_files = self.get_object_files()
 478
 479                 for item in self.rrd_graph:
 480                         try:
 481                                 args.append(item % object_files)
 482                         except TypeError:
 483                                 args.append(item)
 484
 485                         self.log.debug("  %s" % args[-1])
 486
 487                 # Convert arguments to string
 488                 args = [str(e) for e in args]
 489
 490                 graph = rrdtool.graphv("-", *args)
 491
 492                 return graph.get("image")