src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import re
  27 import rrdtool
  28 import tempfile
  29 import threading
  30 import time
  31 import unicodedata
  32
  33 from .. import locales
  34 from .. import util
  35 from ..constants import *
  36 from ..i18n import _
  37
  38 DEF_MATCH = re.compile(r"C?DEF:([A-Za-z0-9_]+)=")
  39
  40 class Environment(object):
  41         """
  42                 Sets the correct environment for rrdtool to create
  43                 localised graphs and graphs in the correct timezone.
  44         """
  45         def __init__(self, timezone, locale):
  46                 # Build the new environment
  47                 self.new_environment = {
  48                         "TZ" : timezone or DEFAULT_TIMEZONE,
  49                 }
  50
  51                 for k in ("LANG", "LC_ALL"):
  52                         self.new_environment[k] = locale or DEFAULT_LOCALE
  53
  54         def __enter__(self):
  55                 # Save the current environment
  56                 self.old_environment = {}
  57                 for k in self.new_environment:
  58                         self.old_environment[k] = os.environ.get(k, None)
  59
  60                 # Apply the new one
  61                 os.environ.update(self.new_environment)
  62
  63         def __exit__(self, type, value, traceback):
  64                 # Roll back to the previous environment
  65                 for k, v in self.old_environment.items():
  66                         if v is None:
  67                                 try:
  68                                         del os.environ[k]
  69                                 except KeyError:
  70                                         pass
  71                         else:
  72                                 os.environ[k] = v
  73
  74
  75 class PluginRegistration(type):
  76         plugins = {}
  77
  78         def __init__(plugin, name, bases, dict):
  79                 type.__init__(plugin, name, bases, dict)
  80
  81                 # The main class from which is inherited is not registered
  82                 # as a plugin.
  83                 if name == "Plugin":
  84                         return
  85
  86                 if not all((plugin.name, plugin.description)):
  87                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  88
  89                 PluginRegistration.plugins[plugin.name] = plugin
  90
  91
  92 def get():
  93         """
  94                 Returns a list with all automatically registered plugins.
  95         """
  96         return PluginRegistration.plugins.values()
  97
  98 class Plugin(object, metaclass=PluginRegistration):
  99         # The name of this plugin.
 100         name = None
 101
 102         # A description for this plugin.
 103         description = None
 104
 105         # Templates which can be used to generate a graph out of
 106         # the data from this data source.
 107         templates = []
 108
 109         # The default interval for all plugins
 110         interval = 60
 111
 112         # Priority
 113         priority = 0
 114
 115         def __init__(self, collecty, **kwargs):
 116                 self.collecty = collecty
 117
 118                 # Check if this plugin was configured correctly.
 119                 assert self.name, "Name of the plugin is not set: %s" % self.name
 120                 assert self.description, "Description of the plugin is not set: %s" % self.description
 121
 122                 # Initialize the logger.
 123                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 124
 125                 self.data = []
 126
 127                 # Run some custom initialization.
 128                 self.init(**kwargs)
 129
 130                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 131
 132         @property
 133         def path(self):
 134                 """
 135                         Returns the name of the sub directory in which all RRD files
 136                         for this plugin should be stored in.
 137                 """
 138                 return self.name
 139
 140         ### Basic methods
 141
 142         def init(self, **kwargs):
 143                 """
 144                         Do some custom initialization stuff here.
 145                 """
 146                 pass
 147
 148         def collect(self):
 149                 """
 150                         Gathers the statistical data, this plugin collects.
 151                 """
 152                 time_start = time.time()
 153
 154                 # Run through all objects of this plugin and call the collect method.
 155                 for o in self.objects:
 156                         now = datetime.datetime.utcnow()
 157                         try:
 158                                 result = o.collect()
 159
 160                                 result = self._format_result(result)
 161                         except:
 162                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 163                                 continue
 164
 165                         if not result:
 166                                 self.log.warning(_("Received empty result: %s") % o)
 167                                 continue
 168
 169                         self.log.debug(_("Collected %s: %s") % (o, result))
 170
 171                         # Add the object to the write queue so that the data is written
 172                         # to the databases later.
 173                         self.collecty.write_queue.add(o, now, result)
 174
 175                 # Returns the time this function took to complete.
 176                 delay = time.time() - time_start
 177
 178                 # Log some warning when a collect method takes too long to return some data
 179                 if delay >= 60:
 180                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 181
 182         @staticmethod
 183         def _format_result(result):
 184                 if not isinstance(result, tuple) and not isinstance(result, list):
 185                         return result
 186
 187                 # Replace all Nones by UNKNOWN
 188                 s = []
 189
 190                 for e in result:
 191                         if e is None:
 192                                 e = "U"
 193
 194                         s.append("%s" % e)
 195
 196                 return ":".join(s)
 197
 198         def get_object(self, id):
 199                 for object in self.objects:
 200                         if not object.id == id:
 201                                 continue
 202
 203                         return object
 204
 205         def get_template(self, template_name, object_id, locale=None, timezone=None):
 206                 for template in self.templates:
 207                         if not template.name == template_name:
 208                                 continue
 209
 210                         return template(self, object_id, locale=locale, timezone=timezone)
 211
 212         def generate_graph(self, template_name, object_id="default",
 213                         timezone=None, locale=None, **kwargs):
 214                 template = self.get_template(template_name, object_id=object_id,
 215                         timezone=timezone, locale=locale)
 216                 if not template:
 217                         raise RuntimeError("Could not find template %s" % template_name)
 218
 219                 time_start = time.time()
 220
 221                 graph = template.generate_graph(**kwargs)
 222
 223                 duration = time.time() - time_start
 224                 self.log.debug(_("Generated graph %s in %.1fms") \
 225                         % (template, duration * 1000))
 226
 227                 return graph
 228
 229         def graph_info(self, template_name, object_id="default",
 230                         timezone=None, locale=None, **kwargs):
 231                 template = self.get_template(template_name, object_id=object_id,
 232                         timezone=timezone, locale=locale)
 233                 if not template:
 234                         raise RuntimeError("Could not find template %s" % template_name)
 235
 236                 return template.graph_info()
 237
 238         def last_update(self, object_id="default"):
 239                 object = self.get_object(object_id)
 240                 if not object:
 241                         raise RuntimeError("Could not find object %s" % object_id)
 242
 243                 return object.last_update()
 244
 245
 246 class Object(object):
 247         # The schema of the RRD database.
 248         rrd_schema = None
 249
 250         # RRA properties.
 251         rra_types     = ("AVERAGE", "MIN", "MAX")
 252         rra_timespans = (
 253                 ("1m", "10d"),
 254                 ("1h", "18M"),
 255                 ("1d",  "5y"),
 256         )
 257
 258         def __init__(self, plugin, *args, **kwargs):
 259                 self.plugin = plugin
 260
 261                 # Indicates if this object has collected its data
 262                 self.collected = False
 263
 264                 # Initialise this object
 265                 self.init(*args, **kwargs)
 266
 267                 # Create the database file.
 268                 self.create()
 269
 270         def __repr__(self):
 271                 return "<%s>" % self.__class__.__name__
 272
 273         def __lt__(self, other):
 274                 return self.id < other.id
 275
 276         @property
 277         def collecty(self):
 278                 return self.plugin.collecty
 279
 280         @property
 281         def log(self):
 282                 return self.plugin.log
 283
 284         @property
 285         def id(self):
 286                 """
 287                         Returns a UNIQUE identifier for this object. As this is incorporated
 288                         into the path of RRD file, it must only contain ASCII characters.
 289                 """
 290                 raise NotImplementedError
 291
 292         @property
 293         def file(self):
 294                 """
 295                         The absolute path to the RRD file of this plugin.
 296                 """
 297                 filename = self._normalise_filename("%s.rrd" % self.id)
 298
 299                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 300
 301         @staticmethod
 302         def _normalise_filename(filename):
 303                 # Convert the filename into ASCII characters only
 304                 filename = unicodedata.normalize("NFKC", filename)
 305
 306                 # Replace any spaces by dashes
 307                 filename = filename.replace(" ", "-")
 308
 309                 return filename
 310
 311         ### Basic methods
 312
 313         def init(self, *args, **kwargs):
 314                 """
 315                         Do some custom initialization stuff here.
 316                 """
 317                 pass
 318
 319         def create(self):
 320                 """
 321                         Creates an empty RRD file with the desired data structures.
 322                 """
 323                 # Skip if the file does already exist.
 324                 if os.path.exists(self.file):
 325                         return
 326
 327                 dirname = os.path.dirname(self.file)
 328                 if not os.path.exists(dirname):
 329                         os.makedirs(dirname)
 330
 331                 # Create argument list.
 332                 args = self.get_rrd_schema()
 333
 334                 rrdtool.create(self.file, *args)
 335
 336                 self.log.debug(_("Created RRD file %s.") % self.file)
 337                 for arg in args:
 338                         self.log.debug("  %s" % arg)
 339
 340         def info(self):
 341                 return rrdtool.info(self.file)
 342
 343         def last_update(self):
 344                 """
 345                         Returns a dictionary with the timestamp and
 346                         data set of the last database update.
 347                 """
 348                 return {
 349                         "dataset"   : self.last_dataset,
 350                         "timestamp" : self.last_updated,
 351                 }
 352
 353         def _last_update(self):
 354                 return rrdtool.lastupdate(self.file)
 355
 356         @property
 357         def last_updated(self):
 358                 """
 359                         Returns the timestamp when this database was last updated
 360                 """
 361                 lu = self._last_update()
 362
 363                 if lu:
 364                         return lu.get("date")
 365
 366         @property
 367         def last_dataset(self):
 368                 """
 369                         Returns the latest dataset in the database
 370                 """
 371                 lu = self._last_update()
 372
 373                 if lu:
 374                         return lu.get("ds")
 375
 376         @property
 377         def stepsize(self):
 378                 return self.plugin.interval
 379
 380         @property
 381         def heartbeat(self):
 382                 return self.stepsize * 2
 383
 384         def get_rrd_schema(self):
 385                 schema = [
 386                         "--step", "%s" % self.stepsize,
 387                 ]
 388                 for line in self.rrd_schema:
 389                         if line.startswith("DS:"):
 390                                 try:
 391                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 392
 393                                         line = ":".join((
 394                                                 prefix,
 395                                                 name,
 396                                                 type,
 397                                                 "%s" % self.heartbeat,
 398                                                 lower_limit,
 399                                                 upper_limit
 400                                         ))
 401                                 except ValueError:
 402                                         pass
 403
 404                         schema.append(line)
 405
 406                 xff = 0.1
 407
 408                 for steps, rows in self.rra_timespans:
 409                         for type in self.rra_types:
 410                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 411
 412                 return schema
 413
 414         @property
 415         def rrd_schema_names(self):
 416                 ret = []
 417
 418                 for line in self.rrd_schema:
 419                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 420                         ret.append(name)
 421
 422                 return ret
 423
 424         def make_rrd_defs(self, prefix=None):
 425                 defs = []
 426
 427                 for name in self.rrd_schema_names:
 428                         if prefix:
 429                                 p = "%s_%s" % (prefix, name)
 430                         else:
 431                                 p = name
 432
 433                         defs += [
 434                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 435                         ]
 436
 437                 return defs
 438
 439         def get_stddev(self, interval=None):
 440                 args = self.make_rrd_defs()
 441
 442                 # Add the correct interval
 443                 args += ["--start", util.make_interval(interval)]
 444
 445                 for name in self.rrd_schema_names:
 446                         args += [
 447                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 448                                 "PRINT:%s_stddev:%%lf" % name,
 449                         ]
 450
 451                 x, y, vals = rrdtool.graph("/dev/null", *args)
 452                 return dict(zip(self.rrd_schema_names, vals))
 453
 454         def execute(self):
 455                 if self.collected:
 456                         raise RuntimeError("This object has already collected its data")
 457
 458                 self.collected = True
 459                 self.now = datetime.datetime.utcnow()
 460
 461                 # Call the collect
 462                 result = self.collect()
 463
 464         def commit(self):
 465                 """
 466                         Will commit the collected data to the database.
 467                 """
 468                 # Make sure that the RRD database has been created
 469                 self.create()
 470
 471                 # Write everything to disk that is in the write queue
 472                 self.collecty.write_queue.commit_file(self.file)
 473
 474
 475 class GraphTemplate(object):
 476         # A unique name to identify this graph template.
 477         name = None
 478
 479         # Headline of the graph image
 480         graph_title = None
 481
 482         # Vertical label of the graph
 483         graph_vertical_label = None
 484
 485         # Limits
 486         lower_limit = None
 487         upper_limit = None
 488
 489         # Instructions how to create the graph.
 490         rrd_graph = None
 491
 492         # Extra arguments passed to rrdgraph.
 493         rrd_graph_args = []
 494
 495         # Default dimensions for this graph
 496         height = GRAPH_DEFAULT_HEIGHT
 497         width  = GRAPH_DEFAULT_WIDTH
 498
 499         def __init__(self, plugin, object_id, locale=None, timezone=None):
 500                 self.plugin = plugin
 501
 502                 # Save localisation parameters
 503                 self.locale = locales.get(locale)
 504                 self.timezone = timezone
 505
 506                 # Get all required RRD objects
 507                 self.object_id = object_id
 508
 509                 # Get the main object
 510                 self.objects = self.get_objects(self.object_id)
 511                 self.objects.sort()
 512
 513         def __repr__(self):
 514                 return "<%s>" % self.__class__.__name__
 515
 516         @property
 517         def collecty(self):
 518                 return self.plugin.collecty
 519
 520         @property
 521         def log(self):
 522                 return self.plugin.log
 523
 524         @property
 525         def object(self):
 526                 """
 527                         Shortcut to the main object
 528                 """
 529                 if len(self.objects) == 1:
 530                         return self.objects[0]
 531
 532         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 533                         width=None, height=None, with_title=True, thumbnail=False):
 534                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 535
 536                 # Set the default dimensions
 537                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 538
 539                 # A thumbnail doesn't have a legend and other labels
 540                 if thumbnail:
 541                         args.append("--only-graph")
 542
 543                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 544                         default_width = THUMBNAIL_DEFAULT_WIDTH
 545
 546                 args += [
 547                         "--imgformat", format,
 548                         "--height", "%s" % (height or default_height),
 549                         "--width", "%s" % (width or default_width),
 550                 ]
 551
 552                 args += self.rrd_graph_args
 553
 554                 # Graph title
 555                 if with_title and self.graph_title:
 556                         args += ["--title", self.graph_title]
 557
 558                 # Vertical label
 559                 if self.graph_vertical_label:
 560                         args += ["--vertical-label", self.graph_vertical_label]
 561
 562                 if self.lower_limit is not None or self.upper_limit is not None:
 563                         # Force to honour the set limits
 564                         args.append("--rigid")
 565
 566                         if self.lower_limit is not None:
 567                                 args += ["--lower-limit", self.lower_limit]
 568
 569                         if self.upper_limit is not None:
 570                                 args += ["--upper-limit", self.upper_limit]
 571
 572                 # Add interval
 573                 args += ["--start", util.make_interval(interval)]
 574
 575                 return args
 576
 577         def _add_defs(self):
 578                 use_prefix = len(self.objects) >= 2
 579
 580                 args = []
 581                 for object in self.objects:
 582                         if use_prefix:
 583                                 args += object.make_rrd_defs(object.id)
 584                         else:
 585                                 args += object.make_rrd_defs()
 586
 587                 return args
 588
 589         def _add_vdefs(self, args):
 590                 ret = []
 591
 592                 for arg in args:
 593                         ret.append(arg)
 594
 595                         # Search for all DEFs and CDEFs
 596                         m = re.match(DEF_MATCH, "%s" % arg)
 597                         if m:
 598                                 name = m.group(1)
 599
 600                                 # Add the VDEFs for minimum, maximum, etc. values
 601                                 ret += [
 602                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 603                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 604                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 605                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 606                                 ]
 607
 608                 return ret
 609
 610         def get_objects(self, *args, **kwargs):
 611                 object = self.plugin.get_object(*args, **kwargs)
 612
 613                 if object:
 614                         return [object,]
 615
 616                 return []
 617
 618         def generate_graph(self, interval=None, **kwargs):
 619                 assert self.objects, "Cannot render graph without any objects"
 620
 621                 # Make sure that all collected data is in the database
 622                 # to get a recent graph image
 623                 for object in self.objects:
 624                         object.commit()
 625
 626                 args = self._make_command_line(interval, **kwargs)
 627
 628                 self.log.info(_("Generating graph %s") % self)
 629
 630                 rrd_graph = self.rrd_graph
 631
 632                 # Add DEFs for all objects
 633                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 634                         args += self._add_defs()
 635
 636                 args += rrd_graph
 637                 args = self._add_vdefs(args)
 638
 639                 # Convert arguments to string
 640                 args = [str(e) for e in args]
 641
 642                 for arg in args:
 643                         self.log.debug("  %s" % arg)
 644
 645                 with Environment(self.timezone, self.locale.lang):
 646                         graph = rrdtool.graphv("-", *args)
 647
 648                 return {
 649                         "image"        : graph.get("image"),
 650                         "image_height" : graph.get("image_height"),
 651                         "image_width"  : graph.get("image_width"),
 652                 }
 653
 654         def graph_info(self):
 655                 """
 656                         Returns a dictionary with useful information
 657                         about this graph.
 658                 """
 659                 return {
 660                         "title"        : self.graph_title,
 661                         "object_id"    : self.object_id or "",
 662                         "template"     : self.name,
 663                 }