src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import re
  27 import rrdtool
  28 import tempfile
  29 import threading
  30 import time
  31 import unicodedata
  32
  33 from .. import locales
  34 from .. import util
  35 from ..constants import *
  36 from ..i18n import _
  37
  38 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  39
  40 class Environment(object):
  41         """
  42                 Sets the correct environment for rrdtool to create
  43                 localised graphs and graphs in the correct timezone.
  44         """
  45         def __init__(self, timezone, locale):
  46                 # Build the new environment
  47                 self.new_environment = {
  48                         "TZ" : timezone or DEFAULT_TIMEZONE,
  49                 }
  50
  51                 for k in ("LANG", "LC_ALL"):
  52                         self.new_environment[k] = locale or DEFAULT_LOCALE
  53
  54         def __enter__(self):
  55                 # Save the current environment
  56                 self.old_environment = {}
  57                 for k in self.new_environment:
  58                         self.old_environment[k] = os.environ.get(k, None)
  59
  60                 # Apply the new one
  61                 os.environ.update(self.new_environment)
  62
  63         def __exit__(self, type, value, traceback):
  64                 # Roll back to the previous environment
  65                 for k, v in self.old_environment.items():
  66                         if v is None:
  67                                 try:
  68                                         del os.environ[k]
  69                                 except KeyError:
  70                                         pass
  71                         else:
  72                                 os.environ[k] = v
  73
  74
  75 class PluginRegistration(type):
  76         plugins = {}
  77
  78         def __init__(plugin, name, bases, dict):
  79                 type.__init__(plugin, name, bases, dict)
  80
  81                 # The main class from which is inherited is not registered
  82                 # as a plugin.
  83                 if name == "Plugin":
  84                         return
  85
  86                 if not all((plugin.name, plugin.description)):
  87                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  88
  89                 PluginRegistration.plugins[plugin.name] = plugin
  90
  91
  92 def get():
  93         """
  94                 Returns a list with all automatically registered plugins.
  95         """
  96         return PluginRegistration.plugins.values()
  97
  98 class Plugin(object, metaclass=PluginRegistration):
  99         # The name of this plugin.
 100         name = None
 101
 102         # A description for this plugin.
 103         description = None
 104
 105         # Templates which can be used to generate a graph out of
 106         # the data from this data source.
 107         templates = []
 108
 109         # The default interval for all plugins
 110         interval = 60
 111
 112         # Priority
 113         priority = 0
 114
 115         def __init__(self, collecty, **kwargs):
 116                 self.collecty = collecty
 117
 118                 # Check if this plugin was configured correctly.
 119                 assert self.name, "Name of the plugin is not set: %s" % self.name
 120                 assert self.description, "Description of the plugin is not set: %s" % self.description
 121
 122                 # Initialize the logger.
 123                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 124
 125                 # Run some custom initialization.
 126                 self.init(**kwargs)
 127
 128                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 129
 130         @property
 131         def path(self):
 132                 """
 133                         Returns the name of the sub directory in which all RRD files
 134                         for this plugin should be stored in.
 135                 """
 136                 return self.name
 137
 138         ### Basic methods
 139
 140         def init(self, **kwargs):
 141                 """
 142                         Do some custom initialization stuff here.
 143                 """
 144                 pass
 145
 146         def collect(self):
 147                 """
 148                         Gathers the statistical data, this plugin collects.
 149                 """
 150                 time_start = time.time()
 151
 152                 # Run through all objects of this plugin and call the collect method.
 153                 for o in self.objects:
 154                         now = datetime.datetime.utcnow()
 155                         try:
 156                                 result = o.collect()
 157
 158                                 result = self._format_result(result)
 159                         except:
 160                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 161                                 continue
 162
 163                         if not result:
 164                                 self.log.warning(_("Received empty result: %s") % o)
 165                                 continue
 166
 167                         self.log.debug(_("Collected %s: %s") % (o, result))
 168
 169                         # Add the object to the write queue so that the data is written
 170                         # to the databases later.
 171                         self.collecty.write_queue.add(o, now, result)
 172
 173                 # Returns the time this function took to complete.
 174                 delay = time.time() - time_start
 175
 176                 # Log some warning when a collect method takes too long to return some data
 177                 if delay >= 60:
 178                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 179
 180         @staticmethod
 181         def _format_result(result):
 182                 if not isinstance(result, tuple) and not isinstance(result, list):
 183                         return result
 184
 185                 # Replace all Nones by UNKNOWN
 186                 s = []
 187
 188                 for e in result:
 189                         if e is None:
 190                                 e = "U"
 191
 192                         s.append("%s" % e)
 193
 194                 return ":".join(s)
 195
 196         def get_object(self, id):
 197                 for object in self.objects:
 198                         if not object.id == id:
 199                                 continue
 200
 201                         return object
 202
 203         def get_template(self, template_name, object_id, locale=None, timezone=None):
 204                 for template in self.templates:
 205                         if not template.name == template_name:
 206                                 continue
 207
 208                         return template(self, object_id, locale=locale, timezone=timezone)
 209
 210         def generate_graph(self, template_name, object_id="default",
 211                         timezone=None, locale=None, **kwargs):
 212                 template = self.get_template(template_name, object_id=object_id,
 213                         timezone=timezone, locale=locale)
 214                 if not template:
 215                         raise RuntimeError("Could not find template %s" % template_name)
 216
 217                 time_start = time.time()
 218
 219                 graph = template.generate_graph(**kwargs)
 220
 221                 duration = time.time() - time_start
 222                 self.log.debug(_("Generated graph %s in %.1fms") \
 223                         % (template, duration * 1000))
 224
 225                 return graph
 226
 227         def graph_info(self, template_name, object_id="default",
 228                         timezone=None, locale=None, **kwargs):
 229                 template = self.get_template(template_name, object_id=object_id,
 230                         timezone=timezone, locale=locale)
 231                 if not template:
 232                         raise RuntimeError("Could not find template %s" % template_name)
 233
 234                 return template.graph_info()
 235
 236         def last_update(self, object_id="default"):
 237                 object = self.get_object(object_id)
 238                 if not object:
 239                         raise RuntimeError("Could not find object %s" % object_id)
 240
 241                 return object.last_update()
 242
 243
 244 class Object(object):
 245         # The schema of the RRD database.
 246         rrd_schema = None
 247
 248         # RRA properties.
 249         rra_types     = ("AVERAGE", "MIN", "MAX")
 250         rra_timespans = (
 251                 ("1m", "10d"),
 252                 ("1h", "18M"),
 253                 ("1d",  "5y"),
 254         )
 255
 256         def __init__(self, plugin, *args, **kwargs):
 257                 self.plugin = plugin
 258
 259                 # Indicates if this object has collected its data
 260                 self.collected = False
 261
 262                 # Initialise this object
 263                 self.init(*args, **kwargs)
 264
 265                 # Create the database file.
 266                 self.create()
 267
 268         def __repr__(self):
 269                 return "<%s>" % self.__class__.__name__
 270
 271         def __lt__(self, other):
 272                 return self.id < other.id
 273
 274         @property
 275         def collecty(self):
 276                 return self.plugin.collecty
 277
 278         @property
 279         def log(self):
 280                 return self.plugin.log
 281
 282         @property
 283         def id(self):
 284                 """
 285                         Returns a UNIQUE identifier for this object. As this is incorporated
 286                         into the path of RRD file, it must only contain ASCII characters.
 287                 """
 288                 raise NotImplementedError
 289
 290         @property
 291         def file(self):
 292                 """
 293                         The absolute path to the RRD file of this plugin.
 294                 """
 295                 filename = self._normalise_filename("%s.rrd" % self.id)
 296
 297                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 298
 299         @staticmethod
 300         def _normalise_filename(filename):
 301                 # Convert the filename into ASCII characters only
 302                 filename = unicodedata.normalize("NFKC", filename)
 303
 304                 # Replace any spaces by dashes
 305                 filename = filename.replace(" ", "-")
 306
 307                 return filename
 308
 309         ### Basic methods
 310
 311         def init(self, *args, **kwargs):
 312                 """
 313                         Do some custom initialization stuff here.
 314                 """
 315                 pass
 316
 317         def create(self):
 318                 """
 319                         Creates an empty RRD file with the desired data structures.
 320                 """
 321                 # Skip if the file does already exist.
 322                 if os.path.exists(self.file):
 323                         return
 324
 325                 dirname = os.path.dirname(self.file)
 326                 if not os.path.exists(dirname):
 327                         os.makedirs(dirname)
 328
 329                 # Create argument list.
 330                 args = self.get_rrd_schema()
 331
 332                 rrdtool.create(self.file, *args)
 333
 334                 self.log.debug(_("Created RRD file %s.") % self.file)
 335                 for arg in args:
 336                         self.log.debug("  %s" % arg)
 337
 338         def info(self):
 339                 return rrdtool.info(self.file)
 340
 341         def last_update(self):
 342                 """
 343                         Returns a dictionary with the timestamp and
 344                         data set of the last database update.
 345                 """
 346                 return {
 347                         "dataset"   : self.last_dataset,
 348                         "timestamp" : self.last_updated,
 349                 }
 350
 351         def _last_update(self):
 352                 return rrdtool.lastupdate(self.file)
 353
 354         @property
 355         def last_updated(self):
 356                 """
 357                         Returns the timestamp when this database was last updated
 358                 """
 359                 lu = self._last_update()
 360
 361                 if lu:
 362                         return lu.get("date")
 363
 364         @property
 365         def last_dataset(self):
 366                 """
 367                         Returns the latest dataset in the database
 368                 """
 369                 lu = self._last_update()
 370
 371                 if lu:
 372                         return lu.get("ds")
 373
 374         @property
 375         def stepsize(self):
 376                 return self.plugin.interval
 377
 378         @property
 379         def heartbeat(self):
 380                 return self.stepsize * 2
 381
 382         def get_rrd_schema(self):
 383                 schema = [
 384                         "--step", "%s" % self.stepsize,
 385                 ]
 386                 for line in self.rrd_schema:
 387                         if line.startswith("DS:"):
 388                                 try:
 389                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 390
 391                                         line = ":".join((
 392                                                 prefix,
 393                                                 name,
 394                                                 type,
 395                                                 "%s" % self.heartbeat,
 396                                                 lower_limit,
 397                                                 upper_limit
 398                                         ))
 399                                 except ValueError:
 400                                         pass
 401
 402                         schema.append(line)
 403
 404                 xff = 0.1
 405
 406                 for steps, rows in self.rra_timespans:
 407                         for type in self.rra_types:
 408                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 409
 410                 return schema
 411
 412         @property
 413         def rrd_schema_names(self):
 414                 ret = []
 415
 416                 for line in self.rrd_schema:
 417                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 418                         ret.append(name)
 419
 420                 return ret
 421
 422         def make_rrd_defs(self, prefix=None):
 423                 defs = []
 424
 425                 for name in self.rrd_schema_names:
 426                         if prefix:
 427                                 p = "%s_%s" % (prefix, name)
 428                         else:
 429                                 p = name
 430
 431                         defs += [
 432                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 433                         ]
 434
 435                 return defs
 436
 437         def get_stddev(self, interval=None):
 438                 args = self.make_rrd_defs()
 439
 440                 # Add the correct interval
 441                 args += ["--start", util.make_interval(interval)]
 442
 443                 for name in self.rrd_schema_names:
 444                         args += [
 445                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 446                                 "PRINT:%s_stddev:%%lf" % name,
 447                         ]
 448
 449                 x, y, vals = rrdtool.graph("/dev/null", *args)
 450                 return dict(zip(self.rrd_schema_names, vals))
 451
 452         def execute(self):
 453                 if self.collected:
 454                         raise RuntimeError("This object has already collected its data")
 455
 456                 self.collected = True
 457                 self.now = datetime.datetime.utcnow()
 458
 459                 # Call the collect
 460                 result = self.collect()
 461
 462         def commit(self):
 463                 """
 464                         Will commit the collected data to the database.
 465                 """
 466                 # Make sure that the RRD database has been created
 467                 self.create()
 468
 469                 # Write everything to disk that is in the write queue
 470                 self.collecty.write_queue.commit_file(self.file)
 471
 472
 473 class GraphTemplate(object):
 474         # A unique name to identify this graph template.
 475         name = None
 476
 477         # Headline of the graph image
 478         graph_title = None
 479
 480         # Vertical label of the graph
 481         graph_vertical_label = None
 482
 483         # Limits
 484         lower_limit = None
 485         upper_limit = None
 486
 487         # Instructions how to create the graph.
 488         rrd_graph = None
 489
 490         # Extra arguments passed to rrdgraph.
 491         rrd_graph_args = []
 492
 493         # Default dimensions for this graph
 494         height = GRAPH_DEFAULT_HEIGHT
 495         width  = GRAPH_DEFAULT_WIDTH
 496
 497         def __init__(self, plugin, object_id, locale=None, timezone=None):
 498                 self.plugin = plugin
 499
 500                 # Save localisation parameters
 501                 self.locale = locales.get(locale)
 502                 self.timezone = timezone
 503
 504                 # Get all required RRD objects
 505                 self.object_id = object_id
 506
 507                 # Get the main object
 508                 self.objects = self.get_objects(self.object_id)
 509                 self.objects.sort()
 510
 511         def __repr__(self):
 512                 return "<%s>" % self.__class__.__name__
 513
 514         @property
 515         def collecty(self):
 516                 return self.plugin.collecty
 517
 518         @property
 519         def log(self):
 520                 return self.plugin.log
 521
 522         @property
 523         def object(self):
 524                 """
 525                         Shortcut to the main object
 526                 """
 527                 if len(self.objects) == 1:
 528                         return self.objects[0]
 529
 530         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 531                         width=None, height=None, with_title=True, thumbnail=False):
 532                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 533
 534                 # Set the default dimensions
 535                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 536
 537                 # A thumbnail doesn't have a legend and other labels
 538                 if thumbnail:
 539                         args.append("--only-graph")
 540
 541                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 542                         default_width = THUMBNAIL_DEFAULT_WIDTH
 543
 544                 args += [
 545                         "--imgformat", format,
 546                         "--height", "%s" % (height or default_height),
 547                         "--width", "%s" % (width or default_width),
 548                 ]
 549
 550                 args += self.rrd_graph_args
 551
 552                 # Graph title
 553                 if with_title and self.graph_title:
 554                         args += ["--title", self.graph_title]
 555
 556                 # Vertical label
 557                 if self.graph_vertical_label:
 558                         args += ["--vertical-label", self.graph_vertical_label]
 559
 560                 if self.lower_limit is not None or self.upper_limit is not None:
 561                         # Force to honour the set limits
 562                         args.append("--rigid")
 563
 564                         if self.lower_limit is not None:
 565                                 args += ["--lower-limit", self.lower_limit]
 566
 567                         if self.upper_limit is not None:
 568                                 args += ["--upper-limit", self.upper_limit]
 569
 570                 # Add interval
 571                 args += ["--start", util.make_interval(interval)]
 572
 573                 return args
 574
 575         def _add_defs(self):
 576                 use_prefix = len(self.objects) >= 2
 577
 578                 args = []
 579                 for object in self.objects:
 580                         if use_prefix:
 581                                 args += object.make_rrd_defs(object.id)
 582                         else:
 583                                 args += object.make_rrd_defs()
 584
 585                 return args
 586
 587         def _add_vdefs(self, args):
 588                 ret = []
 589
 590                 for arg in args:
 591                         ret.append(arg)
 592
 593                         # Search for all DEFs and CDEFs
 594                         m = re.match(DEF_MATCH, "%s" % arg)
 595                         if m:
 596                                 name = m.group(1)
 597
 598                                 # Add the VDEFs for minimum, maximum, etc. values
 599                                 ret += [
 600                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 601                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 602                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 603                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 604                                 ]
 605
 606                 return ret
 607
 608         def get_objects(self, *args, **kwargs):
 609                 object = self.plugin.get_object(*args, **kwargs)
 610
 611                 if object:
 612                         return [object,]
 613
 614                 return []
 615
 616         def generate_graph(self, interval=None, **kwargs):
 617                 assert self.objects, "Cannot render graph without any objects"
 618
 619                 # Make sure that all collected data is in the database
 620                 # to get a recent graph image
 621                 for object in self.objects:
 622                         object.commit()
 623
 624                 args = self._make_command_line(interval, **kwargs)
 625
 626                 self.log.info(_("Generating graph %s") % self)
 627
 628                 rrd_graph = self.rrd_graph
 629
 630                 # Add DEFs for all objects
 631                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 632                         args += self._add_defs()
 633
 634                 args += rrd_graph
 635                 args = self._add_vdefs(args)
 636
 637                 # Convert arguments to string
 638                 args = [str(e) for e in args]
 639
 640                 for arg in args:
 641                         self.log.debug("  %s" % arg)
 642
 643                 with Environment(self.timezone, self.locale.lang):
 644                         graph = rrdtool.graphv("-", *args)
 645
 646                 return {
 647                         "image"        : graph.get("image"),
 648                         "image_height" : graph.get("image_height"),
 649                         "image_width"  : graph.get("image_width"),
 650                 }
 651
 652         def graph_info(self):
 653                 """
 654                         Returns a dictionary with useful information
 655                         about this graph.
 656                 """
 657                 return {
 658                         "title"        : self.graph_title,
 659                         "object_id"    : self.object_id or "",
 660                         "template"     : self.name,
 661                 }