src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import os
  25 import re
  26 import rrdtool
  27 import time
  28 import unicodedata
  29
  30 from .. import locales
  31 from .. import util
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  36
  37 class Environment(object):
  38         """
  39                 Sets the correct environment for rrdtool to create
  40                 localised graphs and graphs in the correct timezone.
  41         """
  42         def __init__(self, timezone, locale):
  43                 # Build the new environment
  44                 self.new_environment = {
  45                         "TZ" : timezone or DEFAULT_TIMEZONE,
  46                 }
  47
  48                 for k in ("LANG", "LC_ALL"):
  49                         self.new_environment[k] = locale or DEFAULT_LOCALE
  50
  51         def __enter__(self):
  52                 # Save the current environment
  53                 self.old_environment = {}
  54                 for k in self.new_environment:
  55                         self.old_environment[k] = os.environ.get(k, None)
  56
  57                 # Apply the new one
  58                 os.environ.update(self.new_environment)
  59
  60         def __exit__(self, type, value, traceback):
  61                 # Roll back to the previous environment
  62                 for k, v in self.old_environment.items():
  63                         if v is None:
  64                                 try:
  65                                         del os.environ[k]
  66                                 except KeyError:
  67                                         pass
  68                         else:
  69                                 os.environ[k] = v
  70
  71
  72 class PluginRegistration(type):
  73         plugins = {}
  74
  75         def __init__(plugin, name, bases, dict):
  76                 type.__init__(plugin, name, bases, dict)
  77
  78                 # The main class from which is inherited is not registered
  79                 # as a plugin.
  80                 if name == "Plugin":
  81                         return
  82
  83                 if not all((plugin.name, plugin.description)):
  84                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  85
  86                 PluginRegistration.plugins[plugin.name] = plugin
  87
  88
  89 def get():
  90         """
  91                 Returns a list with all automatically registered plugins.
  92         """
  93         return PluginRegistration.plugins.values()
  94
  95 class Plugin(object, metaclass=PluginRegistration):
  96         # The name of this plugin.
  97         name = None
  98
  99         # A description for this plugin.
 100         description = None
 101
 102         # Templates which can be used to generate a graph out of
 103         # the data from this data source.
 104         templates = []
 105
 106         # The default interval for all plugins
 107         interval = 60
 108
 109         # Priority
 110         priority = 0
 111
 112         def __init__(self, collecty, **kwargs):
 113                 self.collecty = collecty
 114
 115                 # Check if this plugin was configured correctly.
 116                 assert self.name, "Name of the plugin is not set: %s" % self.name
 117                 assert self.description, "Description of the plugin is not set: %s" % self.description
 118
 119                 # Initialize the logger.
 120                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 126
 127         @property
 128         def path(self):
 129                 """
 130                         Returns the name of the sub directory in which all RRD files
 131                         for this plugin should be stored in.
 132                 """
 133                 return self.name
 134
 135         ### Basic methods
 136
 137         def init(self, **kwargs):
 138                 """
 139                         Do some custom initialization stuff here.
 140                 """
 141                 pass
 142
 143         def collect(self):
 144                 """
 145                         Gathers the statistical data, this plugin collects.
 146                 """
 147                 time_start = time.time()
 148
 149                 # Run through all objects of this plugin and call the collect method.
 150                 for o in self.objects:
 151                         now = datetime.datetime.utcnow()
 152                         try:
 153                                 result = o.collect()
 154
 155                                 result = self._format_result(result)
 156                         except:
 157                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 158                                 continue
 159
 160                         if not result:
 161                                 self.log.warning(_("Received empty result: %s") % o)
 162                                 continue
 163
 164                         self.log.debug(_("Collected %s: %s") % (o, result))
 165
 166                         # Add the object to the write queue so that the data is written
 167                         # to the databases later.
 168                         self.collecty.write_queue.add(o, now, result)
 169
 170                 # Returns the time this function took to complete.
 171                 delay = time.time() - time_start
 172
 173                 # Log some warning when a collect method takes too long to return some data
 174                 if delay >= 60:
 175                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 176
 177         @staticmethod
 178         def _format_result(result):
 179                 if not isinstance(result, tuple) and not isinstance(result, list):
 180                         return result
 181
 182                 # Replace all Nones by UNKNOWN
 183                 s = []
 184
 185                 for e in result:
 186                         if e is None:
 187                                 e = "U"
 188
 189                         s.append("%s" % e)
 190
 191                 return ":".join(s)
 192
 193         def get_object(self, id):
 194                 for object in self.objects:
 195                         if not object.id == id:
 196                                 continue
 197
 198                         return object
 199
 200         def get_template(self, template_name, object_id, locale=None, timezone=None):
 201                 for template in self.templates:
 202                         if not template.name == template_name:
 203                                 continue
 204
 205                         return template(self, object_id, locale=locale, timezone=timezone)
 206
 207         def generate_graph(self, template_name, object_id="default",
 208                         timezone=None, locale=None, **kwargs):
 209                 template = self.get_template(template_name, object_id=object_id,
 210                         timezone=timezone, locale=locale)
 211                 if not template:
 212                         raise RuntimeError("Could not find template %s" % template_name)
 213
 214                 time_start = time.time()
 215
 216                 graph = template.generate_graph(**kwargs)
 217
 218                 duration = time.time() - time_start
 219                 self.log.debug(_("Generated graph %s in %.1fms") \
 220                         % (template, duration * 1000))
 221
 222                 return graph
 223
 224         def graph_info(self, template_name, object_id="default",
 225                         timezone=None, locale=None, **kwargs):
 226                 template = self.get_template(template_name, object_id=object_id,
 227                         timezone=timezone, locale=locale)
 228                 if not template:
 229                         raise RuntimeError("Could not find template %s" % template_name)
 230
 231                 return template.graph_info()
 232
 233         def last_update(self, object_id="default"):
 234                 object = self.get_object(object_id)
 235                 if not object:
 236                         raise RuntimeError("Could not find object %s" % object_id)
 237
 238                 return object.last_update()
 239
 240
 241 class Object(object):
 242         # The schema of the RRD database.
 243         rrd_schema = None
 244
 245         # RRA properties.
 246         rra_types     = ("AVERAGE", "MIN", "MAX")
 247         rra_timespans = (
 248                 ("1m", "10d"),
 249                 ("1h", "18M"),
 250                 ("1d",  "5y"),
 251         )
 252
 253         def __init__(self, plugin, *args, **kwargs):
 254                 self.plugin = plugin
 255
 256                 # Indicates if this object has collected its data
 257                 self.collected = False
 258
 259                 # Initialise this object
 260                 self.init(*args, **kwargs)
 261
 262                 # Create the database file.
 263                 self.create()
 264
 265         def __repr__(self):
 266                 return "<%s %s>" % (self.__class__.__name__, self.id)
 267
 268         def __lt__(self, other):
 269                 return self.id < other.id
 270
 271         @property
 272         def collecty(self):
 273                 return self.plugin.collecty
 274
 275         @property
 276         def log(self):
 277                 return self.plugin.log
 278
 279         @property
 280         def id(self):
 281                 """
 282                         Returns a UNIQUE identifier for this object. As this is incorporated
 283                         into the path of RRD file, it must only contain ASCII characters.
 284                 """
 285                 raise NotImplementedError
 286
 287         @property
 288         def file(self):
 289                 """
 290                         The absolute path to the RRD file of this plugin.
 291                 """
 292                 filename = self._normalise_filename("%s.rrd" % self.id)
 293
 294                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 295
 296         @staticmethod
 297         def _normalise_filename(filename):
 298                 # Convert the filename into ASCII characters only
 299                 filename = unicodedata.normalize("NFKC", filename)
 300
 301                 # Replace any spaces by dashes
 302                 filename = filename.replace(" ", "-")
 303
 304                 return filename
 305
 306         ### Basic methods
 307
 308         def init(self, *args, **kwargs):
 309                 """
 310                         Do some custom initialization stuff here.
 311                 """
 312                 pass
 313
 314         def create(self):
 315                 """
 316                         Creates an empty RRD file with the desired data structures.
 317                 """
 318                 # Skip if the file does already exist.
 319                 if os.path.exists(self.file):
 320                         return
 321
 322                 dirname = os.path.dirname(self.file)
 323                 if not os.path.exists(dirname):
 324                         os.makedirs(dirname)
 325
 326                 # Create argument list.
 327                 args = self.get_rrd_schema()
 328
 329                 rrdtool.create(self.file, *args)
 330
 331                 self.log.debug(_("Created RRD file %s.") % self.file)
 332                 for arg in args:
 333                         self.log.debug("  %s" % arg)
 334
 335         def info(self):
 336                 return rrdtool.info(self.file)
 337
 338         def last_update(self):
 339                 """
 340                         Returns a dictionary with the timestamp and
 341                         data set of the last database update.
 342                 """
 343                 return {
 344                         "dataset"   : self.last_dataset,
 345                         "timestamp" : self.last_updated,
 346                 }
 347
 348         def _last_update(self):
 349                 return rrdtool.lastupdate(self.file)
 350
 351         @property
 352         def last_updated(self):
 353                 """
 354                         Returns the timestamp when this database was last updated
 355                 """
 356                 lu = self._last_update()
 357
 358                 if lu:
 359                         return lu.get("date")
 360
 361         @property
 362         def last_dataset(self):
 363                 """
 364                         Returns the latest dataset in the database
 365                 """
 366                 lu = self._last_update()
 367
 368                 if lu:
 369                         return lu.get("ds")
 370
 371         @property
 372         def stepsize(self):
 373                 return self.plugin.interval
 374
 375         @property
 376         def heartbeat(self):
 377                 return self.stepsize * 2
 378
 379         def get_rrd_schema(self):
 380                 schema = [
 381                         "--step", "%s" % self.stepsize,
 382                 ]
 383                 for line in self.rrd_schema:
 384                         if line.startswith("DS:"):
 385                                 try:
 386                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 387
 388                                         line = ":".join((
 389                                                 prefix,
 390                                                 name,
 391                                                 type,
 392                                                 "%s" % self.heartbeat,
 393                                                 lower_limit,
 394                                                 upper_limit
 395                                         ))
 396                                 except ValueError:
 397                                         pass
 398
 399                         schema.append(line)
 400
 401                 xff = 0.1
 402
 403                 for steps, rows in self.rra_timespans:
 404                         for type in self.rra_types:
 405                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 406
 407                 return schema
 408
 409         @property
 410         def rrd_schema_names(self):
 411                 ret = []
 412
 413                 for line in self.rrd_schema:
 414                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 415                         ret.append(name)
 416
 417                 return ret
 418
 419         def make_rrd_defs(self, prefix=None):
 420                 defs = []
 421
 422                 for name in self.rrd_schema_names:
 423                         if prefix:
 424                                 p = "%s_%s" % (prefix, name)
 425                         else:
 426                                 p = name
 427
 428                         defs += [
 429                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 430                         ]
 431
 432                 return defs
 433
 434         def get_stddev(self, interval=None):
 435                 args = self.make_rrd_defs()
 436
 437                 # Add the correct interval
 438                 args += ["--start", util.make_interval(interval)]
 439
 440                 for name in self.rrd_schema_names:
 441                         args += [
 442                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 443                                 "PRINT:%s_stddev:%%lf" % name,
 444                         ]
 445
 446                 x, y, vals = rrdtool.graph("/dev/null", *args)
 447                 return dict(zip(self.rrd_schema_names, vals))
 448
 449         def execute(self):
 450                 if self.collected:
 451                         raise RuntimeError("This object has already collected its data")
 452
 453                 self.collected = True
 454                 self.now = datetime.datetime.utcnow()
 455
 456                 # Call the collect
 457                 result = self.collect()
 458
 459         def commit(self):
 460                 """
 461                         Will commit the collected data to the database.
 462                 """
 463                 # Make sure that the RRD database has been created
 464                 self.create()
 465
 466                 # Write everything to disk that is in the write queue
 467                 self.collecty.write_queue.commit_file(self.file)
 468
 469         # Convenience functions for plugin authors
 470
 471         def read_file(self, *args, strip=True):
 472                 """
 473                         Reads the content of the given file
 474                 """
 475                 filename = os.path.join(*args)
 476
 477                 with open(filename) as f:
 478                         value = f.read()
 479
 480                 # Strip any excess whitespace
 481                 if strip:
 482                         value = value.strip()
 483
 484                 return value
 485
 486         def read_file_integer(self, filename):
 487                 """
 488                         Reads the content from a file and returns it as an integer
 489                 """
 490                 value = self.read_file(filename)
 491
 492                 try:
 493                         return int(value)
 494                 except ValueError:
 495                         return None
 496
 497         def read_proc_stat(self):
 498                 """
 499                         Reads /proc/stat and returns it as a dictionary
 500                 """
 501                 ret = {}
 502
 503                 with open("/proc/stat") as f:
 504                         for line in f:
 505                                 # Split the key from the rest of the line
 506                                 key, line = line.split(" ", 1)
 507
 508                                 # Remove any line breaks
 509                                 ret[key] = line.rstrip()
 510
 511                 return ret
 512
 513
 514 class GraphTemplate(object):
 515         # A unique name to identify this graph template.
 516         name = None
 517
 518         # Headline of the graph image
 519         graph_title = None
 520
 521         # Vertical label of the graph
 522         graph_vertical_label = None
 523
 524         # Limits
 525         lower_limit = None
 526         upper_limit = None
 527
 528         # Instructions how to create the graph.
 529         rrd_graph = None
 530
 531         # Extra arguments passed to rrdgraph.
 532         rrd_graph_args = []
 533
 534         # Default dimensions for this graph
 535         height = GRAPH_DEFAULT_HEIGHT
 536         width  = GRAPH_DEFAULT_WIDTH
 537
 538         def __init__(self, plugin, object_id, locale=None, timezone=None):
 539                 self.plugin = plugin
 540
 541                 # Save localisation parameters
 542                 self.locale = locales.get(locale)
 543                 self.timezone = timezone
 544
 545                 # Get all required RRD objects
 546                 self.object_id = object_id
 547
 548                 # Get the main object
 549                 self.objects = self.get_objects(self.object_id)
 550                 self.objects.sort()
 551
 552         def __repr__(self):
 553                 return "<%s>" % self.__class__.__name__
 554
 555         @property
 556         def collecty(self):
 557                 return self.plugin.collecty
 558
 559         @property
 560         def log(self):
 561                 return self.plugin.log
 562
 563         @property
 564         def object(self):
 565                 """
 566                         Shortcut to the main object
 567                 """
 568                 if len(self.objects) == 1:
 569                         return self.objects[0]
 570
 571         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 572                         width=None, height=None, with_title=True, thumbnail=False):
 573                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 574
 575                 # Set the default dimensions
 576                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 577
 578                 # A thumbnail doesn't have a legend and other labels
 579                 if thumbnail:
 580                         args.append("--only-graph")
 581
 582                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 583                         default_width = THUMBNAIL_DEFAULT_WIDTH
 584
 585                 args += [
 586                         "--imgformat", format,
 587                         "--height", "%s" % (height or default_height),
 588                         "--width", "%s" % (width or default_width),
 589                 ]
 590
 591                 args += self.rrd_graph_args
 592
 593                 # Graph title
 594                 if with_title and self.graph_title:
 595                         args += ["--title", self.graph_title]
 596
 597                 # Vertical label
 598                 if self.graph_vertical_label:
 599                         args += ["--vertical-label", self.graph_vertical_label]
 600
 601                 if self.lower_limit is not None or self.upper_limit is not None:
 602                         # Force to honour the set limits
 603                         args.append("--rigid")
 604
 605                         if self.lower_limit is not None:
 606                                 args += ["--lower-limit", self.lower_limit]
 607
 608                         if self.upper_limit is not None:
 609                                 args += ["--upper-limit", self.upper_limit]
 610
 611                 # Add interval
 612                 args += ["--start", util.make_interval(interval)]
 613
 614                 return args
 615
 616         def _add_defs(self):
 617                 use_prefix = len(self.objects) >= 2
 618
 619                 args = []
 620                 for object in self.objects:
 621                         if use_prefix:
 622                                 args += object.make_rrd_defs(object.id)
 623                         else:
 624                                 args += object.make_rrd_defs()
 625
 626                 return args
 627
 628         def _add_vdefs(self, args):
 629                 ret = []
 630
 631                 for arg in args:
 632                         ret.append(arg)
 633
 634                         # Search for all DEFs and CDEFs
 635                         m = re.match(DEF_MATCH, "%s" % arg)
 636                         if m:
 637                                 name = m.group(1)
 638
 639                                 # Add the VDEFs for minimum, maximum, etc. values
 640                                 ret += [
 641                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 642                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 643                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 644                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 645                                 ]
 646
 647                 return ret
 648
 649         def get_objects(self, *args, **kwargs):
 650                 object = self.plugin.get_object(*args, **kwargs)
 651
 652                 if object:
 653                         return [object,]
 654
 655                 return []
 656
 657         def generate_graph(self, interval=None, **kwargs):
 658                 assert self.objects, "Cannot render graph without any objects"
 659
 660                 # Make sure that all collected data is in the database
 661                 # to get a recent graph image
 662                 for object in self.objects:
 663                         object.commit()
 664
 665                 args = self._make_command_line(interval, **kwargs)
 666
 667                 self.log.info(_("Generating graph %s") % self)
 668
 669                 rrd_graph = self.rrd_graph
 670
 671                 # Add DEFs for all objects
 672                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 673                         args += self._add_defs()
 674
 675                 args += rrd_graph
 676                 args = self._add_vdefs(args)
 677
 678                 # Convert arguments to string
 679                 args = [str(e) for e in args]
 680
 681                 for arg in args:
 682                         self.log.debug("  %s" % arg)
 683
 684                 with Environment(self.timezone, self.locale.lang):
 685                         graph = rrdtool.graphv("-", *args)
 686
 687                 return {
 688                         "image"        : graph.get("image"),
 689                         "image_height" : graph.get("image_height"),
 690                         "image_width"  : graph.get("image_width"),
 691                 }
 692
 693         def graph_info(self):
 694                 """
 695                         Returns a dictionary with useful information
 696                         about this graph.
 697                 """
 698                 return {
 699                         "title"        : self.graph_title,
 700                         "object_id"    : self.object_id or "",
 701                         "template"     : self.name,
 702                 }