src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import logging
  23 import os
  24 import re
  25 import rrdtool
  26 import time
  27 import unicodedata
  28
  29 from .. import locales
  30 from .. import util
  31 from ..constants import *
  32 from ..i18n import _
  33
  34 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  35
  36 class Environment(object):
  37         """
  38                 Sets the correct environment for rrdtool to create
  39                 localised graphs and graphs in the correct timezone.
  40         """
  41         def __init__(self, timezone, locale):
  42                 # Build the new environment
  43                 self.new_environment = {
  44                         "TZ" : timezone or DEFAULT_TIMEZONE,
  45                 }
  46
  47                 for k in ("LANG", "LC_ALL"):
  48                         self.new_environment[k] = locale or DEFAULT_LOCALE
  49
  50         def __enter__(self):
  51                 # Save the current environment
  52                 self.old_environment = {}
  53                 for k in self.new_environment:
  54                         self.old_environment[k] = os.environ.get(k, None)
  55
  56                 # Apply the new one
  57                 os.environ.update(self.new_environment)
  58
  59         def __exit__(self, type, value, traceback):
  60                 # Roll back to the previous environment
  61                 for k, v in self.old_environment.items():
  62                         if v is None:
  63                                 try:
  64                                         del os.environ[k]
  65                                 except KeyError:
  66                                         pass
  67                         else:
  68                                 os.environ[k] = v
  69
  70
  71 class PluginRegistration(type):
  72         plugins = {}
  73
  74         def __init__(plugin, name, bases, dict):
  75                 type.__init__(plugin, name, bases, dict)
  76
  77                 # The main class from which is inherited is not registered
  78                 # as a plugin.
  79                 if name == "Plugin":
  80                         return
  81
  82                 if not all((plugin.name, plugin.description)):
  83                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  84
  85                 PluginRegistration.plugins[plugin.name] = plugin
  86
  87
  88 def get():
  89         """
  90                 Returns a list with all automatically registered plugins.
  91         """
  92         return PluginRegistration.plugins.values()
  93
  94 class Plugin(object, metaclass=PluginRegistration):
  95         # The name of this plugin.
  96         name = None
  97
  98         # A description for this plugin.
  99         description = None
 100
 101         # Templates which can be used to generate a graph out of
 102         # the data from this data source.
 103         templates = []
 104
 105         # The default interval for all plugins
 106         interval = 60
 107
 108         # Priority
 109         priority = 0
 110
 111         def __init__(self, collecty, **kwargs):
 112                 self.collecty = collecty
 113
 114                 # Check if this plugin was configured correctly.
 115                 assert self.name, "Name of the plugin is not set: %s" % self.name
 116                 assert self.description, "Description of the plugin is not set: %s" % self.description
 117
 118                 # Initialize the logger.
 119                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 120
 121                 # Run some custom initialization.
 122                 self.init(**kwargs)
 123
 124                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 125
 126         @property
 127         def path(self):
 128                 """
 129                         Returns the name of the sub directory in which all RRD files
 130                         for this plugin should be stored in.
 131                 """
 132                 return self.name
 133
 134         ### Basic methods
 135
 136         def init(self, **kwargs):
 137                 """
 138                         Do some custom initialization stuff here.
 139                 """
 140                 pass
 141
 142         def collect(self):
 143                 """
 144                         Gathers the statistical data, this plugin collects.
 145                 """
 146                 time_start = time.time()
 147
 148                 # Run through all objects of this plugin and call the collect method.
 149                 for object in self.objects:
 150                         # Run collection
 151                         try:
 152                                 result = object.collect()
 153
 154                         # Catch any unhandled exceptions
 155                         except Exception as e:
 156                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 157                                 continue
 158
 159                         if not result:
 160                                 self.log.warning(_("Received empty result: %s") % object)
 161                                 continue
 162
 163                         # Add the object to the write queue so that the data is written
 164                         # to the databases later.
 165                         result = self.collecty.write_queue.submit(object, result)
 166
 167                         self.log.debug(_("Collected %s: %s") % (object, result))
 168
 169                 # Returns the time this function took to complete.
 170                 delay = time.time() - time_start
 171
 172                 # Log some warning when a collect method takes too long to return some data
 173                 if delay >= 60:
 174                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 175                 else:
 176                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 177
 178         def get_object(self, id):
 179                 for object in self.objects:
 180                         if not object.id == id:
 181                                 continue
 182
 183                         return object
 184
 185         def get_template(self, template_name, object_id, locale=None, timezone=None):
 186                 for template in self.templates:
 187                         if not template.name == template_name:
 188                                 continue
 189
 190                         return template(self, object_id, locale=locale, timezone=timezone)
 191
 192         def generate_graph(self, template_name, object_id="default",
 193                         timezone=None, locale=None, **kwargs):
 194                 template = self.get_template(template_name, object_id=object_id,
 195                         timezone=timezone, locale=locale)
 196                 if not template:
 197                         raise RuntimeError("Could not find template %s" % template_name)
 198
 199                 time_start = time.time()
 200
 201                 graph = template.generate_graph(**kwargs)
 202
 203                 duration = time.time() - time_start
 204                 self.log.debug(_("Generated graph %s in %.1fms") \
 205                         % (template, duration * 1000))
 206
 207                 return graph
 208
 209         def graph_info(self, template_name, object_id="default",
 210                         timezone=None, locale=None, **kwargs):
 211                 template = self.get_template(template_name, object_id=object_id,
 212                         timezone=timezone, locale=locale)
 213                 if not template:
 214                         raise RuntimeError("Could not find template %s" % template_name)
 215
 216                 return template.graph_info()
 217
 218         def last_update(self, object_id="default"):
 219                 object = self.get_object(object_id)
 220                 if not object:
 221                         raise RuntimeError("Could not find object %s" % object_id)
 222
 223                 return object.last_update()
 224
 225
 226 class Object(object):
 227         # The schema of the RRD database.
 228         rrd_schema = None
 229
 230         # RRA properties.
 231         rra_types     = ("AVERAGE", "MIN", "MAX")
 232         rra_timespans = (
 233                 ("1m", "10d"),
 234                 ("1h", "18M"),
 235                 ("1d",  "5y"),
 236         )
 237
 238         def __init__(self, plugin, *args, **kwargs):
 239                 self.plugin = plugin
 240
 241                 # Initialise this object
 242                 self.init(*args, **kwargs)
 243
 244                 # Create the database file.
 245                 self.create()
 246
 247         def __repr__(self):
 248                 return "<%s %s>" % (self.__class__.__name__, self.id)
 249
 250         def __lt__(self, other):
 251                 return self.id < other.id
 252
 253         @property
 254         def collecty(self):
 255                 return self.plugin.collecty
 256
 257         @property
 258         def log(self):
 259                 return self.plugin.log
 260
 261         @property
 262         def id(self):
 263                 """
 264                         Returns a UNIQUE identifier for this object. As this is incorporated
 265                         into the path of RRD file, it must only contain ASCII characters.
 266                 """
 267                 raise NotImplementedError
 268
 269         @property
 270         def file(self):
 271                 """
 272                         The absolute path to the RRD file of this plugin.
 273                 """
 274                 filename = self._normalise_filename("%s.rrd" % self.id)
 275
 276                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 277
 278         @staticmethod
 279         def _normalise_filename(filename):
 280                 # Convert the filename into ASCII characters only
 281                 filename = unicodedata.normalize("NFKC", filename)
 282
 283                 # Replace any spaces by dashes
 284                 filename = filename.replace(" ", "-")
 285
 286                 return filename
 287
 288         ### Basic methods
 289
 290         def init(self, *args, **kwargs):
 291                 """
 292                         Do some custom initialization stuff here.
 293                 """
 294                 pass
 295
 296         def create(self):
 297                 """
 298                         Creates an empty RRD file with the desired data structures.
 299                 """
 300                 # Skip if the file does already exist.
 301                 if os.path.exists(self.file):
 302                         return
 303
 304                 dirname = os.path.dirname(self.file)
 305                 if not os.path.exists(dirname):
 306                         os.makedirs(dirname)
 307
 308                 # Create argument list.
 309                 args = self.get_rrd_schema()
 310
 311                 rrdtool.create(self.file, *args)
 312
 313                 self.log.debug(_("Created RRD file %s.") % self.file)
 314                 for arg in args:
 315                         self.log.debug("  %s" % arg)
 316
 317         def info(self):
 318                 return rrdtool.info(self.file)
 319
 320         def last_update(self):
 321                 """
 322                         Returns a dictionary with the timestamp and
 323                         data set of the last database update.
 324                 """
 325                 return {
 326                         "dataset"   : self.last_dataset,
 327                         "timestamp" : self.last_updated,
 328                 }
 329
 330         def _last_update(self):
 331                 return rrdtool.lastupdate(self.file)
 332
 333         @property
 334         def last_updated(self):
 335                 """
 336                         Returns the timestamp when this database was last updated
 337                 """
 338                 lu = self._last_update()
 339
 340                 if lu:
 341                         return lu.get("date")
 342
 343         @property
 344         def last_dataset(self):
 345                 """
 346                         Returns the latest dataset in the database
 347                 """
 348                 lu = self._last_update()
 349
 350                 if lu:
 351                         return lu.get("ds")
 352
 353         @property
 354         def stepsize(self):
 355                 return self.plugin.interval
 356
 357         @property
 358         def heartbeat(self):
 359                 return self.stepsize * 2
 360
 361         def get_rrd_schema(self):
 362                 schema = [
 363                         "--step", "%s" % self.stepsize,
 364                 ]
 365                 for line in self.rrd_schema:
 366                         if line.startswith("DS:"):
 367                                 try:
 368                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 369
 370                                         line = ":".join((
 371                                                 prefix,
 372                                                 name,
 373                                                 type,
 374                                                 "%s" % self.heartbeat,
 375                                                 lower_limit,
 376                                                 upper_limit
 377                                         ))
 378                                 except ValueError:
 379                                         pass
 380
 381                         schema.append(line)
 382
 383                 xff = 0.1
 384
 385                 for steps, rows in self.rra_timespans:
 386                         for type in self.rra_types:
 387                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 388
 389                 return schema
 390
 391         @property
 392         def rrd_schema_names(self):
 393                 ret = []
 394
 395                 for line in self.rrd_schema:
 396                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 397                         ret.append(name)
 398
 399                 return ret
 400
 401         def make_rrd_defs(self, prefix=None):
 402                 defs = []
 403
 404                 for name in self.rrd_schema_names:
 405                         if prefix:
 406                                 p = "%s_%s" % (prefix, name)
 407                         else:
 408                                 p = name
 409
 410                         defs += [
 411                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 412                         ]
 413
 414                 return defs
 415
 416         def get_stddev(self, interval=None):
 417                 args = self.make_rrd_defs()
 418
 419                 # Add the correct interval
 420                 args += ["--start", util.make_interval(interval)]
 421
 422                 for name in self.rrd_schema_names:
 423                         args += [
 424                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 425                                 "PRINT:%s_stddev:%%lf" % name,
 426                         ]
 427
 428                 x, y, vals = rrdtool.graph("/dev/null", *args)
 429                 return dict(zip(self.rrd_schema_names, vals))
 430
 431         def commit(self):
 432                 """
 433                         Will commit the collected data to the database.
 434                 """
 435                 # Make sure that the RRD database has been created
 436                 self.create()
 437
 438                 # Write everything to disk that is in the write queue
 439                 self.collecty.write_queue.commit_file(self.file)
 440
 441         # Convenience functions for plugin authors
 442
 443         def read_file(self, *args, strip=True):
 444                 """
 445                         Reads the content of the given file
 446                 """
 447                 filename = os.path.join(*args)
 448
 449                 with open(filename) as f:
 450                         value = f.read()
 451
 452                 # Strip any excess whitespace
 453                 if strip:
 454                         value = value.strip()
 455
 456                 return value
 457
 458         def read_file_integer(self, filename):
 459                 """
 460                         Reads the content from a file and returns it as an integer
 461                 """
 462                 value = self.read_file(filename)
 463
 464                 try:
 465                         return int(value)
 466                 except ValueError:
 467                         return None
 468
 469         def read_proc_stat(self):
 470                 """
 471                         Reads /proc/stat and returns it as a dictionary
 472                 """
 473                 ret = {}
 474
 475                 with open("/proc/stat") as f:
 476                         for line in f:
 477                                 # Split the key from the rest of the line
 478                                 key, line = line.split(" ", 1)
 479
 480                                 # Remove any line breaks
 481                                 ret[key] = line.rstrip()
 482
 483                 return ret
 484
 485         def read_proc_meminfo(self):
 486                 ret = {}
 487
 488                 with open("/proc/meminfo") as f:
 489                         for line in f:
 490                                 # Split the key from the rest of the line
 491                                 key, line = line.split(":", 1)
 492
 493                                 # Remove any whitespace
 494                                 line = line.strip()
 495
 496                                 # Remove any trailing kB
 497                                 if line.endswith(" kB"):
 498                                         line = line[:-3]
 499
 500                                 # Try to convert to integer
 501                                 try:
 502                                         line = int(line)
 503                                 except (TypeError, ValueError):
 504                                         continue
 505
 506                                 ret[key] = line
 507
 508                 return ret
 509
 510
 511 class GraphTemplate(object):
 512         # A unique name to identify this graph template.
 513         name = None
 514
 515         # Headline of the graph image
 516         graph_title = None
 517
 518         # Vertical label of the graph
 519         graph_vertical_label = None
 520
 521         # Limits
 522         lower_limit = None
 523         upper_limit = None
 524
 525         # Instructions how to create the graph.
 526         rrd_graph = None
 527
 528         # Extra arguments passed to rrdgraph.
 529         rrd_graph_args = []
 530
 531         # Default dimensions for this graph
 532         height = GRAPH_DEFAULT_HEIGHT
 533         width  = GRAPH_DEFAULT_WIDTH
 534
 535         def __init__(self, plugin, object_id, locale=None, timezone=None):
 536                 self.plugin = plugin
 537
 538                 # Save localisation parameters
 539                 self.locale = locales.get(locale)
 540                 self.timezone = timezone
 541
 542                 # Get all required RRD objects
 543                 self.object_id = object_id
 544
 545                 # Get the main object
 546                 self.objects = self.get_objects(self.object_id)
 547                 self.objects.sort()
 548
 549         def __repr__(self):
 550                 return "<%s>" % self.__class__.__name__
 551
 552         @property
 553         def collecty(self):
 554                 return self.plugin.collecty
 555
 556         @property
 557         def log(self):
 558                 return self.plugin.log
 559
 560         @property
 561         def object(self):
 562                 """
 563                         Shortcut to the main object
 564                 """
 565                 if len(self.objects) == 1:
 566                         return self.objects[0]
 567
 568         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 569                         width=None, height=None, with_title=True, thumbnail=False):
 570                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 571
 572                 # Set the default dimensions
 573                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 574
 575                 # A thumbnail doesn't have a legend and other labels
 576                 if thumbnail:
 577                         args.append("--only-graph")
 578
 579                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 580                         default_width = THUMBNAIL_DEFAULT_WIDTH
 581
 582                 args += [
 583                         "--imgformat", format,
 584                         "--height", "%s" % (height or default_height),
 585                         "--width", "%s" % (width or default_width),
 586                 ]
 587
 588                 args += self.rrd_graph_args
 589
 590                 # Graph title
 591                 if with_title and self.graph_title:
 592                         args += ["--title", self.graph_title]
 593
 594                 # Vertical label
 595                 if self.graph_vertical_label:
 596                         args += ["--vertical-label", self.graph_vertical_label]
 597
 598                 if self.lower_limit is not None or self.upper_limit is not None:
 599                         # Force to honour the set limits
 600                         args.append("--rigid")
 601
 602                         if self.lower_limit is not None:
 603                                 args += ["--lower-limit", self.lower_limit]
 604
 605                         if self.upper_limit is not None:
 606                                 args += ["--upper-limit", self.upper_limit]
 607
 608                 # Add interval
 609                 args += ["--start", util.make_interval(interval)]
 610
 611                 return args
 612
 613         def _add_defs(self):
 614                 use_prefix = len(self.objects) >= 2
 615
 616                 args = []
 617                 for object in self.objects:
 618                         if use_prefix:
 619                                 args += object.make_rrd_defs(object.id)
 620                         else:
 621                                 args += object.make_rrd_defs()
 622
 623                 return args
 624
 625         def _add_vdefs(self, args):
 626                 ret = []
 627
 628                 for arg in args:
 629                         ret.append(arg)
 630
 631                         # Search for all DEFs and CDEFs
 632                         m = re.match(DEF_MATCH, "%s" % arg)
 633                         if m:
 634                                 name = m.group(1)
 635
 636                                 # Add the VDEFs for minimum, maximum, etc. values
 637                                 ret += [
 638                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 639                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 640                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 641                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 642                                 ]
 643
 644                 return ret
 645
 646         def get_objects(self, *args, **kwargs):
 647                 object = self.plugin.get_object(*args, **kwargs)
 648
 649                 if object:
 650                         return [object,]
 651
 652                 return []
 653
 654         def generate_graph(self, interval=None, **kwargs):
 655                 assert self.objects, "Cannot render graph without any objects"
 656
 657                 # Make sure that all collected data is in the database
 658                 # to get a recent graph image
 659                 for object in self.objects:
 660                         object.commit()
 661
 662                 args = self._make_command_line(interval, **kwargs)
 663
 664                 self.log.info(_("Generating graph %s") % self)
 665
 666                 rrd_graph = self.rrd_graph
 667
 668                 # Add DEFs for all objects
 669                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 670                         args += self._add_defs()
 671
 672                 args += rrd_graph
 673                 args = self._add_vdefs(args)
 674
 675                 # Convert arguments to string
 676                 args = [str(e) for e in args]
 677
 678                 for arg in args:
 679                         self.log.debug("  %s" % arg)
 680
 681                 with Environment(self.timezone, self.locale.lang):
 682                         graph = rrdtool.graphv("-", *args)
 683
 684                 return {
 685                         "image"        : graph.get("image"),
 686                         "image_height" : graph.get("image_height"),
 687                         "image_width"  : graph.get("image_width"),
 688                 }
 689
 690         def graph_info(self):
 691                 """
 692                         Returns a dictionary with useful information
 693                         about this graph.
 694                 """
 695                 return {
 696                         "title"        : self.graph_title,
 697                         "object_id"    : self.object_id or "",
 698                         "template"     : self.name,
 699                 }