src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import logging
  23 import os
  24 import re
  25 import rrdtool
  26 import time
  27 import unicodedata
  28
  29 from .. import util
  30 from ..constants import *
  31 from ..i18n import _
  32
  33 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  34
  35 class Environment(object):
  36         """
  37                 Sets the correct environment for rrdtool to create
  38                 localised graphs and graphs in the correct timezone.
  39         """
  40         def __init__(self, timezone="UTC", locale="en_US.utf-8"):
  41                 # Build the new environment
  42                 self.new_environment = {
  43                         "LANGUAGE" : locale,
  44                         "LC_ALL"   : locale,
  45                         "TZ"       : timezone,
  46                 }
  47
  48         def __enter__(self):
  49                 # Save the current environment
  50                 self.old_environment = {}
  51
  52                 for k in self.new_environment:
  53                         self.old_environment[k] = os.environ.get(k, None)
  54
  55                 # Apply the new one
  56                 os.environ.update(self.new_environment)
  57
  58         def __exit__(self, type, value, traceback):
  59                 # Roll back to the previous environment
  60                 for k, v in self.old_environment.items():
  61                         if v is None:
  62                                 try:
  63                                         del os.environ[k]
  64                                 except KeyError:
  65                                         pass
  66                         else:
  67                                 os.environ[k] = v
  68
  69
  70 class PluginRegistration(type):
  71         plugins = {}
  72
  73         def __init__(plugin, name, bases, dict):
  74                 type.__init__(plugin, name, bases, dict)
  75
  76                 # The main class from which is inherited is not registered
  77                 # as a plugin.
  78                 if name == "Plugin":
  79                         return
  80
  81                 if not all((plugin.name, plugin.description)):
  82                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  83
  84                 PluginRegistration.plugins[plugin.name] = plugin
  85
  86
  87 def get():
  88         """
  89                 Returns a list with all automatically registered plugins.
  90         """
  91         return PluginRegistration.plugins.values()
  92
  93 class Plugin(object, metaclass=PluginRegistration):
  94         # The name of this plugin.
  95         name = None
  96
  97         # A description for this plugin.
  98         description = None
  99
 100         # Templates which can be used to generate a graph out of
 101         # the data from this data source.
 102         templates = []
 103
 104         # The default interval for all plugins
 105         interval = 60
 106
 107         # Priority
 108         priority = 0
 109
 110         def __init__(self, collecty, **kwargs):
 111                 self.collecty = collecty
 112
 113                 # Check if this plugin was configured correctly.
 114                 assert self.name, "Name of the plugin is not set: %s" % self.name
 115                 assert self.description, "Description of the plugin is not set: %s" % self.description
 116
 117                 # Initialize the logger.
 118                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 119
 120                 # Run some custom initialization.
 121                 self.init(**kwargs)
 122
 123                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 124
 125         @property
 126         def path(self):
 127                 """
 128                         Returns the name of the sub directory in which all RRD files
 129                         for this plugin should be stored in.
 130                 """
 131                 return self.name
 132
 133         ### Basic methods
 134
 135         def init(self, **kwargs):
 136                 """
 137                         Do some custom initialization stuff here.
 138                 """
 139                 pass
 140
 141         def collect(self):
 142                 """
 143                         Gathers the statistical data, this plugin collects.
 144                 """
 145                 time_start = time.time()
 146
 147                 # Run through all objects of this plugin and call the collect method.
 148                 for object in self.objects:
 149                         # Run collection
 150                         try:
 151                                 result = object.collect()
 152
 153                         # Catch any unhandled exceptions
 154                         except Exception as e:
 155                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 156                                 continue
 157
 158                         if not result:
 159                                 self.log.warning(_("Received empty result: %s") % object)
 160                                 continue
 161
 162                         # Add the object to the write queue so that the data is written
 163                         # to the databases later.
 164                         result = self.collecty.write_queue.submit(object, result)
 165
 166                         self.log.debug(_("Collected %s: %s") % (object, result))
 167
 168                 # Returns the time this function took to complete.
 169                 delay = time.time() - time_start
 170
 171                 # Log some warning when a collect method takes too long to return some data
 172                 if delay >= 60:
 173                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 174                 else:
 175                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 176
 177         def get_object(self, id):
 178                 for object in self.objects:
 179                         if not object.id == id:
 180                                 continue
 181
 182                         return object
 183
 184         def get_template(self, template_name, object_id, locale=None, timezone=None):
 185                 for template in self.templates:
 186                         if not template.name == template_name:
 187                                 continue
 188
 189                         return template(self, object_id, locale=locale, timezone=timezone)
 190
 191         def generate_graph(self, template_name, object_id="default",
 192                         timezone=None, locale=None, **kwargs):
 193                 template = self.get_template(template_name, object_id=object_id,
 194                         timezone=timezone, locale=locale)
 195                 if not template:
 196                         raise RuntimeError("Could not find template %s" % template_name)
 197
 198                 time_start = time.time()
 199
 200                 with Environment(timezone=timezone, locale=locale):
 201                         graph = template.generate_graph(**kwargs)
 202
 203                 duration = time.time() - time_start
 204                 self.log.debug(_("Generated graph %s in %.1fms") \
 205                         % (template, duration * 1000))
 206
 207                 return graph
 208
 209         def graph_info(self, template_name, object_id="default",
 210                         timezone=None, locale=None, **kwargs):
 211                 template = self.get_template(template_name, object_id=object_id,
 212                         timezone=timezone, locale=locale)
 213                 if not template:
 214                         raise RuntimeError("Could not find template %s" % template_name)
 215
 216                 return template.graph_info()
 217
 218         def last_update(self, object_id="default"):
 219                 object = self.get_object(object_id)
 220                 if not object:
 221                         raise RuntimeError("Could not find object %s" % object_id)
 222
 223                 return object.last_update()
 224
 225
 226 class Object(object):
 227         # The schema of the RRD database.
 228         rrd_schema = None
 229
 230         # RRA properties.
 231         rra_types     = ("AVERAGE", "MIN", "MAX")
 232         rra_timespans = (
 233                 ("1m", "10d"),
 234                 ("1h", "18M"),
 235                 ("1d",  "5y"),
 236         )
 237
 238         def __init__(self, plugin, *args, **kwargs):
 239                 self.plugin = plugin
 240
 241                 # Initialise this object
 242                 self.init(*args, **kwargs)
 243
 244                 # Create the database file.
 245                 self.create()
 246
 247         def __repr__(self):
 248                 return "<%s %s>" % (self.__class__.__name__, self.id)
 249
 250         def __lt__(self, other):
 251                 return self.id < other.id
 252
 253         @property
 254         def collecty(self):
 255                 return self.plugin.collecty
 256
 257         @property
 258         def log(self):
 259                 return self.plugin.log
 260
 261         @property
 262         def id(self):
 263                 """
 264                         Returns a UNIQUE identifier for this object. As this is incorporated
 265                         into the path of RRD file, it must only contain ASCII characters.
 266                 """
 267                 raise NotImplementedError
 268
 269         @property
 270         def file(self):
 271                 """
 272                         The absolute path to the RRD file of this plugin.
 273                 """
 274                 filename = self._normalise_filename("%s.rrd" % self.id)
 275
 276                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 277
 278         @staticmethod
 279         def _normalise_filename(filename):
 280                 # Convert the filename into ASCII characters only
 281                 filename = unicodedata.normalize("NFKC", filename)
 282
 283                 # Replace any spaces by dashes
 284                 filename = filename.replace(" ", "-")
 285
 286                 return filename
 287
 288         ### Basic methods
 289
 290         def init(self, *args, **kwargs):
 291                 """
 292                         Do some custom initialization stuff here.
 293                 """
 294                 pass
 295
 296         def create(self):
 297                 """
 298                         Creates an empty RRD file with the desired data structures.
 299                 """
 300                 # Skip if the file does already exist.
 301                 if os.path.exists(self.file):
 302                         return
 303
 304                 dirname = os.path.dirname(self.file)
 305                 if not os.path.exists(dirname):
 306                         os.makedirs(dirname)
 307
 308                 # Create argument list.
 309                 args = self.get_rrd_schema()
 310
 311                 rrdtool.create(self.file, *args)
 312
 313                 self.log.debug(_("Created RRD file %s.") % self.file)
 314                 for arg in args:
 315                         self.log.debug("  %s" % arg)
 316
 317         def info(self):
 318                 return rrdtool.info(self.file)
 319
 320         def last_update(self):
 321                 """
 322                         Returns a dictionary with the timestamp and
 323                         data set of the last database update.
 324                 """
 325                 return {
 326                         "dataset"   : self.last_dataset,
 327                         "timestamp" : self.last_updated,
 328                 }
 329
 330         def _last_update(self):
 331                 return rrdtool.lastupdate(self.file)
 332
 333         @property
 334         def last_updated(self):
 335                 """
 336                         Returns the timestamp when this database was last updated
 337                 """
 338                 lu = self._last_update()
 339
 340                 if lu:
 341                         return lu.get("date")
 342
 343         @property
 344         def last_dataset(self):
 345                 """
 346                         Returns the latest dataset in the database
 347                 """
 348                 lu = self._last_update()
 349
 350                 if lu:
 351                         return lu.get("ds")
 352
 353         @property
 354         def stepsize(self):
 355                 return self.plugin.interval
 356
 357         @property
 358         def heartbeat(self):
 359                 return self.stepsize * 2
 360
 361         def get_rrd_schema(self):
 362                 schema = [
 363                         "--step", "%s" % self.stepsize,
 364                 ]
 365                 for line in self.rrd_schema:
 366                         if line.startswith("DS:"):
 367                                 try:
 368                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 369
 370                                         line = ":".join((
 371                                                 prefix,
 372                                                 name,
 373                                                 type,
 374                                                 "%s" % self.heartbeat,
 375                                                 lower_limit,
 376                                                 upper_limit
 377                                         ))
 378                                 except ValueError:
 379                                         pass
 380
 381                         schema.append(line)
 382
 383                 xff = 0.1
 384
 385                 for steps, rows in self.rra_timespans:
 386                         for type in self.rra_types:
 387                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 388
 389                 return schema
 390
 391         @property
 392         def rrd_schema_names(self):
 393                 ret = []
 394
 395                 for line in self.rrd_schema:
 396                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 397                         ret.append(name)
 398
 399                 return ret
 400
 401         def make_rrd_defs(self, prefix=None):
 402                 defs = []
 403
 404                 for name in self.rrd_schema_names:
 405                         if prefix:
 406                                 p = "%s_%s" % (prefix, name)
 407                         else:
 408                                 p = name
 409
 410                         defs += [
 411                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 412                         ]
 413
 414                 return defs
 415
 416         def get_stddev(self, interval=None):
 417                 args = self.make_rrd_defs()
 418
 419                 # Add the correct interval
 420                 args += ["--start", util.make_interval(interval)]
 421
 422                 for name in self.rrd_schema_names:
 423                         args += [
 424                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 425                                 "PRINT:%s_stddev:%%lf" % name,
 426                         ]
 427
 428                 x, y, vals = rrdtool.graph("/dev/null", *args)
 429                 return dict(zip(self.rrd_schema_names, vals))
 430
 431         def commit(self):
 432                 """
 433                         Will commit the collected data to the database.
 434                 """
 435                 # Make sure that the RRD database has been created
 436                 self.create()
 437
 438                 # Write everything to disk that is in the write queue
 439                 self.collecty.write_queue.commit_file(self.file)
 440
 441         # Convenience functions for plugin authors
 442
 443         def read_file(self, *args, strip=True):
 444                 """
 445                         Reads the content of the given file
 446                 """
 447                 filename = os.path.join(*args)
 448
 449                 with open(filename) as f:
 450                         value = f.read()
 451
 452                 # Strip any excess whitespace
 453                 if strip:
 454                         value = value.strip()
 455
 456                 return value
 457
 458         def read_file_integer(self, filename):
 459                 """
 460                         Reads the content from a file and returns it as an integer
 461                 """
 462                 value = self.read_file(filename)
 463
 464                 try:
 465                         return int(value)
 466                 except ValueError:
 467                         return None
 468
 469         def read_proc_stat(self):
 470                 """
 471                         Reads /proc/stat and returns it as a dictionary
 472                 """
 473                 ret = {}
 474
 475                 with open("/proc/stat") as f:
 476                         for line in f:
 477                                 # Split the key from the rest of the line
 478                                 key, line = line.split(" ", 1)
 479
 480                                 # Remove any line breaks
 481                                 ret[key] = line.rstrip()
 482
 483                 return ret
 484
 485         def read_proc_meminfo(self):
 486                 ret = {}
 487
 488                 with open("/proc/meminfo") as f:
 489                         for line in f:
 490                                 # Split the key from the rest of the line
 491                                 key, line = line.split(":", 1)
 492
 493                                 # Remove any whitespace
 494                                 line = line.strip()
 495
 496                                 # Remove any trailing kB
 497                                 if line.endswith(" kB"):
 498                                         line = line[:-3]
 499
 500                                 # Try to convert to integer
 501                                 try:
 502                                         line = int(line)
 503                                 except (TypeError, ValueError):
 504                                         continue
 505
 506                                 ret[key] = line
 507
 508                 return ret
 509
 510
 511 class GraphTemplate(object):
 512         # A unique name to identify this graph template.
 513         name = None
 514
 515         # Headline of the graph image
 516         graph_title = None
 517
 518         # Vertical label of the graph
 519         graph_vertical_label = None
 520
 521         # Limits
 522         lower_limit = None
 523         upper_limit = None
 524
 525         # Instructions how to create the graph.
 526         rrd_graph = None
 527
 528         # Extra arguments passed to rrdgraph.
 529         rrd_graph_args = []
 530
 531         def __init__(self, plugin, object_id, locale=None, timezone=None):
 532                 self.plugin = plugin
 533
 534                 # Save localisation parameters
 535                 self.locale = locale
 536                 self.timezone = timezone
 537
 538                 # Get all required RRD objects
 539                 self.object_id = object_id
 540
 541                 # Get the main object
 542                 self.objects = self.get_objects(self.object_id)
 543                 self.objects.sort()
 544
 545         def __repr__(self):
 546                 return "<%s>" % self.__class__.__name__
 547
 548         @property
 549         def collecty(self):
 550                 return self.plugin.collecty
 551
 552         @property
 553         def log(self):
 554                 return self.plugin.log
 555
 556         @property
 557         def object(self):
 558                 """
 559                         Shortcut to the main object
 560                 """
 561                 if len(self.objects) == 1:
 562                         return self.objects[0]
 563
 564         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 565                         width=None, height=None, with_title=True, thumbnail=False):
 566                 args = [
 567                         # Change the background colour
 568                         "--color", "BACK#FFFFFFFF",
 569
 570                         # Disable the border around the image
 571                         "--border", "0",
 572
 573                         # Let's width and height define the size of the entire image
 574                         "--full-size-mode",
 575
 576                         # Gives the curves a more organic look
 577                         "--slope-mode",
 578
 579                         # Show nicer labels
 580                         "--dynamic-labels",
 581
 582                         # Brand all generated graphs
 583                         "--watermark", _("Created by collecty"),
 584                 ]
 585
 586                 # Set the default dimensions
 587                 default_width, default_height = 960, 480
 588
 589                 # A thumbnail doesn't have a legend and other labels
 590                 if thumbnail:
 591                         args.append("--only-graph")
 592
 593                         default_width, default_height = 80, 20
 594
 595                 args += [
 596                         "--imgformat", format,
 597                         "--height", "%s" % (height or default_height),
 598                         "--width", "%s" % (width or default_width),
 599                 ]
 600
 601                 args += self.rrd_graph_args
 602
 603                 # Graph title
 604                 if with_title and self.graph_title:
 605                         args += ["--title", self.graph_title]
 606
 607                 # Vertical label
 608                 if self.graph_vertical_label:
 609                         args += ["--vertical-label", self.graph_vertical_label]
 610
 611                 if self.lower_limit is not None or self.upper_limit is not None:
 612                         # Force to honour the set limits
 613                         args.append("--rigid")
 614
 615                         if self.lower_limit is not None:
 616                                 args += ["--lower-limit", self.lower_limit]
 617
 618                         if self.upper_limit is not None:
 619                                 args += ["--upper-limit", self.upper_limit]
 620
 621                 # Add interval
 622                 args += ["--start", util.make_interval(interval)]
 623
 624                 return args
 625
 626         def _add_defs(self):
 627                 use_prefix = len(self.objects) >= 2
 628
 629                 args = []
 630                 for object in self.objects:
 631                         if use_prefix:
 632                                 args += object.make_rrd_defs(object.id)
 633                         else:
 634                                 args += object.make_rrd_defs()
 635
 636                 return args
 637
 638         def _add_vdefs(self, args):
 639                 ret = []
 640
 641                 for arg in args:
 642                         ret.append(arg)
 643
 644                         # Search for all DEFs and CDEFs
 645                         m = re.match(DEF_MATCH, "%s" % arg)
 646                         if m:
 647                                 name = m.group(1)
 648
 649                                 # Add the VDEFs for minimum, maximum, etc. values
 650                                 ret += [
 651                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 652                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 653                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 654                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 655                                 ]
 656
 657                 return ret
 658
 659         def get_objects(self, *args, **kwargs):
 660                 object = self.plugin.get_object(*args, **kwargs)
 661
 662                 if object:
 663                         return [object,]
 664
 665                 return []
 666
 667         def generate_graph(self, interval=None, **kwargs):
 668                 assert self.objects, "Cannot render graph without any objects"
 669
 670                 # Make sure that all collected data is in the database
 671                 # to get a recent graph image
 672                 for object in self.objects:
 673                         object.commit()
 674
 675                 args = self._make_command_line(interval, **kwargs)
 676
 677                 self.log.info(_("Generating graph %s") % self)
 678
 679                 rrd_graph = self.rrd_graph
 680
 681                 # Add DEFs for all objects
 682                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 683                         args += self._add_defs()
 684
 685                 args += rrd_graph
 686                 args = self._add_vdefs(args)
 687
 688                 # Convert arguments to string
 689                 args = [str(e) for e in args]
 690
 691                 for arg in args:
 692                         self.log.debug("  %s" % arg)
 693
 694                 graph = rrdtool.graphv("-", *args)
 695
 696                 return {
 697                         "image"        : graph.get("image"),
 698                         "image_height" : graph.get("image_height"),
 699                         "image_width"  : graph.get("image_width"),
 700                 }
 701
 702         def graph_info(self):
 703                 """
 704                         Returns a dictionary with useful information
 705                         about this graph.
 706                 """
 707                 return {
 708                         "title"        : self.graph_title,
 709                         "object_id"    : self.object_id or "",
 710                         "template"     : self.name,
 711                 }