src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import logging
  23 import os
  24 import re
  25 import rrdtool
  26 import time
  27 import unicodedata
  28
  29 from .. import locales
  30 from .. import util
  31 from ..constants import *
  32 from ..i18n import _
  33
  34 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  35
  36 class Environment(object):
  37         """
  38                 Sets the correct environment for rrdtool to create
  39                 localised graphs and graphs in the correct timezone.
  40         """
  41         def __init__(self, timezone, locale):
  42                 # Build the new environment
  43                 self.new_environment = {
  44                         "TZ" : timezone or "UTC",
  45                 }
  46
  47                 for k in ("LANG", "LC_ALL"):
  48                         self.new_environment[k] = locale or "en_US.utf-8"
  49
  50         def __enter__(self):
  51                 # Save the current environment
  52                 self.old_environment = {}
  53
  54                 for k in self.new_environment:
  55                         self.old_environment[k] = os.environ.get(k, None)
  56
  57                 # Apply the new one
  58                 os.environ.update(self.new_environment)
  59
  60         def __exit__(self, type, value, traceback):
  61                 # Roll back to the previous environment
  62                 for k, v in self.old_environment.items():
  63                         if v is None:
  64                                 try:
  65                                         del os.environ[k]
  66                                 except KeyError:
  67                                         pass
  68                         else:
  69                                 os.environ[k] = v
  70
  71
  72 class PluginRegistration(type):
  73         plugins = {}
  74
  75         def __init__(plugin, name, bases, dict):
  76                 type.__init__(plugin, name, bases, dict)
  77
  78                 # The main class from which is inherited is not registered
  79                 # as a plugin.
  80                 if name == "Plugin":
  81                         return
  82
  83                 if not all((plugin.name, plugin.description)):
  84                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  85
  86                 PluginRegistration.plugins[plugin.name] = plugin
  87
  88
  89 def get():
  90         """
  91                 Returns a list with all automatically registered plugins.
  92         """
  93         return PluginRegistration.plugins.values()
  94
  95 class Plugin(object, metaclass=PluginRegistration):
  96         # The name of this plugin.
  97         name = None
  98
  99         # A description for this plugin.
 100         description = None
 101
 102         # Templates which can be used to generate a graph out of
 103         # the data from this data source.
 104         templates = []
 105
 106         # The default interval for all plugins
 107         interval = 60
 108
 109         # Priority
 110         priority = 0
 111
 112         def __init__(self, collecty, **kwargs):
 113                 self.collecty = collecty
 114
 115                 # Check if this plugin was configured correctly.
 116                 assert self.name, "Name of the plugin is not set: %s" % self.name
 117                 assert self.description, "Description of the plugin is not set: %s" % self.description
 118
 119                 # Initialize the logger.
 120                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 126
 127         @property
 128         def path(self):
 129                 """
 130                         Returns the name of the sub directory in which all RRD files
 131                         for this plugin should be stored in.
 132                 """
 133                 return self.name
 134
 135         ### Basic methods
 136
 137         def init(self, **kwargs):
 138                 """
 139                         Do some custom initialization stuff here.
 140                 """
 141                 pass
 142
 143         def collect(self):
 144                 """
 145                         Gathers the statistical data, this plugin collects.
 146                 """
 147                 time_start = time.time()
 148
 149                 # Run through all objects of this plugin and call the collect method.
 150                 for object in self.objects:
 151                         # Run collection
 152                         try:
 153                                 result = object.collect()
 154
 155                         # Catch any unhandled exceptions
 156                         except Exception as e:
 157                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 158                                 continue
 159
 160                         if not result:
 161                                 self.log.warning(_("Received empty result: %s") % object)
 162                                 continue
 163
 164                         # Add the object to the write queue so that the data is written
 165                         # to the databases later.
 166                         result = self.collecty.write_queue.submit(object, result)
 167
 168                         self.log.debug(_("Collected %s: %s") % (object, result))
 169
 170                 # Returns the time this function took to complete.
 171                 delay = time.time() - time_start
 172
 173                 # Log some warning when a collect method takes too long to return some data
 174                 if delay >= 60:
 175                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 176                 else:
 177                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 178
 179         def get_object(self, id):
 180                 for object in self.objects:
 181                         if not object.id == id:
 182                                 continue
 183
 184                         return object
 185
 186         def get_template(self, template_name, object_id, locale=None, timezone=None):
 187                 for template in self.templates:
 188                         if not template.name == template_name:
 189                                 continue
 190
 191                         return template(self, object_id, locale=locale, timezone=timezone)
 192
 193         def generate_graph(self, template_name, object_id="default",
 194                         timezone=None, locale=None, **kwargs):
 195                 template = self.get_template(template_name, object_id=object_id,
 196                         timezone=timezone, locale=locale)
 197                 if not template:
 198                         raise RuntimeError("Could not find template %s" % template_name)
 199
 200                 time_start = time.time()
 201
 202                 graph = template.generate_graph(**kwargs)
 203
 204                 duration = time.time() - time_start
 205                 self.log.debug(_("Generated graph %s in %.1fms") \
 206                         % (template, duration * 1000))
 207
 208                 return graph
 209
 210         def graph_info(self, template_name, object_id="default",
 211                         timezone=None, locale=None, **kwargs):
 212                 template = self.get_template(template_name, object_id=object_id,
 213                         timezone=timezone, locale=locale)
 214                 if not template:
 215                         raise RuntimeError("Could not find template %s" % template_name)
 216
 217                 return template.graph_info()
 218
 219         def last_update(self, object_id="default"):
 220                 object = self.get_object(object_id)
 221                 if not object:
 222                         raise RuntimeError("Could not find object %s" % object_id)
 223
 224                 return object.last_update()
 225
 226
 227 class Object(object):
 228         # The schema of the RRD database.
 229         rrd_schema = None
 230
 231         # RRA properties.
 232         rra_types     = ("AVERAGE", "MIN", "MAX")
 233         rra_timespans = (
 234                 ("1m", "10d"),
 235                 ("1h", "18M"),
 236                 ("1d",  "5y"),
 237         )
 238
 239         def __init__(self, plugin, *args, **kwargs):
 240                 self.plugin = plugin
 241
 242                 # Initialise this object
 243                 self.init(*args, **kwargs)
 244
 245                 # Create the database file.
 246                 self.create()
 247
 248         def __repr__(self):
 249                 return "<%s %s>" % (self.__class__.__name__, self.id)
 250
 251         def __lt__(self, other):
 252                 return self.id < other.id
 253
 254         @property
 255         def collecty(self):
 256                 return self.plugin.collecty
 257
 258         @property
 259         def log(self):
 260                 return self.plugin.log
 261
 262         @property
 263         def id(self):
 264                 """
 265                         Returns a UNIQUE identifier for this object. As this is incorporated
 266                         into the path of RRD file, it must only contain ASCII characters.
 267                 """
 268                 raise NotImplementedError
 269
 270         @property
 271         def file(self):
 272                 """
 273                         The absolute path to the RRD file of this plugin.
 274                 """
 275                 filename = self._normalise_filename("%s.rrd" % self.id)
 276
 277                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 278
 279         @staticmethod
 280         def _normalise_filename(filename):
 281                 # Convert the filename into ASCII characters only
 282                 filename = unicodedata.normalize("NFKC", filename)
 283
 284                 # Replace any spaces by dashes
 285                 filename = filename.replace(" ", "-")
 286
 287                 return filename
 288
 289         ### Basic methods
 290
 291         def init(self, *args, **kwargs):
 292                 """
 293                         Do some custom initialization stuff here.
 294                 """
 295                 pass
 296
 297         def create(self):
 298                 """
 299                         Creates an empty RRD file with the desired data structures.
 300                 """
 301                 # Skip if the file does already exist.
 302                 if os.path.exists(self.file):
 303                         return
 304
 305                 dirname = os.path.dirname(self.file)
 306                 if not os.path.exists(dirname):
 307                         os.makedirs(dirname)
 308
 309                 # Create argument list.
 310                 args = self.get_rrd_schema()
 311
 312                 rrdtool.create(self.file, *args)
 313
 314                 self.log.debug(_("Created RRD file %s.") % self.file)
 315                 for arg in args:
 316                         self.log.debug("  %s" % arg)
 317
 318         def info(self):
 319                 return rrdtool.info(self.file)
 320
 321         def last_update(self):
 322                 """
 323                         Returns a dictionary with the timestamp and
 324                         data set of the last database update.
 325                 """
 326                 return {
 327                         "dataset"   : self.last_dataset,
 328                         "timestamp" : self.last_updated,
 329                 }
 330
 331         def _last_update(self):
 332                 return rrdtool.lastupdate(self.file)
 333
 334         @property
 335         def last_updated(self):
 336                 """
 337                         Returns the timestamp when this database was last updated
 338                 """
 339                 lu = self._last_update()
 340
 341                 if lu:
 342                         return lu.get("date")
 343
 344         @property
 345         def last_dataset(self):
 346                 """
 347                         Returns the latest dataset in the database
 348                 """
 349                 lu = self._last_update()
 350
 351                 if lu:
 352                         return lu.get("ds")
 353
 354         @property
 355         def stepsize(self):
 356                 return self.plugin.interval
 357
 358         @property
 359         def heartbeat(self):
 360                 return self.stepsize * 2
 361
 362         def get_rrd_schema(self):
 363                 schema = [
 364                         "--step", "%s" % self.stepsize,
 365                 ]
 366                 for line in self.rrd_schema:
 367                         if line.startswith("DS:"):
 368                                 try:
 369                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 370
 371                                         line = ":".join((
 372                                                 prefix,
 373                                                 name,
 374                                                 type,
 375                                                 "%s" % self.heartbeat,
 376                                                 lower_limit,
 377                                                 upper_limit
 378                                         ))
 379                                 except ValueError:
 380                                         pass
 381
 382                         schema.append(line)
 383
 384                 xff = 0.1
 385
 386                 for steps, rows in self.rra_timespans:
 387                         for type in self.rra_types:
 388                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 389
 390                 return schema
 391
 392         @property
 393         def rrd_schema_names(self):
 394                 ret = []
 395
 396                 for line in self.rrd_schema:
 397                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 398                         ret.append(name)
 399
 400                 return ret
 401
 402         def make_rrd_defs(self, prefix=None):
 403                 defs = []
 404
 405                 for name in self.rrd_schema_names:
 406                         if prefix:
 407                                 p = "%s_%s" % (prefix, name)
 408                         else:
 409                                 p = name
 410
 411                         defs += [
 412                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 413                         ]
 414
 415                 return defs
 416
 417         def get_stddev(self, interval=None):
 418                 args = self.make_rrd_defs()
 419
 420                 # Add the correct interval
 421                 args += ["--start", util.make_interval(interval)]
 422
 423                 for name in self.rrd_schema_names:
 424                         args += [
 425                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 426                                 "PRINT:%s_stddev:%%lf" % name,
 427                         ]
 428
 429                 x, y, vals = rrdtool.graph("/dev/null", *args)
 430                 return dict(zip(self.rrd_schema_names, vals))
 431
 432         def commit(self):
 433                 """
 434                         Will commit the collected data to the database.
 435                 """
 436                 # Make sure that the RRD database has been created
 437                 self.create()
 438
 439                 # Write everything to disk that is in the write queue
 440                 self.collecty.write_queue.commit_file(self.file)
 441
 442         # Convenience functions for plugin authors
 443
 444         def read_file(self, *args, strip=True):
 445                 """
 446                         Reads the content of the given file
 447                 """
 448                 filename = os.path.join(*args)
 449
 450                 with open(filename) as f:
 451                         value = f.read()
 452
 453                 # Strip any excess whitespace
 454                 if strip:
 455                         value = value.strip()
 456
 457                 return value
 458
 459         def read_file_integer(self, filename):
 460                 """
 461                         Reads the content from a file and returns it as an integer
 462                 """
 463                 value = self.read_file(filename)
 464
 465                 try:
 466                         return int(value)
 467                 except ValueError:
 468                         return None
 469
 470         def read_proc_stat(self):
 471                 """
 472                         Reads /proc/stat and returns it as a dictionary
 473                 """
 474                 ret = {}
 475
 476                 with open("/proc/stat") as f:
 477                         for line in f:
 478                                 # Split the key from the rest of the line
 479                                 key, line = line.split(" ", 1)
 480
 481                                 # Remove any line breaks
 482                                 ret[key] = line.rstrip()
 483
 484                 return ret
 485
 486         def read_proc_meminfo(self):
 487                 ret = {}
 488
 489                 with open("/proc/meminfo") as f:
 490                         for line in f:
 491                                 # Split the key from the rest of the line
 492                                 key, line = line.split(":", 1)
 493
 494                                 # Remove any whitespace
 495                                 line = line.strip()
 496
 497                                 # Remove any trailing kB
 498                                 if line.endswith(" kB"):
 499                                         line = line[:-3]
 500
 501                                 # Try to convert to integer
 502                                 try:
 503                                         line = int(line)
 504                                 except (TypeError, ValueError):
 505                                         continue
 506
 507                                 ret[key] = line
 508
 509                 return ret
 510
 511
 512 class GraphTemplate(object):
 513         # A unique name to identify this graph template.
 514         name = None
 515
 516         # Headline of the graph image
 517         graph_title = None
 518
 519         # Vertical label of the graph
 520         graph_vertical_label = None
 521
 522         # Limits
 523         lower_limit = None
 524         upper_limit = None
 525
 526         # Instructions how to create the graph.
 527         rrd_graph = None
 528
 529         # Extra arguments passed to rrdgraph.
 530         rrd_graph_args = []
 531
 532         def __init__(self, plugin, object_id, locale=None, timezone=None):
 533                 self.plugin = plugin
 534
 535                 # Save localisation parameters
 536                 self.locale = locales.get(locale)
 537                 self.timezone = timezone
 538
 539                 # Get all required RRD objects
 540                 self.object_id = object_id
 541
 542                 # Get the main object
 543                 self.objects = self.get_objects(self.object_id)
 544                 self.objects.sort()
 545
 546         def __repr__(self):
 547                 return "<%s>" % self.__class__.__name__
 548
 549         @property
 550         def collecty(self):
 551                 return self.plugin.collecty
 552
 553         @property
 554         def log(self):
 555                 return self.plugin.log
 556
 557         @property
 558         def object(self):
 559                 """
 560                         Shortcut to the main object
 561                 """
 562                 if len(self.objects) == 1:
 563                         return self.objects[0]
 564
 565         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 566                         width=None, height=None, with_title=True, thumbnail=False):
 567                 args = [
 568                         # Change the background colour
 569                         "--color", "BACK#FFFFFFFF",
 570
 571                         # Disable the border around the image
 572                         "--border", "0",
 573
 574                         # Let's width and height define the size of the entire image
 575                         "--full-size-mode",
 576
 577                         # Gives the curves a more organic look
 578                         "--slope-mode",
 579
 580                         # Show nicer labels
 581                         "--dynamic-labels",
 582
 583                         # Brand all generated graphs
 584                         "--watermark", _("Created by collecty"),
 585                 ]
 586
 587                 # Set the default dimensions
 588                 default_height, default_width = 960, 480
 589
 590                 # A thumbnail doesn't have a legend and other labels
 591                 if thumbnail:
 592                         args.append("--only-graph")
 593
 594                         default_height, default_width = 80, 20
 595
 596                 args += [
 597                         "--imgformat", format,
 598                         "--height", "%s" % (height or default_height),
 599                         "--width", "%s" % (width or default_width),
 600                 ]
 601
 602                 args += self.rrd_graph_args
 603
 604                 # Graph title
 605                 if with_title and self.graph_title:
 606                         args += ["--title", self.graph_title]
 607
 608                 # Vertical label
 609                 if self.graph_vertical_label:
 610                         args += ["--vertical-label", self.graph_vertical_label]
 611
 612                 if self.lower_limit is not None or self.upper_limit is not None:
 613                         # Force to honour the set limits
 614                         args.append("--rigid")
 615
 616                         if self.lower_limit is not None:
 617                                 args += ["--lower-limit", self.lower_limit]
 618
 619                         if self.upper_limit is not None:
 620                                 args += ["--upper-limit", self.upper_limit]
 621
 622                 # Add interval
 623                 args += ["--start", util.make_interval(interval)]
 624
 625                 return args
 626
 627         def _add_defs(self):
 628                 use_prefix = len(self.objects) >= 2
 629
 630                 args = []
 631                 for object in self.objects:
 632                         if use_prefix:
 633                                 args += object.make_rrd_defs(object.id)
 634                         else:
 635                                 args += object.make_rrd_defs()
 636
 637                 return args
 638
 639         def _add_vdefs(self, args):
 640                 ret = []
 641
 642                 for arg in args:
 643                         ret.append(arg)
 644
 645                         # Search for all DEFs and CDEFs
 646                         m = re.match(DEF_MATCH, "%s" % arg)
 647                         if m:
 648                                 name = m.group(1)
 649
 650                                 # Add the VDEFs for minimum, maximum, etc. values
 651                                 ret += [
 652                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 653                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 654                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 655                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 656                                 ]
 657
 658                 return ret
 659
 660         def get_objects(self, *args, **kwargs):
 661                 object = self.plugin.get_object(*args, **kwargs)
 662
 663                 if object:
 664                         return [object,]
 665
 666                 return []
 667
 668         def generate_graph(self, interval=None, **kwargs):
 669                 assert self.objects, "Cannot render graph without any objects"
 670
 671                 # Make sure that all collected data is in the database
 672                 # to get a recent graph image
 673                 for object in self.objects:
 674                         object.commit()
 675
 676                 args = self._make_command_line(interval, **kwargs)
 677
 678                 self.log.info(_("Generating graph %s") % self)
 679
 680                 rrd_graph = self.rrd_graph
 681
 682                 # Add DEFs for all objects
 683                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 684                         args += self._add_defs()
 685
 686                 args += rrd_graph
 687                 args = self._add_vdefs(args)
 688
 689                 # Convert arguments to string
 690                 args = [str(e) for e in args]
 691
 692                 for arg in args:
 693                         self.log.debug("  %s" % arg)
 694
 695                 with Environment(self.timezone, self.locale.lang):
 696                         graph = rrdtool.graphv("-", *args)
 697
 698                 return {
 699                         "image"        : graph.get("image"),
 700                         "image_height" : graph.get("image_height"),
 701                         "image_width"  : graph.get("image_width"),
 702                 }
 703
 704         def graph_info(self):
 705                 """
 706                         Returns a dictionary with useful information
 707                         about this graph.
 708                 """
 709                 return {
 710                         "title"        : self.graph_title,
 711                         "object_id"    : self.object_id or "",
 712                         "template"     : self.name,
 713                 }