src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import os
  25 import re
  26 import rrdtool
  27 import time
  28 import unicodedata
  29
  30 from .. import locales
  31 from .. import util
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  36
  37 class Environment(object):
  38         """
  39                 Sets the correct environment for rrdtool to create
  40                 localised graphs and graphs in the correct timezone.
  41         """
  42         def __init__(self, timezone, locale):
  43                 # Build the new environment
  44                 self.new_environment = {
  45                         "TZ" : timezone or DEFAULT_TIMEZONE,
  46                 }
  47
  48                 for k in ("LANG", "LC_ALL"):
  49                         self.new_environment[k] = locale or DEFAULT_LOCALE
  50
  51         def __enter__(self):
  52                 # Save the current environment
  53                 self.old_environment = {}
  54                 for k in self.new_environment:
  55                         self.old_environment[k] = os.environ.get(k, None)
  56
  57                 # Apply the new one
  58                 os.environ.update(self.new_environment)
  59
  60         def __exit__(self, type, value, traceback):
  61                 # Roll back to the previous environment
  62                 for k, v in self.old_environment.items():
  63                         if v is None:
  64                                 try:
  65                                         del os.environ[k]
  66                                 except KeyError:
  67                                         pass
  68                         else:
  69                                 os.environ[k] = v
  70
  71
  72 class PluginRegistration(type):
  73         plugins = {}
  74
  75         def __init__(plugin, name, bases, dict):
  76                 type.__init__(plugin, name, bases, dict)
  77
  78                 # The main class from which is inherited is not registered
  79                 # as a plugin.
  80                 if name == "Plugin":
  81                         return
  82
  83                 if not all((plugin.name, plugin.description)):
  84                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  85
  86                 PluginRegistration.plugins[plugin.name] = plugin
  87
  88
  89 def get():
  90         """
  91                 Returns a list with all automatically registered plugins.
  92         """
  93         return PluginRegistration.plugins.values()
  94
  95 class Plugin(object, metaclass=PluginRegistration):
  96         # The name of this plugin.
  97         name = None
  98
  99         # A description for this plugin.
 100         description = None
 101
 102         # Templates which can be used to generate a graph out of
 103         # the data from this data source.
 104         templates = []
 105
 106         # The default interval for all plugins
 107         interval = 60
 108
 109         # Priority
 110         priority = 0
 111
 112         def __init__(self, collecty, **kwargs):
 113                 self.collecty = collecty
 114
 115                 # Check if this plugin was configured correctly.
 116                 assert self.name, "Name of the plugin is not set: %s" % self.name
 117                 assert self.description, "Description of the plugin is not set: %s" % self.description
 118
 119                 # Initialize the logger.
 120                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 126
 127         @property
 128         def path(self):
 129                 """
 130                         Returns the name of the sub directory in which all RRD files
 131                         for this plugin should be stored in.
 132                 """
 133                 return self.name
 134
 135         ### Basic methods
 136
 137         def init(self, **kwargs):
 138                 """
 139                         Do some custom initialization stuff here.
 140                 """
 141                 pass
 142
 143         def collect(self):
 144                 """
 145                         Gathers the statistical data, this plugin collects.
 146                 """
 147                 time_start = time.time()
 148
 149                 # Run through all objects of this plugin and call the collect method.
 150                 for object in self.objects:
 151                         now = datetime.datetime.utcnow()
 152
 153                         # Run collection
 154                         try:
 155                                 result = object.collect()
 156
 157                         # Catch any unhandled exceptions
 158                         except Exception as e:
 159                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 160                                 continue
 161
 162                         if not result:
 163                                 self.log.warning(_("Received empty result: %s") % object)
 164                                 continue
 165
 166                         # Format the result for RRDtool
 167                         result = self._format_result(result)
 168
 169                         self.log.debug(_("Collected %s: %s") % (object, result))
 170
 171                         # Add the object to the write queue so that the data is written
 172                         # to the databases later.
 173                         self.collecty.write_queue.add(object, now, result)
 174
 175                 # Returns the time this function took to complete.
 176                 delay = time.time() - time_start
 177
 178                 # Log some warning when a collect method takes too long to return some data
 179                 if delay >= 60:
 180                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 181                 else:
 182                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 183
 184         @staticmethod
 185         def _format_result(result):
 186                 if not isinstance(result, tuple) and not isinstance(result, list):
 187                         return result
 188
 189                 # Replace all Nones by UNKNOWN
 190                 s = []
 191
 192                 for e in result:
 193                         if e is None:
 194                                 e = "U"
 195
 196                         s.append("%s" % e)
 197
 198                 return ":".join(s)
 199
 200         def get_object(self, id):
 201                 for object in self.objects:
 202                         if not object.id == id:
 203                                 continue
 204
 205                         return object
 206
 207         def get_template(self, template_name, object_id, locale=None, timezone=None):
 208                 for template in self.templates:
 209                         if not template.name == template_name:
 210                                 continue
 211
 212                         return template(self, object_id, locale=locale, timezone=timezone)
 213
 214         def generate_graph(self, template_name, object_id="default",
 215                         timezone=None, locale=None, **kwargs):
 216                 template = self.get_template(template_name, object_id=object_id,
 217                         timezone=timezone, locale=locale)
 218                 if not template:
 219                         raise RuntimeError("Could not find template %s" % template_name)
 220
 221                 time_start = time.time()
 222
 223                 graph = template.generate_graph(**kwargs)
 224
 225                 duration = time.time() - time_start
 226                 self.log.debug(_("Generated graph %s in %.1fms") \
 227                         % (template, duration * 1000))
 228
 229                 return graph
 230
 231         def graph_info(self, template_name, object_id="default",
 232                         timezone=None, locale=None, **kwargs):
 233                 template = self.get_template(template_name, object_id=object_id,
 234                         timezone=timezone, locale=locale)
 235                 if not template:
 236                         raise RuntimeError("Could not find template %s" % template_name)
 237
 238                 return template.graph_info()
 239
 240         def last_update(self, object_id="default"):
 241                 object = self.get_object(object_id)
 242                 if not object:
 243                         raise RuntimeError("Could not find object %s" % object_id)
 244
 245                 return object.last_update()
 246
 247
 248 class Object(object):
 249         # The schema of the RRD database.
 250         rrd_schema = None
 251
 252         # RRA properties.
 253         rra_types     = ("AVERAGE", "MIN", "MAX")
 254         rra_timespans = (
 255                 ("1m", "10d"),
 256                 ("1h", "18M"),
 257                 ("1d",  "5y"),
 258         )
 259
 260         def __init__(self, plugin, *args, **kwargs):
 261                 self.plugin = plugin
 262
 263                 # Indicates if this object has collected its data
 264                 self.collected = False
 265
 266                 # Initialise this object
 267                 self.init(*args, **kwargs)
 268
 269                 # Create the database file.
 270                 self.create()
 271
 272         def __repr__(self):
 273                 return "<%s %s>" % (self.__class__.__name__, self.id)
 274
 275         def __lt__(self, other):
 276                 return self.id < other.id
 277
 278         @property
 279         def collecty(self):
 280                 return self.plugin.collecty
 281
 282         @property
 283         def log(self):
 284                 return self.plugin.log
 285
 286         @property
 287         def id(self):
 288                 """
 289                         Returns a UNIQUE identifier for this object. As this is incorporated
 290                         into the path of RRD file, it must only contain ASCII characters.
 291                 """
 292                 raise NotImplementedError
 293
 294         @property
 295         def file(self):
 296                 """
 297                         The absolute path to the RRD file of this plugin.
 298                 """
 299                 filename = self._normalise_filename("%s.rrd" % self.id)
 300
 301                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 302
 303         @staticmethod
 304         def _normalise_filename(filename):
 305                 # Convert the filename into ASCII characters only
 306                 filename = unicodedata.normalize("NFKC", filename)
 307
 308                 # Replace any spaces by dashes
 309                 filename = filename.replace(" ", "-")
 310
 311                 return filename
 312
 313         ### Basic methods
 314
 315         def init(self, *args, **kwargs):
 316                 """
 317                         Do some custom initialization stuff here.
 318                 """
 319                 pass
 320
 321         def create(self):
 322                 """
 323                         Creates an empty RRD file with the desired data structures.
 324                 """
 325                 # Skip if the file does already exist.
 326                 if os.path.exists(self.file):
 327                         return
 328
 329                 dirname = os.path.dirname(self.file)
 330                 if not os.path.exists(dirname):
 331                         os.makedirs(dirname)
 332
 333                 # Create argument list.
 334                 args = self.get_rrd_schema()
 335
 336                 rrdtool.create(self.file, *args)
 337
 338                 self.log.debug(_("Created RRD file %s.") % self.file)
 339                 for arg in args:
 340                         self.log.debug("  %s" % arg)
 341
 342         def info(self):
 343                 return rrdtool.info(self.file)
 344
 345         def last_update(self):
 346                 """
 347                         Returns a dictionary with the timestamp and
 348                         data set of the last database update.
 349                 """
 350                 return {
 351                         "dataset"   : self.last_dataset,
 352                         "timestamp" : self.last_updated,
 353                 }
 354
 355         def _last_update(self):
 356                 return rrdtool.lastupdate(self.file)
 357
 358         @property
 359         def last_updated(self):
 360                 """
 361                         Returns the timestamp when this database was last updated
 362                 """
 363                 lu = self._last_update()
 364
 365                 if lu:
 366                         return lu.get("date")
 367
 368         @property
 369         def last_dataset(self):
 370                 """
 371                         Returns the latest dataset in the database
 372                 """
 373                 lu = self._last_update()
 374
 375                 if lu:
 376                         return lu.get("ds")
 377
 378         @property
 379         def stepsize(self):
 380                 return self.plugin.interval
 381
 382         @property
 383         def heartbeat(self):
 384                 return self.stepsize * 2
 385
 386         def get_rrd_schema(self):
 387                 schema = [
 388                         "--step", "%s" % self.stepsize,
 389                 ]
 390                 for line in self.rrd_schema:
 391                         if line.startswith("DS:"):
 392                                 try:
 393                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 394
 395                                         line = ":".join((
 396                                                 prefix,
 397                                                 name,
 398                                                 type,
 399                                                 "%s" % self.heartbeat,
 400                                                 lower_limit,
 401                                                 upper_limit
 402                                         ))
 403                                 except ValueError:
 404                                         pass
 405
 406                         schema.append(line)
 407
 408                 xff = 0.1
 409
 410                 for steps, rows in self.rra_timespans:
 411                         for type in self.rra_types:
 412                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 413
 414                 return schema
 415
 416         @property
 417         def rrd_schema_names(self):
 418                 ret = []
 419
 420                 for line in self.rrd_schema:
 421                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 422                         ret.append(name)
 423
 424                 return ret
 425
 426         def make_rrd_defs(self, prefix=None):
 427                 defs = []
 428
 429                 for name in self.rrd_schema_names:
 430                         if prefix:
 431                                 p = "%s_%s" % (prefix, name)
 432                         else:
 433                                 p = name
 434
 435                         defs += [
 436                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 437                         ]
 438
 439                 return defs
 440
 441         def get_stddev(self, interval=None):
 442                 args = self.make_rrd_defs()
 443
 444                 # Add the correct interval
 445                 args += ["--start", util.make_interval(interval)]
 446
 447                 for name in self.rrd_schema_names:
 448                         args += [
 449                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 450                                 "PRINT:%s_stddev:%%lf" % name,
 451                         ]
 452
 453                 x, y, vals = rrdtool.graph("/dev/null", *args)
 454                 return dict(zip(self.rrd_schema_names, vals))
 455
 456         def execute(self):
 457                 if self.collected:
 458                         raise RuntimeError("This object has already collected its data")
 459
 460                 self.collected = True
 461                 self.now = datetime.datetime.utcnow()
 462
 463                 # Call the collect
 464                 result = self.collect()
 465
 466         def commit(self):
 467                 """
 468                         Will commit the collected data to the database.
 469                 """
 470                 # Make sure that the RRD database has been created
 471                 self.create()
 472
 473                 # Write everything to disk that is in the write queue
 474                 self.collecty.write_queue.commit_file(self.file)
 475
 476         # Convenience functions for plugin authors
 477
 478         def read_file(self, *args, strip=True):
 479                 """
 480                         Reads the content of the given file
 481                 """
 482                 filename = os.path.join(*args)
 483
 484                 with open(filename) as f:
 485                         value = f.read()
 486
 487                 # Strip any excess whitespace
 488                 if strip:
 489                         value = value.strip()
 490
 491                 return value
 492
 493         def read_file_integer(self, filename):
 494                 """
 495                         Reads the content from a file and returns it as an integer
 496                 """
 497                 value = self.read_file(filename)
 498
 499                 try:
 500                         return int(value)
 501                 except ValueError:
 502                         return None
 503
 504         def read_proc_stat(self):
 505                 """
 506                         Reads /proc/stat and returns it as a dictionary
 507                 """
 508                 ret = {}
 509
 510                 with open("/proc/stat") as f:
 511                         for line in f:
 512                                 # Split the key from the rest of the line
 513                                 key, line = line.split(" ", 1)
 514
 515                                 # Remove any line breaks
 516                                 ret[key] = line.rstrip()
 517
 518                 return ret
 519
 520         def read_proc_meminfo(self):
 521                 ret = {}
 522
 523                 with open("/proc/meminfo") as f:
 524                         for line in f:
 525                                 # Split the key from the rest of the line
 526                                 key, line = line.split(":", 1)
 527
 528                                 # Remove any whitespace
 529                                 line = line.strip()
 530
 531                                 # Remove any trailing kB
 532                                 if line.endswith(" kB"):
 533                                         line = line[:-3]
 534
 535                                 # Try to convert to integer
 536                                 try:
 537                                         line = int(line)
 538                                 except (TypeError, ValueError):
 539                                         continue
 540
 541                                 ret[key] = line
 542
 543                 return ret
 544
 545
 546 class GraphTemplate(object):
 547         # A unique name to identify this graph template.
 548         name = None
 549
 550         # Headline of the graph image
 551         graph_title = None
 552
 553         # Vertical label of the graph
 554         graph_vertical_label = None
 555
 556         # Limits
 557         lower_limit = None
 558         upper_limit = None
 559
 560         # Instructions how to create the graph.
 561         rrd_graph = None
 562
 563         # Extra arguments passed to rrdgraph.
 564         rrd_graph_args = []
 565
 566         # Default dimensions for this graph
 567         height = GRAPH_DEFAULT_HEIGHT
 568         width  = GRAPH_DEFAULT_WIDTH
 569
 570         def __init__(self, plugin, object_id, locale=None, timezone=None):
 571                 self.plugin = plugin
 572
 573                 # Save localisation parameters
 574                 self.locale = locales.get(locale)
 575                 self.timezone = timezone
 576
 577                 # Get all required RRD objects
 578                 self.object_id = object_id
 579
 580                 # Get the main object
 581                 self.objects = self.get_objects(self.object_id)
 582                 self.objects.sort()
 583
 584         def __repr__(self):
 585                 return "<%s>" % self.__class__.__name__
 586
 587         @property
 588         def collecty(self):
 589                 return self.plugin.collecty
 590
 591         @property
 592         def log(self):
 593                 return self.plugin.log
 594
 595         @property
 596         def object(self):
 597                 """
 598                         Shortcut to the main object
 599                 """
 600                 if len(self.objects) == 1:
 601                         return self.objects[0]
 602
 603         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 604                         width=None, height=None, with_title=True, thumbnail=False):
 605                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 606
 607                 # Set the default dimensions
 608                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 609
 610                 # A thumbnail doesn't have a legend and other labels
 611                 if thumbnail:
 612                         args.append("--only-graph")
 613
 614                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 615                         default_width = THUMBNAIL_DEFAULT_WIDTH
 616
 617                 args += [
 618                         "--imgformat", format,
 619                         "--height", "%s" % (height or default_height),
 620                         "--width", "%s" % (width or default_width),
 621                 ]
 622
 623                 args += self.rrd_graph_args
 624
 625                 # Graph title
 626                 if with_title and self.graph_title:
 627                         args += ["--title", self.graph_title]
 628
 629                 # Vertical label
 630                 if self.graph_vertical_label:
 631                         args += ["--vertical-label", self.graph_vertical_label]
 632
 633                 if self.lower_limit is not None or self.upper_limit is not None:
 634                         # Force to honour the set limits
 635                         args.append("--rigid")
 636
 637                         if self.lower_limit is not None:
 638                                 args += ["--lower-limit", self.lower_limit]
 639
 640                         if self.upper_limit is not None:
 641                                 args += ["--upper-limit", self.upper_limit]
 642
 643                 # Add interval
 644                 args += ["--start", util.make_interval(interval)]
 645
 646                 return args
 647
 648         def _add_defs(self):
 649                 use_prefix = len(self.objects) >= 2
 650
 651                 args = []
 652                 for object in self.objects:
 653                         if use_prefix:
 654                                 args += object.make_rrd_defs(object.id)
 655                         else:
 656                                 args += object.make_rrd_defs()
 657
 658                 return args
 659
 660         def _add_vdefs(self, args):
 661                 ret = []
 662
 663                 for arg in args:
 664                         ret.append(arg)
 665
 666                         # Search for all DEFs and CDEFs
 667                         m = re.match(DEF_MATCH, "%s" % arg)
 668                         if m:
 669                                 name = m.group(1)
 670
 671                                 # Add the VDEFs for minimum, maximum, etc. values
 672                                 ret += [
 673                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 674                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 675                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 676                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 677                                 ]
 678
 679                 return ret
 680
 681         def get_objects(self, *args, **kwargs):
 682                 object = self.plugin.get_object(*args, **kwargs)
 683
 684                 if object:
 685                         return [object,]
 686
 687                 return []
 688
 689         def generate_graph(self, interval=None, **kwargs):
 690                 assert self.objects, "Cannot render graph without any objects"
 691
 692                 # Make sure that all collected data is in the database
 693                 # to get a recent graph image
 694                 for object in self.objects:
 695                         object.commit()
 696
 697                 args = self._make_command_line(interval, **kwargs)
 698
 699                 self.log.info(_("Generating graph %s") % self)
 700
 701                 rrd_graph = self.rrd_graph
 702
 703                 # Add DEFs for all objects
 704                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 705                         args += self._add_defs()
 706
 707                 args += rrd_graph
 708                 args = self._add_vdefs(args)
 709
 710                 # Convert arguments to string
 711                 args = [str(e) for e in args]
 712
 713                 for arg in args:
 714                         self.log.debug("  %s" % arg)
 715
 716                 with Environment(self.timezone, self.locale.lang):
 717                         graph = rrdtool.graphv("-", *args)
 718
 719                 return {
 720                         "image"        : graph.get("image"),
 721                         "image_height" : graph.get("image_height"),
 722                         "image_width"  : graph.get("image_width"),
 723                 }
 724
 725         def graph_info(self):
 726                 """
 727                         Returns a dictionary with useful information
 728                         about this graph.
 729                 """
 730                 return {
 731                         "title"        : self.graph_title,
 732                         "object_id"    : self.object_id or "",
 733                         "template"     : self.name,
 734                 }