src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import logging
  23 import os
  24 import re
  25 import rrdtool
  26 import time
  27 import unicodedata
  28
  29 from .. import util
  30 from ..constants import *
  31 from ..i18n import _
  32
  33 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  34
  35 class Environment(object):
  36         """
  37                 Sets the correct environment for rrdtool to create
  38                 localised graphs and graphs in the correct timezone.
  39         """
  40         def __init__(self, timezone="UTC", locale="en_US.utf-8"):
  41                 # Build the new environment
  42                 self.new_environment = {
  43                         "LANGUAGE" : locale,
  44                         "LC_ALL"   : locale,
  45                         "TZ"       : timezone,
  46                 }
  47
  48         def __enter__(self):
  49                 # Save the current environment
  50                 self.old_environment = {}
  51
  52                 for k in self.new_environment:
  53                         # Store the old value
  54                         self.old_environment[k] = os.environ.get(k, None)
  55
  56                         # Apply the new one
  57                         if self.new_environment[k]:
  58                                 os.environ.update(self.new_environment[k])
  59
  60         def __exit__(self, type, value, traceback):
  61                 # Roll back to the previous environment
  62                 for k, v in self.old_environment.items():
  63                         if v is None:
  64                                 try:
  65                                         del os.environ[k]
  66                                 except KeyError:
  67                                         pass
  68                         else:
  69                                 os.environ[k] = v
  70
  71
  72 class PluginRegistration(type):
  73         plugins = {}
  74
  75         def __init__(plugin, name, bases, dict):
  76                 type.__init__(plugin, name, bases, dict)
  77
  78                 # The main class from which is inherited is not registered
  79                 # as a plugin.
  80                 if name == "Plugin":
  81                         return
  82
  83                 if not all((plugin.name, plugin.description)):
  84                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  85
  86                 PluginRegistration.plugins[plugin.name] = plugin
  87
  88
  89 def get():
  90         """
  91                 Returns a list with all automatically registered plugins.
  92         """
  93         return PluginRegistration.plugins.values()
  94
  95 class Plugin(object, metaclass=PluginRegistration):
  96         # The name of this plugin.
  97         name = None
  98
  99         # A description for this plugin.
 100         description = None
 101
 102         # Templates which can be used to generate a graph out of
 103         # the data from this data source.
 104         templates = []
 105
 106         # The default interval for all plugins
 107         interval = 60
 108
 109         # Priority
 110         priority = 0
 111
 112         def __init__(self, collecty, **kwargs):
 113                 self.collecty = collecty
 114
 115                 # Check if this plugin was configured correctly.
 116                 assert self.name, "Name of the plugin is not set: %s" % self.name
 117                 assert self.description, "Description of the plugin is not set: %s" % self.description
 118
 119                 # Initialize the logger.
 120                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 126
 127         @property
 128         def path(self):
 129                 """
 130                         Returns the name of the sub directory in which all RRD files
 131                         for this plugin should be stored in.
 132                 """
 133                 return self.name
 134
 135         ### Basic methods
 136
 137         def init(self, **kwargs):
 138                 """
 139                         Do some custom initialization stuff here.
 140                 """
 141                 pass
 142
 143         def collect(self):
 144                 """
 145                         Gathers the statistical data, this plugin collects.
 146                 """
 147                 time_start = time.time()
 148
 149                 # Run through all objects of this plugin and call the collect method.
 150                 for object in self.objects:
 151                         # Run collection
 152                         try:
 153                                 result = object.collect()
 154
 155                         # Catch any unhandled exceptions
 156                         except Exception as e:
 157                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 158                                 continue
 159
 160                         if not result:
 161                                 self.log.warning(_("Received empty result: %s") % object)
 162                                 continue
 163
 164                         # Add the object to the write queue so that the data is written
 165                         # to the databases later.
 166                         result = self.collecty.write_queue.submit(object, result)
 167
 168                         self.log.debug(_("Collected %s: %s") % (object, result))
 169
 170                 # Returns the time this function took to complete.
 171                 delay = time.time() - time_start
 172
 173                 # Log some warning when a collect method takes too long to return some data
 174                 if delay >= 60:
 175                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 176                 else:
 177                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 178
 179         def get_object(self, id):
 180                 for object in self.objects:
 181                         if not object.id == id:
 182                                 continue
 183
 184                         return object
 185
 186         def get_template(self, template_name, object_id, locale=None, timezone=None):
 187                 for template in self.templates:
 188                         if not template.name == template_name:
 189                                 continue
 190
 191                         return template(self, object_id, locale=locale, timezone=timezone)
 192
 193         def generate_graph(self, template_name, object_id="default",
 194                         timezone=None, locale=None, **kwargs):
 195                 template = self.get_template(template_name, object_id=object_id,
 196                         timezone=timezone, locale=locale)
 197                 if not template:
 198                         raise RuntimeError("Could not find template %s" % template_name)
 199
 200                 time_start = time.time()
 201
 202                 with Environment(timezone=timezone, locale=locale):
 203                         graph = template.generate_graph(**kwargs)
 204
 205                 duration = time.time() - time_start
 206                 self.log.debug(_("Generated graph %s in %.1fms") \
 207                         % (template, duration * 1000))
 208
 209                 return graph
 210
 211         def graph_info(self, template_name, object_id="default",
 212                         timezone=None, locale=None, **kwargs):
 213                 template = self.get_template(template_name, object_id=object_id,
 214                         timezone=timezone, locale=locale)
 215                 if not template:
 216                         raise RuntimeError("Could not find template %s" % template_name)
 217
 218                 return template.graph_info()
 219
 220         def last_update(self, object_id="default"):
 221                 object = self.get_object(object_id)
 222                 if not object:
 223                         raise RuntimeError("Could not find object %s" % object_id)
 224
 225                 return object.last_update()
 226
 227
 228 class Object(object):
 229         # The schema of the RRD database.
 230         rrd_schema = None
 231
 232         # RRA properties.
 233         rra_types     = ("AVERAGE", "MIN", "MAX")
 234         rra_timespans = (
 235                 ("1m", "10d"),
 236                 ("1h", "18M"),
 237                 ("1d",  "5y"),
 238         )
 239
 240         def __init__(self, plugin, *args, **kwargs):
 241                 self.plugin = plugin
 242
 243                 # Initialise this object
 244                 self.init(*args, **kwargs)
 245
 246                 # Create the database file.
 247                 self.create()
 248
 249         def __repr__(self):
 250                 return "<%s %s>" % (self.__class__.__name__, self.id)
 251
 252         def __lt__(self, other):
 253                 return self.id < other.id
 254
 255         @property
 256         def collecty(self):
 257                 return self.plugin.collecty
 258
 259         @property
 260         def log(self):
 261                 return self.plugin.log
 262
 263         @property
 264         def id(self):
 265                 """
 266                         Returns a UNIQUE identifier for this object. As this is incorporated
 267                         into the path of RRD file, it must only contain ASCII characters.
 268                 """
 269                 raise NotImplementedError
 270
 271         @property
 272         def file(self):
 273                 """
 274                         The absolute path to the RRD file of this plugin.
 275                 """
 276                 filename = self._normalise_filename("%s.rrd" % self.id)
 277
 278                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 279
 280         @staticmethod
 281         def _normalise_filename(filename):
 282                 # Convert the filename into ASCII characters only
 283                 filename = unicodedata.normalize("NFKC", filename)
 284
 285                 # Replace any spaces by dashes
 286                 filename = filename.replace(" ", "-")
 287
 288                 return filename
 289
 290         ### Basic methods
 291
 292         def init(self, *args, **kwargs):
 293                 """
 294                         Do some custom initialization stuff here.
 295                 """
 296                 pass
 297
 298         def create(self):
 299                 """
 300                         Creates an empty RRD file with the desired data structures.
 301                 """
 302                 # Skip if the file does already exist.
 303                 if os.path.exists(self.file):
 304                         return
 305
 306                 dirname = os.path.dirname(self.file)
 307                 if not os.path.exists(dirname):
 308                         os.makedirs(dirname)
 309
 310                 # Create argument list.
 311                 args = self.get_rrd_schema()
 312
 313                 rrdtool.create(self.file, *args)
 314
 315                 self.log.debug(_("Created RRD file %s.") % self.file)
 316                 for arg in args:
 317                         self.log.debug("  %s" % arg)
 318
 319         def info(self):
 320                 return rrdtool.info(self.file)
 321
 322         def last_update(self):
 323                 """
 324                         Returns a dictionary with the timestamp and
 325                         data set of the last database update.
 326                 """
 327                 return {
 328                         "dataset"   : self.last_dataset,
 329                         "timestamp" : self.last_updated,
 330                 }
 331
 332         def _last_update(self):
 333                 return rrdtool.lastupdate(self.file)
 334
 335         @property
 336         def last_updated(self):
 337                 """
 338                         Returns the timestamp when this database was last updated
 339                 """
 340                 lu = self._last_update()
 341
 342                 if lu:
 343                         return lu.get("date")
 344
 345         @property
 346         def last_dataset(self):
 347                 """
 348                         Returns the latest dataset in the database
 349                 """
 350                 lu = self._last_update()
 351
 352                 if lu:
 353                         return lu.get("ds")
 354
 355         @property
 356         def stepsize(self):
 357                 return self.plugin.interval
 358
 359         @property
 360         def heartbeat(self):
 361                 return self.stepsize * 2
 362
 363         def get_rrd_schema(self):
 364                 schema = [
 365                         "--step", "%s" % self.stepsize,
 366                 ]
 367                 for line in self.rrd_schema:
 368                         if line.startswith("DS:"):
 369                                 try:
 370                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 371
 372                                         line = ":".join((
 373                                                 prefix,
 374                                                 name,
 375                                                 type,
 376                                                 "%s" % self.heartbeat,
 377                                                 lower_limit,
 378                                                 upper_limit
 379                                         ))
 380                                 except ValueError:
 381                                         pass
 382
 383                         schema.append(line)
 384
 385                 xff = 0.1
 386
 387                 for steps, rows in self.rra_timespans:
 388                         for type in self.rra_types:
 389                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 390
 391                 return schema
 392
 393         @property
 394         def rrd_schema_names(self):
 395                 ret = []
 396
 397                 for line in self.rrd_schema:
 398                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 399                         ret.append(name)
 400
 401                 return ret
 402
 403         def make_rrd_defs(self, prefix=None):
 404                 defs = []
 405
 406                 for name in self.rrd_schema_names:
 407                         if prefix:
 408                                 p = "%s_%s" % (prefix, name)
 409                         else:
 410                                 p = name
 411
 412                         defs += [
 413                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 414                         ]
 415
 416                 return defs
 417
 418         def get_stddev(self, interval=None):
 419                 args = self.make_rrd_defs()
 420
 421                 # Add the correct interval
 422                 args += ["--start", util.make_interval(interval)]
 423
 424                 for name in self.rrd_schema_names:
 425                         args += [
 426                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 427                                 "PRINT:%s_stddev:%%lf" % name,
 428                         ]
 429
 430                 x, y, vals = rrdtool.graph("/dev/null", *args)
 431                 return dict(zip(self.rrd_schema_names, vals))
 432
 433         def commit(self):
 434                 """
 435                         Will commit the collected data to the database.
 436                 """
 437                 # Make sure that the RRD database has been created
 438                 self.create()
 439
 440                 # Write everything to disk that is in the write queue
 441                 self.collecty.write_queue.commit_file(self.file)
 442
 443         # Convenience functions for plugin authors
 444
 445         def read_file(self, *args, strip=True):
 446                 """
 447                         Reads the content of the given file
 448                 """
 449                 filename = os.path.join(*args)
 450
 451                 try:
 452                         with open(filename) as f:
 453                                 value = f.read()
 454                 except FileNotFoundError as e:
 455                         return None
 456
 457                 # Strip any excess whitespace
 458                 if strip:
 459                         value = value.strip()
 460
 461                 return value
 462
 463         def read_file_integer(self, filename):
 464                 """
 465                         Reads the content from a file and returns it as an integer
 466                 """
 467                 value = self.read_file(filename)
 468
 469                 try:
 470                         return int(value)
 471                 except (TypeError, ValueError):
 472                         return None
 473
 474         def read_proc_stat(self):
 475                 """
 476                         Reads /proc/stat and returns it as a dictionary
 477                 """
 478                 ret = {}
 479
 480                 with open("/proc/stat") as f:
 481                         for line in f:
 482                                 # Split the key from the rest of the line
 483                                 key, line = line.split(" ", 1)
 484
 485                                 # Remove any line breaks
 486                                 ret[key] = line.rstrip()
 487
 488                 return ret
 489
 490         def read_proc_meminfo(self):
 491                 ret = {}
 492
 493                 with open("/proc/meminfo") as f:
 494                         for line in f:
 495                                 # Split the key from the rest of the line
 496                                 key, line = line.split(":", 1)
 497
 498                                 # Remove any whitespace
 499                                 line = line.strip()
 500
 501                                 # Remove any trailing kB
 502                                 if line.endswith(" kB"):
 503                                         line = line[:-3]
 504
 505                                 # Try to convert to integer
 506                                 try:
 507                                         line = int(line)
 508                                 except (TypeError, ValueError):
 509                                         continue
 510
 511                                 ret[key] = line
 512
 513                 return ret
 514
 515
 516 class GraphTemplate(object):
 517         # A unique name to identify this graph template.
 518         name = None
 519
 520         # Headline of the graph image
 521         graph_title = None
 522
 523         # Vertical label of the graph
 524         graph_vertical_label = None
 525
 526         # Limits
 527         lower_limit = None
 528         upper_limit = None
 529
 530         # Instructions how to create the graph.
 531         rrd_graph = None
 532
 533         # Extra arguments passed to rrdgraph.
 534         rrd_graph_args = []
 535
 536         def __init__(self, plugin, object_id, locale=None, timezone=None):
 537                 self.plugin = plugin
 538
 539                 # Save localisation parameters
 540                 self.locale = locale
 541                 self.timezone = timezone
 542
 543                 # Get all required RRD objects
 544                 self.object_id = object_id
 545
 546                 # Get the main object
 547                 self.objects = self.get_objects(self.object_id)
 548                 self.objects.sort()
 549
 550         def __repr__(self):
 551                 return "<%s>" % self.__class__.__name__
 552
 553         @property
 554         def collecty(self):
 555                 return self.plugin.collecty
 556
 557         @property
 558         def log(self):
 559                 return self.plugin.log
 560
 561         @property
 562         def object(self):
 563                 """
 564                         Shortcut to the main object
 565                 """
 566                 if len(self.objects) == 1:
 567                         return self.objects[0]
 568
 569         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 570                         width=None, height=None, with_title=True, thumbnail=False):
 571                 args = [
 572                         # Change the background colour
 573                         "--color", "BACK#FFFFFFFF",
 574
 575                         # Disable the border around the image
 576                         "--border", "0",
 577
 578                         # Let's width and height define the size of the entire image
 579                         "--full-size-mode",
 580
 581                         # Gives the curves a more organic look
 582                         "--slope-mode",
 583
 584                         # Show nicer labels
 585                         "--dynamic-labels",
 586
 587                         # Brand all generated graphs
 588                         "--watermark", _("Created by collecty"),
 589                 ]
 590
 591                 # Set the default dimensions
 592                 default_width, default_height = 960, 480
 593
 594                 # A thumbnail doesn't have a legend and other labels
 595                 if thumbnail:
 596                         args.append("--only-graph")
 597
 598                         default_width, default_height = 80, 20
 599
 600                 args += [
 601                         "--imgformat", format,
 602                         "--height", "%s" % (height or default_height),
 603                         "--width", "%s" % (width or default_width),
 604                 ]
 605
 606                 args += self.rrd_graph_args
 607
 608                 # Graph title
 609                 if with_title and self.graph_title:
 610                         args += ["--title", self.graph_title]
 611
 612                 # Vertical label
 613                 if self.graph_vertical_label:
 614                         args += ["--vertical-label", self.graph_vertical_label]
 615
 616                 if self.lower_limit is not None or self.upper_limit is not None:
 617                         # Force to honour the set limits
 618                         args.append("--rigid")
 619
 620                         if self.lower_limit is not None:
 621                                 args += ["--lower-limit", self.lower_limit]
 622
 623                         if self.upper_limit is not None:
 624                                 args += ["--upper-limit", self.upper_limit]
 625
 626                 # Add interval
 627                 args += ["--start", util.make_interval(interval)]
 628
 629                 return args
 630
 631         def _add_defs(self):
 632                 use_prefix = len(self.objects) >= 2
 633
 634                 args = []
 635                 for object in self.objects:
 636                         if use_prefix:
 637                                 args += object.make_rrd_defs(object.id)
 638                         else:
 639                                 args += object.make_rrd_defs()
 640
 641                 return args
 642
 643         def _add_vdefs(self, args):
 644                 ret = []
 645
 646                 for arg in args:
 647                         ret.append(arg)
 648
 649                         # Search for all DEFs and CDEFs
 650                         m = re.match(DEF_MATCH, "%s" % arg)
 651                         if m:
 652                                 name = m.group(1)
 653
 654                                 # Add the VDEFs for minimum, maximum, etc. values
 655                                 ret += [
 656                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 657                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 658                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 659                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 660                                 ]
 661
 662                 return ret
 663
 664         def get_objects(self, *args, **kwargs):
 665                 object = self.plugin.get_object(*args, **kwargs)
 666
 667                 if object:
 668                         return [object,]
 669
 670                 return []
 671
 672         def generate_graph(self, interval=None, **kwargs):
 673                 assert self.objects, "Cannot render graph without any objects"
 674
 675                 # Make sure that all collected data is in the database
 676                 # to get a recent graph image
 677                 for object in self.objects:
 678                         object.commit()
 679
 680                 args = self._make_command_line(interval, **kwargs)
 681
 682                 self.log.info(_("Generating graph %s") % self)
 683
 684                 rrd_graph = self.rrd_graph
 685
 686                 # Add DEFs for all objects
 687                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 688                         args += self._add_defs()
 689
 690                 args += rrd_graph
 691                 args = self._add_vdefs(args)
 692
 693                 # Convert arguments to string
 694                 args = [str(e) for e in args]
 695
 696                 for arg in args:
 697                         self.log.debug("  %s" % arg)
 698
 699                 graph = rrdtool.graphv("-", *args)
 700
 701                 return {
 702                         "image"        : graph.get("image"),
 703                         "image_height" : graph.get("image_height"),
 704                         "image_width"  : graph.get("image_width"),
 705                 }
 706
 707         def graph_info(self):
 708                 """
 709                         Returns a dictionary with useful information
 710                         about this graph.
 711                 """
 712                 return {
 713                         "title"        : self.graph_title,
 714                         "object_id"    : self.object_id or "",
 715                         "template"     : self.name,
 716                 }