src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import logging
  23 import os
  24 import re
  25 import rrdtool
  26 import time
  27 import unicodedata
  28
  29 from .. import util
  30 from ..constants import *
  31 from ..i18n import _
  32
  33 DEF_MATCH = r"C?DEF:([A-Za-z0-9_]+)="
  34
  35 class Environment(object):
  36         """
  37                 Sets the correct environment for rrdtool to create
  38                 localised graphs and graphs in the correct timezone.
  39         """
  40         def __init__(self, timezone="UTC", locale="en_US.utf-8"):
  41                 # Build the new environment
  42                 self.new_environment = {
  43                         "LANGUAGE" : locale,
  44                         "LC_ALL"   : locale,
  45                         "TZ"       : timezone,
  46                 }
  47
  48         def __enter__(self):
  49                 # Save the current environment
  50                 self.old_environment = {}
  51
  52                 for k in self.new_environment:
  53                         # Store the old value
  54                         self.old_environment[k] = os.environ.get(k, None)
  55
  56                         # Apply the new one
  57                         if self.new_environment[k]:
  58                                 os.environ.update(self.new_environment[k])
  59
  60         def __exit__(self, type, value, traceback):
  61                 # Roll back to the previous environment
  62                 for k, v in self.old_environment.items():
  63                         if v is None:
  64                                 try:
  65                                         del os.environ[k]
  66                                 except KeyError:
  67                                         pass
  68                         else:
  69                                 os.environ[k] = v
  70
  71
  72 class PluginRegistration(type):
  73         plugins = {}
  74
  75         def __init__(plugin, name, bases, dict):
  76                 type.__init__(plugin, name, bases, dict)
  77
  78                 # The main class from which is inherited is not registered
  79                 # as a plugin.
  80                 if name == "Plugin":
  81                         return
  82
  83                 if not all((plugin.name, plugin.description)):
  84                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  85
  86                 PluginRegistration.plugins[plugin.name] = plugin
  87
  88
  89 def get():
  90         """
  91                 Returns a list with all automatically registered plugins.
  92         """
  93         return PluginRegistration.plugins.values()
  94
  95 class Plugin(object, metaclass=PluginRegistration):
  96         # The name of this plugin.
  97         name = None
  98
  99         # A description for this plugin.
 100         description = None
 101
 102         # Templates which can be used to generate a graph out of
 103         # the data from this data source.
 104         templates = []
 105
 106         # The default interval for all plugins
 107         interval = 60
 108
 109         # Priority
 110         priority = 0
 111
 112         def __init__(self, collecty, **kwargs):
 113                 self.collecty = collecty
 114
 115                 # Check if this plugin was configured correctly.
 116                 assert self.name, "Name of the plugin is not set: %s" % self.name
 117                 assert self.description, "Description of the plugin is not set: %s" % self.description
 118
 119                 # Initialize the logger.
 120                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 121
 122                 # Run some custom initialization.
 123                 self.init(**kwargs)
 124
 125                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 126
 127         @property
 128         def path(self):
 129                 """
 130                         Returns the name of the sub directory in which all RRD files
 131                         for this plugin should be stored in.
 132                 """
 133                 return self.name
 134
 135         ### Basic methods
 136
 137         def init(self, **kwargs):
 138                 """
 139                         Do some custom initialization stuff here.
 140                 """
 141                 pass
 142
 143         def collect(self):
 144                 """
 145                         Gathers the statistical data, this plugin collects.
 146                 """
 147                 time_start = time.time()
 148
 149                 # Run through all objects of this plugin and call the collect method.
 150                 for object in self.objects:
 151                         # Run collection
 152                         try:
 153                                 result = object.collect()
 154
 155                         # Catch any unhandled exceptions
 156                         except Exception as e:
 157                                 self.log.warning(_("Unhandled exception in %s.collect()") % object, exc_info=True)
 158                                 continue
 159
 160                         if not result:
 161                                 self.log.warning(_("Received empty result: %s") % object)
 162                                 continue
 163
 164                         # Add the object to the write queue so that the data is written
 165                         # to the databases later.
 166                         result = self.collecty.write_queue.submit(object, result)
 167
 168                         self.log.debug(_("Collected %s: %s") % (object, result))
 169
 170                 # Returns the time this function took to complete.
 171                 delay = time.time() - time_start
 172
 173                 # Log some warning when a collect method takes too long to return some data
 174                 if delay >= 60:
 175                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 176                 else:
 177                         self.log.debug(_("Collection finished in %.2fms") % (delay * 1000))
 178
 179         def get_object(self, id):
 180                 for object in self.objects:
 181                         if not object.id == id:
 182                                 continue
 183
 184                         return object
 185
 186         def get_template(self, template_name, object_id, locale=None, timezone=None):
 187                 for template in self.templates:
 188                         if not template.name == template_name:
 189                                 continue
 190
 191                         return template(self, object_id, locale=locale, timezone=timezone)
 192
 193         def generate_graph(self, template_name, object_id="default",
 194                         timezone=None, locale=None, **kwargs):
 195                 template = self.get_template(template_name, object_id=object_id,
 196                         timezone=timezone, locale=locale)
 197                 if not template:
 198                         raise RuntimeError("Could not find template %s" % template_name)
 199
 200                 time_start = time.time()
 201
 202                 with Environment(timezone=timezone, locale=locale):
 203                         graph = template.generate_graph(**kwargs)
 204
 205                 duration = time.time() - time_start
 206                 self.log.debug(_("Generated graph %s in %.1fms") \
 207                         % (template, duration * 1000))
 208
 209                 return graph
 210
 211         def graph_info(self, template_name, object_id="default",
 212                         timezone=None, locale=None, **kwargs):
 213                 template = self.get_template(template_name, object_id=object_id,
 214                         timezone=timezone, locale=locale)
 215                 if not template:
 216                         raise RuntimeError("Could not find template %s" % template_name)
 217
 218                 return template.graph_info()
 219
 220         def last_update(self, object_id="default"):
 221                 object = self.get_object(object_id)
 222                 if not object:
 223                         raise RuntimeError("Could not find object %s" % object_id)
 224
 225                 return object.last_update()
 226
 227
 228 class Object(object):
 229         # The schema of the RRD database.
 230         rrd_schema = None
 231
 232         # RRA properties.
 233         rra_types     = ("AVERAGE", "MIN", "MAX")
 234         rra_timespans = (
 235                 ("1m", "10d"),
 236                 ("1h", "18M"),
 237                 ("1d",  "5y"),
 238         )
 239
 240         def __init__(self, plugin, *args, **kwargs):
 241                 self.plugin = plugin
 242
 243                 # Initialise this object
 244                 self.init(*args, **kwargs)
 245
 246                 # Create the database file.
 247                 self.create()
 248
 249         def __repr__(self):
 250                 return "<%s %s>" % (self.__class__.__name__, self.id)
 251
 252         def __lt__(self, other):
 253                 return self.id < other.id
 254
 255         @property
 256         def collecty(self):
 257                 return self.plugin.collecty
 258
 259         @property
 260         def log(self):
 261                 return self.plugin.log
 262
 263         @property
 264         def id(self):
 265                 """
 266                         Returns a UNIQUE identifier for this object. As this is incorporated
 267                         into the path of RRD file, it must only contain ASCII characters.
 268                 """
 269                 raise NotImplementedError
 270
 271         @property
 272         def file(self):
 273                 """
 274                         The absolute path to the RRD file of this plugin.
 275                 """
 276                 filename = self._normalise_filename("%s.rrd" % self.id)
 277
 278                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 279
 280         @staticmethod
 281         def _normalise_filename(filename):
 282                 # Convert the filename into ASCII characters only
 283                 filename = unicodedata.normalize("NFKC", filename)
 284
 285                 # Replace any spaces by dashes
 286                 filename = filename.replace(" ", "-")
 287
 288                 return filename
 289
 290         ### Basic methods
 291
 292         def init(self, *args, **kwargs):
 293                 """
 294                         Do some custom initialization stuff here.
 295                 """
 296                 pass
 297
 298         def create(self):
 299                 """
 300                         Creates an empty RRD file with the desired data structures.
 301                 """
 302                 # Skip if the file does already exist.
 303                 if os.path.exists(self.file):
 304                         return
 305
 306                 dirname = os.path.dirname(self.file)
 307                 if not os.path.exists(dirname):
 308                         os.makedirs(dirname)
 309
 310                 # Create argument list.
 311                 args = self.get_rrd_schema()
 312
 313                 rrdtool.create(self.file, *args)
 314
 315                 self.log.debug(_("Created RRD file %s.") % self.file)
 316                 for arg in args:
 317                         self.log.debug("  %s" % arg)
 318
 319         def info(self):
 320                 return rrdtool.info(self.file)
 321
 322         def last_update(self):
 323                 """
 324                         Returns a dictionary with the timestamp and
 325                         data set of the last database update.
 326                 """
 327                 return {
 328                         "dataset"   : self.last_dataset,
 329                         "timestamp" : self.last_updated,
 330                 }
 331
 332         def _last_update(self):
 333                 return rrdtool.lastupdate(self.file)
 334
 335         @property
 336         def last_updated(self):
 337                 """
 338                         Returns the timestamp when this database was last updated
 339                 """
 340                 lu = self._last_update()
 341
 342                 if lu:
 343                         return lu.get("date")
 344
 345         @property
 346         def last_dataset(self):
 347                 """
 348                         Returns the latest dataset in the database
 349                 """
 350                 lu = self._last_update()
 351
 352                 if lu:
 353                         return lu.get("ds")
 354
 355         @property
 356         def stepsize(self):
 357                 return self.plugin.interval
 358
 359         @property
 360         def heartbeat(self):
 361                 return self.stepsize * 2
 362
 363         def get_rrd_schema(self):
 364                 schema = [
 365                         "--step", "%s" % self.stepsize,
 366                 ]
 367                 for line in self.rrd_schema:
 368                         if line.startswith("DS:"):
 369                                 try:
 370                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 371
 372                                         line = ":".join((
 373                                                 prefix,
 374                                                 name,
 375                                                 type,
 376                                                 "%s" % self.heartbeat,
 377                                                 lower_limit,
 378                                                 upper_limit
 379                                         ))
 380                                 except ValueError:
 381                                         pass
 382
 383                         schema.append(line)
 384
 385                 xff = 0.1
 386
 387                 for steps, rows in self.rra_timespans:
 388                         for type in self.rra_types:
 389                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 390
 391                 return schema
 392
 393         @property
 394         def rrd_schema_names(self):
 395                 ret = []
 396
 397                 for line in self.rrd_schema:
 398                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 399                         ret.append(name)
 400
 401                 return ret
 402
 403         def make_rrd_defs(self, prefix=None):
 404                 defs = []
 405
 406                 for name in self.rrd_schema_names:
 407                         if prefix:
 408                                 p = "%s_%s" % (prefix, name)
 409                         else:
 410                                 p = name
 411
 412                         defs += [
 413                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 414                         ]
 415
 416                 return defs
 417
 418         def get_stddev(self, interval=None):
 419                 args = self.make_rrd_defs()
 420
 421                 # Add the correct interval
 422                 args += ["--start", util.make_interval(interval)]
 423
 424                 for name in self.rrd_schema_names:
 425                         args += [
 426                                 "VDEF:%s_stddev=%s,STDEV" % (name, name),
 427                                 "PRINT:%s_stddev:%%lf" % name,
 428                         ]
 429
 430                 x, y, vals = rrdtool.graph("/dev/null", *args)
 431                 return dict(zip(self.rrd_schema_names, vals))
 432
 433         def commit(self):
 434                 """
 435                         Will commit the collected data to the database.
 436                 """
 437                 # Make sure that the RRD database has been created
 438                 self.create()
 439
 440                 # Write everything to disk that is in the write queue
 441                 self.collecty.write_queue.commit_file(self.file)
 442
 443         # Convenience functions for plugin authors
 444
 445         def read_file(self, *args, strip=True):
 446                 """
 447                         Reads the content of the given file
 448                 """
 449                 filename = os.path.join(*args)
 450
 451                 with open(filename) as f:
 452                         value = f.read()
 453
 454                 # Strip any excess whitespace
 455                 if strip:
 456                         value = value.strip()
 457
 458                 return value
 459
 460         def read_file_integer(self, filename):
 461                 """
 462                         Reads the content from a file and returns it as an integer
 463                 """
 464                 value = self.read_file(filename)
 465
 466                 try:
 467                         return int(value)
 468                 except ValueError:
 469                         return None
 470
 471         def read_proc_stat(self):
 472                 """
 473                         Reads /proc/stat and returns it as a dictionary
 474                 """
 475                 ret = {}
 476
 477                 with open("/proc/stat") as f:
 478                         for line in f:
 479                                 # Split the key from the rest of the line
 480                                 key, line = line.split(" ", 1)
 481
 482                                 # Remove any line breaks
 483                                 ret[key] = line.rstrip()
 484
 485                 return ret
 486
 487         def read_proc_meminfo(self):
 488                 ret = {}
 489
 490                 with open("/proc/meminfo") as f:
 491                         for line in f:
 492                                 # Split the key from the rest of the line
 493                                 key, line = line.split(":", 1)
 494
 495                                 # Remove any whitespace
 496                                 line = line.strip()
 497
 498                                 # Remove any trailing kB
 499                                 if line.endswith(" kB"):
 500                                         line = line[:-3]
 501
 502                                 # Try to convert to integer
 503                                 try:
 504                                         line = int(line)
 505                                 except (TypeError, ValueError):
 506                                         continue
 507
 508                                 ret[key] = line
 509
 510                 return ret
 511
 512
 513 class GraphTemplate(object):
 514         # A unique name to identify this graph template.
 515         name = None
 516
 517         # Headline of the graph image
 518         graph_title = None
 519
 520         # Vertical label of the graph
 521         graph_vertical_label = None
 522
 523         # Limits
 524         lower_limit = None
 525         upper_limit = None
 526
 527         # Instructions how to create the graph.
 528         rrd_graph = None
 529
 530         # Extra arguments passed to rrdgraph.
 531         rrd_graph_args = []
 532
 533         def __init__(self, plugin, object_id, locale=None, timezone=None):
 534                 self.plugin = plugin
 535
 536                 # Save localisation parameters
 537                 self.locale = locale
 538                 self.timezone = timezone
 539
 540                 # Get all required RRD objects
 541                 self.object_id = object_id
 542
 543                 # Get the main object
 544                 self.objects = self.get_objects(self.object_id)
 545                 self.objects.sort()
 546
 547         def __repr__(self):
 548                 return "<%s>" % self.__class__.__name__
 549
 550         @property
 551         def collecty(self):
 552                 return self.plugin.collecty
 553
 554         @property
 555         def log(self):
 556                 return self.plugin.log
 557
 558         @property
 559         def object(self):
 560                 """
 561                         Shortcut to the main object
 562                 """
 563                 if len(self.objects) == 1:
 564                         return self.objects[0]
 565
 566         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 567                         width=None, height=None, with_title=True, thumbnail=False):
 568                 args = [
 569                         # Change the background colour
 570                         "--color", "BACK#FFFFFFFF",
 571
 572                         # Disable the border around the image
 573                         "--border", "0",
 574
 575                         # Let's width and height define the size of the entire image
 576                         "--full-size-mode",
 577
 578                         # Gives the curves a more organic look
 579                         "--slope-mode",
 580
 581                         # Show nicer labels
 582                         "--dynamic-labels",
 583
 584                         # Brand all generated graphs
 585                         "--watermark", _("Created by collecty"),
 586                 ]
 587
 588                 # Set the default dimensions
 589                 default_width, default_height = 960, 480
 590
 591                 # A thumbnail doesn't have a legend and other labels
 592                 if thumbnail:
 593                         args.append("--only-graph")
 594
 595                         default_width, default_height = 80, 20
 596
 597                 args += [
 598                         "--imgformat", format,
 599                         "--height", "%s" % (height or default_height),
 600                         "--width", "%s" % (width or default_width),
 601                 ]
 602
 603                 args += self.rrd_graph_args
 604
 605                 # Graph title
 606                 if with_title and self.graph_title:
 607                         args += ["--title", self.graph_title]
 608
 609                 # Vertical label
 610                 if self.graph_vertical_label:
 611                         args += ["--vertical-label", self.graph_vertical_label]
 612
 613                 if self.lower_limit is not None or self.upper_limit is not None:
 614                         # Force to honour the set limits
 615                         args.append("--rigid")
 616
 617                         if self.lower_limit is not None:
 618                                 args += ["--lower-limit", self.lower_limit]
 619
 620                         if self.upper_limit is not None:
 621                                 args += ["--upper-limit", self.upper_limit]
 622
 623                 # Add interval
 624                 args += ["--start", util.make_interval(interval)]
 625
 626                 return args
 627
 628         def _add_defs(self):
 629                 use_prefix = len(self.objects) >= 2
 630
 631                 args = []
 632                 for object in self.objects:
 633                         if use_prefix:
 634                                 args += object.make_rrd_defs(object.id)
 635                         else:
 636                                 args += object.make_rrd_defs()
 637
 638                 return args
 639
 640         def _add_vdefs(self, args):
 641                 ret = []
 642
 643                 for arg in args:
 644                         ret.append(arg)
 645
 646                         # Search for all DEFs and CDEFs
 647                         m = re.match(DEF_MATCH, "%s" % arg)
 648                         if m:
 649                                 name = m.group(1)
 650
 651                                 # Add the VDEFs for minimum, maximum, etc. values
 652                                 ret += [
 653                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 654                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 655                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 656                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 657                                 ]
 658
 659                 return ret
 660
 661         def get_objects(self, *args, **kwargs):
 662                 object = self.plugin.get_object(*args, **kwargs)
 663
 664                 if object:
 665                         return [object,]
 666
 667                 return []
 668
 669         def generate_graph(self, interval=None, **kwargs):
 670                 assert self.objects, "Cannot render graph without any objects"
 671
 672                 # Make sure that all collected data is in the database
 673                 # to get a recent graph image
 674                 for object in self.objects:
 675                         object.commit()
 676
 677                 args = self._make_command_line(interval, **kwargs)
 678
 679                 self.log.info(_("Generating graph %s") % self)
 680
 681                 rrd_graph = self.rrd_graph
 682
 683                 # Add DEFs for all objects
 684                 if not any((e.startswith("DEF:") for e in rrd_graph)):
 685                         args += self._add_defs()
 686
 687                 args += rrd_graph
 688                 args = self._add_vdefs(args)
 689
 690                 # Convert arguments to string
 691                 args = [str(e) for e in args]
 692
 693                 for arg in args:
 694                         self.log.debug("  %s" % arg)
 695
 696                 graph = rrdtool.graphv("-", *args)
 697
 698                 return {
 699                         "image"        : graph.get("image"),
 700                         "image_height" : graph.get("image_height"),
 701                         "image_width"  : graph.get("image_width"),
 702                 }
 703
 704         def graph_info(self):
 705                 """
 706                         Returns a dictionary with useful information
 707                         about this graph.
 708                 """
 709                 return {
 710                         "title"        : self.graph_title,
 711                         "object_id"    : self.object_id or "",
 712                         "template"     : self.name,
 713                 }