src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import re
  27 import rrdtool
  28 import tempfile
  29 import threading
  30 import time
  31 import unicodedata
  32
  33 from .. import locales
  34 from ..constants import *
  35 from ..i18n import _
  36
  37 DEF_MATCH = re.compile(r"C?DEF:([A-Za-z0-9_]+)=")
  38
  39 class Timer(object):
  40         def __init__(self, timeout, heartbeat=1):
  41                 self.timeout = timeout
  42                 self.heartbeat = heartbeat
  43
  44                 self.delay = 0
  45
  46                 self.reset()
  47
  48         def reset(self, delay=0):
  49                 # Save start time.
  50                 self.start = time.time()
  51
  52                 self.delay = delay
  53
  54                 # Has this timer been killed?
  55                 self.killed = False
  56
  57         @property
  58         def elapsed(self):
  59                 return time.time() - self.start - self.delay
  60
  61         def cancel(self):
  62                 self.killed = True
  63
  64         def wait(self):
  65                 while self.elapsed < self.timeout and not self.killed:
  66                         time.sleep(self.heartbeat)
  67
  68                 return self.elapsed > self.timeout
  69
  70
  71 class Environment(object):
  72         """
  73                 Sets the correct environment for rrdtool to create
  74                 localised graphs and graphs in the correct timezone.
  75         """
  76         def __init__(self, timezone, locale):
  77                 # Build the new environment
  78                 self.new_environment = {
  79                         "TZ" : timezone or DEFAULT_TIMEZONE,
  80                 }
  81
  82                 for k in ("LANG", "LC_ALL"):
  83                         self.new_environment[k] = locale or DEFAULT_LOCALE
  84
  85         def __enter__(self):
  86                 # Save the current environment
  87                 self.old_environment = {}
  88                 for k in self.new_environment:
  89                         self.old_environment[k] = os.environ.get(k, None)
  90
  91                 # Apply the new one
  92                 os.environ.update(self.new_environment)
  93
  94         def __exit__(self, type, value, traceback):
  95                 # Roll back to the previous environment
  96                 for k, v in self.old_environment.items():
  97                         if v is None:
  98                                 try:
  99                                         del os.environ[k]
 100                                 except KeyError:
 101                                         pass
 102                         else:
 103                                 os.environ[k] = v
 104
 105
 106 class PluginRegistration(type):
 107         plugins = {}
 108
 109         def __init__(plugin, name, bases, dict):
 110                 type.__init__(plugin, name, bases, dict)
 111
 112                 # The main class from which is inherited is not registered
 113                 # as a plugin.
 114                 if name == "Plugin":
 115                         return
 116
 117                 if not all((plugin.name, plugin.description)):
 118                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
 119
 120                 PluginRegistration.plugins[plugin.name] = plugin
 121
 122
 123 def get():
 124         """
 125                 Returns a list with all automatically registered plugins.
 126         """
 127         return PluginRegistration.plugins.values()
 128
 129 class Plugin(object, metaclass=PluginRegistration):
 130         # The name of this plugin.
 131         name = None
 132
 133         # A description for this plugin.
 134         description = None
 135
 136         # Templates which can be used to generate a graph out of
 137         # the data from this data source.
 138         templates = []
 139
 140         # The default interval for all plugins
 141         interval = 60
 142
 143         def __init__(self, collecty, **kwargs):
 144                 self.collecty = collecty
 145
 146                 # Check if this plugin was configured correctly.
 147                 assert self.name, "Name of the plugin is not set: %s" % self.name
 148                 assert self.description, "Description of the plugin is not set: %s" % self.description
 149
 150                 # Initialize the logger.
 151                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 152                 self.log.propagate = 1
 153
 154                 self.data = []
 155
 156                 # Run some custom initialization.
 157                 self.init(**kwargs)
 158
 159                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 160
 161         @property
 162         def path(self):
 163                 """
 164                         Returns the name of the sub directory in which all RRD files
 165                         for this plugin should be stored in.
 166                 """
 167                 return self.name
 168
 169         ### Basic methods
 170
 171         def init(self, **kwargs):
 172                 """
 173                         Do some custom initialization stuff here.
 174                 """
 175                 pass
 176
 177         def collect(self):
 178                 """
 179                         Gathers the statistical data, this plugin collects.
 180                 """
 181                 time_start = time.time()
 182
 183                 # Run through all objects of this plugin and call the collect method.
 184                 for o in self.objects:
 185                         now = datetime.datetime.utcnow()
 186                         try:
 187                                 result = o.collect()
 188
 189                                 result = self._format_result(result)
 190                         except:
 191                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 192                                 continue
 193
 194                         if not result:
 195                                 self.log.warning(_("Received empty result: %s") % o)
 196                                 continue
 197
 198                         self.log.debug(_("Collected %s: %s") % (o, result))
 199
 200                         # Add the object to the write queue so that the data is written
 201                         # to the databases later.
 202                         self.collecty.write_queue.add(o, now, result)
 203
 204                 # Returns the time this function took to complete.
 205                 delay = time.time() - time_start
 206
 207                 # Log some warning when a collect method takes too long to return some data
 208                 if delay >= 60:
 209                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 210
 211         @staticmethod
 212         def _format_result(result):
 213                 if not isinstance(result, tuple) and not isinstance(result, list):
 214                         return result
 215
 216                 # Replace all Nones by NaN
 217                 s = []
 218
 219                 for e in result:
 220                         if e is None:
 221                                 e = "NaN"
 222
 223                         # Format as string
 224                         e = "%s" % e
 225
 226                         s.append(e)
 227
 228                 return ":".join(s)
 229
 230         def get_object(self, id):
 231                 for object in self.objects:
 232                         if not object.id == id:
 233                                 continue
 234
 235                         return object
 236
 237         def get_template(self, template_name, object_id, locale=None, timezone=None):
 238                 for template in self.templates:
 239                         if not template.name == template_name:
 240                                 continue
 241
 242                         return template(self, object_id, locale=locale, timezone=timezone)
 243
 244         def generate_graph(self, template_name, object_id="default",
 245                         timezone=None, locale=None, **kwargs):
 246                 template = self.get_template(template_name, object_id=object_id,
 247                         timezone=timezone, locale=locale)
 248                 if not template:
 249                         raise RuntimeError("Could not find template %s" % template_name)
 250
 251                 time_start = time.time()
 252
 253                 graph = template.generate_graph(**kwargs)
 254
 255                 duration = time.time() - time_start
 256                 self.log.debug(_("Generated graph %s in %.1fms") \
 257                         % (template, duration * 1000))
 258
 259                 return graph
 260
 261         def graph_info(self, template_name, object_id="default",
 262                         timezone=None, locale=None, **kwargs):
 263                 template = self.get_template(template_name, object_id=object_id,
 264                         timezone=timezone, locale=locale)
 265                 if not template:
 266                         raise RuntimeError("Could not find template %s" % template_name)
 267
 268                 return template.graph_info()
 269
 270         def last_update(self, object_id="default"):
 271                 object = self.get_object(object_id)
 272                 if not object:
 273                         raise RuntimeError("Could not find object %s" % object_id)
 274
 275                 return object.last_update()
 276
 277
 278 class Object(object):
 279         # The schema of the RRD database.
 280         rrd_schema = None
 281
 282         # RRA properties.
 283         rra_types     = ("AVERAGE", "MIN", "MAX")
 284         rra_timespans = (
 285                 ("1m", "10d"),
 286                 ("1h", "18M"),
 287                 ("1d",  "5y"),
 288         )
 289
 290         def __init__(self, plugin, *args, **kwargs):
 291                 self.plugin = plugin
 292
 293                 # Indicates if this object has collected its data
 294                 self.collected = False
 295
 296                 # Initialise this object
 297                 self.init(*args, **kwargs)
 298
 299                 # Create the database file.
 300                 self.create()
 301
 302         def __repr__(self):
 303                 return "<%s>" % self.__class__.__name__
 304
 305         @property
 306         def collecty(self):
 307                 return self.plugin.collecty
 308
 309         @property
 310         def log(self):
 311                 return self.plugin.log
 312
 313         @property
 314         def id(self):
 315                 """
 316                         Returns a UNIQUE identifier for this object. As this is incorporated
 317                         into the path of RRD file, it must only contain ASCII characters.
 318                 """
 319                 raise NotImplementedError
 320
 321         @property
 322         def file(self):
 323                 """
 324                         The absolute path to the RRD file of this plugin.
 325                 """
 326                 filename = self._normalise_filename("%s.rrd" % self.id)
 327
 328                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 329
 330         @staticmethod
 331         def _normalise_filename(filename):
 332                 # Convert the filename into ASCII characters only
 333                 filename = unicodedata.normalize("NFKC", filename)
 334
 335                 # Replace any spaces by dashes
 336                 filename = filename.replace(" ", "-")
 337
 338                 return filename
 339
 340         ### Basic methods
 341
 342         def init(self, *args, **kwargs):
 343                 """
 344                         Do some custom initialization stuff here.
 345                 """
 346                 pass
 347
 348         def create(self):
 349                 """
 350                         Creates an empty RRD file with the desired data structures.
 351                 """
 352                 # Skip if the file does already exist.
 353                 if os.path.exists(self.file):
 354                         return
 355
 356                 dirname = os.path.dirname(self.file)
 357                 if not os.path.exists(dirname):
 358                         os.makedirs(dirname)
 359
 360                 # Create argument list.
 361                 args = self.get_rrd_schema()
 362
 363                 rrdtool.create(self.file, *args)
 364
 365                 self.log.debug(_("Created RRD file %s.") % self.file)
 366                 for arg in args:
 367                         self.log.debug("  %s" % arg)
 368
 369         def info(self):
 370                 return rrdtool.info(self.file)
 371
 372         def last_update(self):
 373                 """
 374                         Returns a dictionary with the timestamp and
 375                         data set of the last database update.
 376                 """
 377                 return {
 378                         "dataset"   : self.last_dataset,
 379                         "timestamp" : self.last_updated,
 380                 }
 381
 382         def _last_update(self):
 383                 return rrdtool.lastupdate(self.file)
 384
 385         @property
 386         def last_updated(self):
 387                 """
 388                         Returns the timestamp when this database was last updated
 389                 """
 390                 lu = self._last_update()
 391
 392                 if lu:
 393                         return lu.get("date")
 394
 395         @property
 396         def last_dataset(self):
 397                 """
 398                         Returns the latest dataset in the database
 399                 """
 400                 lu = self._last_update()
 401
 402                 if lu:
 403                         return lu.get("ds")
 404
 405         @property
 406         def stepsize(self):
 407                 return self.plugin.interval
 408
 409         @property
 410         def heartbeat(self):
 411                 return self.stepsize * 2
 412
 413         def get_rrd_schema(self):
 414                 schema = [
 415                         "--step", "%s" % self.stepsize,
 416                 ]
 417                 for line in self.rrd_schema:
 418                         if line.startswith("DS:"):
 419                                 try:
 420                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 421
 422                                         line = ":".join((
 423                                                 prefix,
 424                                                 name,
 425                                                 type,
 426                                                 "%s" % self.heartbeat,
 427                                                 lower_limit,
 428                                                 upper_limit
 429                                         ))
 430                                 except ValueError:
 431                                         pass
 432
 433                         schema.append(line)
 434
 435                 xff = 0.1
 436
 437                 for steps, rows in self.rra_timespans:
 438                         for type in self.rra_types:
 439                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 440
 441                 return schema
 442
 443         @property
 444         def rrd_schema_names(self):
 445                 ret = []
 446
 447                 for line in self.rrd_schema:
 448                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 449                         ret.append(name)
 450
 451                 return ret
 452
 453         def make_rrd_defs(self, prefix=None):
 454                 defs = []
 455
 456                 for name in self.rrd_schema_names:
 457                         if prefix:
 458                                 p = "%s_%s" % (prefix, name)
 459                         else:
 460                                 p = name
 461
 462                         defs += [
 463                                 "DEF:%s=%s:%s:AVERAGE" % (p, self.file, name),
 464                         ]
 465
 466                 return defs
 467
 468         def execute(self):
 469                 if self.collected:
 470                         raise RuntimeError("This object has already collected its data")
 471
 472                 self.collected = True
 473                 self.now = datetime.datetime.utcnow()
 474
 475                 # Call the collect
 476                 result = self.collect()
 477
 478         def commit(self):
 479                 """
 480                         Will commit the collected data to the database.
 481                 """
 482                 # Make sure that the RRD database has been created
 483                 self.create()
 484
 485                 # Write everything to disk that is in the write queue
 486                 self.collecty.write_queue.commit_file(self.file)
 487
 488
 489 class GraphTemplate(object):
 490         # A unique name to identify this graph template.
 491         name = None
 492
 493         # Headline of the graph image
 494         graph_title = None
 495
 496         # Vertical label of the graph
 497         graph_vertical_label = None
 498
 499         # Limits
 500         lower_limit = None
 501         upper_limit = None
 502
 503         # Instructions how to create the graph.
 504         rrd_graph = None
 505
 506         # Extra arguments passed to rrdgraph.
 507         rrd_graph_args = []
 508
 509         intervals = {
 510                 None   : "-3h",
 511                 "hour" : "-1h",
 512                 "day"  : "-25h",
 513                 "month": "-30d",
 514                 "week" : "-360h",
 515                 "year" : "-365d",
 516         }
 517
 518         # Default dimensions for this graph
 519         height = GRAPH_DEFAULT_HEIGHT
 520         width  = GRAPH_DEFAULT_WIDTH
 521
 522         def __init__(self, plugin, object_id, locale=None, timezone=None):
 523                 self.plugin = plugin
 524
 525                 # Save localisation parameters
 526                 self.locale = locales.get(locale)
 527                 self.timezone = timezone
 528
 529                 # Get all required RRD objects
 530                 self.object_id = object_id
 531
 532                 # Get the main object
 533                 self.object = self.get_object(self.object_id)
 534
 535         def __repr__(self):
 536                 return "<%s>" % self.__class__.__name__
 537
 538         @property
 539         def collecty(self):
 540                 return self.plugin.collecty
 541
 542         @property
 543         def log(self):
 544                 return self.plugin.log
 545
 546         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 547                         width=None, height=None, with_title=True, thumbnail=False):
 548                 args = [e for e in GRAPH_DEFAULT_ARGUMENTS]
 549
 550                 # Set the default dimensions
 551                 default_height, default_width = GRAPH_DEFAULT_HEIGHT, GRAPH_DEFAULT_WIDTH
 552
 553                 # A thumbnail doesn't have a legend and other labels
 554                 if thumbnail:
 555                         args.append("--only-graph")
 556
 557                         default_height = THUMBNAIL_DEFAULT_HEIGHT
 558                         default_width = THUMBNAIL_DEFAULT_WIDTH
 559
 560                 args += [
 561                         "--imgformat", format,
 562                         "--height", "%s" % (height or default_height),
 563                         "--width", "%s" % (width or default_width),
 564                 ]
 565
 566                 args += self.rrd_graph_args
 567
 568                 # Graph title
 569                 if with_title and self.graph_title:
 570                         args += ["--title", self.graph_title]
 571
 572                 # Vertical label
 573                 if self.graph_vertical_label:
 574                         args += ["--vertical-label", self.graph_vertical_label]
 575
 576                 if self.lower_limit is not None or self.upper_limit is not None:
 577                         # Force to honour the set limits
 578                         args.append("--rigid")
 579
 580                         if self.lower_limit is not None:
 581                                 args += ["--lower-limit", self.lower_limit]
 582
 583                         if self.upper_limit is not None:
 584                                 args += ["--upper-limit", self.upper_limit]
 585
 586                 try:
 587                         interval = self.intervals[interval]
 588                 except KeyError:
 589                         interval = "end-%s" % interval
 590
 591                 # Add interval
 592                 args += ["--start", interval]
 593
 594                 return args
 595
 596         def _add_vdefs(self, args):
 597                 ret = []
 598
 599                 for arg in args:
 600                         ret.append(arg)
 601
 602                         # Search for all DEFs and CDEFs
 603                         m = re.match(DEF_MATCH, "%s" % arg)
 604                         if m:
 605                                 name = m.group(1)
 606
 607                                 # Add the VDEFs for minimum, maximum, etc. values
 608                                 ret += [
 609                                         "VDEF:%s_cur=%s,LAST" % (name, name),
 610                                         "VDEF:%s_avg=%s,AVERAGE" % (name, name),
 611                                         "VDEF:%s_max=%s,MAXIMUM" % (name, name),
 612                                         "VDEF:%s_min=%s,MINIMUM" % (name, name),
 613                                 ]
 614
 615                 return ret
 616
 617         def get_object(self, *args, **kwargs):
 618                 return self.plugin.get_object(*args, **kwargs)
 619
 620         def get_object_table(self):
 621                 return {
 622                         "file" : self.object,
 623                 }
 624
 625         @property
 626         def object_table(self):
 627                 if not hasattr(self, "_object_table"):
 628                         self._object_table = self.get_object_table()
 629
 630                 return self._object_table
 631
 632         def get_object_files(self):
 633                 files = {}
 634
 635                 for id, obj in self.object_table.items():
 636                         files[id] = obj.file
 637
 638                 return files
 639
 640         def generate_graph(self, interval=None, **kwargs):
 641                 # Make sure that all collected data is in the database
 642                 # to get a recent graph image
 643                 if self.object:
 644                         self.object.commit()
 645
 646                 args = self._make_command_line(interval, **kwargs)
 647
 648                 self.log.info(_("Generating graph %s") % self)
 649
 650                 #object_files = self.get_object_files()
 651
 652                 if self.object:
 653                         args += self.object.make_rrd_defs()
 654
 655                 args += self.rrd_graph
 656                 args = self._add_vdefs(args)
 657
 658                 # Convert arguments to string
 659                 args = [str(e) for e in args]
 660
 661                 for arg in args:
 662                         self.log.debug("  %s" % arg)
 663
 664                 with Environment(self.timezone, self.locale.lang):
 665                         graph = rrdtool.graphv("-", *args)
 666
 667                 return {
 668                         "image"        : graph.get("image"),
 669                         "image_height" : graph.get("image_height"),
 670                         "image_width"  : graph.get("image_width"),
 671                 }
 672
 673         def graph_info(self):
 674                 """
 675                         Returns a dictionary with useful information
 676                         about this graph.
 677                 """
 678                 return {
 679                         "title"        : self.graph_title,
 680                         "object_id"    : self.object_id or "",
 681                         "template"     : self.name,
 682                 }