src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import rrdtool
  27 import tempfile
  28 import threading
  29 import time
  30 import unicodedata
  31
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 class Timer(object):
  36         def __init__(self, timeout, heartbeat=1):
  37                 self.timeout = timeout
  38                 self.heartbeat = heartbeat
  39
  40                 self.delay = 0
  41
  42                 self.reset()
  43
  44         def reset(self, delay=0):
  45                 # Save start time.
  46                 self.start = time.time()
  47
  48                 self.delay = delay
  49
  50                 # Has this timer been killed?
  51                 self.killed = False
  52
  53         @property
  54         def elapsed(self):
  55                 return time.time() - self.start - self.delay
  56
  57         def cancel(self):
  58                 self.killed = True
  59
  60         def wait(self):
  61                 while self.elapsed < self.timeout and not self.killed:
  62                         time.sleep(self.heartbeat)
  63
  64                 return self.elapsed > self.timeout
  65
  66
  67 class Environment(object):
  68         """
  69                 Sets the correct environment for rrdtool to create
  70                 localised graphs and graphs in the correct timezone.
  71         """
  72         def __init__(self, timezone, locale):
  73                 # Build the new environment
  74                 self.new_environment = {
  75                         "TZ" : timezone or DEFAULT_TIMEZONE,
  76                 }
  77
  78                 for k in ("LANG", "LC_ALL"):
  79                         self.new_environment[k] = locale or DEFAULT_LOCALE
  80
  81         def __enter__(self):
  82                 # Save the current environment
  83                 self.old_environment = {}
  84                 for k in self.new_environment:
  85                         self.old_environment[k] = os.environ.get(k, None)
  86
  87                 # Apply the new one
  88                 os.environ.update(self.new_environment)
  89
  90         def __exit__(self, type, value, traceback):
  91                 # Roll back to the previous environment
  92                 for k, v in self.old_environment.items():
  93                         if v is None:
  94                                 try:
  95                                         del os.environ[k]
  96                                 except KeyError:
  97                                         pass
  98                         else:
  99                                 os.environ[k] = v
 100
 101
 102 class PluginRegistration(type):
 103         plugins = {}
 104
 105         def __init__(plugin, name, bases, dict):
 106                 type.__init__(plugin, name, bases, dict)
 107
 108                 # The main class from which is inherited is not registered
 109                 # as a plugin.
 110                 if name == "Plugin":
 111                         return
 112
 113                 if not all((plugin.name, plugin.description)):
 114                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
 115
 116                 PluginRegistration.plugins[plugin.name] = plugin
 117
 118
 119 def get():
 120         """
 121                 Returns a list with all automatically registered plugins.
 122         """
 123         return PluginRegistration.plugins.values()
 124
 125 class Plugin(object, metaclass=PluginRegistration):
 126         # The name of this plugin.
 127         name = None
 128
 129         # A description for this plugin.
 130         description = None
 131
 132         # Templates which can be used to generate a graph out of
 133         # the data from this data source.
 134         templates = []
 135
 136         # The default interval for all plugins
 137         interval = 60
 138
 139         def __init__(self, collecty, **kwargs):
 140                 self.collecty = collecty
 141
 142                 # Check if this plugin was configured correctly.
 143                 assert self.name, "Name of the plugin is not set: %s" % self.name
 144                 assert self.description, "Description of the plugin is not set: %s" % self.description
 145
 146                 # Initialize the logger.
 147                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 148                 self.log.propagate = 1
 149
 150                 self.data = []
 151
 152                 # Run some custom initialization.
 153                 self.init(**kwargs)
 154
 155                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 156
 157         @property
 158         def path(self):
 159                 """
 160                         Returns the name of the sub directory in which all RRD files
 161                         for this plugin should be stored in.
 162                 """
 163                 return self.name
 164
 165         ### Basic methods
 166
 167         def init(self, **kwargs):
 168                 """
 169                         Do some custom initialization stuff here.
 170                 """
 171                 pass
 172
 173         def collect(self):
 174                 """
 175                         Gathers the statistical data, this plugin collects.
 176                 """
 177                 time_start = time.time()
 178
 179                 # Run through all objects of this plugin and call the collect method.
 180                 for o in self.objects:
 181                         now = datetime.datetime.utcnow()
 182                         try:
 183                                 result = o.collect()
 184
 185                                 result = self._format_result(result)
 186                         except:
 187                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 188                                 continue
 189
 190                         if not result:
 191                                 self.log.warning(_("Received empty result: %s") % o)
 192                                 continue
 193
 194                         self.log.debug(_("Collected %s: %s") % (o, result))
 195
 196                         # Add the object to the write queue so that the data is written
 197                         # to the databases later.
 198                         self.collecty.write_queue.add(o, now, result)
 199
 200                 # Returns the time this function took to complete.
 201                 delay = time.time() - time_start
 202
 203                 # Log some warning when a collect method takes too long to return some data
 204                 if delay >= 60:
 205                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 206
 207         @staticmethod
 208         def _format_result(result):
 209                 if not isinstance(result, tuple) and not isinstance(result, list):
 210                         return result
 211
 212                 # Replace all Nones by NaN
 213                 s = []
 214
 215                 for e in result:
 216                         if e is None:
 217                                 e = "NaN"
 218
 219                         # Format as string
 220                         e = "%s" % e
 221
 222                         s.append(e)
 223
 224                 return ":".join(s)
 225
 226         def get_object(self, id):
 227                 for object in self.objects:
 228                         if not object.id == id:
 229                                 continue
 230
 231                         return object
 232
 233         def get_template(self, template_name, object_id):
 234                 for template in self.templates:
 235                         if not template.name == template_name:
 236                                 continue
 237
 238                         return template(self, object_id)
 239
 240         def generate_graph(self, template_name, object_id="default", **kwargs):
 241                 template = self.get_template(template_name, object_id=object_id)
 242                 if not template:
 243                         raise RuntimeError("Could not find template %s" % template_name)
 244
 245                 time_start = time.time()
 246
 247                 graph = template.generate_graph(**kwargs)
 248
 249                 duration = time.time() - time_start
 250                 self.log.debug(_("Generated graph %s in %.1fms") \
 251                         % (template, duration * 1000))
 252
 253                 return graph
 254
 255
 256 class Object(object):
 257         # The schema of the RRD database.
 258         rrd_schema = None
 259
 260         # RRA properties.
 261         rra_types     = ("AVERAGE", "MIN", "MAX")
 262         rra_timespans = (
 263                 ("1m", "10d"),
 264                 ("1h", "18M"),
 265                 ("1d",  "5y"),
 266         )
 267
 268         def __init__(self, plugin, *args, **kwargs):
 269                 self.plugin = plugin
 270
 271                 # Indicates if this object has collected its data
 272                 self.collected = False
 273
 274                 # Initialise this object
 275                 self.init(*args, **kwargs)
 276
 277                 # Create the database file.
 278                 self.create()
 279
 280         def __repr__(self):
 281                 return "<%s>" % self.__class__.__name__
 282
 283         @property
 284         def collecty(self):
 285                 return self.plugin.collecty
 286
 287         @property
 288         def log(self):
 289                 return self.plugin.log
 290
 291         @property
 292         def id(self):
 293                 """
 294                         Returns a UNIQUE identifier for this object. As this is incorporated
 295                         into the path of RRD file, it must only contain ASCII characters.
 296                 """
 297                 raise NotImplementedError
 298
 299         @property
 300         def file(self):
 301                 """
 302                         The absolute path to the RRD file of this plugin.
 303                 """
 304                 filename = self._normalise_filename("%s.rrd" % self.id)
 305
 306                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 307
 308         @staticmethod
 309         def _normalise_filename(filename):
 310                 # Convert the filename into ASCII characters only
 311                 filename = unicodedata.normalize("NFKC", filename)
 312
 313                 # Replace any spaces by dashes
 314                 filename = filename.replace(" ", "-")
 315
 316                 return filename
 317
 318         ### Basic methods
 319
 320         def init(self, *args, **kwargs):
 321                 """
 322                         Do some custom initialization stuff here.
 323                 """
 324                 pass
 325
 326         def create(self):
 327                 """
 328                         Creates an empty RRD file with the desired data structures.
 329                 """
 330                 # Skip if the file does already exist.
 331                 if os.path.exists(self.file):
 332                         return
 333
 334                 dirname = os.path.dirname(self.file)
 335                 if not os.path.exists(dirname):
 336                         os.makedirs(dirname)
 337
 338                 # Create argument list.
 339                 args = self.get_rrd_schema()
 340
 341                 rrdtool.create(self.file, *args)
 342
 343                 self.log.debug(_("Created RRD file %s.") % self.file)
 344                 for arg in args:
 345                         self.log.debug("  %s" % arg)
 346
 347         def info(self):
 348                 return rrdtool.info(self.file)
 349
 350         @property
 351         def stepsize(self):
 352                 return self.plugin.interval
 353
 354         @property
 355         def heartbeat(self):
 356                 return self.stepsize * 2
 357
 358         def get_rrd_schema(self):
 359                 schema = [
 360                         "--step", "%s" % self.stepsize,
 361                 ]
 362                 for line in self.rrd_schema:
 363                         if line.startswith("DS:"):
 364                                 try:
 365                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 366
 367                                         line = ":".join((
 368                                                 prefix,
 369                                                 name,
 370                                                 type,
 371                                                 "%s" % self.heartbeat,
 372                                                 lower_limit,
 373                                                 upper_limit
 374                                         ))
 375                                 except ValueError:
 376                                         pass
 377
 378                         schema.append(line)
 379
 380                 xff = 0.1
 381
 382                 for steps, rows in self.rra_timespans:
 383                         for type in self.rra_types:
 384                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 385
 386                 return schema
 387
 388         def execute(self):
 389                 if self.collected:
 390                         raise RuntimeError("This object has already collected its data")
 391
 392                 self.collected = True
 393                 self.now = datetime.datetime.utcnow()
 394
 395                 # Call the collect
 396                 result = self.collect()
 397
 398         def commit(self):
 399                 """
 400                         Will commit the collected data to the database.
 401                 """
 402                 # Make sure that the RRD database has been created
 403                 self.create()
 404
 405
 406 class GraphTemplate(object):
 407         # A unique name to identify this graph template.
 408         name = None
 409
 410         # Headline of the graph image
 411         graph_title = None
 412
 413         # Vertical label of the graph
 414         graph_vertical_label = None
 415
 416         # Limits
 417         lower_limit = None
 418         upper_limit = None
 419
 420         # Instructions how to create the graph.
 421         rrd_graph = None
 422
 423         # Extra arguments passed to rrdgraph.
 424         rrd_graph_args = []
 425
 426         intervals = {
 427                 None   : "-3h",
 428                 "hour" : "-1h",
 429                 "day"  : "-25h",
 430                 "month": "-30d",
 431                 "week" : "-360h",
 432                 "year" : "-365d",
 433         }
 434
 435         # Default dimensions for this graph
 436         height = GRAPH_DEFAULT_HEIGHT
 437         width  = GRAPH_DEFAULT_WIDTH
 438
 439         def __init__(self, plugin, object_id):
 440                 self.plugin = plugin
 441
 442                 # Get all required RRD objects
 443                 self.object_id = object_id
 444
 445                 # Get the main object
 446                 self.object = self.get_object(self.object_id)
 447
 448         def __repr__(self):
 449                 return "<%s>" % self.__class__.__name__
 450
 451         @property
 452         def collecty(self):
 453                 return self.plugin.collecty
 454
 455         @property
 456         def log(self):
 457                 return self.plugin.log
 458
 459         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 460                         width=None, height=None):
 461                 args = []
 462
 463                 args += GRAPH_DEFAULT_ARGUMENTS
 464
 465                 args += [
 466                         "--imgformat", format,
 467                         "--height", "%s" % (height or self.height),
 468                         "--width", "%s" % (width or self.width),
 469                 ]
 470
 471                 args += self.rrd_graph_args
 472
 473                 # Graph title
 474                 if self.graph_title:
 475                         args += ["--title", self.graph_title]
 476
 477                 # Vertical label
 478                 if self.graph_vertical_label:
 479                         args += ["--vertical-label", self.graph_vertical_label]
 480
 481                 if self.lower_limit is not None or self.upper_limit is not None:
 482                         # Force to honour the set limits
 483                         args.append("--rigid")
 484
 485                         if self.lower_limit is not None:
 486                                 args += ["--lower-limit", self.lower_limit]
 487
 488                         if self.upper_limit is not None:
 489                                 args += ["--upper-limit", self.upper_limit]
 490
 491                 try:
 492                         interval = self.intervals[interval]
 493                 except KeyError:
 494                         interval = "end-%s" % interval
 495
 496                 # Add interval
 497                 args += ["--start", interval]
 498
 499                 return args
 500
 501         def get_object(self, *args, **kwargs):
 502                 return self.plugin.get_object(*args, **kwargs)
 503
 504         def get_object_table(self):
 505                 return {
 506                         "file" : self.object,
 507                 }
 508
 509         @property
 510         def object_table(self):
 511                 if not hasattr(self, "_object_table"):
 512                         self._object_table = self.get_object_table()
 513
 514                 return self._object_table
 515
 516         def get_object_files(self):
 517                 files = {}
 518
 519                 for id, obj in self.object_table.items():
 520                         files[id] = obj.file
 521
 522                 return files
 523
 524         def generate_graph(self, interval=None, timezone=None, locale=None, **kwargs):
 525                 args = self._make_command_line(interval, **kwargs)
 526
 527                 self.log.info(_("Generating graph %s") % self)
 528                 self.log.debug("  args: %s" % args)
 529
 530                 object_files = self.get_object_files()
 531
 532                 for item in self.rrd_graph:
 533                         try:
 534                                 args.append(item % object_files)
 535                         except TypeError:
 536                                 args.append(item)
 537
 538                         self.log.debug("  %s" % args[-1])
 539
 540                 # Convert arguments to string
 541                 args = [str(e) for e in args]
 542
 543                 with Environment(timezone, locale):
 544                         graph = rrdtool.graphv("-", *args)
 545
 546                 return graph.get("image")