src/collecty/plugins/base.py

   1 #!/usr/bin/python3
   2 ###############################################################################
   3 #                                                                             #
   4 # collecty - A system statistics collection daemon for IPFire                 #
   5 # Copyright (C) 2012 IPFire development team                                  #
   6 #                                                                             #
   7 # This program is free software: you can redistribute it and/or modify        #
   8 # it under the terms of the GNU General Public License as published by        #
   9 # the Free Software Foundation, either version 3 of the License, or           #
  10 # (at your option) any later version.                                         #
  11 #                                                                             #
  12 # This program is distributed in the hope that it will be useful,             #
  13 # but WITHOUT ANY WARRANTY; without even the implied warranty of              #
  14 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
  15 # GNU General Public License for more details.                                #
  16 #                                                                             #
  17 # You should have received a copy of the GNU General Public License           #
  18 # along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
  19 #                                                                             #
  20 ###############################################################################
  21
  22 import datetime
  23 import logging
  24 import math
  25 import os
  26 import rrdtool
  27 import tempfile
  28 import threading
  29 import time
  30 import unicodedata
  31
  32 from ..constants import *
  33 from ..i18n import _
  34
  35 class Timer(object):
  36         def __init__(self, timeout, heartbeat=1):
  37                 self.timeout = timeout
  38                 self.heartbeat = heartbeat
  39
  40                 self.delay = 0
  41
  42                 self.reset()
  43
  44         def reset(self, delay=0):
  45                 # Save start time.
  46                 self.start = time.time()
  47
  48                 self.delay = delay
  49
  50                 # Has this timer been killed?
  51                 self.killed = False
  52
  53         @property
  54         def elapsed(self):
  55                 return time.time() - self.start - self.delay
  56
  57         def cancel(self):
  58                 self.killed = True
  59
  60         def wait(self):
  61                 while self.elapsed < self.timeout and not self.killed:
  62                         time.sleep(self.heartbeat)
  63
  64                 return self.elapsed > self.timeout
  65
  66
  67 class PluginRegistration(type):
  68         plugins = {}
  69
  70         def __init__(plugin, name, bases, dict):
  71                 type.__init__(plugin, name, bases, dict)
  72
  73                 # The main class from which is inherited is not registered
  74                 # as a plugin.
  75                 if name == "Plugin":
  76                         return
  77
  78                 if not all((plugin.name, plugin.description)):
  79                         raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
  80
  81                 PluginRegistration.plugins[plugin.name] = plugin
  82
  83
  84 def get():
  85         """
  86                 Returns a list with all automatically registered plugins.
  87         """
  88         return PluginRegistration.plugins.values()
  89
  90 class Plugin(object, metaclass=PluginRegistration):
  91         # The name of this plugin.
  92         name = None
  93
  94         # A description for this plugin.
  95         description = None
  96
  97         # Templates which can be used to generate a graph out of
  98         # the data from this data source.
  99         templates = []
 100
 101         # The default interval for all plugins
 102         interval = 60
 103
 104         def __init__(self, collecty, **kwargs):
 105                 self.collecty = collecty
 106
 107                 # Check if this plugin was configured correctly.
 108                 assert self.name, "Name of the plugin is not set: %s" % self.name
 109                 assert self.description, "Description of the plugin is not set: %s" % self.description
 110
 111                 # Initialize the logger.
 112                 self.log = logging.getLogger("collecty.plugins.%s" % self.name)
 113                 self.log.propagate = 1
 114
 115                 self.data = []
 116
 117                 # Run some custom initialization.
 118                 self.init(**kwargs)
 119
 120                 self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
 121
 122         @property
 123         def path(self):
 124                 """
 125                         Returns the name of the sub directory in which all RRD files
 126                         for this plugin should be stored in.
 127                 """
 128                 return self.name
 129
 130         ### Basic methods
 131
 132         def init(self, **kwargs):
 133                 """
 134                         Do some custom initialization stuff here.
 135                 """
 136                 pass
 137
 138         def collect(self):
 139                 """
 140                         Gathers the statistical data, this plugin collects.
 141                 """
 142                 time_start = time.time()
 143
 144                 # Run through all objects of this plugin and call the collect method.
 145                 for o in self.objects:
 146                         now = datetime.datetime.utcnow()
 147                         try:
 148                                 result = o.collect()
 149
 150                                 result = self._format_result(result)
 151                         except:
 152                                 self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
 153                                 continue
 154
 155                         if not result:
 156                                 self.log.warning(_("Received empty result: %s") % o)
 157                                 continue
 158
 159                         self.log.debug(_("Collected %s: %s") % (o, result))
 160
 161                         # Add the object to the write queue so that the data is written
 162                         # to the databases later.
 163                         self.collecty.write_queue.add(o, now, result)
 164
 165                 # Returns the time this function took to complete.
 166                 delay = time.time() - time_start
 167
 168                 # Log some warning when a collect method takes too long to return some data
 169                 if delay >= 60:
 170                         self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
 171
 172         @staticmethod
 173         def _format_result(result):
 174                 if not isinstance(result, tuple) and not isinstance(result, list):
 175                         return result
 176
 177                 # Replace all Nones by NaN
 178                 s = []
 179
 180                 for e in result:
 181                         if e is None:
 182                                 e = "NaN"
 183
 184                         # Format as string
 185                         e = "%s" % e
 186
 187                         s.append(e)
 188
 189                 return ":".join(s)
 190
 191         def get_object(self, id):
 192                 for object in self.objects:
 193                         if not object.id == id:
 194                                 continue
 195
 196                         return object
 197
 198         def get_template(self, template_name, object_id):
 199                 for template in self.templates:
 200                         if not template.name == template_name:
 201                                 continue
 202
 203                         return template(self, object_id)
 204
 205         def generate_graph(self, template_name, object_id="default", **kwargs):
 206                 template = self.get_template(template_name, object_id=object_id)
 207                 if not template:
 208                         raise RuntimeError("Could not find template %s" % template_name)
 209
 210                 time_start = time.time()
 211
 212                 graph = template.generate_graph(**kwargs)
 213
 214                 duration = time.time() - time_start
 215                 self.log.debug(_("Generated graph %s in %.1fms") \
 216                         % (template, duration * 1000))
 217
 218                 return graph
 219
 220
 221 class Object(object):
 222         # The schema of the RRD database.
 223         rrd_schema = None
 224
 225         # RRA properties.
 226         rra_types     = ("AVERAGE", "MIN", "MAX")
 227         rra_timespans = (
 228                 ("1m", "10d"),
 229                 ("1h", "18M"),
 230                 ("1d",  "5y"),
 231         )
 232
 233         def __init__(self, plugin, *args, **kwargs):
 234                 self.plugin = plugin
 235
 236                 # Indicates if this object has collected its data
 237                 self.collected = False
 238
 239                 # Initialise this object
 240                 self.init(*args, **kwargs)
 241
 242                 # Create the database file.
 243                 self.create()
 244
 245         def __repr__(self):
 246                 return "<%s>" % self.__class__.__name__
 247
 248         @property
 249         def collecty(self):
 250                 return self.plugin.collecty
 251
 252         @property
 253         def log(self):
 254                 return self.plugin.log
 255
 256         @property
 257         def id(self):
 258                 """
 259                         Returns a UNIQUE identifier for this object. As this is incorporated
 260                         into the path of RRD file, it must only contain ASCII characters.
 261                 """
 262                 raise NotImplementedError
 263
 264         @property
 265         def file(self):
 266                 """
 267                         The absolute path to the RRD file of this plugin.
 268                 """
 269                 filename = self._normalise_filename("%s.rrd" % self.id)
 270
 271                 return os.path.join(DATABASE_DIR, self.plugin.path, filename)
 272
 273         @staticmethod
 274         def _normalise_filename(filename):
 275                 # Convert the filename into ASCII characters only
 276                 filename = unicodedata.normalize("NFKC", filename)
 277
 278                 # Replace any spaces by dashes
 279                 filename = filename.replace(" ", "-")
 280
 281                 return filename
 282
 283         ### Basic methods
 284
 285         def init(self, *args, **kwargs):
 286                 """
 287                         Do some custom initialization stuff here.
 288                 """
 289                 pass
 290
 291         def create(self):
 292                 """
 293                         Creates an empty RRD file with the desired data structures.
 294                 """
 295                 # Skip if the file does already exist.
 296                 if os.path.exists(self.file):
 297                         return
 298
 299                 dirname = os.path.dirname(self.file)
 300                 if not os.path.exists(dirname):
 301                         os.makedirs(dirname)
 302
 303                 # Create argument list.
 304                 args = self.get_rrd_schema()
 305
 306                 rrdtool.create(self.file, *args)
 307
 308                 self.log.debug(_("Created RRD file %s.") % self.file)
 309                 for arg in args:
 310                         self.log.debug("  %s" % arg)
 311
 312         def info(self):
 313                 return rrdtool.info(self.file)
 314
 315         @property
 316         def stepsize(self):
 317                 return self.plugin.interval
 318
 319         @property
 320         def heartbeat(self):
 321                 return self.stepsize * 2
 322
 323         def get_rrd_schema(self):
 324                 schema = [
 325                         "--step", "%s" % self.stepsize,
 326                 ]
 327                 for line in self.rrd_schema:
 328                         if line.startswith("DS:"):
 329                                 try:
 330                                         (prefix, name, type, lower_limit, upper_limit) = line.split(":")
 331
 332                                         line = ":".join((
 333                                                 prefix,
 334                                                 name,
 335                                                 type,
 336                                                 "%s" % self.heartbeat,
 337                                                 lower_limit,
 338                                                 upper_limit
 339                                         ))
 340                                 except ValueError:
 341                                         pass
 342
 343                         schema.append(line)
 344
 345                 xff = 0.1
 346
 347                 for steps, rows in self.rra_timespans:
 348                         for type in self.rra_types:
 349                                 schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
 350
 351                 return schema
 352
 353         def execute(self):
 354                 if self.collected:
 355                         raise RuntimeError("This object has already collected its data")
 356
 357                 self.collected = True
 358                 self.now = datetime.datetime.utcnow()
 359
 360                 # Call the collect
 361                 result = self.collect()
 362
 363         def commit(self):
 364                 """
 365                         Will commit the collected data to the database.
 366                 """
 367                 # Make sure that the RRD database has been created
 368                 self.create()
 369
 370
 371 class GraphTemplate(object):
 372         # A unique name to identify this graph template.
 373         name = None
 374
 375         # Headline of the graph image
 376         graph_title = None
 377
 378         # Vertical label of the graph
 379         graph_vertical_label = None
 380
 381         # Limits
 382         lower_limit = None
 383         upper_limit = None
 384
 385         # Instructions how to create the graph.
 386         rrd_graph = None
 387
 388         # Extra arguments passed to rrdgraph.
 389         rrd_graph_args = []
 390
 391         intervals = {
 392                 None   : "-3h",
 393                 "hour" : "-1h",
 394                 "day"  : "-25h",
 395                 "month": "-30d",
 396                 "week" : "-360h",
 397                 "year" : "-365d",
 398         }
 399
 400         # Default dimensions for this graph
 401         height = GRAPH_DEFAULT_HEIGHT
 402         width  = GRAPH_DEFAULT_WIDTH
 403
 404         def __init__(self, plugin, object_id):
 405                 self.plugin = plugin
 406
 407                 # Get all required RRD objects
 408                 self.object_id = object_id
 409
 410                 # Get the main object
 411                 self.object = self.get_object(self.object_id)
 412
 413         def __repr__(self):
 414                 return "<%s>" % self.__class__.__name__
 415
 416         @property
 417         def collecty(self):
 418                 return self.plugin.collecty
 419
 420         @property
 421         def log(self):
 422                 return self.plugin.log
 423
 424         def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
 425                         width=None, height=None):
 426                 args = []
 427
 428                 args += GRAPH_DEFAULT_ARGUMENTS
 429
 430                 args += [
 431                         "--imgformat", format,
 432                         "--height", "%s" % (height or self.height),
 433                         "--width", "%s" % (width or self.width),
 434                 ]
 435
 436                 args += self.rrd_graph_args
 437
 438                 # Graph title
 439                 if self.graph_title:
 440                         args += ["--title", self.graph_title]
 441
 442                 # Vertical label
 443                 if self.graph_vertical_label:
 444                         args += ["--vertical-label", self.graph_vertical_label]
 445
 446                 if self.lower_limit is not None or self.upper_limit is not None:
 447                         # Force to honour the set limits
 448                         args.append("--rigid")
 449
 450                         if self.lower_limit is not None:
 451                                 args += ["--lower-limit", self.lower_limit]
 452
 453                         if self.upper_limit is not None:
 454                                 args += ["--upper-limit", self.upper_limit]
 455
 456                 try:
 457                         interval = self.intervals[interval]
 458                 except KeyError:
 459                         interval = "end-%s" % interval
 460
 461                 # Add interval
 462                 args += ["--start", interval]
 463
 464                 return args
 465
 466         def get_object(self, *args, **kwargs):
 467                 return self.plugin.get_object(*args, **kwargs)
 468
 469         def get_object_table(self):
 470                 return {
 471                         "file" : self.object,
 472                 }
 473
 474         @property
 475         def object_table(self):
 476                 if not hasattr(self, "_object_table"):
 477                         self._object_table = self.get_object_table()
 478
 479                 return self._object_table
 480
 481         def get_object_files(self):
 482                 files = {}
 483
 484                 for id, obj in self.object_table.items():
 485                         files[id] = obj.file
 486
 487                 return files
 488
 489         def generate_graph(self, interval=None, **kwargs):
 490                 args = self._make_command_line(interval, **kwargs)
 491
 492                 self.log.info(_("Generating graph %s") % self)
 493                 self.log.debug("  args: %s" % args)
 494
 495                 object_files = self.get_object_files()
 496
 497                 for item in self.rrd_graph:
 498                         try:
 499                                 args.append(item % object_files)
 500                         except TypeError:
 501                                 args.append(item)
 502
 503                         self.log.debug("  %s" % args[-1])
 504
 505                 # Convert arguments to string
 506                 args = [str(e) for e in args]
 507
 508                 graph = rrdtool.graphv("-", *args)
 509
 510                 return graph.get("image")