[collecty.git] / src / collecty / plugins / base.py

#!/usr/bin/python3
###############################################################################
#                                                                             #
# collecty - A system statistics collection daemon for IPFire                 #
# Copyright (C) 2012 IPFire development team                                  #
#                                                                             #
# This program is free software: you can redistribute it and/or modify        #
# it under the terms of the GNU General Public License as published by        #
# the Free Software Foundation, either version 3 of the License, or           #
# (at your option) any later version.                                         #
#                                                                             #
# This program is distributed in the hope that it will be useful,             #
# but WITHOUT ANY WARRANTY; without even the implied warranty of              #
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the               #
# GNU General Public License for more details.                                #
#                                                                             #
# You should have received a copy of the GNU General Public License           #
# along with this program.  If not, see <http://www.gnu.org/licenses/>.       #
#                                                                             #
###############################################################################

import datetime
import logging
import math
import os
import rrdtool
import tempfile
import threading
import time
import unicodedata

from ..constants import *
from ..i18n import _

class Timer(object):
	def __init__(self, timeout, heartbeat=1):
		self.timeout = timeout
		self.heartbeat = heartbeat

		self.delay = 0

		self.reset()

	def reset(self, delay=0):
		# Save start time.
		self.start = time.time()

		self.delay = delay

		# Has this timer been killed?
		self.killed = False

	@property
	def elapsed(self):
		return time.time() - self.start - self.delay

	def cancel(self):
		self.killed = True

	def wait(self):
		while self.elapsed < self.timeout and not self.killed:
			time.sleep(self.heartbeat)

		return self.elapsed > self.timeout


class PluginRegistration(type):
	plugins = {}

	def __init__(plugin, name, bases, dict):
		type.__init__(plugin, name, bases, dict)

		# The main class from which is inherited is not registered
		# as a plugin.
		if name == "Plugin":
			return

		if not all((plugin.name, plugin.description)):
			raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)

		PluginRegistration.plugins[plugin.name] = plugin


def get():
	"""
		Returns a list with all automatically registered plugins.
	"""
	return PluginRegistration.plugins.values()

class Plugin(object, metaclass=PluginRegistration):
	# The name of this plugin.
	name = None

	# A description for this plugin.
	description = None

	# Templates which can be used to generate a graph out of
	# the data from this data source.
	templates = []

	# The default interval for all plugins
	interval = 60

	def __init__(self, collecty, **kwargs):
		self.collecty = collecty

		# Check if this plugin was configured correctly.
		assert self.name, "Name of the plugin is not set: %s" % self.name
		assert self.description, "Description of the plugin is not set: %s" % self.description

		# Initialize the logger.
		self.log = logging.getLogger("collecty.plugins.%s" % self.name)
		self.log.propagate = 1

		self.data = []

		# Run some custom initialization.
		self.init(**kwargs)

		self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)

	@property
	def path(self):
		"""
			Returns the name of the sub directory in which all RRD files
			for this plugin should be stored in.
		"""
		return self.name

	### Basic methods

	def init(self, **kwargs):
		"""
			Do some custom initialization stuff here.
		"""
		pass

	def collect(self):
		"""
			Gathers the statistical data, this plugin collects.
		"""
		time_start = time.time()

		# Run through all objects of this plugin and call the collect method.
		for o in self.objects:
			now = datetime.datetime.utcnow()
			try:
				result = o.collect()

				if isinstance(result, tuple) or isinstance(result, list):
					result = ":".join(("%s" % e for e in result))
			except:
				self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
				continue

			if not result:
				self.log.warning(_("Received empty result: %s") % o)
				continue

			self.log.debug(_("Collected %s: %s") % (o, result))

			# Add the object to the write queue so that the data is written
			# to the databases later.
			self.collecty.write_queue.add(o, now, result)

		# Returns the time this function took to complete.
		delay = time.time() - time_start

		# Log some warning when a collect method takes too long to return some data
		if delay >= 60:
			self.log.warning(_("A worker thread was stalled for %.4fs") % delay)

	def get_object(self, id):
		for object in self.objects:
			if not object.id == id:
				continue

			return object

	def get_template(self, template_name, object_id):
		for template in self.templates:
			if not template.name == template_name:
				continue

			return template(self, object_id)

	def generate_graph(self, template_name, object_id="default", **kwargs):
		template = self.get_template(template_name, object_id=object_id)
		if not template:
			raise RuntimeError("Could not find template %s" % template_name)

		time_start = time.time()

		graph = template.generate_graph(**kwargs)

		duration = time.time() - time_start
		self.log.debug(_("Generated graph %s in %.1fms") \
			% (template, duration * 1000))

		return graph


class Object(object):
	# The schema of the RRD database.
	rrd_schema = None

	# RRA properties.
	rra_types     = ("AVERAGE", "MIN", "MAX")
	rra_timespans = (
		("1m", "10d"),
		("1h", "18M"),
		("1d",  "5y"),
	)

	def __init__(self, plugin, *args, **kwargs):
		self.plugin = plugin

		# Indicates if this object has collected its data
		self.collected = False

		# Initialise this object
		self.init(*args, **kwargs)

		# Create the database file.
		self.create()

	def __repr__(self):
		return "<%s>" % self.__class__.__name__

	@property
	def collecty(self):
		return self.plugin.collecty

	@property
	def log(self):
		return self.plugin.log

	@property
	def id(self):
		"""
			Returns a UNIQUE identifier for this object. As this is incorporated
			into the path of RRD file, it must only contain ASCII characters.
		"""
		raise NotImplementedError

	@property
	def file(self):
		"""
			The absolute path to the RRD file of this plugin.
		"""
		filename = self._normalise_filename("%s.rrd" % self.id)

		return os.path.join(DATABASE_DIR, self.plugin.path, filename)

	@staticmethod
	def _normalise_filename(filename):
		# Convert the filename into ASCII characters only
		filename = unicodedata.normalize("NFKC", filename)

		# Replace any spaces by dashes
		filename = filename.replace(" ", "-")

		return filename

	### Basic methods

	def init(self, *args, **kwargs):
		"""
			Do some custom initialization stuff here.
		"""
		pass

	def create(self):
		"""
			Creates an empty RRD file with the desired data structures.
		"""
		# Skip if the file does already exist.
		if os.path.exists(self.file):
			return

		dirname = os.path.dirname(self.file)
		if not os.path.exists(dirname):
			os.makedirs(dirname)

		# Create argument list.
		args = self.get_rrd_schema()

		rrdtool.create(self.file, *args)

		self.log.debug(_("Created RRD file %s.") % self.file)
		for arg in args:
			self.log.debug("  %s" % arg)

	def info(self):
		return rrdtool.info(self.file)

	@property
	def stepsize(self):
		return self.plugin.interval

	@property
	def heartbeat(self):
		return self.stepsize * 2

	def get_rrd_schema(self):
		schema = [
			"--step", "%s" % self.stepsize,
		]
		for line in self.rrd_schema:
			if line.startswith("DS:"):
				try:
					(prefix, name, type, lower_limit, upper_limit) = line.split(":")

					line = ":".join((
						prefix,
						name,
						type,
						"%s" % self.heartbeat,
						lower_limit,
						upper_limit
					))
				except ValueError:
					pass

			schema.append(line)

		xff = 0.1

		for steps, rows in self.rra_timespans:
			for type in self.rra_types:
				schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))

		return schema

	def execute(self):
		if self.collected:
			raise RuntimeError("This object has already collected its data")

		self.collected = True
		self.now = datetime.datetime.utcnow()

		# Call the collect
		result = self.collect()

	def commit(self):
		"""
			Will commit the collected data to the database.
		"""
		# Make sure that the RRD database has been created
		self.create()


class GraphTemplate(object):
	# A unique name to identify this graph template.
	name = None

	# Headline of the graph image
	graph_title = None

	# Vertical label of the graph
	graph_vertical_label = None

	# Limits
	lower_limit = None
	upper_limit = None

	# Instructions how to create the graph.
	rrd_graph = None

	# Extra arguments passed to rrdgraph.
	rrd_graph_args = []

	intervals = {
		None   : "-3h",
		"hour" : "-1h",
		"day"  : "-25h",
		"week" : "-360h",
		"year" : "-365d",
	}

	# Default dimensions for this graph
	height = GRAPH_DEFAULT_HEIGHT
	width  = GRAPH_DEFAULT_WIDTH

	def __init__(self, plugin, object_id):
		self.plugin = plugin

		# Get all required RRD objects
		self.object_id = object_id

		# Get the main object
		self.object = self.get_object(self.object_id)

	def __repr__(self):
		return "<%s>" % self.__class__.__name__

	@property
	def collecty(self):
		return self.plugin.collecty

	@property
	def log(self):
		return self.plugin.log

	def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
			width=None, height=None):
		args = []

		args += GRAPH_DEFAULT_ARGUMENTS

		args += [
			"--imgformat", format,
			"--height", "%s" % (height or self.height),
			"--width", "%s" % (width or self.width),
		]

		args += self.rrd_graph_args

		# Graph title
		if self.graph_title:
			args += ["--title", self.graph_title]

		# Vertical label
		if self.graph_vertical_label:
			args += ["--vertical-label", self.graph_vertical_label]

		if self.lower_limit is not None or self.upper_limit is not None:
			# Force to honour the set limits
			args.append("--rigid")

			if self.lower_limit is not None:
				args += ["--lower-limit", self.lower_limit]

			if self.upper_limit is not None:
				args += ["--upper-limit", self.upper_limit]

		# Add interval
		args.append("--start")

		try:
			args.append(self.intervals[interval])
		except KeyError:
			args.append(str(interval))

		return args

	def get_object(self, *args, **kwargs):
		return self.plugin.get_object(*args, **kwargs)

	def get_object_table(self):
		return {
			"file" : self.object,
		}

	@property
	def object_table(self):
		if not hasattr(self, "_object_table"):
			self._object_table = self.get_object_table()

		return self._object_table

	def get_object_files(self):
		files = {}

		for id, obj in self.object_table.items():
			files[id] = obj.file

		return files

	def generate_graph(self, interval=None, **kwargs):
		args = self._make_command_line(interval, **kwargs)

		self.log.info(_("Generating graph %s") % self)
		self.log.debug("  args: %s" % args)

		object_files = self.get_object_files()

		for item in self.rrd_graph:
			try:
				args.append(item % object_files)
			except TypeError:
				args.append(item)

			self.log.debug("  %s" % args[-1])

		# Convert arguments to string
		args = [str(e) for e in args]

		graph = rrdtool.graphv("-", *args)

		return graph.get("image")
Commit	Line	Data
	1	#!/usr/bin/python3
	2	###############################################################################
	3	# #
	4	# collecty - A system statistics collection daemon for IPFire #
	5	# Copyright (C) 2012 IPFire development team #
	6	# #
	7	# This program is free software: you can redistribute it and/or modify #
	8	# it under the terms of the GNU General Public License as published by #
	9	# the Free Software Foundation, either version 3 of the License, or #
	10	# (at your option) any later version. #
	11	# #
	12	# This program is distributed in the hope that it will be useful, #
	13	# but WITHOUT ANY WARRANTY; without even the implied warranty of #
	14	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #
	15	# GNU General Public License for more details. #
	16	# #
	17	# You should have received a copy of the GNU General Public License #
	18	# along with this program. If not, see <http://www.gnu.org/licenses/>. #
	19	# #
	20	###############################################################################
	21
	22	import datetime
	23	import logging
	24	import math
	25	import os
	26	import rrdtool
	27	import tempfile
	28	import threading
	29	import time
	30	import unicodedata
	31
	32	from ..constants import *
	33	from ..i18n import _
	34
	35	class Timer(object):
	36	def __init__(self, timeout, heartbeat=1):
	37	self.timeout = timeout
	38	self.heartbeat = heartbeat
	39
	40	self.delay = 0
	41
	42	self.reset()
	43
	44	def reset(self, delay=0):
	45	# Save start time.
	46	self.start = time.time()
	47
	48	self.delay = delay
	49
	50	# Has this timer been killed?
	51	self.killed = False
	52
	53	@property
	54	def elapsed(self):
	55	return time.time() - self.start - self.delay
	56
	57	def cancel(self):
	58	self.killed = True
	59
	60	def wait(self):
	61	while self.elapsed < self.timeout and not self.killed:
	62	time.sleep(self.heartbeat)
	63
	64	return self.elapsed > self.timeout
	65
	66
	67	class PluginRegistration(type):
	68	plugins = {}
	69
	70	def __init__(plugin, name, bases, dict):
	71	type.__init__(plugin, name, bases, dict)
	72
	73	# The main class from which is inherited is not registered
	74	# as a plugin.
	75	if name == "Plugin":
	76	return
	77
	78	if not all((plugin.name, plugin.description)):
	79	raise RuntimeError(_("Plugin is not properly configured: %s") % plugin)
	80
	81	PluginRegistration.plugins[plugin.name] = plugin
	82
	83
	84	def get():
	85	"""
	86	Returns a list with all automatically registered plugins.
	87	"""
	88	return PluginRegistration.plugins.values()
	89
	90	class Plugin(object, metaclass=PluginRegistration):
	91	# The name of this plugin.
	92	name = None
	93
	94	# A description for this plugin.
	95	description = None
	96
	97	# Templates which can be used to generate a graph out of
	98	# the data from this data source.
	99	templates = []
	100
	101	# The default interval for all plugins
	102	interval = 60
	103
	104	def __init__(self, collecty, **kwargs):
	105	self.collecty = collecty
	106
	107	# Check if this plugin was configured correctly.
	108	assert self.name, "Name of the plugin is not set: %s" % self.name
	109	assert self.description, "Description of the plugin is not set: %s" % self.description
	110
	111	# Initialize the logger.
	112	self.log = logging.getLogger("collecty.plugins.%s" % self.name)
	113	self.log.propagate = 1
	114
	115	self.data = []
	116
	117	# Run some custom initialization.
	118	self.init(**kwargs)
	119
	120	self.log.debug(_("Successfully initialized %s") % self.__class__.__name__)
	121
	122	@property
	123	def path(self):
	124	"""
	125	Returns the name of the sub directory in which all RRD files
	126	for this plugin should be stored in.
	127	"""
	128	return self.name
	129
	130	### Basic methods
	131
	132	def init(self, **kwargs):
	133	"""
	134	Do some custom initialization stuff here.
	135	"""
	136	pass
	137
	138	def collect(self):
	139	"""
	140	Gathers the statistical data, this plugin collects.
	141	"""
	142	time_start = time.time()
	143
	144	# Run through all objects of this plugin and call the collect method.
	145	for o in self.objects:
	146	now = datetime.datetime.utcnow()
	147	try:
	148	result = o.collect()
	149
	150	if isinstance(result, tuple) or isinstance(result, list):
	151	result = ":".join(("%s" % e for e in result))
	152	except:
	153	self.log.warning(_("Unhandled exception in %s.collect()") % o, exc_info=True)
	154	continue
	155
	156	if not result:
	157	self.log.warning(_("Received empty result: %s") % o)
	158	continue
	159
	160	self.log.debug(_("Collected %s: %s") % (o, result))
	161
	162	# Add the object to the write queue so that the data is written
	163	# to the databases later.
	164	self.collecty.write_queue.add(o, now, result)
	165
	166	# Returns the time this function took to complete.
	167	delay = time.time() - time_start
	168
	169	# Log some warning when a collect method takes too long to return some data
	170	if delay >= 60:
	171	self.log.warning(_("A worker thread was stalled for %.4fs") % delay)
	172
	173	def get_object(self, id):
	174	for object in self.objects:
	175	if not object.id == id:
	176	continue
	177
	178	return object
	179
	180	def get_template(self, template_name, object_id):
	181	for template in self.templates:
	182	if not template.name == template_name:
	183	continue
	184
	185	return template(self, object_id)
	186
	187	def generate_graph(self, template_name, object_id="default", **kwargs):
	188	template = self.get_template(template_name, object_id=object_id)
	189	if not template:
	190	raise RuntimeError("Could not find template %s" % template_name)
	191
	192	time_start = time.time()
	193
	194	graph = template.generate_graph(**kwargs)
	195
	196	duration = time.time() - time_start
	197	self.log.debug(_("Generated graph %s in %.1fms") \
	198	% (template, duration * 1000))
	199
	200	return graph
	201
	202
	203	class Object(object):
	204	# The schema of the RRD database.
	205	rrd_schema = None
	206
	207	# RRA properties.
	208	rra_types = ("AVERAGE", "MIN", "MAX")
	209	rra_timespans = (
	210	("1m", "10d"),
	211	("1h", "18M"),
	212	("1d", "5y"),
	213	)
	214
	215	def __init__(self, plugin, args, *kwargs):
	216	self.plugin = plugin
	217
	218	# Indicates if this object has collected its data
	219	self.collected = False
	220
	221	# Initialise this object
	222	self.init(args, *kwargs)
	223
	224	# Create the database file.
	225	self.create()
	226
	227	def __repr__(self):
	228	return "<%s>" % self.__class__.__name__
	229
	230	@property
	231	def collecty(self):
	232	return self.plugin.collecty
	233
	234	@property
	235	def log(self):
	236	return self.plugin.log
	237
	238	@property
	239	def id(self):
	240	"""
	241	Returns a UNIQUE identifier for this object. As this is incorporated
	242	into the path of RRD file, it must only contain ASCII characters.
	243	"""
	244	raise NotImplementedError
	245
	246	@property
	247	def file(self):
	248	"""
	249	The absolute path to the RRD file of this plugin.
	250	"""
	251	filename = self._normalise_filename("%s.rrd" % self.id)
	252
	253	return os.path.join(DATABASE_DIR, self.plugin.path, filename)
	254
	255	@staticmethod
	256	def _normalise_filename(filename):
	257	# Convert the filename into ASCII characters only
	258	filename = unicodedata.normalize("NFKC", filename)
	259
	260	# Replace any spaces by dashes
	261	filename = filename.replace(" ", "-")
	262
	263	return filename
	264
	265	### Basic methods
	266
	267	def init(self, args, *kwargs):
	268	"""
	269	Do some custom initialization stuff here.
	270	"""
	271	pass
	272
	273	def create(self):
	274	"""
	275	Creates an empty RRD file with the desired data structures.
	276	"""
	277	# Skip if the file does already exist.
	278	if os.path.exists(self.file):
	279	return
	280
	281	dirname = os.path.dirname(self.file)
	282	if not os.path.exists(dirname):
	283	os.makedirs(dirname)
	284
	285	# Create argument list.
	286	args = self.get_rrd_schema()
	287
	288	rrdtool.create(self.file, *args)
	289
	290	self.log.debug(_("Created RRD file %s.") % self.file)
	291	for arg in args:
	292	self.log.debug(" %s" % arg)
	293
	294	def info(self):
	295	return rrdtool.info(self.file)
	296
	297	@property
	298	def stepsize(self):
	299	return self.plugin.interval
	300
	301	@property
	302	def heartbeat(self):
	303	return self.stepsize * 2
	304
	305	def get_rrd_schema(self):
	306	schema = [
	307	"--step", "%s" % self.stepsize,
	308	]
	309	for line in self.rrd_schema:
	310	if line.startswith("DS:"):
	311	try:
	312	(prefix, name, type, lower_limit, upper_limit) = line.split(":")
	313
	314	line = ":".join((
	315	prefix,
	316	name,
	317	type,
	318	"%s" % self.heartbeat,
	319	lower_limit,
	320	upper_limit
	321	))
	322	except ValueError:
	323	pass
	324
	325	schema.append(line)
	326
	327	xff = 0.1
	328
	329	for steps, rows in self.rra_timespans:
	330	for type in self.rra_types:
	331	schema.append("RRA:%s:%s:%s:%s" % (type, xff, steps, rows))
	332
	333	return schema
	334
	335	def execute(self):
	336	if self.collected:
	337	raise RuntimeError("This object has already collected its data")
	338
	339	self.collected = True
	340	self.now = datetime.datetime.utcnow()
	341
	342	# Call the collect
	343	result = self.collect()
	344
	345	def commit(self):
	346	"""
	347	Will commit the collected data to the database.
	348	"""
	349	# Make sure that the RRD database has been created
	350	self.create()
	351
	352
	353	class GraphTemplate(object):
	354	# A unique name to identify this graph template.
	355	name = None
	356
	357	# Headline of the graph image
	358	graph_title = None
	359
	360	# Vertical label of the graph
	361	graph_vertical_label = None
	362
	363	# Limits
	364	lower_limit = None
	365	upper_limit = None
	366
	367	# Instructions how to create the graph.
	368	rrd_graph = None
	369
	370	# Extra arguments passed to rrdgraph.
	371	rrd_graph_args = []
	372
	373	intervals = {
	374	None : "-3h",
	375	"hour" : "-1h",
	376	"day" : "-25h",
	377	"week" : "-360h",
	378	"year" : "-365d",
	379	}
	380
	381	# Default dimensions for this graph
	382	height = GRAPH_DEFAULT_HEIGHT
	383	width = GRAPH_DEFAULT_WIDTH
	384
	385	def __init__(self, plugin, object_id):
	386	self.plugin = plugin
	387
	388	# Get all required RRD objects
	389	self.object_id = object_id
	390
	391	# Get the main object
	392	self.object = self.get_object(self.object_id)
	393
	394	def __repr__(self):
	395	return "<%s>" % self.__class__.__name__
	396
	397	@property
	398	def collecty(self):
	399	return self.plugin.collecty
	400
	401	@property
	402	def log(self):
	403	return self.plugin.log
	404
	405	def _make_command_line(self, interval, format=DEFAULT_IMAGE_FORMAT,
	406	width=None, height=None):
	407	args = []
	408
	409	args += GRAPH_DEFAULT_ARGUMENTS
	410
	411	args += [
	412	"--imgformat", format,
	413	"--height", "%s" % (height or self.height),
	414	"--width", "%s" % (width or self.width),
	415	]
	416
	417	args += self.rrd_graph_args
	418
	419	# Graph title
	420	if self.graph_title:
	421	args += ["--title", self.graph_title]
	422
	423	# Vertical label
	424	if self.graph_vertical_label:
	425	args += ["--vertical-label", self.graph_vertical_label]
	426
	427	if self.lower_limit is not None or self.upper_limit is not None:
	428	# Force to honour the set limits
	429	args.append("--rigid")
	430
	431	if self.lower_limit is not None:
	432	args += ["--lower-limit", self.lower_limit]
	433
	434	if self.upper_limit is not None:
	435	args += ["--upper-limit", self.upper_limit]
	436
	437	# Add interval
	438	args.append("--start")
	439
	440	try:
	441	args.append(self.intervals[interval])
	442	except KeyError:
	443	args.append(str(interval))
	444
	445	return args
	446
	447	def get_object(self, args, *kwargs):
	448	return self.plugin.get_object(args, *kwargs)
	449
	450	def get_object_table(self):
	451	return {
	452	"file" : self.object,
	453	}
	454
	455	@property
	456	def object_table(self):
	457	if not hasattr(self, "_object_table"):
	458	self._object_table = self.get_object_table()
	459
	460	return self._object_table
	461
	462	def get_object_files(self):
	463	files = {}
	464
	465	for id, obj in self.object_table.items():
	466	files[id] = obj.file
	467
	468	return files
	469
	470	def generate_graph(self, interval=None, **kwargs):
	471	args = self._make_command_line(interval, **kwargs)
	472
	473	self.log.info(_("Generating graph %s") % self)
	474	self.log.debug(" args: %s" % args)
	475
	476	object_files = self.get_object_files()
	477
	478	for item in self.rrd_graph:
	479	try:
	480	args.append(item % object_files)
	481	except TypeError:
	482	args.append(item)
	483
	484	self.log.debug(" %s" % args[-1])
	485
	486	# Convert arguments to string
	487	args = [str(e) for e in args]
	488
	489	graph = rrdtool.graphv("-", *args)
	490
	491	return graph.get("image")