From: Michael Tremer Date: Sun, 24 May 2015 10:38:05 +0000 (+0000) Subject: Add disk plugin X-Git-Tag: 003~26 X-Git-Url: http://git.ipfire.org/?p=collecty.git;a=commitdiff_plain;h=30777a6c8aed027b18b00c495b79b353bacbe923 Add disk plugin The disk plugin collects information about read and written sectors, IO operations, bad sectors and the temperature of the disks. It uses libatasmart to collect that information. --- diff --git a/.gitignore b/.gitignore index 6256b1b..6e3c11c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /Makefile /build-aux +/libtool /man/*.[0-9] /man/*.html /missing @@ -9,11 +10,17 @@ /*.tar.bz2 /*.tar.gz /*.tar.xz +*.a +*.la +*.lo *.log *.cache *.gmo *.mo *~ +.deps +.dirstamp +.libs Makefile.in aclocal.m4 config.log diff --git a/Makefile.am b/Makefile.am index 72c06ad..b9aa841 100644 --- a/Makefile.am +++ b/Makefile.am @@ -87,6 +87,7 @@ collectyplugins_PYTHON = \ src/collecty/plugins/base.py \ src/collecty/plugins/conntrack.py \ src/collecty/plugins/cpu.py \ + src/collecty/plugins/disk.py \ src/collecty/plugins/entropy.py \ src/collecty/plugins/__init__.py \ src/collecty/plugins/interface.py \ @@ -96,6 +97,27 @@ collectyplugins_PYTHON = \ collectypluginsdir = $(collectydir)/plugins +pkgpyexec_LTLIBRARIES = \ + _collecty.la + +_collecty_la_SOURCES = \ + src/_collectymodule.c + +_collecty_la_CFLAGS = \ + $(AM_CFLAGS) \ + $(LIBATASMART_CFLAGS) \ + $(PYTHON_DEVEL_CFLAGS) + +_collecty_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + -shared \ + -module \ + -avoid-version + +_collecty_la_LIBADD = \ + $(LIBATASMART_LIBS) \ + $(PYTHON_DEVEL_LIBS) + dist_dbuspolicy_DATA = \ src/dbus/org.ipfire.collecty1.conf diff --git a/autogen.sh b/autogen.sh index 077e109..212a394 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,4 +1,5 @@ #!/bin/sh +libtoolize intltoolize --force --automake autoreconf --force --install --symlink diff --git a/configure.ac b/configure.ac index 0d943e3..810f693 100644 --- a/configure.ac +++ b/configure.ac @@ -41,6 +41,8 @@ AM_INIT_AUTOMAKE([ subdir-objects ]) AM_SILENT_RULES([yes]) +LT_PREREQ(2.2) +LT_INIT([disable-static]) IT_PROG_INTLTOOL([0.40.0]) @@ -51,10 +53,20 @@ AC_PROG_LN_S AC_PROG_MKDIR_P AC_PROG_SED +# C Compiler +AC_PROG_CC +AC_PROG_CC_C99 +AC_PROG_CC_C_O +AC_PROG_GCC_TRADITIONAL + AC_PATH_PROG([XSLTPROC], [xsltproc]) # Python AM_PATH_PYTHON([2.7]) +PKG_CHECK_MODULES([PYTHON_DEVEL], [python-${PYTHON_VERSION}]) + +# libatasmart +PKG_CHECK_MODULES([LIBATASMART], [libatasmart >= 0.19]) save_LIBS="$LIBS" diff --git a/po/POTFILES.in b/po/POTFILES.in index 61715be..b94ec75 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -9,6 +9,7 @@ src/collecty/ping.py src/collecty/plugins/base.py src/collecty/plugins/conntrack.py src/collecty/plugins/cpu.py +src/collecty/plugins/disk.py src/collecty/plugins/entropy.py src/collecty/plugins/__init__.py src/collecty/plugins/interface.py diff --git a/src/_collectymodule.c b/src/_collectymodule.c new file mode 100644 index 0000000..eefe871 --- /dev/null +++ b/src/_collectymodule.c @@ -0,0 +1,325 @@ +/* + * collecty + * Copyright (C) 2015 IPFire Team (www.ipfire.org) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#define MODEL_SIZE 40 +#define SERIAL_SIZE 20 + +typedef struct { + PyObject_HEAD + char* path; + struct hd_driveid identity; + SkDisk* disk; +} BlockDevice; + +static void BlockDevice_dealloc(BlockDevice* self) { + if (self->disk) + sk_disk_free(self->disk); + + if (self->path) + free(self->path); + + self->ob_type->tp_free((PyObject*)self); +} + +static int BlockDevice_get_identity(BlockDevice* device) { + int fd; + + if ((fd = open(device->path, O_RDONLY | O_NONBLOCK)) < 0) { + return 1; + } + + int r = ioctl(fd, HDIO_GET_IDENTITY, &device->identity); + close(fd); + + if (r) + return 1; + + return 0; +} + +static int BlockDevice_smart_is_available(BlockDevice* device) { + SkBool available = FALSE; + + int r = sk_disk_smart_is_available(device->disk, &available); + if (r) + return -1; + + if (available) + return 0; + + return 1; +} + +static int BlockDevice_check_sleep_mode(BlockDevice* device) { + SkBool awake = FALSE; + + int r = sk_disk_check_sleep_mode(device->disk, &awake); + if (r) + return -1; + + if (awake) + return 0; + + return 1; +} + +static PyObject * BlockDevice_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + BlockDevice* self = (BlockDevice*)type->tp_alloc(type, 0); + + if (self) { + self->path = NULL; + + // libatasmart + self->disk = NULL; + } + + return (PyObject *)self; +} + +static int BlockDevice_init(BlockDevice* self, PyObject* args, PyObject* kwds) { + const char* path = NULL; + + if (!PyArg_ParseTuple(args, "s", &path)) + return -1; + + self->path = strdup(path); + + int r = BlockDevice_get_identity(self); + if (r) { + PyErr_Format(PyExc_OSError, "Could not open block device: %s", path); + return -1; + } + + r = sk_disk_open(path, &self->disk); + if (r == 0) { + if (BlockDevice_smart_is_available(self) == 0) { + if (BlockDevice_check_sleep_mode(self) == 0) { + r = sk_disk_smart_read_data(self->disk); + if (r) { + PyErr_Format(PyExc_OSError, "Could not open block device %s: %s", path, + strerror(errno)); + return -1; + } + } + } + } else { + PyErr_Format(PyExc_OSError, "Could not open block device %s: %s", path, + strerror(errno)); + return -1; + } + + //sk_disk_identify_is_available + + return 0; +} + +static PyObject* BlockDevice_get_path(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + return PyString_FromString(device->path); +} + +static void clean_string(char *s) { + for (char* e = s; *e; e++) { + if (*e < ' ' || *e >= 127) + *e = ' '; + } +} + +static void drop_spaces(char *s) { + char *d = s; + bool prev_space = false; + + s += strspn(s, " "); + + for (; *s; s++) { + if (prev_space) { + if (*s != ' ') { + prev_space = false; + *(d++) = ' '; + *(d++) = *s; + } + } else { + if (*s == ' ') + prev_space = true; + else + *(d++) = *s; + } + } + + *d = 0; +} + +static void copy_string(char* d, const char* s, size_t n) { + // Copy the source buffer to the destination buffer up to n + memcpy(d, s, n); + + // Terminate the destination buffer with NULL + d[n] = '\0'; + + // Clean up the string from non-printable characters + clean_string(d); + drop_spaces(d); +} + +static PyObject* BlockDevice_get_model(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + char model[MODEL_SIZE + 1]; + copy_string(model, device->identity.model, sizeof(model)); + + return PyString_FromString(model); +} + +static PyObject* BlockDevice_get_serial(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + char serial[SERIAL_SIZE + 1]; + copy_string(serial, device->identity.serial_no, sizeof(serial)); + + return PyString_FromString(serial); +} + +static PyObject* BlockDevice_is_smart_supported(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device) == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* BlockDevice_is_awake(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_check_sleep_mode(device) == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* BlockDevice_get_bad_sectors(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device)) { + PyErr_Format(PyExc_OSError, "Device does not support SMART"); + return NULL; + } + + uint64_t bad_sectors; + int r = sk_disk_smart_get_bad(device->disk, &bad_sectors); + if (r) + return NULL; + + return PyLong_FromUnsignedLongLong((unsigned long long)bad_sectors); +} + +static PyObject* BlockDevice_get_temperature(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device)) { + PyErr_Format(PyExc_OSError, "Device does not support SMART"); + return NULL; + } + + uint64_t mkelvin; + int r = sk_disk_smart_get_temperature(device->disk, &mkelvin); + if (r) + return NULL; + + return PyLong_FromUnsignedLongLong((unsigned long long)mkelvin); +} + +static PyGetSetDef BlockDevice_getsetters[] = { + {"path", (getter)BlockDevice_get_path, NULL, NULL, NULL}, + {"model", (getter)BlockDevice_get_model, NULL, NULL, NULL}, + {"serial", (getter)BlockDevice_get_serial, NULL, NULL, NULL}, +}; + +static PyMethodDef BlockDevice_methods[] = { + {"get_bad_sectors", (PyCFunction)BlockDevice_get_bad_sectors, METH_NOARGS, NULL}, + {"get_temperature", (PyCFunction)BlockDevice_get_temperature, METH_NOARGS, NULL}, + {"is_smart_supported", (PyCFunction)BlockDevice_is_smart_supported, METH_NOARGS, NULL}, + {"is_awake", (PyCFunction)BlockDevice_is_awake, METH_NOARGS, NULL}, + {NULL} +}; + +static PyTypeObject BlockDeviceType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "_collecty.BlockDevice", /*tp_name*/ + sizeof(BlockDevice), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BlockDevice_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "BlockDevice objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BlockDevice_methods, /* tp_methods */ + 0, /* tp_members */ + BlockDevice_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BlockDevice_init, /* tp_init */ + 0, /* tp_alloc */ + BlockDevice_new, /* tp_new */ +}; + +static PyMethodDef collecty_module_methods[] = { + {NULL}, +}; + +void init_collecty(void) { + if (PyType_Ready(&BlockDeviceType) < 0) + return; + + PyObject* m = Py_InitModule("_collecty", collecty_module_methods); + + PyModule_AddObject(m, "BlockDevice", (PyObject*)&BlockDeviceType); +} diff --git a/src/collecty/plugins/__init__.py b/src/collecty/plugins/__init__.py index cc7f41c..5f843a8 100644 --- a/src/collecty/plugins/__init__.py +++ b/src/collecty/plugins/__init__.py @@ -24,6 +24,7 @@ from base import Timer, get import base import conntrack import cpu +import disk import entropy import interface import latency diff --git a/src/collecty/plugins/disk.py b/src/collecty/plugins/disk.py new file mode 100644 index 0000000..b5a2346 --- /dev/null +++ b/src/collecty/plugins/disk.py @@ -0,0 +1,315 @@ +#!/usr/bin/python +############################################################################### +# # +# collecty - A system statistics collection daemon for IPFire # +# Copyright (C) 2012 IPFire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +from collecty import _collecty +import os +import re + +import base + +from ..i18n import _ + +class GraphTemplateDiskBadSectors(base.GraphTemplate): + name = "disk-bad-sectors" + + rrd_graph = [ + "DEF:bad_sectors=%(file)s:bad_sectors:AVERAGE", + + "AREA:bad_sectors#ff0000:%s" % _("Bad Sectors"), + + "VDEF:bad_sectors_cur=bad_sectors,LAST", + "VDEF:bad_sectors_max=bad_sectors,MAXIMUM", + "GPRINT:bad_sectors_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:bad_sectors_max:%12s\:" % _("Maximum") + " %9.2lf\\n", + ] + + @property + def graph_title(self): + return _("Bad Sectors of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Pending/Relocated Sectors") + + +class GraphTemplateDiskBytes(base.GraphTemplate): + name = "disk-bytes" + + rrd_graph = [ + "DEF:read_sectors=%(file)s:read_sectors:AVERAGE", + "DEF:write_sectors=%(file)s:write_sectors:AVERAGE", + + "CDEF:read_bytes=read_sectors,512,*", + "CDEF:write_bytes=write_sectors,512,*", + + "LINE1:read_bytes#ff0000:%-15s" % _("Read"), + "VDEF:read_cur=read_bytes,LAST", + "VDEF:read_min=read_bytes,MINIMUM", + "VDEF:read_max=read_bytes,MAXIMUM", + "VDEF:read_avg=read_bytes,AVERAGE", + "GPRINT:read_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:read_max:%12s\:" % _("Maximum") + " %9.2lf", + "GPRINT:read_min:%12s\:" % _("Minimum") + " %9.2lf", + "GPRINT:read_avg:%12s\:" % _("Average") + " %9.2lf\\n", + + "LINE1:write_bytes#00ff00:%-15s" % _("Written"), + "VDEF:write_cur=write_bytes,LAST", + "VDEF:write_min=write_bytes,MINIMUM", + "VDEF:write_max=write_bytes,MAXIMUM", + "VDEF:write_avg=write_bytes,AVERAGE", + "GPRINT:write_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:write_max:%12s\:" % _("Maximum") + " %9.2lf", + "GPRINT:write_min:%12s\:" % _("Minimum") + " %9.2lf", + "GPRINT:write_avg:%12s\:" % _("Average") + " %9.2lf\\n", + ] + + lower_limit = 0 + + @property + def graph_title(self): + return _("Disk Utilisation of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Byte per Second") + + +class GraphTemplateDiskIoOps(base.GraphTemplate): + name = "disk-io-ops" + + rrd_graph = [ + "DEF:read_ios=%(file)s:read_ios:AVERAGE", + "DEF:write_ios=%(file)s:write_ios:AVERAGE", + + "LINE1:read_ios#ff0000:%-15s" % _("Read"), + "VDEF:read_cur=read_ios,LAST", + "VDEF:read_min=read_ios,MINIMUM", + "VDEF:read_max=read_ios,MAXIMUM", + "VDEF:read_avg=read_ios,AVERAGE", + "GPRINT:read_cur:%12s\:" % _("Current") + " %6.2lf", + "GPRINT:read_max:%12s\:" % _("Maximum") + " %6.2lf", + "GPRINT:read_min:%12s\:" % _("Minimum") + " %6.2lf", + "GPRINT:read_avg:%12s\:" % _("Average") + " %6.2lf\\n", + + "LINE1:write_ios#00ff00:%-15s" % _("Written"), + "VDEF:write_cur=write_ios,LAST", + "VDEF:write_min=write_ios,MINIMUM", + "VDEF:write_max=write_ios,MAXIMUM", + "VDEF:write_avg=write_ios,AVERAGE", + "GPRINT:write_cur:%12s\:" % _("Current") + " %6.2lf", + "GPRINT:write_max:%12s\:" % _("Maximum") + " %6.2lf", + "GPRINT:write_min:%12s\:" % _("Minimum") + " %6.2lf", + "GPRINT:write_avg:%12s\:" % _("Average") + " %6.2lf\\n", + ] + + lower_limit = 0 + + @property + def graph_title(self): + return _("Disk IO Operations of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Operations per Second") + + +class GraphTemplateDiskTemperature(base.GraphTemplate): + name = "disk-temperature" + + rrd_graph = [ + "DEF:mkelvin=%(file)s:temperature:AVERAGE", + "CDEF:celsius=mkelvin,1000,/,273.15,-", + + "LINE2:celsius#ff0000:%s" % _("Temperature"), + "VDEF:temp_cur=celsius,LAST", + "VDEF:temp_min=celsius,MINIMUM", + "VDEF:temp_max=celsius,MAXIMUM", + "VDEF:temp_avg=celsius,AVERAGE", + "GPRINT:temp_cur:%12s\:" % _("Current") + " %3.2lf", + "GPRINT:temp_max:%12s\:" % _("Maximum") + " %3.2lf", + "GPRINT:temp_min:%12s\:" % _("Minimum") + " %3.2lf", + "GPRINT:temp_avg:%12s\:" % _("Average") + " %3.2lf\\n", + ] + + @property + def graph_title(self): + return _("Disk Temperature of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("° Celsius") + + @property + def rrd_graph_args(self): + return [ + # Make the y-axis have a decimal + "--left-axis-format", "%3.1lf", + ] + + +class DiskObject(base.Object): + rrd_schema = [ + "DS:awake:GAUGE:0:1", + "DS:read_ios:DERIVE:0:U", + "DS:read_sectors:DERIVE:0:U", + "DS:write_ios:DERIVE:0:U", + "DS:write_sectors:DERIVE:0:U", + "DS:bad_sectors:GAUGE:0:U", + "DS:temperature:GAUGE:U:U", + ] + + def __repr__(self): + return "<%s %s (%s)>" % (self.__class__.__name__, self.sys_path, self.id) + + def init(self, device): + self.dev_path = os.path.join("/dev", device) + self.sys_path = os.path.join("/sys/block", device) + + self.device = _collecty.BlockDevice(self.dev_path) + + @property + def id(self): + return "-".join((self.device.model, self.device.serial)) + + @property + def device_string(self): + return "%s (%s)" % (self.device.model, self.dev_path) + + def collect(self): + stats = self.parse_stats() + + return ":".join(( + self.is_awake(), + stats.get("read_ios"), + stats.get("read_sectors"), + stats.get("write_ios"), + stats.get("write_sectors"), + self.get_bad_sectors(), + self.get_temperature(), + )) + + def parse_stats(self): + """ + https://www.kernel.org/doc/Documentation/block/stat.txt + + Name units description + ---- ----- ----------- + read I/Os requests number of read I/Os processed + read merges requests number of read I/Os merged with in-queue I/O + read sectors sectors number of sectors read + read ticks milliseconds total wait time for read requests + write I/Os requests number of write I/Os processed + write merges requests number of write I/Os merged with in-queue I/O + write sectors sectors number of sectors written + write ticks milliseconds total wait time for write requests + in_flight requests number of I/Os currently in flight + io_ticks milliseconds total time this block device has been active + time_in_queue milliseconds total wait time for all requests + """ + stats_file = os.path.join(self.sys_path, "stat") + + with open(stats_file) as f: + stats = f.read().split() + + return { + "read_ios" : stats[0], + "read_merges" : stats[1], + "read_sectors" : stats[2], + "read_ticks" : stats[3], + "write_ios" : stats[4], + "write_merges" : stats[5], + "write_sectors" : stats[6], + "write_ticks" : stats[7], + "in_flight" : stats[8], + "io_ticks" : stats[9], + "time_in_queue" : stats[10], + } + + def is_smart_supported(self): + """ + We can only query SMART data if SMART is supported by the disk + and when the disk is awake. + """ + return self.device.is_smart_supported() and self.device.is_awake() + + def is_awake(self): + # If SMART is supported we can get the data from the disk + if self.device.is_smart_supported(): + if self.device.is_awake(): + return 1 + else: + return 0 + + # Otherwise we just assume that the disk is awake + return 1 + + def get_temperature(self): + if not self.is_smart_supported(): + return "NaN" + + return self.device.get_temperature() + + def get_bad_sectors(self): + if not self.is_smart_supported(): + return "NaN" + + return self.device.get_bad_sectors() + + +class DiskPlugin(base.Plugin): + name = "disk" + description = "Disk Plugin" + + templates = [ + GraphTemplateDiskBadSectors, + GraphTemplateDiskBytes, + GraphTemplateDiskIoOps, + GraphTemplateDiskTemperature, + ] + + block_device_patterns = [ + re.compile(r"(x?v|s)d[a-z]+"), + re.compile(r"mmcblk[0-9]+"), + ] + + @property + def objects(self): + for dev in self.find_block_devices(): + try: + yield DiskObject(self, dev) + except OSError: + pass + + def find_block_devices(self): + for device in os.listdir("/sys/block"): + # Skip invalid device names + if not self._valid_block_device_name(device): + continue + + yield device + + def _valid_block_device_name(self, name): + # Check if the given name matches any of the valid patterns. + for pattern in self.block_device_patterns: + if pattern.match(name): + return True + + return False