From 30777a6c8aed027b18b00c495b79b353bacbe923 Mon Sep 17 00:00:00 2001 From: Michael Tremer Date: Sun, 24 May 2015 10:38:05 +0000 Subject: [PATCH] Add disk plugin The disk plugin collects information about read and written sectors, IO operations, bad sectors and the temperature of the disks. It uses libatasmart to collect that information. --- .gitignore | 7 + Makefile.am | 22 +++ autogen.sh | 1 + configure.ac | 12 ++ po/POTFILES.in | 1 + src/_collectymodule.c | 325 +++++++++++++++++++++++++++++++ src/collecty/plugins/__init__.py | 1 + src/collecty/plugins/disk.py | 315 ++++++++++++++++++++++++++++++ 8 files changed, 684 insertions(+) create mode 100644 src/_collectymodule.c create mode 100644 src/collecty/plugins/disk.py diff --git a/.gitignore b/.gitignore index 6256b1b..6e3c11c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /Makefile /build-aux +/libtool /man/*.[0-9] /man/*.html /missing @@ -9,11 +10,17 @@ /*.tar.bz2 /*.tar.gz /*.tar.xz +*.a +*.la +*.lo *.log *.cache *.gmo *.mo *~ +.deps +.dirstamp +.libs Makefile.in aclocal.m4 config.log diff --git a/Makefile.am b/Makefile.am index 72c06ad..b9aa841 100644 --- a/Makefile.am +++ b/Makefile.am @@ -87,6 +87,7 @@ collectyplugins_PYTHON = \ src/collecty/plugins/base.py \ src/collecty/plugins/conntrack.py \ src/collecty/plugins/cpu.py \ + src/collecty/plugins/disk.py \ src/collecty/plugins/entropy.py \ src/collecty/plugins/__init__.py \ src/collecty/plugins/interface.py \ @@ -96,6 +97,27 @@ collectyplugins_PYTHON = \ collectypluginsdir = $(collectydir)/plugins +pkgpyexec_LTLIBRARIES = \ + _collecty.la + +_collecty_la_SOURCES = \ + src/_collectymodule.c + +_collecty_la_CFLAGS = \ + $(AM_CFLAGS) \ + $(LIBATASMART_CFLAGS) \ + $(PYTHON_DEVEL_CFLAGS) + +_collecty_la_LDFLAGS = \ + $(AM_LDFLAGS) \ + -shared \ + -module \ + -avoid-version + +_collecty_la_LIBADD = \ + $(LIBATASMART_LIBS) \ + $(PYTHON_DEVEL_LIBS) + dist_dbuspolicy_DATA = \ src/dbus/org.ipfire.collecty1.conf diff --git a/autogen.sh b/autogen.sh index 077e109..212a394 100755 --- a/autogen.sh +++ b/autogen.sh @@ -1,4 +1,5 @@ #!/bin/sh +libtoolize intltoolize --force --automake autoreconf --force --install --symlink diff --git a/configure.ac b/configure.ac index 0d943e3..810f693 100644 --- a/configure.ac +++ b/configure.ac @@ -41,6 +41,8 @@ AM_INIT_AUTOMAKE([ subdir-objects ]) AM_SILENT_RULES([yes]) +LT_PREREQ(2.2) +LT_INIT([disable-static]) IT_PROG_INTLTOOL([0.40.0]) @@ -51,10 +53,20 @@ AC_PROG_LN_S AC_PROG_MKDIR_P AC_PROG_SED +# C Compiler +AC_PROG_CC +AC_PROG_CC_C99 +AC_PROG_CC_C_O +AC_PROG_GCC_TRADITIONAL + AC_PATH_PROG([XSLTPROC], [xsltproc]) # Python AM_PATH_PYTHON([2.7]) +PKG_CHECK_MODULES([PYTHON_DEVEL], [python-${PYTHON_VERSION}]) + +# libatasmart +PKG_CHECK_MODULES([LIBATASMART], [libatasmart >= 0.19]) save_LIBS="$LIBS" diff --git a/po/POTFILES.in b/po/POTFILES.in index 61715be..b94ec75 100644 --- a/po/POTFILES.in +++ b/po/POTFILES.in @@ -9,6 +9,7 @@ src/collecty/ping.py src/collecty/plugins/base.py src/collecty/plugins/conntrack.py src/collecty/plugins/cpu.py +src/collecty/plugins/disk.py src/collecty/plugins/entropy.py src/collecty/plugins/__init__.py src/collecty/plugins/interface.py diff --git a/src/_collectymodule.c b/src/_collectymodule.c new file mode 100644 index 0000000..eefe871 --- /dev/null +++ b/src/_collectymodule.c @@ -0,0 +1,325 @@ +/* + * collecty + * Copyright (C) 2015 IPFire Team (www.ipfire.org) + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include + +#include +#include +#include +#include +#include +#include +#include + +#define MODEL_SIZE 40 +#define SERIAL_SIZE 20 + +typedef struct { + PyObject_HEAD + char* path; + struct hd_driveid identity; + SkDisk* disk; +} BlockDevice; + +static void BlockDevice_dealloc(BlockDevice* self) { + if (self->disk) + sk_disk_free(self->disk); + + if (self->path) + free(self->path); + + self->ob_type->tp_free((PyObject*)self); +} + +static int BlockDevice_get_identity(BlockDevice* device) { + int fd; + + if ((fd = open(device->path, O_RDONLY | O_NONBLOCK)) < 0) { + return 1; + } + + int r = ioctl(fd, HDIO_GET_IDENTITY, &device->identity); + close(fd); + + if (r) + return 1; + + return 0; +} + +static int BlockDevice_smart_is_available(BlockDevice* device) { + SkBool available = FALSE; + + int r = sk_disk_smart_is_available(device->disk, &available); + if (r) + return -1; + + if (available) + return 0; + + return 1; +} + +static int BlockDevice_check_sleep_mode(BlockDevice* device) { + SkBool awake = FALSE; + + int r = sk_disk_check_sleep_mode(device->disk, &awake); + if (r) + return -1; + + if (awake) + return 0; + + return 1; +} + +static PyObject * BlockDevice_new(PyTypeObject* type, PyObject* args, PyObject* kwds) { + BlockDevice* self = (BlockDevice*)type->tp_alloc(type, 0); + + if (self) { + self->path = NULL; + + // libatasmart + self->disk = NULL; + } + + return (PyObject *)self; +} + +static int BlockDevice_init(BlockDevice* self, PyObject* args, PyObject* kwds) { + const char* path = NULL; + + if (!PyArg_ParseTuple(args, "s", &path)) + return -1; + + self->path = strdup(path); + + int r = BlockDevice_get_identity(self); + if (r) { + PyErr_Format(PyExc_OSError, "Could not open block device: %s", path); + return -1; + } + + r = sk_disk_open(path, &self->disk); + if (r == 0) { + if (BlockDevice_smart_is_available(self) == 0) { + if (BlockDevice_check_sleep_mode(self) == 0) { + r = sk_disk_smart_read_data(self->disk); + if (r) { + PyErr_Format(PyExc_OSError, "Could not open block device %s: %s", path, + strerror(errno)); + return -1; + } + } + } + } else { + PyErr_Format(PyExc_OSError, "Could not open block device %s: %s", path, + strerror(errno)); + return -1; + } + + //sk_disk_identify_is_available + + return 0; +} + +static PyObject* BlockDevice_get_path(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + return PyString_FromString(device->path); +} + +static void clean_string(char *s) { + for (char* e = s; *e; e++) { + if (*e < ' ' || *e >= 127) + *e = ' '; + } +} + +static void drop_spaces(char *s) { + char *d = s; + bool prev_space = false; + + s += strspn(s, " "); + + for (; *s; s++) { + if (prev_space) { + if (*s != ' ') { + prev_space = false; + *(d++) = ' '; + *(d++) = *s; + } + } else { + if (*s == ' ') + prev_space = true; + else + *(d++) = *s; + } + } + + *d = 0; +} + +static void copy_string(char* d, const char* s, size_t n) { + // Copy the source buffer to the destination buffer up to n + memcpy(d, s, n); + + // Terminate the destination buffer with NULL + d[n] = '\0'; + + // Clean up the string from non-printable characters + clean_string(d); + drop_spaces(d); +} + +static PyObject* BlockDevice_get_model(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + char model[MODEL_SIZE + 1]; + copy_string(model, device->identity.model, sizeof(model)); + + return PyString_FromString(model); +} + +static PyObject* BlockDevice_get_serial(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + char serial[SERIAL_SIZE + 1]; + copy_string(serial, device->identity.serial_no, sizeof(serial)); + + return PyString_FromString(serial); +} + +static PyObject* BlockDevice_is_smart_supported(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device) == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* BlockDevice_is_awake(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_check_sleep_mode(device) == 0) + Py_RETURN_TRUE; + + Py_RETURN_FALSE; +} + +static PyObject* BlockDevice_get_bad_sectors(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device)) { + PyErr_Format(PyExc_OSError, "Device does not support SMART"); + return NULL; + } + + uint64_t bad_sectors; + int r = sk_disk_smart_get_bad(device->disk, &bad_sectors); + if (r) + return NULL; + + return PyLong_FromUnsignedLongLong((unsigned long long)bad_sectors); +} + +static PyObject* BlockDevice_get_temperature(PyObject* self) { + BlockDevice* device = (BlockDevice*)self; + + if (BlockDevice_smart_is_available(device)) { + PyErr_Format(PyExc_OSError, "Device does not support SMART"); + return NULL; + } + + uint64_t mkelvin; + int r = sk_disk_smart_get_temperature(device->disk, &mkelvin); + if (r) + return NULL; + + return PyLong_FromUnsignedLongLong((unsigned long long)mkelvin); +} + +static PyGetSetDef BlockDevice_getsetters[] = { + {"path", (getter)BlockDevice_get_path, NULL, NULL, NULL}, + {"model", (getter)BlockDevice_get_model, NULL, NULL, NULL}, + {"serial", (getter)BlockDevice_get_serial, NULL, NULL, NULL}, +}; + +static PyMethodDef BlockDevice_methods[] = { + {"get_bad_sectors", (PyCFunction)BlockDevice_get_bad_sectors, METH_NOARGS, NULL}, + {"get_temperature", (PyCFunction)BlockDevice_get_temperature, METH_NOARGS, NULL}, + {"is_smart_supported", (PyCFunction)BlockDevice_is_smart_supported, METH_NOARGS, NULL}, + {"is_awake", (PyCFunction)BlockDevice_is_awake, METH_NOARGS, NULL}, + {NULL} +}; + +static PyTypeObject BlockDeviceType = { + PyObject_HEAD_INIT(NULL) + 0, /*ob_size*/ + "_collecty.BlockDevice", /*tp_name*/ + sizeof(BlockDevice), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)BlockDevice_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_compare*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash */ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, /*tp_flags*/ + "BlockDevice objects", /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + BlockDevice_methods, /* tp_methods */ + 0, /* tp_members */ + BlockDevice_getsetters, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)BlockDevice_init, /* tp_init */ + 0, /* tp_alloc */ + BlockDevice_new, /* tp_new */ +}; + +static PyMethodDef collecty_module_methods[] = { + {NULL}, +}; + +void init_collecty(void) { + if (PyType_Ready(&BlockDeviceType) < 0) + return; + + PyObject* m = Py_InitModule("_collecty", collecty_module_methods); + + PyModule_AddObject(m, "BlockDevice", (PyObject*)&BlockDeviceType); +} diff --git a/src/collecty/plugins/__init__.py b/src/collecty/plugins/__init__.py index cc7f41c..5f843a8 100644 --- a/src/collecty/plugins/__init__.py +++ b/src/collecty/plugins/__init__.py @@ -24,6 +24,7 @@ from base import Timer, get import base import conntrack import cpu +import disk import entropy import interface import latency diff --git a/src/collecty/plugins/disk.py b/src/collecty/plugins/disk.py new file mode 100644 index 0000000..b5a2346 --- /dev/null +++ b/src/collecty/plugins/disk.py @@ -0,0 +1,315 @@ +#!/usr/bin/python +############################################################################### +# # +# collecty - A system statistics collection daemon for IPFire # +# Copyright (C) 2012 IPFire development team # +# # +# This program is free software: you can redistribute it and/or modify # +# it under the terms of the GNU General Public License as published by # +# the Free Software Foundation, either version 3 of the License, or # +# (at your option) any later version. # +# # +# This program is distributed in the hope that it will be useful, # +# but WITHOUT ANY WARRANTY; without even the implied warranty of # +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # +# GNU General Public License for more details. # +# # +# You should have received a copy of the GNU General Public License # +# along with this program. If not, see . # +# # +############################################################################### + +from collecty import _collecty +import os +import re + +import base + +from ..i18n import _ + +class GraphTemplateDiskBadSectors(base.GraphTemplate): + name = "disk-bad-sectors" + + rrd_graph = [ + "DEF:bad_sectors=%(file)s:bad_sectors:AVERAGE", + + "AREA:bad_sectors#ff0000:%s" % _("Bad Sectors"), + + "VDEF:bad_sectors_cur=bad_sectors,LAST", + "VDEF:bad_sectors_max=bad_sectors,MAXIMUM", + "GPRINT:bad_sectors_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:bad_sectors_max:%12s\:" % _("Maximum") + " %9.2lf\\n", + ] + + @property + def graph_title(self): + return _("Bad Sectors of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Pending/Relocated Sectors") + + +class GraphTemplateDiskBytes(base.GraphTemplate): + name = "disk-bytes" + + rrd_graph = [ + "DEF:read_sectors=%(file)s:read_sectors:AVERAGE", + "DEF:write_sectors=%(file)s:write_sectors:AVERAGE", + + "CDEF:read_bytes=read_sectors,512,*", + "CDEF:write_bytes=write_sectors,512,*", + + "LINE1:read_bytes#ff0000:%-15s" % _("Read"), + "VDEF:read_cur=read_bytes,LAST", + "VDEF:read_min=read_bytes,MINIMUM", + "VDEF:read_max=read_bytes,MAXIMUM", + "VDEF:read_avg=read_bytes,AVERAGE", + "GPRINT:read_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:read_max:%12s\:" % _("Maximum") + " %9.2lf", + "GPRINT:read_min:%12s\:" % _("Minimum") + " %9.2lf", + "GPRINT:read_avg:%12s\:" % _("Average") + " %9.2lf\\n", + + "LINE1:write_bytes#00ff00:%-15s" % _("Written"), + "VDEF:write_cur=write_bytes,LAST", + "VDEF:write_min=write_bytes,MINIMUM", + "VDEF:write_max=write_bytes,MAXIMUM", + "VDEF:write_avg=write_bytes,AVERAGE", + "GPRINT:write_cur:%12s\:" % _("Current") + " %9.2lf", + "GPRINT:write_max:%12s\:" % _("Maximum") + " %9.2lf", + "GPRINT:write_min:%12s\:" % _("Minimum") + " %9.2lf", + "GPRINT:write_avg:%12s\:" % _("Average") + " %9.2lf\\n", + ] + + lower_limit = 0 + + @property + def graph_title(self): + return _("Disk Utilisation of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Byte per Second") + + +class GraphTemplateDiskIoOps(base.GraphTemplate): + name = "disk-io-ops" + + rrd_graph = [ + "DEF:read_ios=%(file)s:read_ios:AVERAGE", + "DEF:write_ios=%(file)s:write_ios:AVERAGE", + + "LINE1:read_ios#ff0000:%-15s" % _("Read"), + "VDEF:read_cur=read_ios,LAST", + "VDEF:read_min=read_ios,MINIMUM", + "VDEF:read_max=read_ios,MAXIMUM", + "VDEF:read_avg=read_ios,AVERAGE", + "GPRINT:read_cur:%12s\:" % _("Current") + " %6.2lf", + "GPRINT:read_max:%12s\:" % _("Maximum") + " %6.2lf", + "GPRINT:read_min:%12s\:" % _("Minimum") + " %6.2lf", + "GPRINT:read_avg:%12s\:" % _("Average") + " %6.2lf\\n", + + "LINE1:write_ios#00ff00:%-15s" % _("Written"), + "VDEF:write_cur=write_ios,LAST", + "VDEF:write_min=write_ios,MINIMUM", + "VDEF:write_max=write_ios,MAXIMUM", + "VDEF:write_avg=write_ios,AVERAGE", + "GPRINT:write_cur:%12s\:" % _("Current") + " %6.2lf", + "GPRINT:write_max:%12s\:" % _("Maximum") + " %6.2lf", + "GPRINT:write_min:%12s\:" % _("Minimum") + " %6.2lf", + "GPRINT:write_avg:%12s\:" % _("Average") + " %6.2lf\\n", + ] + + lower_limit = 0 + + @property + def graph_title(self): + return _("Disk IO Operations of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("Operations per Second") + + +class GraphTemplateDiskTemperature(base.GraphTemplate): + name = "disk-temperature" + + rrd_graph = [ + "DEF:mkelvin=%(file)s:temperature:AVERAGE", + "CDEF:celsius=mkelvin,1000,/,273.15,-", + + "LINE2:celsius#ff0000:%s" % _("Temperature"), + "VDEF:temp_cur=celsius,LAST", + "VDEF:temp_min=celsius,MINIMUM", + "VDEF:temp_max=celsius,MAXIMUM", + "VDEF:temp_avg=celsius,AVERAGE", + "GPRINT:temp_cur:%12s\:" % _("Current") + " %3.2lf", + "GPRINT:temp_max:%12s\:" % _("Maximum") + " %3.2lf", + "GPRINT:temp_min:%12s\:" % _("Minimum") + " %3.2lf", + "GPRINT:temp_avg:%12s\:" % _("Average") + " %3.2lf\\n", + ] + + @property + def graph_title(self): + return _("Disk Temperature of %s") % self.object.device_string + + @property + def graph_vertical_label(self): + return _("° Celsius") + + @property + def rrd_graph_args(self): + return [ + # Make the y-axis have a decimal + "--left-axis-format", "%3.1lf", + ] + + +class DiskObject(base.Object): + rrd_schema = [ + "DS:awake:GAUGE:0:1", + "DS:read_ios:DERIVE:0:U", + "DS:read_sectors:DERIVE:0:U", + "DS:write_ios:DERIVE:0:U", + "DS:write_sectors:DERIVE:0:U", + "DS:bad_sectors:GAUGE:0:U", + "DS:temperature:GAUGE:U:U", + ] + + def __repr__(self): + return "<%s %s (%s)>" % (self.__class__.__name__, self.sys_path, self.id) + + def init(self, device): + self.dev_path = os.path.join("/dev", device) + self.sys_path = os.path.join("/sys/block", device) + + self.device = _collecty.BlockDevice(self.dev_path) + + @property + def id(self): + return "-".join((self.device.model, self.device.serial)) + + @property + def device_string(self): + return "%s (%s)" % (self.device.model, self.dev_path) + + def collect(self): + stats = self.parse_stats() + + return ":".join(( + self.is_awake(), + stats.get("read_ios"), + stats.get("read_sectors"), + stats.get("write_ios"), + stats.get("write_sectors"), + self.get_bad_sectors(), + self.get_temperature(), + )) + + def parse_stats(self): + """ + https://www.kernel.org/doc/Documentation/block/stat.txt + + Name units description + ---- ----- ----------- + read I/Os requests number of read I/Os processed + read merges requests number of read I/Os merged with in-queue I/O + read sectors sectors number of sectors read + read ticks milliseconds total wait time for read requests + write I/Os requests number of write I/Os processed + write merges requests number of write I/Os merged with in-queue I/O + write sectors sectors number of sectors written + write ticks milliseconds total wait time for write requests + in_flight requests number of I/Os currently in flight + io_ticks milliseconds total time this block device has been active + time_in_queue milliseconds total wait time for all requests + """ + stats_file = os.path.join(self.sys_path, "stat") + + with open(stats_file) as f: + stats = f.read().split() + + return { + "read_ios" : stats[0], + "read_merges" : stats[1], + "read_sectors" : stats[2], + "read_ticks" : stats[3], + "write_ios" : stats[4], + "write_merges" : stats[5], + "write_sectors" : stats[6], + "write_ticks" : stats[7], + "in_flight" : stats[8], + "io_ticks" : stats[9], + "time_in_queue" : stats[10], + } + + def is_smart_supported(self): + """ + We can only query SMART data if SMART is supported by the disk + and when the disk is awake. + """ + return self.device.is_smart_supported() and self.device.is_awake() + + def is_awake(self): + # If SMART is supported we can get the data from the disk + if self.device.is_smart_supported(): + if self.device.is_awake(): + return 1 + else: + return 0 + + # Otherwise we just assume that the disk is awake + return 1 + + def get_temperature(self): + if not self.is_smart_supported(): + return "NaN" + + return self.device.get_temperature() + + def get_bad_sectors(self): + if not self.is_smart_supported(): + return "NaN" + + return self.device.get_bad_sectors() + + +class DiskPlugin(base.Plugin): + name = "disk" + description = "Disk Plugin" + + templates = [ + GraphTemplateDiskBadSectors, + GraphTemplateDiskBytes, + GraphTemplateDiskIoOps, + GraphTemplateDiskTemperature, + ] + + block_device_patterns = [ + re.compile(r"(x?v|s)d[a-z]+"), + re.compile(r"mmcblk[0-9]+"), + ] + + @property + def objects(self): + for dev in self.find_block_devices(): + try: + yield DiskObject(self, dev) + except OSError: + pass + + def find_block_devices(self): + for device in os.listdir("/sys/block"): + # Skip invalid device names + if not self._valid_block_device_name(device): + continue + + yield device + + def _valid_block_device_name(self, name): + # Check if the given name matches any of the valid patterns. + for pattern in self.block_device_patterns: + if pattern.match(name): + return True + + return False -- 2.39.2