]> git.ipfire.org Git - thirdparty/mdadm.git/blob - Monitor.c
Merge branch 'master' of git://github.com/djbw/mdadm into devel-3.0
[thirdparty/mdadm.git] / Monitor.c
1 /*
2 * mdadm - manage Linux "md" devices aka RAID arrays.
3 *
4 * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
5 *
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 *
21 * Author: Neil Brown
22 * Email: <neilb@suse.de>
23 */
24
25 #include "mdadm.h"
26 #include "md_p.h"
27 #include "md_u.h"
28 #include <sys/wait.h>
29 #include <signal.h>
30 #include <limits.h>
31 #include <syslog.h>
32
33 static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom,
34 char *cmd, int dosyslog);
35
36 static char *percentalerts[] = {
37 "RebuildStarted",
38 "Rebuild20",
39 "Rebuild40",
40 "Rebuild60",
41 "Rebuild80",
42 };
43
44 /* The largest number of disks current arrays can manage is 384
45 * This really should be dynamically, but that will have to wait
46 * At least it isn't MD_SB_DISKS.
47 */
48 #define MaxDisks 384
49 int Monitor(mddev_dev_t devlist,
50 char *mailaddr, char *alert_cmd,
51 int period, int daemonise, int scan, int oneshot,
52 int dosyslog, int test, char* pidfile)
53 {
54 /*
55 * Every few seconds, scan every md device looking for changes
56 * When a change is found, log it, possibly run the alert command,
57 * and possibly send Email
58 *
59 * For each array, we record:
60 * Update time
61 * active/working/failed/spare drives
62 * State of each device.
63 * %rebuilt if rebuilding
64 *
65 * If the update time changes, check out all the data again
66 * It is possible that we cannot get the state of each device
67 * due to bugs in the md kernel module.
68 * We also read /proc/mdstat to get rebuild percent,
69 * and to get state on all active devices incase of kernel bug.
70 *
71 * Events are:
72 * Fail
73 * An active device had Faulty set or Active/Sync removed
74 * FailSpare
75 * A spare device had Faulty set
76 * SpareActive
77 * An active device had a reverse transition
78 * RebuildStarted
79 * percent went from -1 to +ve
80 * Rebuild20 Rebuild40 Rebuild60 Rebuild80
81 * percent went from below to not-below that number
82 * DeviceDisappeared
83 * Couldn't access a device which was previously visible
84 *
85 * if we detect an array with active<raid and spare==0
86 * we look at other arrays that have same spare-group
87 * If we find one with active==raid and spare>0,
88 * and if we can get_disk_info and find a name
89 * Then we hot-remove and hot-add to the other array
90 *
91 * If devlist is NULL, then we can monitor everything because --scan
92 * was given. We get an initial list from config file and add anything
93 * that appears in /proc/mdstat
94 */
95
96 struct state {
97 char *devname;
98 int devnum; /* to sync with mdstat info */
99 long utime;
100 int err;
101 char *spare_group;
102 int active, working, failed, spare, raid;
103 int expected_spares;
104 int devstate[MaxDisks];
105 int devid[MaxDisks];
106 int percent;
107 struct state *next;
108 } *statelist = NULL;
109 int finished = 0;
110 struct mdstat_ent *mdstat = NULL;
111 char *mailfrom = NULL;
112
113 if (!mailaddr) {
114 mailaddr = conf_get_mailaddr();
115 if (mailaddr && ! scan)
116 fprintf(stderr, Name ": Monitor using email address \"%s\" from config file\n",
117 mailaddr);
118 }
119 mailfrom = conf_get_mailfrom();
120
121 if (!alert_cmd) {
122 alert_cmd = conf_get_program();
123 if (alert_cmd && ! scan)
124 fprintf(stderr, Name ": Monitor using program \"%s\" from config file\n",
125 alert_cmd);
126 }
127 if (scan && !mailaddr && !alert_cmd) {
128 fprintf(stderr, Name ": No mail address or alert command - not monitoring.\n");
129 return 1;
130 }
131
132 if (daemonise) {
133 int pid = fork();
134 if (pid > 0) {
135 if (!pidfile)
136 printf("%d\n", pid);
137 else {
138 FILE *pid_file;
139 pid_file=fopen(pidfile, "w");
140 if (!pid_file)
141 perror("cannot create pid file");
142 else {
143 fprintf(pid_file,"%d\n", pid);
144 fclose(pid_file);
145 }
146 }
147 return 0;
148 }
149 if (pid < 0) {
150 perror("daemonise");
151 return 1;
152 }
153 close(0);
154 open("/dev/null", O_RDWR);
155 dup2(0,1);
156 dup2(0,2);
157 setsid();
158 }
159
160 if (devlist == NULL) {
161 mddev_ident_t mdlist = conf_get_ident(NULL);
162 for (; mdlist; mdlist=mdlist->next) {
163 struct state *st;
164 if (mdlist->devname == NULL)
165 continue;
166 if (strcasecmp(mdlist->devname, "<ignore>") == 0)
167 continue;
168 st = malloc(sizeof *st);
169 if (st == NULL)
170 continue;
171 if (mdlist->devname[0] == '/')
172 st->devname = strdup(mdlist->devname);
173 else {
174 st->devname = malloc(8+strlen(mdlist->devname)+1);
175 strcpy(strcpy(st->devname, "/dev/md/"),
176 mdlist->devname);
177 }
178 st->utime = 0;
179 st->next = statelist;
180 st->err = 0;
181 st->devnum = INT_MAX;
182 st->percent = -2;
183 st->expected_spares = mdlist->spare_disks;
184 if (mdlist->spare_group)
185 st->spare_group = strdup(mdlist->spare_group);
186 else
187 st->spare_group = NULL;
188 statelist = st;
189 }
190 } else {
191 mddev_dev_t dv;
192 for (dv=devlist ; dv; dv=dv->next) {
193 mddev_ident_t mdlist = conf_get_ident(dv->devname);
194 struct state *st = malloc(sizeof *st);
195 if (st == NULL)
196 continue;
197 st->devname = strdup(dv->devname);
198 st->utime = 0;
199 st->next = statelist;
200 st->err = 0;
201 st->devnum = INT_MAX;
202 st->percent = -2;
203 st->expected_spares = -1;
204 st->spare_group = NULL;
205 if (mdlist) {
206 st->expected_spares = mdlist->spare_disks;
207 if (mdlist->spare_group)
208 st->spare_group = strdup(mdlist->spare_group);
209 }
210 statelist = st;
211 }
212 }
213
214
215 while (! finished) {
216 int new_found = 0;
217 struct state *st;
218
219 if (mdstat)
220 free_mdstat(mdstat);
221 mdstat = mdstat_read(oneshot?0:1, 0);
222
223 for (st=statelist; st; st=st->next) {
224 struct { int state, major, minor; } info[MaxDisks];
225 mdu_array_info_t array;
226 struct mdstat_ent *mse = NULL, *mse2;
227 char *dev = st->devname;
228 int fd;
229 unsigned int i;
230
231 if (test)
232 alert("TestMessage", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
233 fd = open(dev, O_RDONLY);
234 if (fd < 0) {
235 if (!st->err)
236 alert("DeviceDisappeared", dev, NULL,
237 mailaddr, mailfrom, alert_cmd, dosyslog);
238 /* fprintf(stderr, Name ": cannot open %s: %s\n",
239 dev, strerror(errno));
240 */ st->err=1;
241 continue;
242 }
243 fcntl(fd, F_SETFD, FD_CLOEXEC);
244 if (ioctl(fd, GET_ARRAY_INFO, &array)<0) {
245 if (!st->err)
246 alert("DeviceDisappeared", dev, NULL,
247 mailaddr, mailfrom, alert_cmd, dosyslog);
248 /* fprintf(stderr, Name ": cannot get array info for %s: %s\n",
249 dev, strerror(errno));
250 */ st->err=1;
251 close(fd);
252 continue;
253 }
254 /* It's much easier to list what array levels can't
255 * have a device disappear than all of them that can
256 */
257 if (array.level == 0 || array.level == -1) {
258 if (!st->err)
259 alert("DeviceDisappeared", dev, "Wrong-Level",
260 mailaddr, mailfrom, alert_cmd, dosyslog);
261 st->err = 1;
262 close(fd);
263 continue;
264 }
265 if (st->devnum == INT_MAX) {
266 struct stat stb;
267 if (fstat(fd, &stb) == 0 &&
268 (S_IFMT&stb.st_mode)==S_IFBLK) {
269 if (major(stb.st_rdev) == MD_MAJOR)
270 st->devnum = minor(stb.st_rdev);
271 else
272 st->devnum = -1- (minor(stb.st_rdev)>>6);
273 }
274 }
275
276 for (mse2 = mdstat ; mse2 ; mse2=mse2->next)
277 if (mse2->devnum == st->devnum) {
278 mse2->devnum = INT_MAX; /* flag it as "used" */
279 mse = mse2;
280 }
281
282 if (array.utime == 0)
283 /* external arrays don't update utime */
284 array.utime = time(0);
285
286 if (st->utime == array.utime &&
287 st->failed == array.failed_disks &&
288 st->working == array.working_disks &&
289 st->spare == array.spare_disks &&
290 (mse == NULL || (
291 mse->percent == st->percent
292 ))) {
293 close(fd);
294 st->err = 0;
295 continue;
296 }
297 if (st->utime == 0 && /* new array */
298 mse && /* is in /proc/mdstat */
299 mse->pattern && strchr(mse->pattern, '_') /* degraded */
300 )
301 alert("DegradedArray", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
302
303 if (st->utime == 0 && /* new array */
304 st->expected_spares > 0 &&
305 array.spare_disks < st->expected_spares)
306 alert("SparesMissing", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
307 if (mse &&
308 st->percent == -1 &&
309 mse->percent >= 0)
310 alert("RebuildStarted", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
311 if (mse &&
312 st->percent >= 0 &&
313 mse->percent >= 0 &&
314 (mse->percent / 20) > (st->percent / 20))
315 alert(percentalerts[mse->percent/20],
316 dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
317
318 if (mse &&
319 mse->percent == -1 &&
320 st->percent >= 0) {
321 /* Rebuild/sync/whatever just finished.
322 * If there is a number in /mismatch_cnt,
323 * we should report that.
324 */
325 struct mdinfo *sra =
326 sysfs_read(-1, st->devnum, GET_MISMATCH);
327 if (sra && sra->mismatch_cnt > 0) {
328 char cnt[40];
329 sprintf(cnt, " mismatches found: %d", sra->mismatch_cnt);
330 alert("RebuildFinished", dev, cnt, mailaddr, mailfrom, alert_cmd, dosyslog);
331 } else
332 alert("RebuildFinished", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
333 if (sra)
334 free(sra);
335 }
336
337 if (mse)
338 st->percent = mse->percent;
339
340
341 for (i=0; i<MaxDisks && i <= array.raid_disks + array.nr_disks;
342 i++) {
343 mdu_disk_info_t disc;
344 disc.number = i;
345 if (ioctl(fd, GET_DISK_INFO, &disc) >= 0) {
346 info[i].state = disc.state;
347 info[i].major = disc.major;
348 info[i].minor = disc.minor;
349 } else
350 info[i].major = info[i].minor = 0;
351 }
352 close(fd);
353
354 for (i=0; i<MaxDisks; i++) {
355 mdu_disk_info_t disc = {0};
356 int newstate=0;
357 int change;
358 char *dv = NULL;
359 disc.number = i;
360 if (i > array.raid_disks + array.nr_disks) {
361 newstate = 0;
362 disc.major = disc.minor = 0;
363 } else if (info[i].major || info[i].minor) {
364 newstate = info[i].state;
365 dv = map_dev(info[i].major, info[i].minor, 1);
366 disc.state = newstate;
367 disc.major = info[i].major;
368 disc.minor = info[i].minor;
369 } else if (mse && mse->pattern && i < strlen(mse->pattern)) {
370 switch(mse->pattern[i]) {
371 case 'U': newstate = 6 /* ACTIVE/SYNC */; break;
372 case '_': newstate = 0; break;
373 }
374 disc.major = disc.minor = 0;
375 }
376 if (dv == NULL && st->devid[i])
377 dv = map_dev(major(st->devid[i]),
378 minor(st->devid[i]), 1);
379 change = newstate ^ st->devstate[i];
380 if (st->utime && change && !st->err) {
381 if (i < (unsigned)array.raid_disks &&
382 (((newstate&change)&(1<<MD_DISK_FAULTY)) ||
383 ((st->devstate[i]&change)&(1<<MD_DISK_ACTIVE)) ||
384 ((st->devstate[i]&change)&(1<<MD_DISK_SYNC)))
385 )
386 alert("Fail", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
387 else if (i >= (unsigned)array.raid_disks &&
388 (disc.major || disc.minor) &&
389 st->devid[i] == makedev(disc.major, disc.minor) &&
390 ((newstate&change)&(1<<MD_DISK_FAULTY))
391 )
392 alert("FailSpare", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
393 else if (i < (unsigned)array.raid_disks &&
394 (((st->devstate[i]&change)&(1<<MD_DISK_FAULTY)) ||
395 ((newstate&change)&(1<<MD_DISK_ACTIVE)) ||
396 ((newstate&change)&(1<<MD_DISK_SYNC)))
397 )
398 alert("SpareActive", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
399 }
400 st->devstate[i] = newstate;
401 st->devid[i] = makedev(disc.major, disc.minor);
402 }
403 st->active = array.active_disks;
404 st->working = array.working_disks;
405 st->spare = array.spare_disks;
406 st->failed = array.failed_disks;
407 st->utime = array.utime;
408 st->raid = array.raid_disks;
409 st->err = 0;
410 }
411 /* now check if there are any new devices found in mdstat */
412 if (scan) {
413 struct mdstat_ent *mse;
414 for (mse=mdstat; mse; mse=mse->next)
415 if (mse->devnum != INT_MAX &&
416 mse->level &&
417 (strcmp(mse->level, "raid0")!=0 &&
418 strcmp(mse->level, "linear")!=0)
419 ) {
420 struct state *st = malloc(sizeof *st);
421 mdu_array_info_t array;
422 int fd;
423 if (st == NULL)
424 continue;
425 st->devname = strdup(get_md_name(mse->devnum));
426 if ((fd = open(st->devname, O_RDONLY)) < 0 ||
427 ioctl(fd, GET_ARRAY_INFO, &array)< 0) {
428 /* no such array */
429 if (fd >=0) close(fd);
430 put_md_name(st->devname);
431 free(st->devname);
432 free(st);
433 continue;
434 }
435 close(fd);
436 st->utime = 0;
437 st->next = statelist;
438 st->err = 1;
439 st->devnum = mse->devnum;
440 st->percent = -2;
441 st->spare_group = NULL;
442 st->expected_spares = -1;
443 statelist = st;
444 if (test)
445 alert("TestMessage", st->devname, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
446 alert("NewArray", st->devname, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
447 new_found = 1;
448 }
449 }
450 /* If an array has active < raid && spare == 0 && spare_group != NULL
451 * Look for another array with spare > 0 and active == raid and same spare_group
452 * if found, choose a device and hotremove/hotadd
453 */
454 for (st = statelist; st; st=st->next)
455 if (st->active < st->raid &&
456 st->spare == 0 &&
457 st->spare_group != NULL) {
458 struct state *st2;
459 for (st2=statelist ; st2 ; st2=st2->next)
460 if (st2 != st &&
461 st2->spare > 0 &&
462 st2->active == st2->raid &&
463 st2->spare_group != NULL &&
464 strcmp(st->spare_group, st2->spare_group) == 0) {
465 /* try to remove and add */
466 int fd1 = open(st->devname, O_RDONLY);
467 int fd2 = open(st2->devname, O_RDONLY);
468 int dev = -1;
469 int d;
470 if (fd1 < 0 || fd2 < 0) {
471 if (fd1>=0) close(fd1);
472 if (fd2>=0) close(fd2);
473 continue;
474 }
475 for (d=st2->raid; d < MaxDisks; d++) {
476 if (st2->devid[d] > 0 &&
477 st2->devstate[d] == 0) {
478 dev = st2->devid[d];
479 break;
480 }
481 }
482 if (dev > 0) {
483 struct mddev_dev_s devlist;
484 char devname[20];
485 devlist.next = NULL;
486 devlist.used = 0;
487 devlist.re_add = 0;
488 devlist.writemostly = 0;
489 devlist.devname = devname;
490 sprintf(devname, "%d:%d", major(dev), minor(dev));
491
492 devlist.disposition = 'r';
493 if (Manage_subdevs(st2->devname, fd2, &devlist, -1) == 0) {
494 devlist.disposition = 'a';
495 if (Manage_subdevs(st->devname, fd1, &devlist, -1) == 0) {
496 alert("MoveSpare", st->devname, st2->devname, mailaddr, mailfrom, alert_cmd, dosyslog);
497 close(fd1);
498 close(fd2);
499 break;
500 }
501 else Manage_subdevs(st2->devname, fd2, &devlist, -1);
502 }
503 }
504 close(fd1);
505 close(fd2);
506 }
507 }
508 if (!new_found) {
509 if (oneshot)
510 break;
511 else
512 mdstat_wait(period);
513 }
514 test = 0;
515 }
516 if (pidfile)
517 unlink(pidfile);
518 return 0;
519 }
520
521
522 static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom, char *cmd,
523 int dosyslog)
524 {
525 int priority;
526
527 if (!cmd && !mailaddr) {
528 time_t now = time(0);
529
530 printf("%1.15s: %s on %s %s\n", ctime(&now)+4, event, dev, disc?disc:"unknown device");
531 }
532 if (cmd) {
533 int pid = fork();
534 switch(pid) {
535 default:
536 waitpid(pid, NULL, 0);
537 break;
538 case -1:
539 break;
540 case 0:
541 execl(cmd, cmd, event, dev, disc, NULL);
542 exit(2);
543 }
544 }
545 if (mailaddr &&
546 (strncmp(event, "Fail", 4)==0 ||
547 strncmp(event, "Test", 4)==0 ||
548 strncmp(event, "Spares", 6)==0 ||
549 strncmp(event, "Degrade", 7)==0)) {
550 FILE *mp = popen(Sendmail, "w");
551 if (mp) {
552 FILE *mdstat;
553 char hname[256];
554 gethostname(hname, sizeof(hname));
555 signal(SIGPIPE, SIG_IGN);
556 if (mailfrom)
557 fprintf(mp, "From: %s\n", mailfrom);
558 else
559 fprintf(mp, "From: " Name " monitoring <root>\n");
560 fprintf(mp, "To: %s\n", mailaddr);
561 fprintf(mp, "Subject: %s event on %s:%s\n\n", event, dev, hname);
562
563 fprintf(mp, "This is an automatically generated mail message from " Name "\n");
564 fprintf(mp, "running on %s\n\n", hname);
565
566 fprintf(mp, "A %s event had been detected on md device %s.\n\n", event, dev);
567
568 if (disc && disc[0] != ' ')
569 fprintf(mp, "It could be related to component device %s.\n\n", disc);
570 if (disc && disc[0] == ' ')
571 fprintf(mp, "Extra information:%s.\n\n", disc);
572
573 fprintf(mp, "Faithfully yours, etc.\n");
574
575 mdstat = fopen("/proc/mdstat", "r");
576 if (mdstat) {
577 char buf[8192];
578 int n;
579 fprintf(mp, "\nP.S. The /proc/mdstat file currently contains the following:\n\n");
580 while ( (n=fread(buf, 1, sizeof(buf), mdstat)) > 0)
581 n=fwrite(buf, 1, n, mp); /* yes, i don't care about the result */
582 fclose(mdstat);
583 }
584 fclose(mp);
585 }
586
587 }
588
589 /* log the event to syslog maybe */
590 if (dosyslog) {
591 /* Log at a different severity depending on the event.
592 *
593 * These are the critical events: */
594 if (strncmp(event, "Fail", 4)==0 ||
595 strncmp(event, "Degrade", 7)==0 ||
596 strncmp(event, "DeviceDisappeared", 17)==0)
597 priority = LOG_CRIT;
598 /* Good to know about, but are not failures: */
599 else if (strncmp(event, "Rebuild", 7)==0 ||
600 strncmp(event, "MoveSpare", 9)==0 ||
601 strncmp(event, "Spares", 6) != 0)
602 priority = LOG_WARNING;
603 /* Everything else: */
604 else
605 priority = LOG_INFO;
606
607 if (disc)
608 syslog(priority, "%s event detected on md device %s, component device %s", event, dev, disc);
609 else
610 syslog(priority, "%s event detected on md device %s", event, dev);
611 }
612 }
613
614 /* Not really Monitor but ... */
615 int Wait(char *dev)
616 {
617 struct stat stb;
618 int devnum;
619 int rv = 1;
620
621 if (stat(dev, &stb) != 0) {
622 fprintf(stderr, Name ": Cannot find %s: %s\n", dev,
623 strerror(errno));
624 return 2;
625 }
626 devnum = stat2devnum(&stb);
627
628 while(1) {
629 struct mdstat_ent *ms = mdstat_read(1, 0);
630 struct mdstat_ent *e;
631
632 for (e=ms ; e; e=e->next)
633 if (e->devnum == devnum)
634 break;
635
636 if (!e || e->percent < 0) {
637 if (e && e->metadata_version &&
638 strncmp(e->metadata_version, "external:", 9) == 0) {
639 if (is_subarray(&e->metadata_version[9]))
640 ping_monitor(&e->metadata_version[9]);
641 else
642 ping_monitor(devnum2devname(devnum));
643 }
644 free_mdstat(ms);
645 return rv;
646 }
647 free_mdstat(ms);
648 rv = 0;
649 mdstat_wait(5);
650 }
651 }