]> git.ipfire.org Git - thirdparty/mdadm.git/blob - Monitor.c
mdadm: added --no-sharing option for Monitor mode
[thirdparty/mdadm.git] / Monitor.c
1 /*
2 * mdadm - manage Linux "md" devices aka RAID arrays.
3 *
4 * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
5 *
6 *
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
11 *
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
16 *
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
20 *
21 * Author: Neil Brown
22 * Email: <neilb@suse.de>
23 */
24
25 #include "mdadm.h"
26 #include "md_p.h"
27 #include "md_u.h"
28 #include <sys/wait.h>
29 #include <signal.h>
30 #include <limits.h>
31 #include <syslog.h>
32
33 static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom,
34 char *cmd, int dosyslog);
35
36 /* The largest number of disks current arrays can manage is 384
37 * This really should be dynamically, but that will have to wait
38 * At least it isn't MD_SB_DISKS.
39 */
40 #define MaxDisks 384
41 int Monitor(struct mddev_dev *devlist,
42 char *mailaddr, char *alert_cmd,
43 int period, int daemonise, int scan, int oneshot,
44 int dosyslog, int test, char *pidfile, int increments,
45 int share)
46 {
47 /*
48 * Every few seconds, scan every md device looking for changes
49 * When a change is found, log it, possibly run the alert command,
50 * and possibly send Email
51 *
52 * For each array, we record:
53 * Update time
54 * active/working/failed/spare drives
55 * State of each device.
56 * %rebuilt if rebuilding
57 *
58 * If the update time changes, check out all the data again
59 * It is possible that we cannot get the state of each device
60 * due to bugs in the md kernel module.
61 * We also read /proc/mdstat to get rebuild percent,
62 * and to get state on all active devices incase of kernel bug.
63 *
64 * Events are:
65 * Fail
66 * An active device had Faulty set or Active/Sync removed
67 * FailSpare
68 * A spare device had Faulty set
69 * SpareActive
70 * An active device had a reverse transition
71 * RebuildStarted
72 * percent went from -1 to +ve
73 * RebuildNN
74 * percent went from below to not-below NN%
75 * DeviceDisappeared
76 * Couldn't access a device which was previously visible
77 *
78 * if we detect an array with active<raid and spare==0
79 * we look at other arrays that have same spare-group
80 * If we find one with active==raid and spare>0,
81 * and if we can get_disk_info and find a name
82 * Then we hot-remove and hot-add to the other array
83 *
84 * If devlist is NULL, then we can monitor everything because --scan
85 * was given. We get an initial list from config file and add anything
86 * that appears in /proc/mdstat
87 */
88
89 struct state {
90 char *devname;
91 int devnum; /* to sync with mdstat info */
92 long utime;
93 int err;
94 char *spare_group;
95 int active, working, failed, spare, raid;
96 int expected_spares;
97 int devstate[MaxDisks];
98 unsigned devid[MaxDisks];
99 int percent;
100 struct state *next;
101 } *statelist = NULL;
102 int finished = 0;
103 struct mdstat_ent *mdstat = NULL;
104 char *mailfrom = NULL;
105
106 if (!mailaddr) {
107 mailaddr = conf_get_mailaddr();
108 if (mailaddr && ! scan)
109 fprintf(stderr, Name ": Monitor using email address \"%s\" from config file\n",
110 mailaddr);
111 }
112 mailfrom = conf_get_mailfrom();
113
114 if (!alert_cmd) {
115 alert_cmd = conf_get_program();
116 if (alert_cmd && ! scan)
117 fprintf(stderr, Name ": Monitor using program \"%s\" from config file\n",
118 alert_cmd);
119 }
120 if (scan && !mailaddr && !alert_cmd) {
121 fprintf(stderr, Name ": No mail address or alert command - not monitoring.\n");
122 return 1;
123 }
124
125 if (daemonise) {
126 int pid = fork();
127 if (pid > 0) {
128 if (!pidfile)
129 printf("%d\n", pid);
130 else {
131 FILE *pid_file;
132 pid_file=fopen(pidfile, "w");
133 if (!pid_file)
134 perror("cannot create pid file");
135 else {
136 fprintf(pid_file,"%d\n", pid);
137 fclose(pid_file);
138 }
139 }
140 return 0;
141 }
142 if (pid < 0) {
143 perror("daemonise");
144 return 1;
145 }
146 close(0);
147 open("/dev/null", O_RDWR);
148 dup2(0,1);
149 dup2(0,2);
150 setsid();
151 }
152
153 if (share) {
154 int pid, rv;
155 FILE *fp;
156 char dir[20];
157 struct stat buf;
158 fp = fopen("/var/run/mdadm/autorebuild.pid", "r");
159 if (fp) {
160 fscanf(fp, "%d", &pid);
161 sprintf(dir, "/proc/%d", pid);
162 rv = stat(dir, &buf);
163 if (rv != -1) {
164 if (scan) {
165 fprintf(stderr, Name ": Only one "
166 "autorebuild process allowed"
167 " in scan mode, aborting\n");
168 fclose(fp);
169 return 1;
170 } else {
171 fprintf(stderr, Name ": Warning: One"
172 " autorebuild process already"
173 " running.");
174 }
175 }
176 fclose(fp);
177 }
178 if (scan) {
179 fp = fopen("/var/run/mdadm/autorebuild.pid", "w");
180 if (!fp)
181 fprintf(stderr, Name ": Cannot create"
182 " autorebuild.pid "
183 "file\n");
184 else {
185 pid = getpid();
186 fprintf(fp, "%d\n", pid);
187 fclose(fp);
188 }
189 }
190 }
191
192 if (devlist == NULL) {
193 struct mddev_ident *mdlist = conf_get_ident(NULL);
194 for (; mdlist; mdlist=mdlist->next) {
195 struct state *st;
196 if (mdlist->devname == NULL)
197 continue;
198 if (strcasecmp(mdlist->devname, "<ignore>") == 0)
199 continue;
200 st = malloc(sizeof *st);
201 if (st == NULL)
202 continue;
203 if (mdlist->devname[0] == '/')
204 st->devname = strdup(mdlist->devname);
205 else {
206 st->devname = malloc(8+strlen(mdlist->devname)+1);
207 strcpy(strcpy(st->devname, "/dev/md/"),
208 mdlist->devname);
209 }
210 st->utime = 0;
211 st->next = statelist;
212 st->err = 0;
213 st->devnum = INT_MAX;
214 st->percent = -2;
215 st->expected_spares = mdlist->spare_disks;
216 if (mdlist->spare_group)
217 st->spare_group = strdup(mdlist->spare_group);
218 else
219 st->spare_group = NULL;
220 statelist = st;
221 }
222 } else {
223 struct mddev_dev *dv;
224 for (dv=devlist ; dv; dv=dv->next) {
225 struct mddev_ident *mdlist = conf_get_ident(dv->devname);
226 struct state *st = malloc(sizeof *st);
227 if (st == NULL)
228 continue;
229 st->devname = strdup(dv->devname);
230 st->utime = 0;
231 st->next = statelist;
232 st->err = 0;
233 st->devnum = INT_MAX;
234 st->percent = -2;
235 st->expected_spares = -1;
236 st->spare_group = NULL;
237 if (mdlist) {
238 st->expected_spares = mdlist->spare_disks;
239 if (mdlist->spare_group)
240 st->spare_group = strdup(mdlist->spare_group);
241 }
242 statelist = st;
243 }
244 }
245
246
247 while (! finished) {
248 int new_found = 0;
249 struct state *st;
250
251 if (mdstat)
252 free_mdstat(mdstat);
253 mdstat = mdstat_read(oneshot?0:1, 0);
254
255 for (st=statelist; st; st=st->next) {
256 struct { int state, major, minor; } info[MaxDisks];
257 mdu_array_info_t array;
258 struct mdstat_ent *mse = NULL, *mse2;
259 char *dev = st->devname;
260 int fd;
261 int i;
262
263 if (test)
264 alert("TestMessage", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
265 fd = open(dev, O_RDONLY);
266 if (fd < 0) {
267 if (!st->err)
268 alert("DeviceDisappeared", dev, NULL,
269 mailaddr, mailfrom, alert_cmd, dosyslog);
270 /* fprintf(stderr, Name ": cannot open %s: %s\n",
271 dev, strerror(errno));
272 */ st->err=1;
273 continue;
274 }
275 fcntl(fd, F_SETFD, FD_CLOEXEC);
276 if (ioctl(fd, GET_ARRAY_INFO, &array)<0) {
277 if (!st->err)
278 alert("DeviceDisappeared", dev, NULL,
279 mailaddr, mailfrom, alert_cmd, dosyslog);
280 /* fprintf(stderr, Name ": cannot get array info for %s: %s\n",
281 dev, strerror(errno));
282 */ st->err=1;
283 close(fd);
284 continue;
285 }
286 /* It's much easier to list what array levels can't
287 * have a device disappear than all of them that can
288 */
289 if (array.level == 0 || array.level == -1) {
290 if (!st->err)
291 alert("DeviceDisappeared", dev, "Wrong-Level",
292 mailaddr, mailfrom, alert_cmd, dosyslog);
293 st->err = 1;
294 close(fd);
295 continue;
296 }
297 if (st->devnum == INT_MAX) {
298 struct stat stb;
299 if (fstat(fd, &stb) == 0 &&
300 (S_IFMT&stb.st_mode)==S_IFBLK) {
301 if (major(stb.st_rdev) == MD_MAJOR)
302 st->devnum = minor(stb.st_rdev);
303 else
304 st->devnum = -1- (minor(stb.st_rdev)>>6);
305 }
306 }
307
308 for (mse2 = mdstat ; mse2 ; mse2=mse2->next)
309 if (mse2->devnum == st->devnum) {
310 mse2->devnum = INT_MAX; /* flag it as "used" */
311 mse = mse2;
312 }
313
314 if (!mse) {
315 /* duplicated array in statelist
316 * or re-created after reading mdstat*/
317 st->err = 1;
318 close(fd);
319 continue;
320 }
321 /* this array is in /proc/mdstat */
322 if (array.utime == 0)
323 /* external arrays don't update utime */
324 array.utime = time(0);
325
326 if (st->utime == array.utime &&
327 st->failed == array.failed_disks &&
328 st->working == array.working_disks &&
329 st->spare == array.spare_disks &&
330 (mse == NULL || (
331 mse->percent == st->percent
332 ))) {
333 close(fd);
334 st->err = 0;
335 continue;
336 }
337 if (st->utime == 0 && /* new array */
338 mse->pattern && strchr(mse->pattern, '_') /* degraded */
339 )
340 alert("DegradedArray", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
341
342 if (st->utime == 0 && /* new array */
343 st->expected_spares > 0 &&
344 array.spare_disks < st->expected_spares)
345 alert("SparesMissing", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
346 if (st->percent == -1 &&
347 mse->percent >= 0)
348 alert("RebuildStarted", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
349 if (st->percent >= 0 &&
350 mse->percent >= 0 &&
351 (mse->percent / increments) > (st->percent / increments)) {
352 char percentalert[15]; // "RebuildNN" (10 chars) or "RebuildStarted" (15 chars)
353
354 if((mse->percent / increments) == 0)
355 snprintf(percentalert, sizeof(percentalert), "RebuildStarted");
356 else
357 snprintf(percentalert, sizeof(percentalert), "Rebuild%02d", mse->percent);
358
359 alert(percentalert,
360 dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
361 }
362
363 if (mse->percent == -1 &&
364 st->percent >= 0) {
365 /* Rebuild/sync/whatever just finished.
366 * If there is a number in /mismatch_cnt,
367 * we should report that.
368 */
369 struct mdinfo *sra =
370 sysfs_read(-1, st->devnum, GET_MISMATCH);
371 if (sra && sra->mismatch_cnt > 0) {
372 char cnt[40];
373 sprintf(cnt, " mismatches found: %d", sra->mismatch_cnt);
374 alert("RebuildFinished", dev, cnt, mailaddr, mailfrom, alert_cmd, dosyslog);
375 } else
376 alert("RebuildFinished", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
377 if (sra)
378 free(sra);
379 }
380 st->percent = mse->percent;
381
382 for (i=0; i<MaxDisks && i <= array.raid_disks + array.nr_disks;
383 i++) {
384 mdu_disk_info_t disc;
385 disc.number = i;
386 if (ioctl(fd, GET_DISK_INFO, &disc) >= 0) {
387 info[i].state = disc.state;
388 info[i].major = disc.major;
389 info[i].minor = disc.minor;
390 } else
391 info[i].major = info[i].minor = 0;
392 }
393 close(fd);
394
395 for (i=0; i<MaxDisks; i++) {
396 mdu_disk_info_t disc = {0,0,0,0,0};
397 int newstate=0;
398 int change;
399 char *dv = NULL;
400 disc.number = i;
401 if (i > array.raid_disks + array.nr_disks) {
402 newstate = 0;
403 disc.major = disc.minor = 0;
404 } else if (info[i].major || info[i].minor) {
405 newstate = info[i].state;
406 dv = map_dev(info[i].major, info[i].minor, 1);
407 disc.state = newstate;
408 disc.major = info[i].major;
409 disc.minor = info[i].minor;
410 } else if (mse && mse->pattern && i < (int)strlen(mse->pattern)) {
411 switch(mse->pattern[i]) {
412 case 'U': newstate = 6 /* ACTIVE/SYNC */; break;
413 case '_': newstate = 0; break;
414 }
415 disc.major = disc.minor = 0;
416 }
417 if (dv == NULL && st->devid[i])
418 dv = map_dev(major(st->devid[i]),
419 minor(st->devid[i]), 1);
420 change = newstate ^ st->devstate[i];
421 if (st->utime && change && !st->err) {
422 if (i < array.raid_disks &&
423 (((newstate&change)&(1<<MD_DISK_FAULTY)) ||
424 ((st->devstate[i]&change)&(1<<MD_DISK_ACTIVE)) ||
425 ((st->devstate[i]&change)&(1<<MD_DISK_SYNC)))
426 )
427 alert("Fail", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
428 else if (i >= array.raid_disks &&
429 (disc.major || disc.minor) &&
430 st->devid[i] == makedev(disc.major, disc.minor) &&
431 ((newstate&change)&(1<<MD_DISK_FAULTY))
432 )
433 alert("FailSpare", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
434 else if (i < array.raid_disks &&
435 ! (newstate & (1<<MD_DISK_REMOVED)) &&
436 (((st->devstate[i]&change)&(1<<MD_DISK_FAULTY)) ||
437 ((newstate&change)&(1<<MD_DISK_ACTIVE)) ||
438 ((newstate&change)&(1<<MD_DISK_SYNC)))
439 )
440 alert("SpareActive", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog);
441 }
442 st->devstate[i] = newstate;
443 st->devid[i] = makedev(disc.major, disc.minor);
444 }
445 st->active = array.active_disks;
446 st->working = array.working_disks;
447 st->spare = array.spare_disks;
448 st->failed = array.failed_disks;
449 st->utime = array.utime;
450 st->raid = array.raid_disks;
451 st->err = 0;
452 }
453 /* now check if there are any new devices found in mdstat */
454 if (scan) {
455 struct mdstat_ent *mse;
456 for (mse=mdstat; mse; mse=mse->next)
457 if (mse->devnum != INT_MAX &&
458 mse->level &&
459 (strcmp(mse->level, "raid0")!=0 &&
460 strcmp(mse->level, "linear")!=0)
461 ) {
462 struct state *st = malloc(sizeof *st);
463 mdu_array_info_t array;
464 int fd;
465 if (st == NULL)
466 continue;
467 st->devname = strdup(get_md_name(mse->devnum));
468 if ((fd = open(st->devname, O_RDONLY)) < 0 ||
469 ioctl(fd, GET_ARRAY_INFO, &array)< 0) {
470 /* no such array */
471 if (fd >=0) close(fd);
472 put_md_name(st->devname);
473 free(st->devname);
474 free(st);
475 continue;
476 }
477 close(fd);
478 st->utime = 0;
479 st->next = statelist;
480 st->err = 1;
481 st->devnum = mse->devnum;
482 st->percent = -2;
483 st->spare_group = NULL;
484 st->expected_spares = -1;
485 statelist = st;
486 if (test)
487 alert("TestMessage", st->devname, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
488 alert("NewArray", st->devname, NULL, mailaddr, mailfrom, alert_cmd, dosyslog);
489 new_found = 1;
490 }
491 }
492 /* If an array has active < raid && spare == 0 && spare_group != NULL
493 * Look for another array with spare > 0 and active == raid and same spare_group
494 * if found, choose a device and hotremove/hotadd
495 */
496 if (share) for (st = statelist; st; st=st->next)
497 if (st->active < st->raid &&
498 st->spare == 0 &&
499 st->spare_group != NULL) {
500 struct state *st2;
501 for (st2=statelist ; st2 ; st2=st2->next)
502 if (st2 != st &&
503 st2->spare > 0 &&
504 st2->active == st2->raid &&
505 st2->spare_group != NULL &&
506 strcmp(st->spare_group, st2->spare_group) == 0) {
507 /* try to remove and add */
508 int fd1 = open(st->devname, O_RDONLY);
509 int fd2 = open(st2->devname, O_RDONLY);
510 int dev = -1;
511 int d;
512 if (fd1 < 0 || fd2 < 0) {
513 if (fd1>=0) close(fd1);
514 if (fd2>=0) close(fd2);
515 continue;
516 }
517 for (d=st2->raid; d < MaxDisks; d++) {
518 if (st2->devid[d] > 0 &&
519 st2->devstate[d] == 0) {
520 dev = st2->devid[d];
521 break;
522 }
523 }
524 if (dev > 0) {
525 struct mddev_dev devlist;
526 char devname[20];
527 devlist.next = NULL;
528 devlist.used = 0;
529 devlist.re_add = 0;
530 devlist.writemostly = 0;
531 devlist.devname = devname;
532 sprintf(devname, "%d:%d", major(dev), minor(dev));
533
534 devlist.disposition = 'r';
535 if (Manage_subdevs(st2->devname, fd2, &devlist, -1, 0) == 0) {
536 devlist.disposition = 'a';
537 if (Manage_subdevs(st->devname, fd1, &devlist, -1, 0) == 0) {
538 alert("MoveSpare", st->devname, st2->devname, mailaddr, mailfrom, alert_cmd, dosyslog);
539 close(fd1);
540 close(fd2);
541 break;
542 }
543 else Manage_subdevs(st2->devname, fd2, &devlist, -1, 0);
544 }
545 }
546 close(fd1);
547 close(fd2);
548 }
549 }
550 if (!new_found) {
551 if (oneshot)
552 break;
553 else
554 mdstat_wait(period);
555 }
556 test = 0;
557 }
558 if (pidfile)
559 unlink(pidfile);
560 return 0;
561 }
562
563
564 static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom, char *cmd,
565 int dosyslog)
566 {
567 int priority;
568
569 if (!cmd && !mailaddr) {
570 time_t now = time(0);
571
572 printf("%1.15s: %s on %s %s\n", ctime(&now)+4, event, dev, disc?disc:"unknown device");
573 }
574 if (cmd) {
575 int pid = fork();
576 switch(pid) {
577 default:
578 waitpid(pid, NULL, 0);
579 break;
580 case -1:
581 break;
582 case 0:
583 execl(cmd, cmd, event, dev, disc, NULL);
584 exit(2);
585 }
586 }
587 if (mailaddr &&
588 (strncmp(event, "Fail", 4)==0 ||
589 strncmp(event, "Test", 4)==0 ||
590 strncmp(event, "Spares", 6)==0 ||
591 strncmp(event, "Degrade", 7)==0)) {
592 FILE *mp = popen(Sendmail, "w");
593 if (mp) {
594 FILE *mdstat;
595 char hname[256];
596 gethostname(hname, sizeof(hname));
597 signal(SIGPIPE, SIG_IGN);
598 if (mailfrom)
599 fprintf(mp, "From: %s\n", mailfrom);
600 else
601 fprintf(mp, "From: " Name " monitoring <root>\n");
602 fprintf(mp, "To: %s\n", mailaddr);
603 fprintf(mp, "Subject: %s event on %s:%s\n\n", event, dev, hname);
604
605 fprintf(mp, "This is an automatically generated mail message from " Name "\n");
606 fprintf(mp, "running on %s\n\n", hname);
607
608 fprintf(mp, "A %s event had been detected on md device %s.\n\n", event, dev);
609
610 if (disc && disc[0] != ' ')
611 fprintf(mp, "It could be related to component device %s.\n\n", disc);
612 if (disc && disc[0] == ' ')
613 fprintf(mp, "Extra information:%s.\n\n", disc);
614
615 fprintf(mp, "Faithfully yours, etc.\n");
616
617 mdstat = fopen("/proc/mdstat", "r");
618 if (mdstat) {
619 char buf[8192];
620 int n;
621 fprintf(mp, "\nP.S. The /proc/mdstat file currently contains the following:\n\n");
622 while ( (n=fread(buf, 1, sizeof(buf), mdstat)) > 0)
623 n=fwrite(buf, 1, n, mp); /* yes, i don't care about the result */
624 fclose(mdstat);
625 }
626 pclose(mp);
627 }
628
629 }
630
631 /* log the event to syslog maybe */
632 if (dosyslog) {
633 /* Log at a different severity depending on the event.
634 *
635 * These are the critical events: */
636 if (strncmp(event, "Fail", 4)==0 ||
637 strncmp(event, "Degrade", 7)==0 ||
638 strncmp(event, "DeviceDisappeared", 17)==0)
639 priority = LOG_CRIT;
640 /* Good to know about, but are not failures: */
641 else if (strncmp(event, "Rebuild", 7)==0 ||
642 strncmp(event, "MoveSpare", 9)==0 ||
643 strncmp(event, "Spares", 6) != 0)
644 priority = LOG_WARNING;
645 /* Everything else: */
646 else
647 priority = LOG_INFO;
648
649 if (disc)
650 syslog(priority, "%s event detected on md device %s, component device %s", event, dev, disc);
651 else
652 syslog(priority, "%s event detected on md device %s", event, dev);
653 }
654 }
655
656 /* Not really Monitor but ... */
657 int Wait(char *dev)
658 {
659 struct stat stb;
660 int devnum;
661 int rv = 1;
662
663 if (stat(dev, &stb) != 0) {
664 fprintf(stderr, Name ": Cannot find %s: %s\n", dev,
665 strerror(errno));
666 return 2;
667 }
668 devnum = stat2devnum(&stb);
669
670 while(1) {
671 struct mdstat_ent *ms = mdstat_read(1, 0);
672 struct mdstat_ent *e;
673
674 for (e=ms ; e; e=e->next)
675 if (e->devnum == devnum)
676 break;
677
678 if (!e || e->percent < 0) {
679 if (e && e->metadata_version &&
680 strncmp(e->metadata_version, "external:", 9) == 0) {
681 if (is_subarray(&e->metadata_version[9]))
682 ping_monitor(&e->metadata_version[9]);
683 else
684 ping_monitor(devnum2devname(devnum));
685 }
686 free_mdstat(ms);
687 return rv;
688 }
689 free_mdstat(ms);
690 rv = 0;
691 mdstat_wait(5);
692 }
693 }