]>
git.ipfire.org Git - thirdparty/mdadm.git/blob - Monitor.c
2 * mdadm - manage Linux "md" devices aka RAID arrays.
4 * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 * Email: <neilb@suse.de>
33 /* The largest number of disks current arrays can manage is 384
34 * This really should be dynamically, but that will have to wait
35 * At least it isn't MD_SB_DISKS.
40 int devnum
; /* to sync with mdstat info */
44 int active
, working
, failed
, spare
, raid
;
46 int devstate
[MaxDisks
];
47 unsigned devid
[MaxDisks
];
49 int parent_dev
; /* For subarray, devnum of parent.
52 struct supertype
*metadata
;
53 struct state
*subarray
;/* for a container it is a link to first subarray
54 * for a subarray it is a link to next subarray
55 * in the same container */
56 struct state
*parent
; /* for a subarray it is a link to its container
61 static int make_daemon(char *pidfile
);
62 static int check_one_sharer(int scan
);
63 static void alert(char *event
, char *dev
, char *disc
, char *mailaddr
, char *mailfrom
,
64 char *cmd
, int dosyslog
);
65 static int check_array(struct state
*st
, struct mdstat_ent
*mdstat
,
66 int test
, char *mailaddr
,
67 char *mailfrom
, char *alert_cmd
, int dosyslog
,
69 static int add_new_arrays(struct mdstat_ent
*mdstat
, struct state
*statelist
,
70 int test
, char *mailaddr
, char *mailfrom
,
71 char *alert_cmd
, int dosyslog
);
72 static void try_spare_migration(struct state
*statelist
,
73 char *mailaddr
, char *mailfrom
,
74 char *alert_cmd
, int dosyslog
);
75 static void link_containers_with_subarrays(struct state
*list
);
77 int Monitor(struct mddev_dev
*devlist
,
78 char *mailaddr
, char *alert_cmd
,
79 int period
, int daemonise
, int scan
, int oneshot
,
80 int dosyslog
, int test
, char *pidfile
, int increments
,
84 * Every few seconds, scan every md device looking for changes
85 * When a change is found, log it, possibly run the alert command,
86 * and possibly send Email
88 * For each array, we record:
90 * active/working/failed/spare drives
91 * State of each device.
92 * %rebuilt if rebuilding
94 * If the update time changes, check out all the data again
95 * It is possible that we cannot get the state of each device
96 * due to bugs in the md kernel module.
97 * We also read /proc/mdstat to get rebuild percent,
98 * and to get state on all active devices incase of kernel bug.
102 * An active device had Faulty set or Active/Sync removed
104 * A spare device had Faulty set
106 * An active device had a reverse transition
108 * percent went from -1 to +ve
110 * percent went from below to not-below NN%
112 * Couldn't access a device which was previously visible
114 * if we detect an array with active<raid and spare==0
115 * we look at other arrays that have same spare-group
116 * If we find one with active==raid and spare>0,
117 * and if we can get_disk_info and find a name
118 * Then we hot-remove and hot-add to the other array
120 * If devlist is NULL, then we can monitor everything because --scan
121 * was given. We get an initial list from config file and add anything
122 * that appears in /proc/mdstat
125 struct state
*statelist
= NULL
;
127 struct mdstat_ent
*mdstat
= NULL
;
128 char *mailfrom
= NULL
;
131 mailaddr
= conf_get_mailaddr();
132 if (mailaddr
&& ! scan
)
133 fprintf(stderr
, Name
": Monitor using email address \"%s\" from config file\n",
136 mailfrom
= conf_get_mailfrom();
139 alert_cmd
= conf_get_program();
140 if (alert_cmd
&& ! scan
)
141 fprintf(stderr
, Name
": Monitor using program \"%s\" from config file\n",
144 if (scan
&& !mailaddr
&& !alert_cmd
) {
145 fprintf(stderr
, Name
": No mail address or alert command - not monitoring.\n");
150 if (make_daemon(pidfile
))
154 if (check_one_sharer(scan
))
157 if (devlist
== NULL
) {
158 struct mddev_ident
*mdlist
= conf_get_ident(NULL
);
159 for (; mdlist
; mdlist
=mdlist
->next
) {
161 if (mdlist
->devname
== NULL
)
163 if (strcasecmp(mdlist
->devname
, "<ignore>") == 0)
165 st
= malloc(sizeof *st
);
168 if (mdlist
->devname
[0] == '/')
169 st
->devname
= strdup(mdlist
->devname
);
171 st
->devname
= malloc(8+strlen(mdlist
->devname
)+1);
172 strcpy(strcpy(st
->devname
, "/dev/md/"),
176 st
->next
= statelist
;
178 st
->devnum
= INT_MAX
;
180 st
->expected_spares
= mdlist
->spare_disks
;
181 if (mdlist
->spare_group
)
182 st
->spare_group
= strdup(mdlist
->spare_group
);
184 st
->spare_group
= NULL
;
188 struct mddev_dev
*dv
;
189 for (dv
=devlist
; dv
; dv
=dv
->next
) {
190 struct mddev_ident
*mdlist
= conf_get_ident(dv
->devname
);
191 struct state
*st
= malloc(sizeof *st
);
194 st
->devname
= strdup(dv
->devname
);
196 st
->next
= statelist
;
198 st
->devnum
= INT_MAX
;
200 st
->expected_spares
= -1;
201 st
->spare_group
= NULL
;
203 st
->expected_spares
= mdlist
->spare_disks
;
204 if (mdlist
->spare_group
)
205 st
->spare_group
= strdup(mdlist
->spare_group
);
219 mdstat
= mdstat_read(oneshot
?0:1, 0);
221 for (st
=statelist
; st
; st
=st
->next
)
222 if (check_array(st
, mdstat
, test
, mailaddr
, mailfrom
,
223 alert_cmd
, dosyslog
, increments
))
226 /* now check if there are any new devices found in mdstat */
228 new_found
= add_new_arrays(mdstat
, statelist
, test
,
229 mailaddr
, mailfrom
, alert_cmd
,
232 /* If an array has active < raid && spare == 0 && spare_group != NULL
233 * Look for another array with spare > 0 and active == raid and same spare_group
234 * if found, choose a device and hotremove/hotadd
236 if (share
&& anydegraded
)
237 try_spare_migration(statelist
, mailaddr
, mailfrom
,
238 alert_cmd
, dosyslog
);
252 static int make_daemon(char *pidfile
)
260 pid_file
=fopen(pidfile
, "w");
262 perror("cannot create pid file");
264 fprintf(pid_file
,"%d\n", pid
);
275 open("/dev/null", O_RDWR
);
282 static int check_one_sharer(int scan
)
288 fp
= fopen("/var/run/mdadm/autorebuild.pid", "r");
290 fscanf(fp
, "%d", &pid
);
291 sprintf(dir
, "/proc/%d", pid
);
292 rv
= stat(dir
, &buf
);
295 fprintf(stderr
, Name
": Only one "
296 "autorebuild process allowed"
297 " in scan mode, aborting\n");
301 fprintf(stderr
, Name
": Warning: One"
302 " autorebuild process already"
309 fp
= fopen("/var/run/mdadm/autorebuild.pid", "w");
311 fprintf(stderr
, Name
": Cannot create"
316 fprintf(fp
, "%d\n", pid
);
323 static void alert(char *event
, char *dev
, char *disc
, char *mailaddr
, char *mailfrom
, char *cmd
,
328 if (!cmd
&& !mailaddr
) {
329 time_t now
= time(0);
331 printf("%1.15s: %s on %s %s\n", ctime(&now
)+4, event
, dev
, disc
?disc
:"unknown device");
337 waitpid(pid
, NULL
, 0);
342 execl(cmd
, cmd
, event
, dev
, disc
, NULL
);
347 (strncmp(event
, "Fail", 4)==0 ||
348 strncmp(event
, "Test", 4)==0 ||
349 strncmp(event
, "Spares", 6)==0 ||
350 strncmp(event
, "Degrade", 7)==0)) {
351 FILE *mp
= popen(Sendmail
, "w");
355 gethostname(hname
, sizeof(hname
));
356 signal(SIGPIPE
, SIG_IGN
);
358 fprintf(mp
, "From: %s\n", mailfrom
);
360 fprintf(mp
, "From: " Name
" monitoring <root>\n");
361 fprintf(mp
, "To: %s\n", mailaddr
);
362 fprintf(mp
, "Subject: %s event on %s:%s\n\n", event
, dev
, hname
);
364 fprintf(mp
, "This is an automatically generated mail message from " Name
"\n");
365 fprintf(mp
, "running on %s\n\n", hname
);
367 fprintf(mp
, "A %s event had been detected on md device %s.\n\n", event
, dev
);
369 if (disc
&& disc
[0] != ' ')
370 fprintf(mp
, "It could be related to component device %s.\n\n", disc
);
371 if (disc
&& disc
[0] == ' ')
372 fprintf(mp
, "Extra information:%s.\n\n", disc
);
374 fprintf(mp
, "Faithfully yours, etc.\n");
376 mdstat
= fopen("/proc/mdstat", "r");
380 fprintf(mp
, "\nP.S. The /proc/mdstat file currently contains the following:\n\n");
381 while ( (n
=fread(buf
, 1, sizeof(buf
), mdstat
)) > 0)
382 n
=fwrite(buf
, 1, n
, mp
); /* yes, i don't care about the result */
390 /* log the event to syslog maybe */
392 /* Log at a different severity depending on the event.
394 * These are the critical events: */
395 if (strncmp(event
, "Fail", 4)==0 ||
396 strncmp(event
, "Degrade", 7)==0 ||
397 strncmp(event
, "DeviceDisappeared", 17)==0)
399 /* Good to know about, but are not failures: */
400 else if (strncmp(event
, "Rebuild", 7)==0 ||
401 strncmp(event
, "MoveSpare", 9)==0 ||
402 strncmp(event
, "Spares", 6) != 0)
403 priority
= LOG_WARNING
;
404 /* Everything else: */
409 syslog(priority
, "%s event detected on md device %s, component device %s", event
, dev
, disc
);
411 syslog(priority
, "%s event detected on md device %s", event
, dev
);
415 static int check_array(struct state
*st
, struct mdstat_ent
*mdstat
,
416 int test
, char *mailaddr
,
417 char *mailfrom
, char *alert_cmd
, int dosyslog
,
420 struct { int state
, major
, minor
; } info
[MaxDisks
];
421 mdu_array_info_t array
;
422 struct mdstat_ent
*mse
= NULL
, *mse2
;
423 char *dev
= st
->devname
;
428 alert("TestMessage", dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
429 fd
= open(dev
, O_RDONLY
);
432 alert("DeviceDisappeared", dev
, NULL
,
433 mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
434 /* fprintf(stderr, Name ": cannot open %s: %s\n",
435 dev, strerror(errno));
439 fcntl(fd
, F_SETFD
, FD_CLOEXEC
);
440 if (ioctl(fd
, GET_ARRAY_INFO
, &array
)<0) {
442 alert("DeviceDisappeared", dev
, NULL
,
443 mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
444 /* fprintf(stderr, Name ": cannot get array info for %s: %s\n",
445 dev, strerror(errno));
450 /* It's much easier to list what array levels can't
451 * have a device disappear than all of them that can
453 if (array
.level
== 0 || array
.level
== -1) {
455 alert("DeviceDisappeared", dev
, "Wrong-Level",
456 mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
461 if (st
->devnum
== INT_MAX
) {
463 if (fstat(fd
, &stb
) == 0 &&
464 (S_IFMT
&stb
.st_mode
)==S_IFBLK
) {
465 if (major(stb
.st_rdev
) == MD_MAJOR
)
466 st
->devnum
= minor(stb
.st_rdev
);
468 st
->devnum
= -1- (minor(stb
.st_rdev
)>>6);
472 for (mse2
= mdstat
; mse2
; mse2
=mse2
->next
)
473 if (mse2
->devnum
== st
->devnum
) {
474 mse2
->devnum
= INT_MAX
; /* flag it as "used" */
479 /* duplicated array in statelist
480 * or re-created after reading mdstat*/
485 /* this array is in /proc/mdstat */
486 if (array
.utime
== 0)
487 /* external arrays don't update utime, so
488 * just make sure it is always different. */
489 array
.utime
= st
->utime
+ 1;;
491 if (st
->utime
== array
.utime
&&
492 st
->failed
== array
.failed_disks
&&
493 st
->working
== array
.working_disks
&&
494 st
->spare
== array
.spare_disks
&&
496 mse
->percent
== st
->percent
502 if (st
->utime
== 0 && /* new array */
503 mse
->pattern
&& strchr(mse
->pattern
, '_') /* degraded */
505 alert("DegradedArray", dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
507 if (st
->utime
== 0 && /* new array */
508 st
->expected_spares
> 0 &&
509 array
.spare_disks
< st
->expected_spares
)
510 alert("SparesMissing", dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
511 if (st
->percent
== -1 &&
513 alert("RebuildStarted", dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
514 if (st
->percent
>= 0 &&
516 (mse
->percent
/ increments
) > (st
->percent
/ increments
)) {
517 char percentalert
[15]; // "RebuildNN" (10 chars) or "RebuildStarted" (15 chars)
519 if((mse
->percent
/ increments
) == 0)
520 snprintf(percentalert
, sizeof(percentalert
), "RebuildStarted");
522 snprintf(percentalert
, sizeof(percentalert
), "Rebuild%02d", mse
->percent
);
525 dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
528 if (mse
->percent
== -1 &&
530 /* Rebuild/sync/whatever just finished.
531 * If there is a number in /mismatch_cnt,
532 * we should report that.
535 sysfs_read(-1, st
->devnum
, GET_MISMATCH
);
536 if (sra
&& sra
->mismatch_cnt
> 0) {
538 sprintf(cnt
, " mismatches found: %d", sra
->mismatch_cnt
);
539 alert("RebuildFinished", dev
, cnt
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
541 alert("RebuildFinished", dev
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
545 st
->percent
= mse
->percent
;
547 for (i
=0; i
<MaxDisks
&& i
<= array
.raid_disks
+ array
.nr_disks
;
549 mdu_disk_info_t disc
;
551 if (ioctl(fd
, GET_DISK_INFO
, &disc
) >= 0) {
552 info
[i
].state
= disc
.state
;
553 info
[i
].major
= disc
.major
;
554 info
[i
].minor
= disc
.minor
;
556 info
[i
].major
= info
[i
].minor
= 0;
559 if (strncmp(mse
->metadata_version
, "external:", 9) == 0 &&
560 is_subarray(mse
->metadata_version
+9))
562 devname2devnum(mse
->metadata_version
+10);
564 st
->parent_dev
= NoMdDev
;
565 if (st
->metadata
== NULL
&&
566 st
->parent_dev
== NoMdDev
)
567 st
->metadata
= super_by_fd(fd
, NULL
);
571 for (i
=0; i
<MaxDisks
; i
++) {
572 mdu_disk_info_t disc
= {0,0,0,0,0};
577 if (i
> array
.raid_disks
+ array
.nr_disks
) {
579 disc
.major
= disc
.minor
= 0;
580 } else if (info
[i
].major
|| info
[i
].minor
) {
581 newstate
= info
[i
].state
;
582 dv
= map_dev(info
[i
].major
, info
[i
].minor
, 1);
583 disc
.state
= newstate
;
584 disc
.major
= info
[i
].major
;
585 disc
.minor
= info
[i
].minor
;
586 } else if (mse
&& mse
->pattern
&& i
< (int)strlen(mse
->pattern
)) {
587 switch(mse
->pattern
[i
]) {
588 case 'U': newstate
= 6 /* ACTIVE/SYNC */; break;
589 case '_': newstate
= 0; break;
591 disc
.major
= disc
.minor
= 0;
593 if (dv
== NULL
&& st
->devid
[i
])
594 dv
= map_dev(major(st
->devid
[i
]),
595 minor(st
->devid
[i
]), 1);
596 change
= newstate
^ st
->devstate
[i
];
597 if (st
->utime
&& change
&& !st
->err
) {
598 if (i
< array
.raid_disks
&&
599 (((newstate
&change
)&(1<<MD_DISK_FAULTY
)) ||
600 ((st
->devstate
[i
]&change
)&(1<<MD_DISK_ACTIVE
)) ||
601 ((st
->devstate
[i
]&change
)&(1<<MD_DISK_SYNC
)))
603 alert("Fail", dev
, dv
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
604 else if (i
>= array
.raid_disks
&&
605 (disc
.major
|| disc
.minor
) &&
606 st
->devid
[i
] == makedev(disc
.major
, disc
.minor
) &&
607 ((newstate
&change
)&(1<<MD_DISK_FAULTY
))
609 alert("FailSpare", dev
, dv
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
610 else if (i
< array
.raid_disks
&&
611 ! (newstate
& (1<<MD_DISK_REMOVED
)) &&
612 (((st
->devstate
[i
]&change
)&(1<<MD_DISK_FAULTY
)) ||
613 ((newstate
&change
)&(1<<MD_DISK_ACTIVE
)) ||
614 ((newstate
&change
)&(1<<MD_DISK_SYNC
)))
616 alert("SpareActive", dev
, dv
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
618 st
->devstate
[i
] = newstate
;
619 st
->devid
[i
] = makedev(disc
.major
, disc
.minor
);
621 st
->active
= array
.active_disks
;
622 st
->working
= array
.working_disks
;
623 st
->spare
= array
.spare_disks
;
624 st
->failed
= array
.failed_disks
;
625 st
->utime
= array
.utime
;
626 st
->raid
= array
.raid_disks
;
628 if ((st
->active
< st
->raid
) && st
->spare
== 0)
633 static int add_new_arrays(struct mdstat_ent
*mdstat
, struct state
*statelist
,
634 int test
, char *mailaddr
, char *mailfrom
,
635 char *alert_cmd
, int dosyslog
)
637 struct mdstat_ent
*mse
;
640 for (mse
=mdstat
; mse
; mse
=mse
->next
)
641 if (mse
->devnum
!= INT_MAX
&&
642 (!mse
->level
|| /* retrieve containers */
643 (strcmp(mse
->level
, "raid0") != 0 &&
644 strcmp(mse
->level
, "linear") != 0))
646 struct state
*st
= malloc(sizeof *st
);
647 mdu_array_info_t array
;
651 st
->devname
= strdup(get_md_name(mse
->devnum
));
652 if ((fd
= open(st
->devname
, O_RDONLY
)) < 0 ||
653 ioctl(fd
, GET_ARRAY_INFO
, &array
)< 0) {
655 if (fd
>=0) close(fd
);
656 put_md_name(st
->devname
);
659 st
->metadata
->ss
->free_super(st
->metadata
);
667 st
->next
= statelist
;
669 st
->devnum
= mse
->devnum
;
671 st
->spare_group
= NULL
;
672 st
->expected_spares
= -1;
673 if (strncmp(mse
->metadata_version
, "external:", 9) == 0 &&
674 is_subarray(mse
->metadata_version
+9))
676 devname2devnum(mse
->metadata_version
+10);
678 st
->parent_dev
= NoMdDev
;
682 alert("TestMessage", st
->devname
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
683 alert("NewArray", st
->devname
, NULL
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
689 static void try_spare_migration(struct state
*statelist
,
690 char *mailaddr
, char *mailfrom
,
691 char *alert_cmd
, int dosyslog
)
695 link_containers_with_subarrays(statelist
);
696 for (st
= statelist
; st
; st
=st
->next
)
697 if (st
->active
< st
->raid
&&
699 st
->spare_group
!= NULL
) {
701 for (st2
=statelist
; st2
; st2
=st2
->next
)
704 st2
->active
== st2
->raid
&&
705 st2
->spare_group
!= NULL
&&
706 strcmp(st
->spare_group
, st2
->spare_group
) == 0) {
707 /* try to remove and add */
708 int fd1
= open(st
->devname
, O_RDONLY
);
709 int fd2
= open(st2
->devname
, O_RDONLY
);
712 if (fd1
< 0 || fd2
< 0) {
713 if (fd1
>=0) close(fd1
);
714 if (fd2
>=0) close(fd2
);
717 for (d
=st2
->raid
; d
< MaxDisks
; d
++) {
718 if (st2
->devid
[d
] > 0 &&
719 st2
->devstate
[d
] == 0) {
725 struct mddev_dev devlist
;
730 devlist
.writemostly
= 0;
731 devlist
.devname
= devname
;
732 sprintf(devname
, "%d:%d", major(dev
), minor(dev
));
734 devlist
.disposition
= 'r';
735 if (Manage_subdevs(st2
->devname
, fd2
, &devlist
, -1, 0) == 0) {
736 devlist
.disposition
= 'a';
737 if (Manage_subdevs(st
->devname
, fd1
, &devlist
, -1, 0) == 0) {
738 alert("MoveSpare", st
->devname
, st2
->devname
, mailaddr
, mailfrom
, alert_cmd
, dosyslog
);
743 else Manage_subdevs(st2
->devname
, fd2
, &devlist
, -1, 0);
752 /* search the statelist to connect external
753 * metadata subarrays with their containers
754 * We always completely rebuild the tree from scratch as
755 * that is safest considering the possibility of entries
756 * disappearing or changing.
758 static void link_containers_with_subarrays(struct state
*list
)
762 for (st
= list
; st
; st
= st
->next
) {
766 for (st
= list
; st
; st
= st
->next
)
767 if (st
->parent_dev
!= NoMdDev
)
768 for (cont
= list
; cont
; cont
= cont
->next
)
770 cont
->parent_dev
== NoMdDev
&&
771 cont
->devnum
== st
->parent_dev
) {
773 st
->subarray
= cont
->subarray
;
779 /* Not really Monitor but ... */
786 if (stat(dev
, &stb
) != 0) {
787 fprintf(stderr
, Name
": Cannot find %s: %s\n", dev
,
791 devnum
= stat2devnum(&stb
);
794 struct mdstat_ent
*ms
= mdstat_read(1, 0);
795 struct mdstat_ent
*e
;
797 for (e
=ms
; e
; e
=e
->next
)
798 if (e
->devnum
== devnum
)
801 if (!e
|| e
->percent
< 0) {
802 if (e
&& e
->metadata_version
&&
803 strncmp(e
->metadata_version
, "external:", 9) == 0) {
804 if (is_subarray(&e
->metadata_version
[9]))
805 ping_monitor(&e
->metadata_version
[9]);
807 ping_monitor(devnum2devname(devnum
));