]>
Commit | Line | Data |
---|---|---|
1 | /* | |
2 | * mdadm - manage Linux "md" devices aka RAID arrays. | |
3 | * | |
4 | * Copyright (C) 2001-2006 Neil Brown <neilb@suse.de> | |
5 | * | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License as published by | |
9 | * the Free Software Foundation; either version 2 of the License, or | |
10 | * (at your option) any later version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program; if not, write to the Free Software | |
19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | * | |
21 | * Author: Neil Brown | |
22 | * Email: <neilb@cse.unsw.edu.au> | |
23 | * Paper: Neil Brown | |
24 | * School of Computer Science and Engineering | |
25 | * The University of New South Wales | |
26 | * Sydney, 2052 | |
27 | * Australia | |
28 | */ | |
29 | ||
30 | #include "mdadm.h" | |
31 | #include "md_p.h" | |
32 | #include "md_u.h" | |
33 | #include <sys/wait.h> | |
34 | #include <sys/signal.h> | |
35 | #include <values.h> | |
36 | #include <syslog.h> | |
37 | ||
38 | static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom, | |
39 | char *cmd, int dosyslog); | |
40 | ||
41 | static char *percentalerts[] = { | |
42 | "RebuildStarted", | |
43 | "Rebuild20", | |
44 | "Rebuild40", | |
45 | "Rebuild60", | |
46 | "Rebuild80", | |
47 | }; | |
48 | ||
49 | /* The largest number of disks current arrays can manage is 384 | |
50 | * This really should be dynamically, but that will have to wait | |
51 | * At least it isn't MD_SB_DISKS. | |
52 | */ | |
53 | #define MaxDisks 384 | |
54 | int Monitor(mddev_dev_t devlist, | |
55 | char *mailaddr, char *alert_cmd, | |
56 | int period, int daemonise, int scan, int oneshot, | |
57 | int dosyslog, int test, char* pidfile) | |
58 | { | |
59 | /* | |
60 | * Every few seconds, scan every md device looking for changes | |
61 | * When a change is found, log it, possibly run the alert command, | |
62 | * and possibly send Email | |
63 | * | |
64 | * For each array, we record: | |
65 | * Update time | |
66 | * active/working/failed/spare drives | |
67 | * State of each device. | |
68 | * %rebuilt if rebuilding | |
69 | * | |
70 | * If the update time changes, check out all the data again | |
71 | * It is possible that we cannot get the state of each device | |
72 | * due to bugs in the md kernel module. | |
73 | * We also read /proc/mdstat to get rebuild percent, | |
74 | * and to get state on all active devices incase of kernel bug. | |
75 | * | |
76 | * Events are: | |
77 | * Fail | |
78 | * An active device had Faulty set or Active/Sync removed | |
79 | * FailSpare | |
80 | * A spare device had Faulty set | |
81 | * SpareActive | |
82 | * An active device had a reverse transition | |
83 | * RebuildStarted | |
84 | * percent went from -1 to +ve | |
85 | * Rebuild20 Rebuild40 Rebuild60 Rebuild80 | |
86 | * percent went from below to not-below that number | |
87 | * DeviceDisappeared | |
88 | * Couldn't access a device which was previously visible | |
89 | * | |
90 | * if we detect an array with active<raid and spare==0 | |
91 | * we look at other arrays that have same spare-group | |
92 | * If we find one with active==raid and spare>0, | |
93 | * and if we can get_disk_info and find a name | |
94 | * Then we hot-remove and hot-add to the other array | |
95 | * | |
96 | * If devlist is NULL, then we can monitor everything because --scan | |
97 | * was given. We get an initial list from config file and add anything | |
98 | * that appears in /proc/mdstat | |
99 | */ | |
100 | ||
101 | struct state { | |
102 | char *devname; | |
103 | int devnum; /* to sync with mdstat info */ | |
104 | long utime; | |
105 | int err; | |
106 | char *spare_group; | |
107 | int active, working, failed, spare, raid; | |
108 | int expected_spares; | |
109 | int devstate[MaxDisks]; | |
110 | int devid[MaxDisks]; | |
111 | int percent; | |
112 | struct state *next; | |
113 | } *statelist = NULL; | |
114 | int finished = 0; | |
115 | struct mdstat_ent *mdstat = NULL; | |
116 | char *mailfrom = NULL; | |
117 | ||
118 | if (!mailaddr) { | |
119 | mailaddr = conf_get_mailaddr(); | |
120 | if (mailaddr && ! scan) | |
121 | fprintf(stderr, Name ": Monitor using email address \"%s\" from config file\n", | |
122 | mailaddr); | |
123 | } | |
124 | mailfrom = conf_get_mailfrom(); | |
125 | ||
126 | if (!alert_cmd) { | |
127 | alert_cmd = conf_get_program(); | |
128 | if (alert_cmd && ! scan) | |
129 | fprintf(stderr, Name ": Monitor using program \"%s\" from config file\n", | |
130 | alert_cmd); | |
131 | } | |
132 | if (scan && !mailaddr && !alert_cmd) { | |
133 | fprintf(stderr, Name ": No mail address or alert command - not monitoring.\n"); | |
134 | return 1; | |
135 | } | |
136 | ||
137 | if (daemonise) { | |
138 | int pid = fork(); | |
139 | if (pid > 0) { | |
140 | if (!pidfile) | |
141 | printf("%d\n", pid); | |
142 | else { | |
143 | FILE *pid_file; | |
144 | pid_file=fopen(pidfile, "w"); | |
145 | if (!pid_file) | |
146 | perror("cannot create pid file"); | |
147 | else { | |
148 | fprintf(pid_file,"%d\n", pid); | |
149 | fclose(pid_file); | |
150 | } | |
151 | } | |
152 | return 0; | |
153 | } | |
154 | if (pid < 0) { | |
155 | perror("daemonise"); | |
156 | return 1; | |
157 | } | |
158 | close(0); | |
159 | open("/dev/null", 3); | |
160 | dup2(0,1); | |
161 | dup2(0,2); | |
162 | setsid(); | |
163 | } | |
164 | ||
165 | if (devlist == NULL) { | |
166 | mddev_ident_t mdlist = conf_get_ident(NULL); | |
167 | for (; mdlist; mdlist=mdlist->next) { | |
168 | struct state *st = malloc(sizeof *st); | |
169 | if (st == NULL) | |
170 | continue; | |
171 | st->devname = strdup(mdlist->devname); | |
172 | st->utime = 0; | |
173 | st->next = statelist; | |
174 | st->err = 0; | |
175 | st->devnum = MAXINT; | |
176 | st->percent = -2; | |
177 | st->expected_spares = mdlist->spare_disks; | |
178 | if (mdlist->spare_group) | |
179 | st->spare_group = strdup(mdlist->spare_group); | |
180 | else | |
181 | st->spare_group = NULL; | |
182 | statelist = st; | |
183 | } | |
184 | } else { | |
185 | mddev_dev_t dv; | |
186 | for (dv=devlist ; dv; dv=dv->next) { | |
187 | mddev_ident_t mdlist = conf_get_ident(dv->devname); | |
188 | struct state *st = malloc(sizeof *st); | |
189 | if (st == NULL) | |
190 | continue; | |
191 | st->devname = strdup(dv->devname); | |
192 | st->utime = 0; | |
193 | st->next = statelist; | |
194 | st->err = 0; | |
195 | st->devnum = MAXINT; | |
196 | st->percent = -2; | |
197 | st->expected_spares = -1; | |
198 | st->spare_group = NULL; | |
199 | if (mdlist) { | |
200 | st->expected_spares = mdlist->spare_disks; | |
201 | if (mdlist->spare_group) | |
202 | st->spare_group = strdup(mdlist->spare_group); | |
203 | } | |
204 | statelist = st; | |
205 | } | |
206 | } | |
207 | ||
208 | ||
209 | while (! finished) { | |
210 | int new_found = 0; | |
211 | struct state *st; | |
212 | ||
213 | if (mdstat) | |
214 | free_mdstat(mdstat); | |
215 | mdstat = mdstat_read(oneshot?0:1, 0); | |
216 | ||
217 | for (st=statelist; st; st=st->next) { | |
218 | mdu_array_info_t array; | |
219 | struct mdstat_ent *mse = NULL, *mse2; | |
220 | char *dev = st->devname; | |
221 | int fd; | |
222 | unsigned int i; | |
223 | ||
224 | if (test) | |
225 | alert("TestMessage", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
226 | fd = open(dev, O_RDONLY); | |
227 | if (fd < 0) { | |
228 | if (!st->err) | |
229 | alert("DeviceDisappeared", dev, NULL, | |
230 | mailaddr, mailfrom, alert_cmd, dosyslog); | |
231 | /* fprintf(stderr, Name ": cannot open %s: %s\n", | |
232 | dev, strerror(errno)); | |
233 | */ st->err=1; | |
234 | continue; | |
235 | } | |
236 | if (ioctl(fd, GET_ARRAY_INFO, &array)<0) { | |
237 | if (!st->err) | |
238 | alert("DeviceDisappeared", dev, NULL, | |
239 | mailaddr, mailfrom, alert_cmd, dosyslog); | |
240 | /* fprintf(stderr, Name ": cannot get array info for %s: %s\n", | |
241 | dev, strerror(errno)); | |
242 | */ st->err=1; | |
243 | close(fd); | |
244 | continue; | |
245 | } | |
246 | if (array.level != 1 && array.level != 5 && array.level != -4 && | |
247 | array.level != 6 && array.level != 10) { | |
248 | if (!st->err) | |
249 | alert("DeviceDisappeared", dev, "Wrong-Level", | |
250 | mailaddr, mailfrom, alert_cmd, dosyslog); | |
251 | st->err = 1; | |
252 | close(fd); | |
253 | continue; | |
254 | } | |
255 | if (st->devnum == MAXINT) { | |
256 | struct stat stb; | |
257 | if (fstat(fd, &stb) == 0 && | |
258 | (S_IFMT&stb.st_mode)==S_IFBLK) { | |
259 | if (major(stb.st_rdev) == MD_MAJOR) | |
260 | st->devnum = minor(stb.st_rdev); | |
261 | else | |
262 | st->devnum = -1- (minor(stb.st_rdev)>>6); | |
263 | } | |
264 | } | |
265 | ||
266 | for (mse2 = mdstat ; mse2 ; mse2=mse2->next) | |
267 | if (mse2->devnum == st->devnum) { | |
268 | mse2->devnum = MAXINT; /* flag it as "used" */ | |
269 | mse = mse2; | |
270 | } | |
271 | ||
272 | if (st->utime == array.utime && | |
273 | st->failed == array.failed_disks && | |
274 | st->working == array.working_disks && | |
275 | st->spare == array.spare_disks && | |
276 | (mse == NULL || ( | |
277 | mse->percent == st->percent | |
278 | ))) { | |
279 | close(fd); | |
280 | st->err = 0; | |
281 | continue; | |
282 | } | |
283 | if (st->utime == 0 && /* new array */ | |
284 | mse && /* is in /proc/mdstat */ | |
285 | mse->pattern && strchr(mse->pattern, '_') /* degraded */ | |
286 | ) | |
287 | alert("DegradedArray", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
288 | ||
289 | if (st->utime == 0 && /* new array */ | |
290 | st->expected_spares > 0 && | |
291 | array.spare_disks < st->expected_spares) | |
292 | alert("SparesMissing", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
293 | if (mse && | |
294 | st->percent == -1 && | |
295 | mse->percent >= 0) | |
296 | alert("RebuildStarted", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
297 | if (mse && | |
298 | st->percent >= 0 && | |
299 | mse->percent >= 0 && | |
300 | (mse->percent / 20) > (st->percent / 20)) | |
301 | alert(percentalerts[mse->percent/20], | |
302 | dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
303 | ||
304 | if (mse && | |
305 | mse->percent == -1 && | |
306 | st->percent >= 0) | |
307 | alert("RebuildFinished", dev, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
308 | ||
309 | if (mse) | |
310 | st->percent = mse->percent; | |
311 | ||
312 | for (i=0; i<MaxDisks; i++) { | |
313 | mdu_disk_info_t disc; | |
314 | int newstate=0; | |
315 | int change; | |
316 | char *dv = NULL; | |
317 | disc.number = i; | |
318 | if (i > array.raid_disks + array.nr_disks) { | |
319 | newstate = 0; | |
320 | disc.major = disc.minor = 0; | |
321 | } else if (ioctl(fd, GET_DISK_INFO, &disc)>= 0) { | |
322 | newstate = disc.state; | |
323 | dv = map_dev(disc.major, disc.minor, 1); | |
324 | } else if (mse && mse->pattern && i < strlen(mse->pattern)) { | |
325 | switch(mse->pattern[i]) { | |
326 | case 'U': newstate = 6 /* ACTIVE/SYNC */; break; | |
327 | case '_': newstate = 0; break; | |
328 | } | |
329 | disc.major = disc.minor = 0; | |
330 | } | |
331 | if (dv == NULL && st->devid[i]) | |
332 | dv = map_dev(major(st->devid[i]), | |
333 | minor(st->devid[i]), 1); | |
334 | change = newstate ^ st->devstate[i]; | |
335 | if (st->utime && change && !st->err) { | |
336 | if (i < (unsigned)array.raid_disks && | |
337 | (((newstate&change)&(1<<MD_DISK_FAULTY)) || | |
338 | ((st->devstate[i]&change)&(1<<MD_DISK_ACTIVE)) || | |
339 | ((st->devstate[i]&change)&(1<<MD_DISK_SYNC))) | |
340 | ) | |
341 | alert("Fail", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog); | |
342 | else if (i >= (unsigned)array.raid_disks && | |
343 | (disc.major || disc.minor) && | |
344 | st->devid[i] == makedev(disc.major, disc.minor) && | |
345 | ((newstate&change)&(1<<MD_DISK_FAULTY)) | |
346 | ) | |
347 | alert("FailSpare", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog); | |
348 | else if (i < (unsigned)array.raid_disks && | |
349 | (((st->devstate[i]&change)&(1<<MD_DISK_FAULTY)) || | |
350 | ((newstate&change)&(1<<MD_DISK_ACTIVE)) || | |
351 | ((newstate&change)&(1<<MD_DISK_SYNC))) | |
352 | ) | |
353 | alert("SpareActive", dev, dv, mailaddr, mailfrom, alert_cmd, dosyslog); | |
354 | } | |
355 | st->devstate[i] = disc.state; | |
356 | st->devid[i] = makedev(disc.major, disc.minor); | |
357 | } | |
358 | close(fd); | |
359 | st->active = array.active_disks; | |
360 | st->working = array.working_disks; | |
361 | st->spare = array.spare_disks; | |
362 | st->failed = array.failed_disks; | |
363 | st->utime = array.utime; | |
364 | st->raid = array.raid_disks; | |
365 | st->err = 0; | |
366 | } | |
367 | /* now check if there are any new devices found in mdstat */ | |
368 | if (scan) { | |
369 | struct mdstat_ent *mse; | |
370 | for (mse=mdstat; mse; mse=mse->next) | |
371 | if (mse->devnum != MAXINT && | |
372 | (strcmp(mse->level, "raid1")==0 || | |
373 | strcmp(mse->level, "raid5")==0 || | |
374 | strcmp(mse->level, "multipath")==0) | |
375 | ) { | |
376 | struct state *st = malloc(sizeof *st); | |
377 | mdu_array_info_t array; | |
378 | int fd; | |
379 | if (st == NULL) | |
380 | continue; | |
381 | st->devname = strdup(get_md_name(mse->devnum)); | |
382 | if ((fd = open(st->devname, O_RDONLY)) < 0 || | |
383 | ioctl(fd, GET_ARRAY_INFO, &array)< 0) { | |
384 | /* no such array */ | |
385 | if (fd >=0) close(fd); | |
386 | put_md_name(st->devname); | |
387 | free(st->devname); | |
388 | free(st); | |
389 | continue; | |
390 | } | |
391 | close(fd); | |
392 | st->utime = 0; | |
393 | st->next = statelist; | |
394 | st->err = 1; | |
395 | st->devnum = mse->devnum; | |
396 | st->percent = -2; | |
397 | st->spare_group = NULL; | |
398 | st->expected_spares = -1; | |
399 | statelist = st; | |
400 | alert("NewArray", st->devname, NULL, mailaddr, mailfrom, alert_cmd, dosyslog); | |
401 | new_found = 1; | |
402 | } | |
403 | } | |
404 | /* If an array has active < raid && spare == 0 && spare_group != NULL | |
405 | * Look for another array with spare > 0 and active == raid and same spare_group | |
406 | * if found, choose a device and hotremove/hotadd | |
407 | */ | |
408 | for (st = statelist; st; st=st->next) | |
409 | if (st->active < st->raid && | |
410 | st->spare == 0 && | |
411 | st->spare_group != NULL) { | |
412 | struct state *st2; | |
413 | for (st2=statelist ; st2 ; st2=st2->next) | |
414 | if (st2 != st && | |
415 | st2->spare > 0 && | |
416 | st2->active == st2->raid && | |
417 | st2->spare_group != NULL && | |
418 | strcmp(st->spare_group, st2->spare_group) == 0) { | |
419 | /* try to remove and add */ | |
420 | int fd1 = open(st->devname, O_RDONLY); | |
421 | int fd2 = open(st2->devname, O_RDONLY); | |
422 | int dev = -1; | |
423 | int d; | |
424 | if (fd1 < 0 || fd2 < 0) { | |
425 | if (fd1>=0) close(fd1); | |
426 | if (fd2>=0) close(fd2); | |
427 | continue; | |
428 | } | |
429 | for (d=st2->raid; d < MaxDisks; d++) { | |
430 | if (st2->devid[d] > 0 && | |
431 | st2->devstate[d] == 0) { | |
432 | dev = st2->devid[d]; | |
433 | break; | |
434 | } | |
435 | } | |
436 | if (dev > 0) { | |
437 | if (ioctl(fd2, HOT_REMOVE_DISK, | |
438 | (unsigned long)dev) == 0) { | |
439 | if (ioctl(fd1, HOT_ADD_DISK, | |
440 | (unsigned long)dev) == 0) { | |
441 | alert("MoveSpare", st->devname, st2->devname, mailaddr, mailfrom, alert_cmd, dosyslog); | |
442 | close(fd1); | |
443 | close(fd2); | |
444 | break; | |
445 | } | |
446 | else ioctl(fd2, HOT_ADD_DISK, (unsigned long) dev); | |
447 | } | |
448 | } | |
449 | close(fd1); | |
450 | close(fd2); | |
451 | } | |
452 | } | |
453 | if (!new_found) { | |
454 | if (oneshot) | |
455 | break; | |
456 | else | |
457 | mdstat_wait(period); | |
458 | } | |
459 | test = 0; | |
460 | } | |
461 | if (pidfile) | |
462 | unlink(pidfile); | |
463 | return 0; | |
464 | } | |
465 | ||
466 | ||
467 | static void alert(char *event, char *dev, char *disc, char *mailaddr, char *mailfrom, char *cmd, | |
468 | int dosyslog) | |
469 | { | |
470 | int priority; | |
471 | ||
472 | if (!cmd && !mailaddr) { | |
473 | time_t now = time(0); | |
474 | ||
475 | printf("%1.15s: %s on %s %s\n", ctime(&now)+4, event, dev, disc?disc:"unknown device"); | |
476 | } | |
477 | if (cmd) { | |
478 | int pid = fork(); | |
479 | switch(pid) { | |
480 | default: | |
481 | waitpid(pid, NULL, 0); | |
482 | break; | |
483 | case -1: | |
484 | break; | |
485 | case 0: | |
486 | execl(cmd, cmd, event, dev, disc, NULL); | |
487 | exit(2); | |
488 | } | |
489 | } | |
490 | if (mailaddr && | |
491 | (strncmp(event, "Fail", 4)==0 || | |
492 | strncmp(event, "Test", 4)==0 || | |
493 | strncmp(event, "Spares", 6)==0 || | |
494 | strncmp(event, "Degrade", 7)==0)) { | |
495 | FILE *mp = popen(Sendmail, "w"); | |
496 | if (mp) { | |
497 | FILE *mdstat; | |
498 | char hname[256]; | |
499 | gethostname(hname, sizeof(hname)); | |
500 | signal(SIGPIPE, SIG_IGN); | |
501 | if (mailfrom) | |
502 | fprintf(mp, "From: %s\n", mailfrom); | |
503 | else | |
504 | fprintf(mp, "From: " Name " monitoring <root>\n"); | |
505 | fprintf(mp, "To: %s\n", mailaddr); | |
506 | fprintf(mp, "Subject: %s event on %s:%s\n\n", event, dev, hname); | |
507 | ||
508 | fprintf(mp, "This is an automatically generated mail message from " Name "\n"); | |
509 | fprintf(mp, "running on %s\n\n", hname); | |
510 | ||
511 | fprintf(mp, "A %s event had been detected on md device %s.\n\n", event, dev); | |
512 | ||
513 | if (disc) | |
514 | fprintf(mp, "It could be related to component device %s.\n\n", disc); | |
515 | ||
516 | fprintf(mp, "Faithfully yours, etc.\n"); | |
517 | ||
518 | mdstat = fopen("/proc/mdstat", "r"); | |
519 | if (mdstat) { | |
520 | char buf[8192]; | |
521 | int n; | |
522 | fprintf(mp, "\nP.S. The /proc/mdstat file currently contains the following:\n\n"); | |
523 | while ( (n=fread(buf, 1, sizeof(buf), mdstat)) > 0) | |
524 | n=fwrite(buf, 1, n, mp); /* yes, i don't care about the result */ | |
525 | fclose(mdstat); | |
526 | } | |
527 | fclose(mp); | |
528 | } | |
529 | ||
530 | } | |
531 | ||
532 | /* log the event to syslog maybe */ | |
533 | if (dosyslog) { | |
534 | /* Log at a different severity depending on the event. | |
535 | * | |
536 | * These are the critical events: */ | |
537 | if (strncmp(event, "Fail", 4)==0 || | |
538 | strncmp(event, "Degrade", 7)==0 || | |
539 | strncmp(event, "DeviceDisappeared", 17)==0) | |
540 | priority = LOG_CRIT; | |
541 | /* Good to know about, but are not failures: */ | |
542 | else if (strncmp(event, "Rebuild", 7)==0 || | |
543 | strncmp(event, "MoveSpare", 9)==0 || | |
544 | strncmp(event, "Spares", 6) != 0) | |
545 | priority = LOG_WARNING; | |
546 | /* Everything else: */ | |
547 | else | |
548 | priority = LOG_INFO; | |
549 | ||
550 | if (disc) | |
551 | syslog(priority, "%s event detected on md device %s, component device %s", event, dev, disc); | |
552 | else | |
553 | syslog(priority, "%s event detected on md device %s", event, dev); | |
554 | } | |
555 | } |