]>
Commit | Line | Data |
---|---|---|
52826846 | 1 | /* |
9a9dab36 | 2 | * mdadm - manage Linux "md" devices aka RAID arrays. |
52826846 | 3 | * |
cd29a5c8 | 4 | * Copyright (C) 2001-2002 Neil Brown <neilb@cse.unsw.edu.au> |
52826846 NB |
5 | * |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License as published by | |
9 | * the Free Software Foundation; either version 2 of the License, or | |
10 | * (at your option) any later version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program; if not, write to the Free Software | |
19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | * | |
21 | * Author: Neil Brown | |
22 | * Email: <neilb@cse.unsw.edu.au> | |
23 | * Paper: Neil Brown | |
24 | * School of Computer Science and Engineering | |
25 | * The University of New South Wales | |
26 | * Sydney, 2052 | |
27 | * Australia | |
28 | */ | |
29 | ||
9a9dab36 | 30 | #include "mdadm.h" |
52826846 NB |
31 | #include "md_p.h" |
32 | #include "md_u.h" | |
e0d19036 | 33 | #include <sys/wait.h> |
52826846 NB |
34 | #include <sys/signal.h> |
35 | ||
36 | static void alert(char *event, char *dev, char *disc, char *mailaddr, char *cmd); | |
37 | ||
e0d19036 NB |
38 | static char *percentalerts[] = { |
39 | "RebuildStarted", | |
40 | "Rebuild20", | |
41 | "Rebuild40", | |
42 | "Rebuild60", | |
43 | "Rebuild80", | |
44 | }; | |
45 | ||
cd29a5c8 | 46 | int Monitor(mddev_dev_t devlist, |
52826846 | 47 | char *mailaddr, char *alert_cmd, |
e0d19036 | 48 | int period, int scan, |
52826846 NB |
49 | char *config) |
50 | { | |
51 | /* | |
52 | * Every few seconds, scan every md device looking for changes | |
53 | * When a change is found, log it, possibly run the alert command, | |
54 | * and possibly send Email | |
55 | * | |
56 | * For each array, we record: | |
57 | * Update time | |
58 | * active/working/failed/spare drives | |
59 | * State of each device. | |
e0d19036 | 60 | * %rebuilt if rebuilding |
52826846 NB |
61 | * |
62 | * If the update time changes, check out all the data again | |
63 | * It is possible that we cannot get the state of each device | |
64 | * due to bugs in the md kernel module. | |
e0d19036 NB |
65 | * We also read /proc/mdstat to get rebuild percent, |
66 | * and to get state on all active devices incase of kernel bug. | |
52826846 | 67 | * |
e0d19036 NB |
68 | * Events are: |
69 | * Fail | |
70 | * An active device had Faulty set or Active/Sync removed | |
71 | * FailSpare | |
72 | * A spare device had Faulty set | |
73 | * SpareActive | |
74 | * An active device had a reverse transition | |
75 | * RebuildStarted | |
76 | * percent went from -1 to +ve | |
77 | * Rebuild20 Rebuild40 Rebuild60 Rebuild80 | |
78 | * percent went from below to not-below that number | |
79 | * DeviceDisappeared | |
80 | * Couldn't access a device which was previously visible | |
52826846 NB |
81 | * |
82 | * if we detect an array with active<raid and spare==0 | |
83 | * we look at other arrays that have same spare-group | |
84 | * If we find one with active==raid and spare>0, | |
85 | * and if we can get_disk_info and find a name | |
86 | * Then we hot-remove and hot-add to the other array | |
87 | * | |
e0d19036 NB |
88 | * If devlist is NULL, then we can monitor everything because --scan |
89 | * was given. We get an initial list from config file and add anything | |
90 | * that appears in /proc/mdstat | |
52826846 NB |
91 | */ |
92 | ||
93 | struct state { | |
94 | char *devname; | |
e0d19036 | 95 | int devnum; /* to sync with mdstat info */ |
52826846 NB |
96 | long utime; |
97 | int err; | |
e0d19036 NB |
98 | char *spare_group; |
99 | int active, working, failed, spare, raid; | |
52826846 | 100 | int devstate[MD_SB_DISKS]; |
e0d19036 NB |
101 | int devid[MD_SB_DISKS]; |
102 | int percent; | |
52826846 NB |
103 | struct state *next; |
104 | } *statelist = NULL; | |
105 | int finished = 0; | |
e0d19036 NB |
106 | struct mdstat_ent *mdstat = NULL; |
107 | ||
108 | if (!mailaddr && scan) | |
109 | mailaddr = conf_get_mailaddr(config); | |
110 | if (!alert_cmd && scan) | |
111 | alert_cmd = conf_get_program(config); | |
112 | if (scan && !mailaddr && !alert_cmd) | |
113 | return 0; | |
114 | ||
115 | if (devlist == NULL) { | |
116 | mddev_ident_t mdlist = conf_get_ident(config, NULL); | |
117 | for (; mdlist; mdlist=mdlist->next) { | |
118 | struct state *st = malloc(sizeof *st); | |
119 | if (st == NULL) | |
120 | continue; | |
121 | st->devname = strdup(mdlist->devname); | |
122 | st->utime = 0; | |
123 | st->next = statelist; | |
124 | st->err = 1; | |
125 | st->devnum = -1; | |
126 | st->percent = -2; | |
127 | if (mdlist->spare_group) | |
128 | st->spare_group = strdup(mdlist->spare_group); | |
129 | else | |
130 | st->spare_group = NULL; | |
131 | statelist = st; | |
132 | } | |
133 | } else { | |
cd29a5c8 | 134 | mddev_dev_t dv; |
e0d19036 NB |
135 | for (dv=devlist ; dv; dv=dv->next) { |
136 | struct state *st = malloc(sizeof *st); | |
137 | if (st == NULL) | |
138 | continue; | |
139 | st->devname = strdup(dv->devname); | |
140 | st->utime = 0; | |
141 | st->next = statelist; | |
142 | st->err = 1; | |
143 | st->devnum = -1; | |
144 | st->percent = -2; | |
145 | st->spare_group = NULL; | |
146 | statelist = st; | |
147 | } | |
148 | } | |
149 | ||
150 | ||
151 | while (! finished) { | |
152 | struct state *st; | |
153 | ||
154 | if (mdstat) | |
155 | free_mdstat(mdstat); | |
156 | mdstat = mdstat_read(); | |
157 | ||
158 | for (st=statelist; st; st=st->next) { | |
52826846 | 159 | mdu_array_info_t array; |
e0d19036 NB |
160 | struct mdstat_ent *mse; |
161 | char *dev = st->devname; | |
52826846 | 162 | int fd; |
52826846 | 163 | int i; |
e0d19036 | 164 | |
52826846 NB |
165 | fd = open(dev, O_RDONLY); |
166 | if (fd < 0) { | |
167 | if (!st->err) | |
e0d19036 NB |
168 | alert("DeviceDisappeared", dev, NULL, |
169 | mailaddr, alert_cmd); | |
170 | /* fprintf(stderr, Name ": cannot open %s: %s\n", | |
52826846 | 171 | dev, strerror(errno)); |
e0d19036 | 172 | */ st->err=1; |
52826846 NB |
173 | continue; |
174 | } | |
175 | if (ioctl(fd, GET_ARRAY_INFO, &array)<0) { | |
176 | if (!st->err) | |
e0d19036 NB |
177 | alert("DeviceDisappeared", dev, NULL, |
178 | mailaddr, alert_cmd); | |
179 | /* fprintf(stderr, Name ": cannot get array info for %s: %s\n", | |
52826846 | 180 | dev, strerror(errno)); |
e0d19036 | 181 | */ st->err=1; |
52826846 NB |
182 | close(fd); |
183 | continue; | |
184 | } | |
e0d19036 NB |
185 | if (st->devnum < 0) { |
186 | struct stat stb; | |
187 | if (fstat(fd, &stb) == 0 && | |
188 | (S_IFMT&stb.st_mode)==S_IFBLK) | |
189 | st->devnum = MINOR(stb.st_rdev); | |
190 | } | |
191 | ||
192 | for (mse = mdstat ; mse ; mse=mse->next) | |
193 | if (mse->devnum == st->devnum) { | |
194 | mse->devnum = -1; /* flag it as "used" */ | |
195 | break; | |
196 | } | |
197 | ||
52826846 | 198 | if (st->utime == array.utime && |
e0d19036 NB |
199 | st->failed == array.failed_disks && |
200 | st->working == array.working_disks && | |
201 | st->spare == array.spare_disks && | |
202 | (mse == NULL || ( | |
203 | mse->percent == st->percent | |
204 | ))) { | |
52826846 | 205 | close(fd); |
e0d19036 | 206 | st->err = 0; |
52826846 NB |
207 | continue; |
208 | } | |
e0d19036 NB |
209 | if (mse && |
210 | st->percent == -1 && | |
211 | mse->percent >= 0) | |
212 | alert("RebuildStarted", dev, NULL, mailaddr, alert_cmd); | |
213 | if (mse && | |
214 | st->percent >= 0 && | |
215 | mse->percent >= 0 && | |
216 | (mse->percent / 20) > (st->percent / 20)) | |
217 | alert(percentalerts[mse->percent/20], | |
218 | dev, NULL, mailaddr, alert_cmd); | |
219 | ||
220 | if (mse) | |
221 | st->percent = mse->percent; | |
222 | ||
223 | for (i=0; i<MD_SB_DISKS; i++) { | |
52826846 | 224 | mdu_disk_info_t disc; |
e0d19036 NB |
225 | int newstate=0; |
226 | int change; | |
227 | char *dv = NULL; | |
52826846 NB |
228 | disc.number = i; |
229 | if (ioctl(fd, GET_DISK_INFO, &disc)>= 0) { | |
e0d19036 NB |
230 | newstate = disc.state; |
231 | dv = map_dev(disc.major, disc.minor); | |
232 | } else if (mse && i < strlen(mse->pattern)) | |
233 | switch(mse->pattern[i]) { | |
234 | case 'U': newstate = 6 /* ACTIVE/SYNC */; break; | |
235 | case '_': newstate = 0; break; | |
52826846 | 236 | } |
e0d19036 NB |
237 | change = newstate ^ st->devstate[i]; |
238 | if (st->utime && change && !st->err) { | |
239 | if (i < array.raid_disks && | |
240 | (((newstate&change)&(1<<MD_DISK_FAULTY)) || | |
241 | ((st->devstate[i]&change)&(1<<MD_DISK_ACTIVE)) || | |
242 | ((st->devstate[i]&change)&(1<<MD_DISK_SYNC))) | |
243 | ) | |
244 | alert("Fail", dev, dv, mailaddr, alert_cmd); | |
245 | else if (i>=array.raid_disks && | |
246 | (disc.major || disc.minor) && | |
247 | st->devid[i] == MKDEV(disc.major, disc.minor) && | |
248 | ((newstate&change)&(1<<MD_DISK_FAULTY)) | |
249 | ) | |
250 | alert("FailSpare", dev, dv, mailaddr, alert_cmd); | |
251 | else if (i < array.raid_disks && | |
252 | (((st->devstate[i]&change)&(1<<MD_DISK_FAULTY)) || | |
253 | ((newstate&change)&(1<<MD_DISK_ACTIVE)) || | |
254 | ((newstate&change)&(1<<MD_DISK_SYNC))) | |
255 | ) | |
256 | alert("SpareActive", dev, dv, mailaddr, alert_cmd); | |
52826846 | 257 | } |
e0d19036 NB |
258 | st->devstate[i] = disc.state; |
259 | st->devid[i] = MKDEV(disc.major, disc.minor); | |
52826846 NB |
260 | } |
261 | close(fd); | |
262 | st->active = array.active_disks; | |
263 | st->working = array.working_disks; | |
264 | st->spare = array.spare_disks; | |
265 | st->failed = array.failed_disks; | |
266 | st->utime = array.utime; | |
e0d19036 NB |
267 | st->raid = array.raid_disks; |
268 | st->err = 0; | |
52826846 | 269 | } |
e0d19036 NB |
270 | /* now check if there are any new devices found in mdstat */ |
271 | if (scan) { | |
272 | struct mdstat_ent *mse; | |
273 | for (mse=mdstat; mse; mse=mse->next) | |
274 | if (mse->devnum > 0) { | |
275 | struct state *st = malloc(sizeof *st); | |
276 | if (st == NULL) | |
277 | continue; | |
278 | st->devname = strdup(get_md_name(mse->devnum)); | |
279 | st->utime = 0; | |
280 | st->next = statelist; | |
281 | st->err = 1; | |
282 | st->devnum = mse->devnum; | |
283 | st->percent = -2; | |
284 | st->spare_group = NULL; | |
285 | statelist = st; | |
286 | alert("NewArray", st->devname, NULL, mailaddr, alert_cmd); | |
287 | } | |
288 | } | |
289 | /* If an array has active < raid && spare == 0 && spare_group != NULL | |
290 | * Look for another array with spare > 0 and active == raid and same spare_group | |
291 | * if found, choose a device and hotremove/hotadd | |
292 | */ | |
293 | for (st = statelist; st; st=st->next) | |
294 | if (st->active < st->raid && | |
295 | st->spare == 0 && | |
296 | st->spare_group != NULL) { | |
297 | struct state *st2; | |
298 | for (st2=statelist ; st2 ; st2=st2->next) | |
299 | if (st2 != st && | |
300 | st2->spare > 0 && | |
301 | st2->active == st2->raid && | |
302 | st2->spare_group != NULL && | |
303 | strcmp(st->spare_group, st2->spare_group) == 0) { | |
304 | /* try to remove and add */ | |
305 | int fd1 = open(st->devname, O_RDONLY); | |
306 | int fd2 = open(st2->devname, O_RDONLY); | |
307 | int dev = -1; | |
308 | int d; | |
309 | if (fd1 < 0 || fd2 < 0) { | |
310 | if (fd1>=0) close(fd1); | |
311 | if (fd2>=0) close(fd2); | |
312 | continue; | |
313 | } | |
314 | for (d=st2->raid; d<MD_SB_DISKS; d++) { | |
315 | if (st2->devid[d] > 0 && | |
316 | st2->devstate[d] == 0) { | |
317 | dev = st2->devid[d]; | |
318 | break; | |
319 | } | |
320 | } | |
321 | if (dev > 0) { | |
322 | if (ioctl(fd2, HOT_REMOVE_DISK, | |
323 | (unsigned long)dev) == 0) { | |
324 | if (ioctl(fd1, HOT_ADD_DISK, | |
325 | (unsigned long)dev) == 0) { | |
326 | alert("MoveSpare", st->devname, st2->devname, mailaddr, alert_cmd); | |
327 | close(fd1); | |
328 | close(fd2); | |
329 | break; | |
330 | } | |
331 | else ioctl(fd2, HOT_ADD_DISK, (unsigned long) dev); | |
332 | } | |
333 | } | |
334 | close(fd1); | |
335 | close(fd2); | |
336 | } | |
337 | } | |
338 | ||
52826846 NB |
339 | sleep(period); |
340 | } | |
341 | return 0; | |
342 | } | |
343 | ||
344 | ||
345 | static void alert(char *event, char *dev, char *disc, char *mailaddr, char *cmd) | |
346 | { | |
cd29a5c8 NB |
347 | if (!cmd && !mailaddr) { |
348 | time_t now = time(0); | |
349 | ||
e0d19036 | 350 | printf("%1.15s: %s on %s %s\n", ctime(&now)+4, event, dev, disc?disc:"unknown device"); |
cd29a5c8 | 351 | } |
52826846 NB |
352 | if (cmd) { |
353 | int pid = fork(); | |
354 | switch(pid) { | |
355 | default: | |
356 | waitpid(pid, NULL, 0); | |
357 | break; | |
358 | case -1: | |
359 | break; | |
360 | case 0: | |
361 | execl(cmd, cmd, event, dev, disc, NULL); | |
362 | exit(2); | |
363 | } | |
364 | } | |
365 | if (mailaddr && strncmp(event, "Fail", 4)==0) { | |
366 | FILE *mp = popen(Sendmail, "w"); | |
367 | if (mp) { | |
368 | char hname[256]; | |
369 | gethostname(hname, sizeof(hname)); | |
370 | signal(SIGPIPE, SIG_IGN); | |
371 | fprintf(mp, "From: " Name " monitoring <root>\n"); | |
372 | fprintf(mp, "To: %s\n", mailaddr); | |
373 | fprintf(mp, "Subject: %s event on %s:%s\n\n", event, dev, hname); | |
374 | ||
375 | fprintf(mp, "This is an automatically generated mail message from " Name "\n"); | |
376 | fprintf(mp, "running on %s\n\n", hname); | |
377 | ||
378 | fprintf(mp, "A %s event had been detected on md device %s.\n\n", event, dev); | |
379 | ||
380 | if (disc) | |
b83d95f3 | 381 | fprintf(mp, "It could be related to componenet device %s.\n\n", disc); |
52826846 NB |
382 | |
383 | fprintf(mp, "Faithfully yours, etc.\n"); | |
384 | fclose(mp); | |
385 | } | |
386 | ||
387 | } | |
388 | /* FIXME log the event to syslog maybe */ | |
389 | } |