]> git.ipfire.org Git - thirdparty/mdadm.git/blob - managemon.c
Add scaffolding for handling md/dev-XXX/recovery_start
[thirdparty/mdadm.git] / managemon.c
1 /*
2 * mdmon - monitor external metadata arrays
3 *
4 * Copyright (C) 2007-2009 Neil Brown <neilb@suse.de>
5 * Copyright (C) 2007-2009 Intel Corporation
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
19 */
20
21 /*
22 * The management thread for monitoring active md arrays.
23 * This thread does things which might block such as memory
24 * allocation.
25 * In particular:
26 *
27 * - Find out about new arrays in this container.
28 * Allocate the data structures and open the files.
29 *
30 * For this we watch /proc/mdstat and find new arrays with
31 * metadata type that confirms sharing. e.g. "md4"
32 * When we find a new array we slip it into the list of
33 * arrays and signal 'monitor' by writing to a pipe.
34 *
35 * - Respond to reshape requests by allocating new data structures
36 * and opening new files.
37 *
38 * These come as a change to raid_disks. We allocate a new
39 * version of the data structures and slip it into the list.
40 * 'monitor' will notice and release the old version.
41 * Changes to level, chunksize, layout.. do not need re-allocation.
42 * Reductions in raid_disks don't really either, but we handle
43 * them the same way for consistency.
44 *
45 * - When a device is added to the container, we add it to the metadata
46 * as a spare.
47 *
48 * - Deal with degraded array
49 * We only do this when first noticing the array is degraded.
50 * This can be when we first see the array, when sync completes or
51 * when recovery completes.
52 *
53 * Check if number of failed devices suggests recovery is needed, and
54 * skip if not.
55 * Ask metadata to allocate a spare device
56 * Add device as not in_sync and give a role
57 * Update metadata.
58 * Open sysfs files and pass to monitor.
59 * Make sure that monitor Starts recovery....
60 *
61 * - Pass on metadata updates from external programs such as
62 * mdadm creating a new array.
63 *
64 * This is most-messy.
65 * It might involve adding a new array or changing the status of
66 * a spare, or any reconfig that the kernel doesn't get involved in.
67 *
68 * The required updates are received via a named pipe. There will
69 * be one named pipe for each container. Each message contains a
70 * sync marker: 0x5a5aa5a5, A byte count, and the message. This is
71 * passed to the metadata handler which will interpret and process it.
72 * For 'DDF' messages are internal data blocks with the leading
73 * 'magic number' signifying what sort of data it is.
74 *
75 */
76
77 /*
78 * We select on /proc/mdstat and the named pipe.
79 * We create new arrays or updated version of arrays and slip
80 * them into the head of the list, then signal 'monitor' via a pipe write.
81 * 'monitor' will notice and place the old array on a return list.
82 * Metadata updates are placed on a queue just like they arrive
83 * from the named pipe.
84 *
85 * When new arrays are found based on correct metadata string, we
86 * need to identify them with an entry in the metadata. Maybe we require
87 * the metadata to be mdX/NN when NN is the index into an appropriate table.
88 *
89 */
90
91 /*
92 * List of tasks:
93 * - Watch for spares to be added to the container, and write updated
94 * metadata to them.
95 * - Watch for new arrays using this container, confirm they match metadata
96 * and if so, start monitoring them
97 * - Watch for spares being added to monitored arrays. This shouldn't
98 * happen, as we should do all the adding. Just remove them.
99 * - Watch for change in raid-disks, chunk-size, etc. Update metadata and
100 * start a reshape.
101 */
102 #ifndef _GNU_SOURCE
103 #define _GNU_SOURCE
104 #endif
105 #include "mdadm.h"
106 #include "mdmon.h"
107 #include <sys/syscall.h>
108 #include <sys/socket.h>
109 #include <signal.h>
110
111 static void close_aa(struct active_array *aa)
112 {
113 struct mdinfo *d;
114
115 for (d = aa->info.devs; d; d = d->next) {
116 close(d->recovery_fd);
117 close(d->state_fd);
118 }
119
120 close(aa->action_fd);
121 close(aa->info.state_fd);
122 close(aa->resync_start_fd);
123 }
124
125 static void free_aa(struct active_array *aa)
126 {
127 /* Note that this doesn't close fds if they are being used
128 * by a clone. ->container will be set for a clone
129 */
130 dprintf("%s: devnum: %d\n", __func__, aa->devnum);
131 if (!aa->container)
132 close_aa(aa);
133 while (aa->info.devs) {
134 struct mdinfo *d = aa->info.devs;
135 aa->info.devs = d->next;
136 free(d);
137 }
138 free(aa);
139 }
140
141 static struct active_array *duplicate_aa(struct active_array *aa)
142 {
143 struct active_array *newa = malloc(sizeof(*newa));
144 struct mdinfo **dp1, **dp2;
145
146 *newa = *aa;
147 newa->next = NULL;
148 newa->replaces = NULL;
149 newa->info.next = NULL;
150
151 dp2 = &newa->info.devs;
152
153 for (dp1 = &aa->info.devs; *dp1; dp1 = &(*dp1)->next) {
154 struct mdinfo *d;
155 if ((*dp1)->state_fd < 0)
156 continue;
157
158 d = malloc(sizeof(*d));
159 *d = **dp1;
160 *dp2 = d;
161 dp2 = & d->next;
162 }
163 *dp2 = NULL;
164
165 return newa;
166 }
167
168 static void wakeup_monitor(void)
169 {
170 /* tgkill(getpid(), mon_tid, SIGUSR1); */
171 int pid = getpid();
172 syscall(SYS_tgkill, pid, mon_tid, SIGUSR1);
173 }
174
175 static void remove_old(void)
176 {
177 if (discard_this) {
178 discard_this->next = NULL;
179 free_aa(discard_this);
180 if (pending_discard == discard_this)
181 pending_discard = NULL;
182 discard_this = NULL;
183 wakeup_monitor();
184 }
185 }
186
187 static void replace_array(struct supertype *container,
188 struct active_array *old,
189 struct active_array *new)
190 {
191 /* To replace an array, we add it to the top of the list
192 * marked with ->replaces to point to the original.
193 * 'monitor' will take the original out of the list
194 * and put it on 'discard_this'. We take it from there
195 * and discard it.
196 */
197 remove_old();
198 while (pending_discard) {
199 while (discard_this == NULL)
200 sleep(1);
201 remove_old();
202 }
203 pending_discard = old;
204 new->replaces = old;
205 new->next = container->arrays;
206 container->arrays = new;
207 wakeup_monitor();
208 }
209
210 struct metadata_update *update_queue = NULL;
211 struct metadata_update *update_queue_handled = NULL;
212 struct metadata_update *update_queue_pending = NULL;
213
214 static void free_updates(struct metadata_update **update)
215 {
216 while (*update) {
217 struct metadata_update *this = *update;
218
219 *update = this->next;
220 free(this->buf);
221 free(this->space);
222 free(this);
223 }
224 }
225
226 void check_update_queue(struct supertype *container)
227 {
228 free_updates(&update_queue_handled);
229
230 if (update_queue == NULL &&
231 update_queue_pending) {
232 update_queue = update_queue_pending;
233 update_queue_pending = NULL;
234 wakeup_monitor();
235 }
236 }
237
238 static void queue_metadata_update(struct metadata_update *mu)
239 {
240 struct metadata_update **qp;
241
242 qp = &update_queue_pending;
243 while (*qp)
244 qp = & ((*qp)->next);
245 *qp = mu;
246 }
247
248 static void add_disk_to_container(struct supertype *st, struct mdinfo *sd)
249 {
250 int dfd;
251 char nm[20];
252 struct supertype *st2;
253 struct metadata_update *update = NULL;
254 struct mdinfo info;
255 mdu_disk_info_t dk = {
256 .number = -1,
257 .major = sd->disk.major,
258 .minor = sd->disk.minor,
259 .raid_disk = -1,
260 .state = 0,
261 };
262
263 dprintf("%s: add %d:%d to container\n",
264 __func__, sd->disk.major, sd->disk.minor);
265
266 sd->next = st->devs;
267 st->devs = sd;
268
269 sprintf(nm, "%d:%d", sd->disk.major, sd->disk.minor);
270 dfd = dev_open(nm, O_RDWR);
271 if (dfd < 0)
272 return;
273
274 /* Check the metadata and see if it is already part of this
275 * array
276 */
277 st2 = dup_super(st);
278 if (st2->ss->load_super(st2, dfd, NULL) == 0) {
279 st2->ss->getinfo_super(st, &info);
280 if (st->ss->compare_super(st, st2) == 0 &&
281 info.disk.raid_disk >= 0) {
282 /* Looks like a good member of array.
283 * Just accept it.
284 * mdadm will incorporate any parts into
285 * active arrays.
286 */
287 st2->ss->free_super(st2);
288 return;
289 }
290 }
291 st2->ss->free_super(st2);
292
293 st->update_tail = &update;
294 st->ss->add_to_super(st, &dk, dfd, NULL);
295 st->ss->write_init_super(st);
296 queue_metadata_update(update);
297 st->update_tail = NULL;
298 }
299
300 static void manage_container(struct mdstat_ent *mdstat,
301 struct supertype *container)
302 {
303 /* The only thing of interest here is if a new device
304 * has been added to the container. We add it to the
305 * array ignoring any metadata on it.
306 * FIXME should we look for compatible metadata and take hints
307 * about spare assignment.... probably not.
308 */
309 if (mdstat->devcnt != container->devcnt) {
310 struct mdinfo **cdp, *cd, *di, *mdi;
311 int found;
312
313 /* read /sys/block/NAME/md/dev-??/block/dev to find out
314 * what is there, and compare with container->info.devs
315 * To see what is removed and what is added.
316 * These need to be remove from, or added to, the array
317 */
318 mdi = sysfs_read(-1, mdstat->devnum, GET_DEVS|SKIP_GONE_DEVS);
319 if (!mdi) {
320 /* invalidate the current count so we can try again */
321 container->devcnt = -1;
322 return;
323 }
324
325 /* check for removals */
326 for (cdp = &container->devs; *cdp; ) {
327 found = 0;
328 for (di = mdi->devs; di; di = di->next)
329 if (di->disk.major == (*cdp)->disk.major &&
330 di->disk.minor == (*cdp)->disk.minor) {
331 found = 1;
332 break;
333 }
334 if (!found) {
335 cd = *cdp;
336 *cdp = (*cdp)->next;
337 free(cd);
338 } else
339 cdp = &(*cdp)->next;
340 }
341
342 /* check for additions */
343 for (di = mdi->devs; di; di = di->next) {
344 for (cd = container->devs; cd; cd = cd->next)
345 if (di->disk.major == cd->disk.major &&
346 di->disk.minor == cd->disk.minor)
347 break;
348 if (!cd) {
349 struct mdinfo *newd = malloc(sizeof(*newd));
350
351 if (!newd) {
352 container->devcnt = -1;
353 continue;
354 }
355 *newd = *di;
356 add_disk_to_container(container, newd);
357 }
358 }
359 sysfs_free(mdi);
360 container->devcnt = mdstat->devcnt;
361 }
362 }
363
364 static void manage_member(struct mdstat_ent *mdstat,
365 struct active_array *a)
366 {
367 /* Compare mdstat info with known state of member array.
368 * We do not need to look for device state changes here, that
369 * is dealt with by the monitor.
370 *
371 * We just look for changes which suggest that a reshape is
372 * being requested.
373 * Unfortunately decreases in raid_disks don't show up in
374 * mdstat until the reshape completes FIXME.
375 *
376 * Actually, we also want to handle degraded arrays here by
377 * trying to find and assign a spare.
378 * We do that whenever the monitor tells us too.
379 */
380 // FIXME
381 a->info.array.raid_disks = mdstat->raid_disks;
382 a->info.array.chunk_size = mdstat->chunk_size;
383 // MORE
384
385 if (a->check_degraded) {
386 struct metadata_update *updates = NULL;
387 struct mdinfo *newdev = NULL;
388 struct active_array *newa;
389 struct mdinfo *d;
390
391 a->check_degraded = 0;
392
393 /* The array may not be degraded, this is just a good time
394 * to check.
395 */
396 newdev = a->container->ss->activate_spare(a, &updates);
397 if (!newdev)
398 return;
399
400 newa = duplicate_aa(a);
401 if (!newa)
402 goto out;
403 /* Cool, we can add a device or several. */
404
405 /* Add device to array and set offset/size/slot.
406 * and open files for each newdev */
407 for (d = newdev; d ; d = d->next) {
408 struct mdinfo *newd;
409
410 newd = malloc(sizeof(*newd));
411 if (!newd)
412 continue;
413 if (sysfs_add_disk(&newa->info, d, 0) < 0) {
414 free(newd);
415 continue;
416 }
417 *newd = *d;
418 newd->next = newa->info.devs;
419 newa->info.devs = newd;
420
421 newd->state_fd = sysfs_open(a->devnum, newd->sys_name,
422 "state");
423 newd->prev_state = read_dev_state(newd->state_fd);
424 newd->curr_state = newd->prev_state;
425 }
426 queue_metadata_update(updates);
427 updates = NULL;
428 replace_array(a->container, a, newa);
429 sysfs_set_str(&a->info, NULL, "sync_action", "recover");
430 out:
431 while (newdev) {
432 d = newdev->next;
433 free(newdev);
434 newdev = d;
435 }
436 free_updates(&updates);
437 }
438 }
439
440 static int aa_ready(struct active_array *aa)
441 {
442 struct mdinfo *d;
443 int level = aa->info.array.level;
444
445 for (d = aa->info.devs; d; d = d->next)
446 if (d->state_fd < 0)
447 return 0;
448
449 if (aa->info.state_fd < 0)
450 return 0;
451
452 if (level > 0 && (aa->action_fd < 0 || aa->resync_start_fd < 0))
453 return 0;
454
455 if (!aa->container)
456 return 0;
457
458 return 1;
459 }
460
461 static void manage_new(struct mdstat_ent *mdstat,
462 struct supertype *container,
463 struct active_array *victim)
464 {
465 /* A new array has appeared in this container.
466 * Hopefully it is already recorded in the metadata.
467 * Check, then create the new array to report it to
468 * the monitor.
469 */
470
471 struct active_array *new;
472 struct mdinfo *mdi, *di;
473 char *inst;
474 int i;
475 int failed = 0;
476
477 /* check if array is ready to be monitored */
478 if (!mdstat->active)
479 return;
480
481 mdi = sysfs_read(-1, mdstat->devnum,
482 GET_LEVEL|GET_CHUNK|GET_DISKS|GET_COMPONENT|
483 GET_DEGRADED|GET_DEVS|GET_OFFSET|GET_SIZE|GET_STATE);
484
485 new = malloc(sizeof(*new));
486
487 if (!new || !mdi) {
488 if (mdi)
489 sysfs_free(mdi);
490 if (new)
491 free(new);
492 return;
493 }
494 memset(new, 0, sizeof(*new));
495
496 new->devnum = mdstat->devnum;
497 strcpy(new->info.sys_name, devnum2devname(new->devnum));
498
499 new->prev_state = new->curr_state = new->next_state = inactive;
500 new->prev_action= new->curr_action= new->next_action= idle;
501
502 new->container = container;
503
504 inst = &mdstat->metadata_version[10+strlen(container->devname)+1];
505
506 new->info.array = mdi->array;
507 new->info.component_size = mdi->component_size;
508
509 for (i = 0; i < new->info.array.raid_disks; i++) {
510 struct mdinfo *newd = malloc(sizeof(*newd));
511
512 for (di = mdi->devs; di; di = di->next)
513 if (i == di->disk.raid_disk)
514 break;
515
516 if (di && newd) {
517 memcpy(newd, di, sizeof(*newd));
518
519 newd->state_fd = sysfs_open(new->devnum,
520 newd->sys_name,
521 "state");
522 newd->recovery_fd = sysfs_open(new->devnum,
523 newd->sys_name,
524 "recovery_start");
525
526 newd->prev_state = read_dev_state(newd->state_fd);
527 newd->curr_state = newd->prev_state;
528 } else {
529 if (newd)
530 free(newd);
531
532 failed++;
533 if (failed > new->info.array.failed_disks) {
534 /* we cannot properly monitor without all working disks */
535 new->container = NULL;
536 break;
537 }
538 continue;
539 }
540 sprintf(newd->sys_name, "rd%d", i);
541 newd->next = new->info.devs;
542 new->info.devs = newd;
543 }
544
545 new->action_fd = sysfs_open(new->devnum, NULL, "sync_action");
546 new->info.state_fd = sysfs_open(new->devnum, NULL, "array_state");
547 new->resync_start_fd = sysfs_open(new->devnum, NULL, "resync_start");
548 new->metadata_fd = sysfs_open(new->devnum, NULL, "metadata_version");
549 dprintf("%s: inst: %d action: %d state: %d\n", __func__, atoi(inst),
550 new->action_fd, new->info.state_fd);
551
552 sysfs_free(mdi);
553
554 /* if everything checks out tell the metadata handler we want to
555 * manage this instance
556 */
557 if (!aa_ready(new) || container->ss->open_new(container, new, inst) < 0) {
558 fprintf(stderr, "mdmon: failed to monitor %s\n",
559 mdstat->metadata_version);
560 new->container = NULL;
561 free_aa(new);
562 } else {
563 replace_array(container, victim, new);
564 if (failed) {
565 new->check_degraded = 1;
566 manage_member(mdstat, new);
567 }
568 }
569 }
570
571 void manage(struct mdstat_ent *mdstat, struct supertype *container)
572 {
573 /* We have just read mdstat and need to compare it with
574 * the known active arrays.
575 * Arrays with the wrong metadata are ignored.
576 */
577
578 for ( ; mdstat ; mdstat = mdstat->next) {
579 struct active_array *a;
580 if (mdstat->devnum == container->devnum) {
581 manage_container(mdstat, container);
582 continue;
583 }
584 if (!is_container_member(mdstat, container->devname))
585 /* Not for this array */
586 continue;
587 /* Looks like a member of this container */
588 for (a = container->arrays; a; a = a->next) {
589 if (mdstat->devnum == a->devnum) {
590 if (a->container)
591 manage_member(mdstat, a);
592 break;
593 }
594 }
595 if (a == NULL || !a->container)
596 manage_new(mdstat, container, a);
597 }
598 }
599
600 static void handle_message(struct supertype *container, struct metadata_update *msg)
601 {
602 /* queue this metadata update through to the monitor */
603
604 struct metadata_update *mu;
605
606 if (msg->len <= 0)
607 while (update_queue_pending || update_queue) {
608 check_update_queue(container);
609 usleep(15*1000);
610 }
611
612 if (msg->len == 0) { /* ping_monitor */
613 int cnt;
614
615 cnt = monitor_loop_cnt;
616 if (cnt & 1)
617 cnt += 2; /* wait until next pselect */
618 else
619 cnt += 3; /* wait for 2 pselects */
620 wakeup_monitor();
621
622 while (monitor_loop_cnt - cnt < 0)
623 usleep(10 * 1000);
624 } else if (msg->len == -1) { /* ping_manager */
625 struct mdstat_ent *mdstat = mdstat_read(1, 0);
626
627 manage(mdstat, container);
628 free_mdstat(mdstat);
629 } else if (!sigterm) {
630 mu = malloc(sizeof(*mu));
631 mu->len = msg->len;
632 mu->buf = msg->buf;
633 msg->buf = NULL;
634 mu->space = NULL;
635 mu->next = NULL;
636 if (container->ss->prepare_update)
637 container->ss->prepare_update(container, mu);
638 queue_metadata_update(mu);
639 }
640 }
641
642 void read_sock(struct supertype *container)
643 {
644 int fd;
645 struct metadata_update msg;
646 int terminate = 0;
647 long fl;
648 int tmo = 3; /* 3 second timeout before hanging up the socket */
649
650 fd = accept(container->sock, NULL, NULL);
651 if (fd < 0)
652 return;
653
654 fl = fcntl(fd, F_GETFL, 0);
655 fl |= O_NONBLOCK;
656 fcntl(fd, F_SETFL, fl);
657
658 do {
659 msg.buf = NULL;
660
661 /* read and validate the message */
662 if (receive_message(fd, &msg, tmo) == 0) {
663 handle_message(container, &msg);
664 if (ack(fd, tmo) < 0)
665 terminate = 1;
666 } else
667 terminate = 1;
668
669 } while (!terminate);
670
671 close(fd);
672 }
673
674 int exit_now = 0;
675 int manager_ready = 0;
676 void do_manager(struct supertype *container)
677 {
678 struct mdstat_ent *mdstat;
679 sigset_t set;
680 int proc_fd;
681
682 sigprocmask(SIG_UNBLOCK, NULL, &set);
683 sigdelset(&set, SIGUSR1);
684 sigdelset(&set, SIGHUP);
685 sigdelset(&set, SIGALRM);
686 sigdelset(&set, SIGTERM);
687 proc_fd = open("/proc/mounts", O_RDONLY);
688
689 do {
690
691 if (exit_now)
692 exit(0);
693
694 /* Can only 'manage' things if 'monitor' is not making
695 * structural changes to metadata, so need to check
696 * update_queue
697 */
698 if (update_queue == NULL) {
699 mdstat = mdstat_read(1, 0);
700
701 manage(mdstat, container);
702
703 read_sock(container);
704
705 if (container->sock < 0 || socket_hup_requested) {
706 /* If this fails, we hope it already exists
707 * pid file lives in /var/run/mdadm/mdXX.pid
708 */
709 mkdir("/var", 0600);
710 mkdir("/var/run", 0600);
711 mkdir("/var/run/mdadm", 0600);
712 close(container->sock);
713 container->sock = make_control_sock(container->devname);
714 make_pidfile(container->devname, 0);
715 socket_hup_requested = 0;
716 }
717 if (container->sock < 0)
718 alarm(30);
719
720 free_mdstat(mdstat);
721 }
722 remove_old();
723
724 check_update_queue(container);
725
726 manager_ready = 1;
727
728 if (sigterm)
729 wakeup_monitor();
730
731 if (update_queue == NULL) {
732 if (container->sock < 0)
733 mdstat_wait_fd(proc_fd, &set);
734 else
735 mdstat_wait_fd(container->sock, &set);
736 } else
737 /* If an update is happening, just wait for signal */
738 pselect(0, NULL, NULL, NULL, NULL, &set);
739 } while(1);
740 }