]>
git.ipfire.org Git - thirdparty/mdadm.git/blob - managemon.c
2 * mdmon - monitor external metadata arrays
4 * Copyright (C) 2007-2009 Neil Brown <neilb@suse.de>
5 * Copyright (C) 2007-2009 Intel Corporation
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 * You should have received a copy of the GNU General Public License along with
17 * this program; if not, write to the Free Software Foundation, Inc.,
18 * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
22 * The management thread for monitoring active md arrays.
23 * This thread does things which might block such as memory
27 * - Find out about new arrays in this container.
28 * Allocate the data structures and open the files.
30 * For this we watch /proc/mdstat and find new arrays with
31 * metadata type that confirms sharing. e.g. "md4"
32 * When we find a new array we slip it into the list of
33 * arrays and signal 'monitor' by writing to a pipe.
35 * - Respond to reshape requests by allocating new data structures
36 * and opening new files.
38 * These come as a change to raid_disks. We allocate a new
39 * version of the data structures and slip it into the list.
40 * 'monitor' will notice and release the old version.
41 * Changes to level, chunksize, layout.. do not need re-allocation.
42 * Reductions in raid_disks don't really either, but we handle
43 * them the same way for consistency.
45 * - When a device is added to the container, we add it to the metadata
48 * - Deal with degraded array
49 * We only do this when first noticing the array is degraded.
50 * This can be when we first see the array, when sync completes or
51 * when recovery completes.
53 * Check if number of failed devices suggests recovery is needed, and
55 * Ask metadata to allocate a spare device
56 * Add device as not in_sync and give a role
58 * Open sysfs files and pass to monitor.
59 * Make sure that monitor Starts recovery....
61 * - Pass on metadata updates from external programs such as
62 * mdadm creating a new array.
65 * It might involve adding a new array or changing the status of
66 * a spare, or any reconfig that the kernel doesn't get involved in.
68 * The required updates are received via a named pipe. There will
69 * be one named pipe for each container. Each message contains a
70 * sync marker: 0x5a5aa5a5, A byte count, and the message. This is
71 * passed to the metadata handler which will interpret and process it.
72 * For 'DDF' messages are internal data blocks with the leading
73 * 'magic number' signifying what sort of data it is.
78 * We select on /proc/mdstat and the named pipe.
79 * We create new arrays or updated version of arrays and slip
80 * them into the head of the list, then signal 'monitor' via a pipe write.
81 * 'monitor' will notice and place the old array on a return list.
82 * Metadata updates are placed on a queue just like they arrive
83 * from the named pipe.
85 * When new arrays are found based on correct metadata string, we
86 * need to identify them with an entry in the metadata. Maybe we require
87 * the metadata to be mdX/NN when NN is the index into an appropriate table.
93 * - Watch for spares to be added to the container, and write updated
95 * - Watch for new arrays using this container, confirm they match metadata
96 * and if so, start monitoring them
97 * - Watch for spares being added to monitored arrays. This shouldn't
98 * happen, as we should do all the adding. Just remove them.
99 * - Watch for change in raid-disks, chunk-size, etc. Update metadata and
107 #include <sys/syscall.h>
108 #include <sys/socket.h>
111 static void close_aa(struct active_array
*aa
)
115 for (d
= aa
->info
.devs
; d
; d
= d
->next
) {
116 close(d
->recovery_fd
);
120 if (aa
->action_fd
>= 0)
121 close(aa
->action_fd
);
122 if (aa
->info
.state_fd
>= 0)
123 close(aa
->info
.state_fd
);
124 if (aa
->resync_start_fd
>= 0)
125 close(aa
->resync_start_fd
);
126 if (aa
->metadata_fd
>= 0)
127 close(aa
->metadata_fd
);
128 if (aa
->sync_completed_fd
>= 0)
129 close(aa
->sync_completed_fd
);
132 static void free_aa(struct active_array
*aa
)
134 /* Note that this doesn't close fds if they are being used
135 * by a clone. ->container will be set for a clone
137 dprintf("%s: devnum: %d\n", __func__
, aa
->devnum
);
140 while (aa
->info
.devs
) {
141 struct mdinfo
*d
= aa
->info
.devs
;
142 aa
->info
.devs
= d
->next
;
148 static struct active_array
*duplicate_aa(struct active_array
*aa
)
150 struct active_array
*newa
= malloc(sizeof(*newa
));
151 struct mdinfo
**dp1
, **dp2
;
155 newa
->replaces
= NULL
;
156 newa
->info
.next
= NULL
;
158 dp2
= &newa
->info
.devs
;
160 for (dp1
= &aa
->info
.devs
; *dp1
; dp1
= &(*dp1
)->next
) {
162 if ((*dp1
)->state_fd
< 0)
165 d
= malloc(sizeof(*d
));
175 static void wakeup_monitor(void)
177 /* tgkill(getpid(), mon_tid, SIGUSR1); */
179 syscall(SYS_tgkill
, pid
, mon_tid
, SIGUSR1
);
182 static void remove_old(void)
185 discard_this
->next
= NULL
;
186 free_aa(discard_this
);
187 if (pending_discard
== discard_this
)
188 pending_discard
= NULL
;
194 static void replace_array(struct supertype
*container
,
195 struct active_array
*old
,
196 struct active_array
*new)
198 /* To replace an array, we add it to the top of the list
199 * marked with ->replaces to point to the original.
200 * 'monitor' will take the original out of the list
201 * and put it on 'discard_this'. We take it from there
205 while (pending_discard
) {
206 while (discard_this
== NULL
)
210 pending_discard
= old
;
212 new->next
= container
->arrays
;
213 container
->arrays
= new;
217 struct metadata_update
*update_queue
= NULL
;
218 struct metadata_update
*update_queue_handled
= NULL
;
219 struct metadata_update
*update_queue_pending
= NULL
;
221 static void free_updates(struct metadata_update
**update
)
224 struct metadata_update
*this = *update
;
225 void **space_list
= this->space_list
;
227 *update
= this->next
;
231 void *space
= space_list
;
232 space_list
= *space_list
;
239 void check_update_queue(struct supertype
*container
)
241 free_updates(&update_queue_handled
);
243 if (update_queue
== NULL
&&
244 update_queue_pending
) {
245 update_queue
= update_queue_pending
;
246 update_queue_pending
= NULL
;
251 static void queue_metadata_update(struct metadata_update
*mu
)
253 struct metadata_update
**qp
;
255 qp
= &update_queue_pending
;
257 qp
= & ((*qp
)->next
);
261 static void add_disk_to_container(struct supertype
*st
, struct mdinfo
*sd
)
265 struct supertype
*st2
;
266 struct metadata_update
*update
= NULL
;
268 mdu_disk_info_t dk
= {
270 .major
= sd
->disk
.major
,
271 .minor
= sd
->disk
.minor
,
276 dprintf("%s: add %d:%d to container\n",
277 __func__
, sd
->disk
.major
, sd
->disk
.minor
);
282 sprintf(nm
, "%d:%d", sd
->disk
.major
, sd
->disk
.minor
);
283 dfd
= dev_open(nm
, O_RDWR
);
287 /* Check the metadata and see if it is already part of this
291 if (st2
->ss
->load_super(st2
, dfd
, NULL
) == 0) {
292 st2
->ss
->getinfo_super(st
, &info
, NULL
);
293 if (st
->ss
->compare_super(st
, st2
) == 0 &&
294 info
.disk
.raid_disk
>= 0) {
295 /* Looks like a good member of array.
297 * mdadm will incorporate any parts into
300 st2
->ss
->free_super(st2
);
304 st2
->ss
->free_super(st2
);
306 st
->update_tail
= &update
;
307 st
->ss
->add_to_super(st
, &dk
, dfd
, NULL
);
308 st
->ss
->write_init_super(st
);
309 queue_metadata_update(update
);
310 st
->update_tail
= NULL
;
314 * Create and queue update structure about the removed disks.
315 * The update is prepared by super type handler and passed to the monitor
318 static void remove_disk_from_container(struct supertype
*st
, struct mdinfo
*sd
)
320 struct metadata_update
*update
= NULL
;
321 mdu_disk_info_t dk
= {
323 .major
= sd
->disk
.major
,
324 .minor
= sd
->disk
.minor
,
328 dprintf("%s: remove %d:%d from container\n",
329 __func__
, sd
->disk
.major
, sd
->disk
.minor
);
331 st
->update_tail
= &update
;
332 st
->ss
->remove_from_super(st
, &dk
);
333 /* FIXME this write_init_super shouldn't be here.
334 * We have it after add_to_super to write to new device,
335 * but with 'remove' we don't ant to write to that device!
337 st
->ss
->write_init_super(st
);
338 queue_metadata_update(update
);
339 st
->update_tail
= NULL
;
342 static void manage_container(struct mdstat_ent
*mdstat
,
343 struct supertype
*container
)
345 /* Of interest here are:
346 * - if a new device has been added to the container, we
347 * add it to the array ignoring any metadata on it.
348 * - if a device has been removed from the container, we
349 * remove it from the device list and update the metadata.
350 * FIXME should we look for compatible metadata and take hints
351 * about spare assignment.... probably not.
353 if (mdstat
->devcnt
!= container
->devcnt
) {
354 struct mdinfo
**cdp
, *cd
, *di
, *mdi
;
357 /* read /sys/block/NAME/md/dev-??/block/dev to find out
358 * what is there, and compare with container->info.devs
359 * To see what is removed and what is added.
360 * These need to be remove from, or added to, the array
362 mdi
= sysfs_read(-1, mdstat
->devnum
, GET_DEVS
);
364 /* invalidate the current count so we can try again */
365 container
->devcnt
= -1;
369 /* check for removals */
370 for (cdp
= &container
->devs
; *cdp
; ) {
372 for (di
= mdi
->devs
; di
; di
= di
->next
)
373 if (di
->disk
.major
== (*cdp
)->disk
.major
&&
374 di
->disk
.minor
== (*cdp
)->disk
.minor
) {
381 remove_disk_from_container(container
, cd
);
387 /* check for additions */
388 for (di
= mdi
->devs
; di
; di
= di
->next
) {
389 for (cd
= container
->devs
; cd
; cd
= cd
->next
)
390 if (di
->disk
.major
== cd
->disk
.major
&&
391 di
->disk
.minor
== cd
->disk
.minor
)
394 struct mdinfo
*newd
= malloc(sizeof(*newd
));
397 container
->devcnt
= -1;
401 add_disk_to_container(container
, newd
);
405 container
->devcnt
= mdstat
->devcnt
;
409 static int disk_init_and_add(struct mdinfo
*disk
, struct mdinfo
*clone
,
410 struct active_array
*aa
)
416 disk
->recovery_fd
= sysfs_open(aa
->devnum
, disk
->sys_name
, "recovery_start");
417 if (disk
->recovery_fd
< 0)
419 disk
->state_fd
= sysfs_open(aa
->devnum
, disk
->sys_name
, "state");
420 if (disk
->state_fd
< 0) {
421 close(disk
->recovery_fd
);
424 disk
->prev_state
= read_dev_state(disk
->state_fd
);
425 disk
->curr_state
= disk
->prev_state
;
426 disk
->next
= aa
->info
.devs
;
427 aa
->info
.devs
= disk
;
432 static void manage_member(struct mdstat_ent
*mdstat
,
433 struct active_array
*a
)
435 /* Compare mdstat info with known state of member array.
436 * We do not need to look for device state changes here, that
437 * is dealt with by the monitor.
439 * If a reshape is being requested, monitor will have noticed
440 * that sync_action changed and will have set check_reshape.
441 * We just need to see if new devices have appeared. All metadata
442 * updates will already have been processed.
444 * We also want to handle degraded arrays here by
445 * trying to find and assign a spare.
446 * We do that whenever the monitor tells us too.
450 struct supertype
*container
= a
->container
;
452 if (container
== NULL
)
453 /* Raced with something */
457 a
->info
.array
.raid_disks
= mdstat
->raid_disks
;
461 if (sysfs_get_str(&a
->info
, NULL
, "metadata_version", buf
, sizeof(buf
)) > 0)
462 frozen
= buf
[9] == '-';
464 frozen
= 1; /* can't read metadata_version assume the worst */
466 /* If sync_action is not 'idle' then don't try recovery now */
468 && sysfs_get_str(&a
->info
, NULL
, "sync_action", buf
, sizeof(buf
)) > 0
469 && strncmp(buf
, "idle", 4) != 0)
473 int level
= map_name(pers
, mdstat
->level
);
474 if (level
== 0 || level
== LEVEL_LINEAR
) {
479 else if (a
->info
.array
.level
!= level
&& level
> 0) {
480 struct active_array
*newa
= duplicate_aa(a
);
482 newa
->info
.array
.level
= level
;
483 replace_array(container
, a
, newa
);
489 /* we are after monitor kick,
490 * so container field can be cleared - check it again
492 if (a
->container
== NULL
)
495 /* We don't check the array while any update is pending, as it
496 * might container a change (such as a spare assignment) which
497 * could affect our decisions.
499 if (a
->check_degraded
&& !frozen
&&
500 update_queue
== NULL
&& update_queue_pending
== NULL
) {
501 struct metadata_update
*updates
= NULL
;
502 struct mdinfo
*newdev
= NULL
;
503 struct active_array
*newa
;
506 a
->check_degraded
= 0;
508 /* The array may not be degraded, this is just a good time
511 newdev
= container
->ss
->activate_spare(a
, &updates
);
515 newa
= duplicate_aa(a
);
518 /* prevent the kernel from activating the disk(s) before we
521 sysfs_set_str(&a
->info
, NULL
, "sync_action", "frozen");
523 /* Add device to array and set offset/size/slot.
524 * and open files for each newdev */
525 for (d
= newdev
; d
; d
= d
->next
) {
528 newd
= malloc(sizeof(*newd
));
531 if (sysfs_add_disk(&newa
->info
, d
, 0) < 0) {
535 disk_init_and_add(newd
, d
, newa
);
537 queue_metadata_update(updates
);
539 replace_array(container
, a
, newa
);
540 sysfs_set_str(&a
->info
, NULL
, "sync_action", "recover");
547 free_updates(&updates
);
550 if (a
->check_reshape
) {
551 /* mdadm might have added some devices to the array.
552 * We want to disk_init_and_add any such device to a
553 * duplicate_aa and replace a with that.
554 * mdstat doesn't have enough info so we sysfs_read
555 * and look for new stuff.
557 struct mdinfo
*info
, *d
, *d2
, *newd
;
558 unsigned long long array_size
;
559 struct active_array
*newa
= NULL
;
560 a
->check_reshape
= 0;
561 info
= sysfs_read(-1, mdstat
->devnum
,
562 GET_DEVS
|GET_OFFSET
|GET_SIZE
|GET_STATE
);
565 for (d
= info
->devs
; d
; d
= d
->next
) {
566 if (d
->disk
.raid_disk
< 0)
568 for (d2
= a
->info
.devs
; d2
; d2
= d2
->next
)
569 if (d2
->disk
.raid_disk
==
573 /* already have this one */
576 newa
= duplicate_aa(a
);
580 newd
= malloc(sizeof(*newd
));
583 disk_init_and_add(newd
, d
, newa
);
585 if (sysfs_get_ll(info
, NULL
, "array_size", &array_size
) == 0
586 && a
->info
.custom_array_size
> array_size
*2) {
587 sysfs_set_num(info
, NULL
, "array_size",
588 a
->info
.custom_array_size
/2);
593 replace_array(container
, a
, newa
);
597 static int aa_ready(struct active_array
*aa
)
600 int level
= aa
->info
.array
.level
;
602 for (d
= aa
->info
.devs
; d
; d
= d
->next
)
606 if (aa
->info
.state_fd
< 0)
609 if (level
> 0 && (aa
->action_fd
< 0 || aa
->resync_start_fd
< 0))
618 static void manage_new(struct mdstat_ent
*mdstat
,
619 struct supertype
*container
,
620 struct active_array
*victim
)
622 /* A new array has appeared in this container.
623 * Hopefully it is already recorded in the metadata.
624 * Check, then create the new array to report it to
628 struct active_array
*new;
629 struct mdinfo
*mdi
, *di
;
635 /* check if array is ready to be monitored */
636 if (!mdstat
->active
|| !mdstat
->level
)
638 if (strcmp(mdstat
->level
, "raid0") == 0 ||
639 strcmp(mdstat
->level
, "linear") == 0)
642 mdi
= sysfs_read(-1, mdstat
->devnum
,
643 GET_LEVEL
|GET_CHUNK
|GET_DISKS
|GET_COMPONENT
|
644 GET_DEGRADED
|GET_DEVS
|GET_OFFSET
|GET_SIZE
|GET_STATE
);
646 new = malloc(sizeof(*new));
655 memset(new, 0, sizeof(*new));
657 new->devnum
= mdstat
->devnum
;
658 strcpy(new->info
.sys_name
, devnum2devname(new->devnum
));
660 new->prev_state
= new->curr_state
= new->next_state
= inactive
;
661 new->prev_action
= new->curr_action
= new->next_action
= idle
;
663 new->container
= container
;
665 inst
= to_subarray(mdstat
, container
->devname
);
667 new->info
.array
= mdi
->array
;
668 new->info
.component_size
= mdi
->component_size
;
670 for (i
= 0; i
< new->info
.array
.raid_disks
; i
++) {
671 struct mdinfo
*newd
= malloc(sizeof(*newd
));
673 for (di
= mdi
->devs
; di
; di
= di
->next
)
674 if (i
== di
->disk
.raid_disk
)
677 if (disk_init_and_add(newd
, di
, new) != 0) {
682 if (failed
> new->info
.array
.failed_disks
) {
683 /* we cannot properly monitor without all working disks */
684 new->container
= NULL
;
690 new->action_fd
= sysfs_open(new->devnum
, NULL
, "sync_action");
691 new->info
.state_fd
= sysfs_open(new->devnum
, NULL
, "array_state");
692 new->resync_start_fd
= sysfs_open(new->devnum
, NULL
, "resync_start");
693 new->metadata_fd
= sysfs_open(new->devnum
, NULL
, "metadata_version");
694 new->sync_completed_fd
= sysfs_open(new->devnum
, NULL
, "sync_completed");
695 dprintf("%s: inst: %d action: %d state: %d\n", __func__
, atoi(inst
),
696 new->action_fd
, new->info
.state_fd
);
698 /* reshape_position is set by mdadm in sysfs
699 * read this information for new arrays only (empty victim)
701 if ((victim
== NULL
) &&
702 (sysfs_get_str(mdi
, NULL
, "sync_action", buf
, 40) > 0) &&
703 (strncmp(buf
, "reshape", 7) == 0)) {
704 if (sysfs_get_ll(mdi
, NULL
, "reshape_position",
705 &new->last_checkpoint
) != 0)
706 new->last_checkpoint
= 0;
708 int data_disks
= mdi
->array
.raid_disks
;
709 if (mdi
->array
.level
== 4 || mdi
->array
.level
== 5)
711 if (mdi
->array
.level
== 6)
714 new->last_checkpoint
/= data_disks
;
716 dprintf("mdmon: New monitored array is under reshape.\n"
717 " Last checkpoint is: %llu\n",
718 new->last_checkpoint
);
723 /* if everything checks out tell the metadata handler we want to
724 * manage this instance
726 if (!aa_ready(new) || container
->ss
->open_new(container
, new, inst
) < 0) {
727 fprintf(stderr
, "mdmon: failed to monitor %s\n",
728 mdstat
->metadata_version
);
729 new->container
= NULL
;
732 replace_array(container
, victim
, new);
734 new->check_degraded
= 1;
735 manage_member(mdstat
, new);
740 void manage(struct mdstat_ent
*mdstat
, struct supertype
*container
)
742 /* We have just read mdstat and need to compare it with
743 * the known active arrays.
744 * Arrays with the wrong metadata are ignored.
747 for ( ; mdstat
; mdstat
= mdstat
->next
) {
748 struct active_array
*a
;
749 if (mdstat
->devnum
== container
->devnum
) {
750 manage_container(mdstat
, container
);
753 if (!is_container_member(mdstat
, container
->devname
))
754 /* Not for this array */
756 /* Looks like a member of this container */
757 for (a
= container
->arrays
; a
; a
= a
->next
) {
758 if (mdstat
->devnum
== a
->devnum
) {
759 if (a
->container
&& a
->to_remove
== 0)
760 manage_member(mdstat
, a
);
764 if (a
== NULL
|| !a
->container
)
765 manage_new(mdstat
, container
, a
);
769 static void handle_message(struct supertype
*container
, struct metadata_update
*msg
)
771 /* queue this metadata update through to the monitor */
773 struct metadata_update
*mu
;
776 while (update_queue_pending
|| update_queue
) {
777 check_update_queue(container
);
781 if (msg
->len
== 0) { /* ping_monitor */
784 cnt
= monitor_loop_cnt
;
786 cnt
+= 2; /* wait until next pselect */
788 cnt
+= 3; /* wait for 2 pselects */
791 while (monitor_loop_cnt
- cnt
< 0)
793 } else if (msg
->len
== -1) { /* ping_manager */
794 struct mdstat_ent
*mdstat
= mdstat_read(1, 0);
796 manage(mdstat
, container
);
798 } else if (!sigterm
) {
799 mu
= malloc(sizeof(*mu
));
804 mu
->space_list
= NULL
;
806 if (container
->ss
->prepare_update
)
807 container
->ss
->prepare_update(container
, mu
);
808 queue_metadata_update(mu
);
812 void read_sock(struct supertype
*container
)
815 struct metadata_update msg
;
818 int tmo
= 3; /* 3 second timeout before hanging up the socket */
820 fd
= accept(container
->sock
, NULL
, NULL
);
824 fl
= fcntl(fd
, F_GETFL
, 0);
826 fcntl(fd
, F_SETFL
, fl
);
831 /* read and validate the message */
832 if (receive_message(fd
, &msg
, tmo
) == 0) {
833 handle_message(container
, &msg
);
835 /* ping reply with version */
837 msg
.len
= strlen(Version
) + 1;
838 if (send_message(fd
, &msg
, tmo
) < 0)
840 } else if (ack(fd
, tmo
) < 0)
845 } while (!terminate
);
851 int manager_ready
= 0;
852 void do_manager(struct supertype
*container
)
854 struct mdstat_ent
*mdstat
;
857 sigprocmask(SIG_UNBLOCK
, NULL
, &set
);
858 sigdelset(&set
, SIGUSR1
);
859 sigdelset(&set
, SIGTERM
);
866 /* Can only 'manage' things if 'monitor' is not making
867 * structural changes to metadata, so need to check
870 if (update_queue
== NULL
) {
871 mdstat
= mdstat_read(1, 0);
873 manage(mdstat
, container
);
875 read_sock(container
);
881 check_update_queue(container
);
888 if (update_queue
== NULL
)
889 mdstat_wait_fd(container
->sock
, &set
);
891 /* If an update is happening, just wait for signal */
892 pselect(0, NULL
, NULL
, NULL
, NULL
, &set
);