X-Git-Url: http://git.ipfire.org/?a=blobdiff_plain;f=managemon.c;h=269d06ed58d8bfd2ac7183f8b5e564d6fd2f4ea4;hb=b4f8e38b94dc10d7754e19dcc85f88a92c2a4387;hp=4b06778e356403332f5bdaae28ed01c35a140900;hpb=cba0191bad1617bdbc43c4a9dc2ad8c0a10f5d02;p=thirdparty%2Fmdadm.git diff --git a/managemon.c b/managemon.c index 4b06778e..269d06ed 100644 --- a/managemon.c +++ b/managemon.c @@ -1,3 +1,22 @@ +/* + * mdmon - monitor external metadata arrays + * + * Copyright (C) 2007-2009 Neil Brown + * Copyright (C) 2007-2009 Intel Corporation + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., + * 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA. + */ /* * The management thread for monitoring active md arrays. @@ -26,7 +45,18 @@ * - When a device is added to the container, we add it to the metadata * as a spare. * - * - assist with activating spares by opening relevant sysfs file. + * - Deal with degraded array + * We only do this when first noticing the array is degraded. + * This can be when we first see the array, when sync completes or + * when recovery completes. + * + * Check if number of failed devices suggests recovery is needed, and + * skip if not. + * Ask metadata to allocate a spare device + * Add device as not in_sync and give a role + * Update metadata. + * Open sysfs files and pass to monitor. + * Make sure that monitor Starts recovery.... * * - Pass on metadata updates from external programs such as * mdadm creating a new array. @@ -74,6 +104,7 @@ #endif #include "mdadm.h" #include "mdmon.h" +#include #include #include @@ -81,12 +112,16 @@ static void close_aa(struct active_array *aa) { struct mdinfo *d; - for (d = aa->info.devs; d; d = d->next) + for (d = aa->info.devs; d; d = d->next) { + close(d->recovery_fd); close(d->state_fd); + } close(aa->action_fd); close(aa->info.state_fd); close(aa->resync_start_fd); + close(aa->metadata_fd); + close(aa->sync_completed_fd); } static void free_aa(struct active_array *aa) @@ -94,6 +129,7 @@ static void free_aa(struct active_array *aa) /* Note that this doesn't close fds if they are being used * by a clone. ->container will be set for a clone */ + dprintf("%s: devnum: %d\n", __func__, aa->devnum); if (!aa->container) close_aa(aa); while (aa->info.devs) { @@ -104,16 +140,38 @@ static void free_aa(struct active_array *aa) free(aa); } -static void write_wakeup(struct supertype *c) +static struct active_array *duplicate_aa(struct active_array *aa) { - static struct md_generic_cmd cmd = { .action = md_action_ping_monitor }; - int err; + struct active_array *newa = malloc(sizeof(*newa)); + struct mdinfo **dp1, **dp2; - active_cmd = &cmd; + *newa = *aa; + newa->next = NULL; + newa->replaces = NULL; + newa->info.next = NULL; - /* send the monitor thread a pointer to the ping action */ - write(c->mgr_pipe[1], &err, 1); - read(c->mon_pipe[0], &err, 1); + dp2 = &newa->info.devs; + + for (dp1 = &aa->info.devs; *dp1; dp1 = &(*dp1)->next) { + struct mdinfo *d; + if ((*dp1)->state_fd < 0) + continue; + + d = malloc(sizeof(*d)); + *d = **dp1; + *dp2 = d; + dp2 = & d->next; + } + *dp2 = NULL; + + return newa; +} + +static void wakeup_monitor(void) +{ + /* tgkill(getpid(), mon_tid, SIGUSR1); */ + int pid = getpid(); + syscall(SYS_tgkill, pid, mon_tid, SIGUSR1); } static void remove_old(void) @@ -124,6 +182,7 @@ static void remove_old(void) if (pending_discard == discard_this) pending_discard = NULL; discard_this = NULL; + wakeup_monitor(); } } @@ -139,7 +198,6 @@ static void replace_array(struct supertype *container, */ remove_old(); while (pending_discard) { - write_wakeup(container); while (discard_this == NULL) sleep(1); remove_old(); @@ -148,30 +206,219 @@ static void replace_array(struct supertype *container, new->replaces = old; new->next = container->arrays; container->arrays = new; - write_wakeup(container); + wakeup_monitor(); +} + +struct metadata_update *update_queue = NULL; +struct metadata_update *update_queue_handled = NULL; +struct metadata_update *update_queue_pending = NULL; + +static void free_updates(struct metadata_update **update) +{ + while (*update) { + struct metadata_update *this = *update; + void **space_list = this->space_list; + + *update = this->next; + free(this->buf); + free(this->space); + while (space_list) { + void *space = space_list; + space_list = *space_list; + free(space); + } + free(this); + } +} + +void check_update_queue(struct supertype *container) +{ + free_updates(&update_queue_handled); + + if (update_queue == NULL && + update_queue_pending) { + update_queue = update_queue_pending; + update_queue_pending = NULL; + wakeup_monitor(); + } +} + +static void queue_metadata_update(struct metadata_update *mu) +{ + struct metadata_update **qp; + + qp = &update_queue_pending; + while (*qp) + qp = & ((*qp)->next); + *qp = mu; +} + +static void add_disk_to_container(struct supertype *st, struct mdinfo *sd) +{ + int dfd; + char nm[20]; + struct supertype *st2; + struct metadata_update *update = NULL; + struct mdinfo info; + mdu_disk_info_t dk = { + .number = -1, + .major = sd->disk.major, + .minor = sd->disk.minor, + .raid_disk = -1, + .state = 0, + }; + + dprintf("%s: add %d:%d to container\n", + __func__, sd->disk.major, sd->disk.minor); + + sd->next = st->devs; + st->devs = sd; + + sprintf(nm, "%d:%d", sd->disk.major, sd->disk.minor); + dfd = dev_open(nm, O_RDWR); + if (dfd < 0) + return; + + /* Check the metadata and see if it is already part of this + * array + */ + st2 = dup_super(st); + if (st2->ss->load_super(st2, dfd, NULL) == 0) { + st2->ss->getinfo_super(st, &info, NULL); + if (st->ss->compare_super(st, st2) == 0 && + info.disk.raid_disk >= 0) { + /* Looks like a good member of array. + * Just accept it. + * mdadm will incorporate any parts into + * active arrays. + */ + st2->ss->free_super(st2); + return; + } + } + st2->ss->free_super(st2); + + st->update_tail = &update; + st->ss->add_to_super(st, &dk, dfd, NULL); + st->ss->write_init_super(st); + queue_metadata_update(update); + st->update_tail = NULL; +} + +/* + * Create and queue update structure about the removed disks. + * The update is prepared by super type handler and passed to the monitor + * thread. + */ +static void remove_disk_from_container(struct supertype *st, struct mdinfo *sd) +{ + struct metadata_update *update = NULL; + mdu_disk_info_t dk = { + .number = -1, + .major = sd->disk.major, + .minor = sd->disk.minor, + .raid_disk = -1, + .state = 0, + }; + /* nothing to do if super type handler does not support + * remove disk primitive + */ + if (!st->ss->remove_from_super) + return; + dprintf("%s: remove %d:%d from container\n", + __func__, sd->disk.major, sd->disk.minor); + + st->update_tail = &update; + st->ss->remove_from_super(st, &dk); + st->ss->write_init_super(st); + queue_metadata_update(update); + st->update_tail = NULL; } static void manage_container(struct mdstat_ent *mdstat, struct supertype *container) { - /* The only thing of interest here is if a new device - * has been added to the container. We add it to the - * array ignoring any metadata on it. + /* Of interest here are: + * - if a new device has been added to the container, we + * add it to the array ignoring any metadata on it. + * - if a device has been removed from the container, we + * remove it from the device list and update the metadata. * FIXME should we look for compatible metadata and take hints * about spare assignment.... probably not. - * */ if (mdstat->devcnt != container->devcnt) { + struct mdinfo **cdp, *cd, *di, *mdi; + int found; + /* read /sys/block/NAME/md/dev-??/block/dev to find out * what is there, and compare with container->info.devs * To see what is removed and what is added. * These need to be remove from, or added to, the array */ - // FIXME + mdi = sysfs_read(-1, mdstat->devnum, GET_DEVS); + if (!mdi) { + /* invalidate the current count so we can try again */ + container->devcnt = -1; + return; + } + + /* check for removals */ + for (cdp = &container->devs; *cdp; ) { + found = 0; + for (di = mdi->devs; di; di = di->next) + if (di->disk.major == (*cdp)->disk.major && + di->disk.minor == (*cdp)->disk.minor) { + found = 1; + break; + } + if (!found) { + cd = *cdp; + *cdp = (*cdp)->next; + remove_disk_from_container(container, cd); + free(cd); + } else + cdp = &(*cdp)->next; + } + + /* check for additions */ + for (di = mdi->devs; di; di = di->next) { + for (cd = container->devs; cd; cd = cd->next) + if (di->disk.major == cd->disk.major && + di->disk.minor == cd->disk.minor) + break; + if (!cd) { + struct mdinfo *newd = malloc(sizeof(*newd)); + + if (!newd) { + container->devcnt = -1; + continue; + } + *newd = *di; + add_disk_to_container(container, newd); + } + } + sysfs_free(mdi); container->devcnt = mdstat->devcnt; } } +static int disk_init_and_add(struct mdinfo *disk, struct mdinfo *clone, + struct active_array *aa) +{ + if (!disk || !clone) + return -1; + + *disk = *clone; + disk->recovery_fd = sysfs_open(aa->devnum, disk->sys_name, "recovery_start"); + disk->state_fd = sysfs_open(aa->devnum, disk->sys_name, "state"); + disk->prev_state = read_dev_state(disk->state_fd); + disk->curr_state = disk->prev_state; + disk->next = aa->info.devs; + aa->info.devs = disk; + + return 0; +} + static void manage_member(struct mdstat_ent *mdstat, struct active_array *a) { @@ -179,16 +426,153 @@ static void manage_member(struct mdstat_ent *mdstat, * We do not need to look for device state changes here, that * is dealt with by the monitor. * - * We just look for changes which suggest that a reshape is - * being requested. - * Unfortunately decreases in raid_disks don't show up in - * mdstat until the reshape completes FIXME. + * If a reshape is being requested, monitor will have noticed + * that sync_action changed and will have set check_reshape. + * We just need to see if new devices have appeared. All metadata + * updates will already have been processed. + * + * We also want to handle degraded arrays here by + * trying to find and assign a spare. + * We do that whenever the monitor tells us too. */ + char buf[64]; + int frozen; + // FIXME a->info.array.raid_disks = mdstat->raid_disks; - a->info.array.chunk_size = mdstat->chunk_size; // MORE + /* honor 'frozen' */ + if (sysfs_get_str(&a->info, NULL, "metadata_version", buf, sizeof(buf)) > 0) + frozen = buf[9] == '-'; + else + frozen = 1; /* can't read metadata_version assume the worst */ + + if (mdstat->level) { + int level = map_name(pers, mdstat->level); + if (a->info.array.level != level && level >= 0) { + struct active_array *newa = duplicate_aa(a); + if (newa) { + newa->info.array.level = level; + replace_array(a->container, a, newa); + a = newa; + } + } + } + + if (a->check_degraded && !frozen) { + struct metadata_update *updates = NULL; + struct mdinfo *newdev = NULL; + struct active_array *newa; + struct mdinfo *d; + + a->check_degraded = 0; + + /* The array may not be degraded, this is just a good time + * to check. + */ + newdev = a->container->ss->activate_spare(a, &updates); + if (!newdev) + return; + + newa = duplicate_aa(a); + if (!newa) + goto out; + /* Cool, we can add a device or several. */ + + /* Add device to array and set offset/size/slot. + * and open files for each newdev */ + for (d = newdev; d ; d = d->next) { + struct mdinfo *newd; + + newd = malloc(sizeof(*newd)); + if (!newd) + continue; + if (sysfs_add_disk(&newa->info, d, 0) < 0) { + free(newd); + continue; + } + disk_init_and_add(newd, d, newa); + } + queue_metadata_update(updates); + updates = NULL; + replace_array(a->container, a, newa); + sysfs_set_str(&a->info, NULL, "sync_action", "recover"); + out: + while (newdev) { + d = newdev->next; + free(newdev); + newdev = d; + } + free_updates(&updates); + } + + if (a->check_reshape) { + /* mdadm might have added some devices to the array. + * We want to disk_init_and_add any such device to a + * duplicate_aa and replace a with that. + * mdstat doesn't have enough info so we sysfs_read + * and look for new stuff. + */ + struct mdinfo *info, *d, *d2, *newd; + unsigned long long array_size; + struct active_array *newa = NULL; + a->check_reshape = 0; + info = sysfs_read(-1, mdstat->devnum, + GET_DEVS|GET_OFFSET|GET_SIZE|GET_STATE); + if (!info) + goto out2; + for (d = info->devs; d; d = d->next) { + if (d->disk.raid_disk < 0) + continue; + for (d2 = a->info.devs; d2; d2 = d2->next) + if (d2->disk.raid_disk == + d->disk.raid_disk) + break; + if (d2) + /* already have this one */ + continue; + if (!newa) { + newa = duplicate_aa(a); + if (!newa) + break; + } + newd = malloc(sizeof(*newd)); + if (!newd) + continue; + disk_init_and_add(newd, d, newa); + } + if (sysfs_get_ll(info, NULL, "array_size", &array_size) == 0 + && a->info.custom_array_size > array_size) { + sysfs_set_num(info, NULL, "array_size", + a->info.custom_array_size); + } + out2: + sysfs_free(info); + if (newa) + replace_array(a->container, a, newa); + } +} + +static int aa_ready(struct active_array *aa) +{ + struct mdinfo *d; + int level = aa->info.array.level; + + for (d = aa->info.devs; d; d = d->next) + if (d->state_fd < 0) + return 0; + + if (aa->info.state_fd < 0) + return 0; + + if (level > 0 && (aa->action_fd < 0 || aa->resync_start_fd < 0)) + return 0; + + if (!aa->container) + return 0; + + return 1; } static void manage_new(struct mdstat_ent *mdstat, @@ -205,33 +589,39 @@ static void manage_new(struct mdstat_ent *mdstat, struct mdinfo *mdi, *di; char *inst; int i; + int failed = 0; + + /* check if array is ready to be monitored */ + if (!mdstat->active) + return; + + mdi = sysfs_read(-1, mdstat->devnum, + GET_LEVEL|GET_CHUNK|GET_DISKS|GET_COMPONENT| + GET_DEGRADED|GET_DEVS|GET_OFFSET|GET_SIZE|GET_STATE); new = malloc(sizeof(*new)); + if (!new || !mdi) { + if (mdi) + sysfs_free(mdi); + if (new) + free(new); + return; + } memset(new, 0, sizeof(*new)); new->devnum = mdstat->devnum; + strcpy(new->info.sys_name, devnum2devname(new->devnum)); new->prev_state = new->curr_state = new->next_state = inactive; new->prev_action= new->curr_action= new->next_action= idle; new->container = container; - inst = &mdstat->metadata_version[10+strlen(container->devname)+1]; - - mdi = sysfs_read(-1, new->devnum, - GET_LEVEL|GET_CHUNK|GET_DISKS| - GET_DEVS|GET_OFFSET|GET_SIZE|GET_STATE); - if (!mdi) { - /* Eeek. Cannot monitor this array. - * Mark it to be ignored by setting container to NULL - */ - new->container = NULL; - replace_array(container, victim, new); - return; - } + inst = to_subarray(mdstat, container->devname); new->info.array = mdi->array; + new->info.component_size = mdi->component_size; for (i = 0; i < new->info.array.raid_disks; i++) { struct mdinfo *newd = malloc(sizeof(*newd)); @@ -240,38 +630,44 @@ static void manage_new(struct mdstat_ent *mdstat, if (i == di->disk.raid_disk) break; - if (di) { - memcpy(newd, di, sizeof(*newd)); + if (disk_init_and_add(newd, di, new) != 0) { + if (newd) + free(newd); - sprintf(newd->sys_name, "rd%d", i); - - newd->state_fd = sysfs_open(new->devnum, - newd->sys_name, - "state"); - - newd->prev_state = read_dev_state(newd->state_fd); - newd->curr_state = newd->curr_state; - } else { - newd->state_fd = -1; + failed++; + if (failed > new->info.array.failed_disks) { + /* we cannot properly monitor without all working disks */ + new->container = NULL; + break; + } } - newd->next = new->info.devs; - new->info.devs = newd; } + new->action_fd = sysfs_open(new->devnum, NULL, "sync_action"); new->info.state_fd = sysfs_open(new->devnum, NULL, "array_state"); new->resync_start_fd = sysfs_open(new->devnum, NULL, "resync_start"); - new->resync_start = 0; + new->metadata_fd = sysfs_open(new->devnum, NULL, "metadata_version"); + new->sync_completed_fd = sysfs_open(new->devnum, NULL, "sync_completed"); + dprintf("%s: inst: %d action: %d state: %d\n", __func__, atoi(inst), + new->action_fd, new->info.state_fd); sysfs_free(mdi); - // finds and compares. - if (container->ss->open_new(container, new, inst) < 0) { - // FIXME close all those files + + /* if everything checks out tell the metadata handler we want to + * manage this instance + */ + if (!aa_ready(new) || container->ss->open_new(container, new, inst) < 0) { + fprintf(stderr, "mdmon: failed to monitor %s\n", + mdstat->metadata_version); new->container = NULL; + free_aa(new); + } else { replace_array(container, victim, new); - return; + if (failed) { + new->check_degraded = 1; + manage_member(mdstat, new); + } } - replace_array(container, victim, new); - return; } void manage(struct mdstat_ent *mdstat, struct supertype *container) @@ -287,12 +683,7 @@ void manage(struct mdstat_ent *mdstat, struct supertype *container) manage_container(mdstat, container); continue; } - if (mdstat->metadata_version == NULL || - strncmp(mdstat->metadata_version, "external:/", 10) != 0 || - strncmp(mdstat->metadata_version+10, container->devname, - strlen(container->devname)) != 0 || - mdstat->metadata_version[10+strlen(container->devname)] - != '/') + if (!is_container_member(mdstat, container->devname)) /* Not for this array */ continue; /* Looks like a member of this container */ @@ -308,32 +699,53 @@ void manage(struct mdstat_ent *mdstat, struct supertype *container) } } -static int handle_message(struct supertype *container, struct md_message *msg) +static void handle_message(struct supertype *container, struct metadata_update *msg) { - int err; - struct md_generic_cmd *cmd = msg->buf; + /* queue this metadata update through to the monitor */ - if (!cmd) - return 0; + struct metadata_update *mu; - switch (cmd->action) { - case md_action_remove_device: + if (msg->len <= 0) + while (update_queue_pending || update_queue) { + check_update_queue(container); + usleep(15*1000); + } - /* forward to the monitor */ - active_cmd = cmd; - write(container->mgr_pipe[1], &err, 1); - read(container->mon_pipe[0], &err, 1); - return err; + if (msg->len == 0) { /* ping_monitor */ + int cnt; + + cnt = monitor_loop_cnt; + if (cnt & 1) + cnt += 2; /* wait until next pselect */ + else + cnt += 3; /* wait for 2 pselects */ + wakeup_monitor(); + + while (monitor_loop_cnt - cnt < 0) + usleep(10 * 1000); + } else if (msg->len == -1) { /* ping_manager */ + struct mdstat_ent *mdstat = mdstat_read(1, 0); - default: - return -1; + manage(mdstat, container); + free_mdstat(mdstat); + } else if (!sigterm) { + mu = malloc(sizeof(*mu)); + mu->len = msg->len; + mu->buf = msg->buf; + msg->buf = NULL; + mu->space = NULL; + mu->space_list = NULL; + mu->next = NULL; + if (container->ss->prepare_update) + container->ss->prepare_update(container, mu); + queue_metadata_update(mu); } } void read_sock(struct supertype *container) { int fd; - struct md_message msg; + struct metadata_update msg; int terminate = 0; long fl; int tmo = 3; /* 3 second timeout before hanging up the socket */ @@ -347,67 +759,69 @@ void read_sock(struct supertype *container) fcntl(fd, F_SETFL, fl); do { - int err; - msg.buf = NULL; /* read and validate the message */ if (receive_message(fd, &msg, tmo) == 0) { - err = handle_message(container, &msg); - if (!err) - ack(fd, msg.seq, tmo); - else - nack(fd, err, tmo); - } else { + handle_message(container, &msg); + if (msg.len == 0) { + /* ping reply with version */ + msg.buf = Version; + msg.len = strlen(Version) + 1; + if (send_message(fd, &msg, tmo) < 0) + terminate = 1; + } else if (ack(fd, tmo) < 0) + terminate = 1; + } else terminate = 1; - nack(fd, -1, tmo); - } - if (msg.buf) - free(msg.buf); } while (!terminate); close(fd); } -static int woke = 0; -void wake_me(int sig) -{ - woke = 1; -} - int exit_now = 0; int manager_ready = 0; void do_manager(struct supertype *container) { struct mdstat_ent *mdstat; - sigset_t block, orig; - - sigemptyset(&block); - sigaddset(&block, SIGUSR1); + sigset_t set; - signal(SIGUSR1, wake_me); + sigprocmask(SIG_UNBLOCK, NULL, &set); + sigdelset(&set, SIGUSR1); + sigdelset(&set, SIGTERM); do { - woke = 0; if (exit_now) exit(0); - mdstat = mdstat_read(1, 0); - - manage(mdstat, container); + /* Can only 'manage' things if 'monitor' is not making + * structural changes to metadata, so need to check + * update_queue + */ + if (update_queue == NULL) { + mdstat = mdstat_read(1, 0); - read_sock(container); + manage(mdstat, container); - free_mdstat(mdstat); + read_sock(container); + free_mdstat(mdstat); + } remove_old(); + check_update_queue(container); + manager_ready = 1; - sigprocmask(SIG_SETMASK, &block, &orig); - if (woke == 0) - mdstat_wait_fd(container->sock, &orig); - sigprocmask(SIG_SETMASK, &orig, NULL); + + if (sigterm) + wakeup_monitor(); + + if (update_queue == NULL) + mdstat_wait_fd(container->sock, &set); + else + /* If an update is happening, just wait for signal */ + pselect(0, NULL, NULL, NULL, NULL, &set); } while(1); }