managemon.c

   1
   2 /*
   3  * The management thread for monitoring active md arrays.
   4  * This thread does things which might block such as memory
   5  * allocation.
   6  * In particular:
   7  *
   8  * - Find out about new arrays in this container.
   9  *   Allocate the data structures and open the files.
  10  *
  11  *   For this we watch /proc/mdstat and find new arrays with
  12  *   metadata type that confirms sharing. e.g. "md4"
  13  *   When we find a new array we slip it into the list of
  14  *   arrays and signal 'monitor' by writing to a pipe.
  15  *
  16  * - Respond to reshape requests by allocating new data structures
  17  *   and opening new files.
  18  *
  19  *   These come as a change to raid_disks.  We allocate a new
  20  *   version of the data structures and slip it into the list.
  21  *   'monitor' will notice and release the old version.
  22  *   Changes to level, chunksize, layout.. do not need re-allocation.
  23  *   Reductions in raid_disks don't really either, but we handle
  24  *   them the same way for consistency.
  25  *
  26  * - When a device is added to the container, we add it to the metadata
  27  *   as a spare.
  28  *
  29  * - assist with activating spares by opening relevant sysfs file.
  30  *
  31  * - Pass on metadata updates from external programs such as
  32  *   mdadm creating a new array.
  33  *
  34  *   This is most-messy.
  35  *   It might involve adding a new array or changing the status of
  36  *   a spare, or any reconfig that the kernel doesn't get involved in.
  37  *
  38  *   The required updates are received via a named pipe.  There will
  39  *   be one named pipe for each container. Each message contains a
  40  *   sync marker: 0x5a5aa5a5, A byte count, and the message.  This is
  41  *   passed to the metadata handler which will interpret and process it.
  42  *   For 'DDF' messages are internal data blocks with the leading
  43  *   'magic number' signifying what sort of data it is.
  44  *
  45  */
  46
  47 /*
  48  * We select on /proc/mdstat and the named pipe.
  49  * We create new arrays or updated version of arrays and slip
  50  * them into the head of the list, then signal 'monitor' via a pipe write.
  51  * 'monitor' will notice and place the old array on a return list.
  52  * Metadata updates are placed on a queue just like they arrive
  53  * from the named pipe.
  54  *
  55  * When new arrays are found based on correct metadata string, we
  56  * need to identify them with an entry in the metadata.  Maybe we require
  57  * the metadata to be mdX/NN  when NN is the index into an appropriate table.
  58  *
  59  */
  60
  61 /*
  62  * List of tasks:
  63  * - Watch for spares to be added to the container, and write updated
  64  *   metadata to them.
  65  * - Watch for new arrays using this container, confirm they match metadata
  66  *   and if so, start monitoring them
  67  * - Watch for spares being added to monitored arrays.  This shouldn't
  68  *   happen, as we should do all the adding.  Just remove them.
  69  * - Watch for change in raid-disks, chunk-size, etc.  Update metadata and
  70  *   start a reshape.
  71  */
  72 #ifndef _GNU_SOURCE
  73 #define _GNU_SOURCE
  74 #endif
  75 #include        "mdadm.h"
  76 #include        "mdmon.h"
  77 #include        <sys/socket.h>
  78 #include        <signal.h>
  79
  80 static void close_aa(struct active_array *aa)
  81 {
  82         struct mdinfo *d;
  83
  84         for (d = aa->info.devs; d; d = d->next)
  85                 close(d->state_fd);
  86
  87         close(aa->action_fd);
  88         close(aa->info.state_fd);
  89         close(aa->resync_start_fd);
  90         close(aa->sync_pos_fd);
  91 }
  92
  93 static void free_aa(struct active_array *aa)
  94 {
  95         /* Note that this doesn't close fds if they are being used
  96          * by a clone.  ->container will be set for a clone
  97          */
  98         if (!aa->container)
  99                 close_aa(aa);
 100         while (aa->info.devs) {
 101                 struct mdinfo *d = aa->info.devs;
 102                 aa->info.devs = d->next;
 103                 free(d);
 104         }
 105         free(aa);
 106 }
 107
 108 static void write_wakeup(struct supertype *c)
 109 {
 110         static struct md_generic_cmd cmd = { .action = md_action_ping_monitor };
 111         int err;
 112
 113         active_cmd = &cmd;
 114
 115         /* send the monitor thread a pointer to the ping action */
 116         write(c->mgr_pipe[1], &err, 1);
 117         read(c->mon_pipe[0], &err, 1);
 118 }
 119
 120 static void remove_old(void)
 121 {
 122         if (discard_this) {
 123                 discard_this->next = NULL;
 124                 free_aa(discard_this);
 125                 if (pending_discard == discard_this)
 126                         pending_discard = NULL;
 127                 discard_this = NULL;
 128         }
 129 }
 130
 131 static void replace_array(struct supertype *container,
 132                           struct active_array *old,
 133                           struct active_array *new)
 134 {
 135         /* To replace an array, we add it to the top of the list
 136          * marked with ->replaces to point to the original.
 137          * 'monitor' will take the original out of the list
 138          * and put it on 'discard_this'.  We take it from there
 139          * and discard it.
 140          */
 141         remove_old();
 142         while (pending_discard) {
 143                 write_wakeup(container);
 144                 while (discard_this == NULL)
 145                         sleep(1);
 146                 remove_old();
 147         }
 148         pending_discard = old;
 149         new->replaces = old;
 150         new->next = container->arrays;
 151         container->arrays = new;
 152         write_wakeup(container);
 153 }
 154
 155 static void manage_container(struct mdstat_ent *mdstat,
 156                              struct supertype *container)
 157 {
 158         /* The only thing of interest here is if a new device
 159          * has been added to the container.  We add it to the
 160          * array ignoring any metadata on it.
 161          * FIXME should we look for compatible metadata and take hints
 162          * about spare assignment.... probably not.
 163          *
 164          */
 165         if (mdstat->devcnt != container->devcnt) {
 166                 /* read /sys/block/NAME/md/dev-??/block/dev to find out
 167                  * what is there, and compare with container->info.devs
 168                  * To see what is removed and what is added.
 169                  * These need to be remove from, or added to, the array
 170                  */
 171                 // FIXME
 172                 container->devcnt = mdstat->devcnt;
 173         }
 174 }
 175
 176 static void manage_member(struct mdstat_ent *mdstat,
 177                           struct active_array *a)
 178 {
 179         /* Compare mdstat info with known state of member array.
 180          * We do not need to look for device state changes here, that
 181          * is dealt with by the monitor.
 182          *
 183          * We just look for changes which suggest that a reshape is
 184          * being requested.
 185          * Unfortunately decreases in raid_disks don't show up in
 186          * mdstat until the reshape completes FIXME.
 187          */
 188         // FIXME
 189         a->info.array.raid_disks = mdstat->raid_disks;
 190         a->info.array.chunk_size = mdstat->chunk_size;
 191         // MORE
 192
 193 }
 194
 195 static void manage_new(struct mdstat_ent *mdstat,
 196                        struct supertype *container,
 197                        struct active_array *victim)
 198 {
 199         /* A new array has appeared in this container.
 200          * Hopefully it is already recorded in the metadata.
 201          * Check, then create the new array to report it to
 202          * the monitor.
 203          */
 204
 205         struct active_array *new;
 206         struct mdinfo *mdi, *di;
 207         char *n;
 208         int inst;
 209         int i;
 210
 211         new = malloc(sizeof(*new));
 212
 213         memset(new, 0, sizeof(*new));
 214
 215         new->devnum = mdstat->devnum;
 216
 217         new->prev_state = new->curr_state = new->next_state = inactive;
 218         new->prev_action= new->curr_action= new->next_action= idle;
 219
 220         new->container = container;
 221
 222         n = &mdstat->metadata_version[10+strlen(container->devname)+1];
 223         inst = atoi(n);
 224         if (inst < 0)
 225                 abort();//FIXME
 226
 227         mdi = sysfs_read(-1, new->devnum,
 228                          GET_LEVEL|GET_CHUNK|GET_DISKS|
 229                          GET_DEVS|GET_OFFSET|GET_SIZE|GET_STATE);
 230         if (!mdi) {
 231                 /* Eeek. Cannot monitor this array.
 232                  * Mark it to be ignored by setting container to NULL
 233                  */
 234                 new->container = NULL;
 235                 replace_array(container, victim, new);
 236                 return;
 237         }
 238
 239         new->info.array = mdi->array;
 240
 241         for (i = 0; i < new->info.array.raid_disks; i++) {
 242                 struct mdinfo *newd = malloc(sizeof(*newd));
 243
 244                 for (di = mdi->devs; di; di = di->next)
 245                         if (i == di->disk.raid_disk)
 246                                 break;
 247
 248                 if (di) {
 249                         memcpy(newd, di, sizeof(*newd));
 250
 251                         sprintf(newd->sys_name, "rd%d", i);
 252
 253                         newd->state_fd = sysfs_open(new->devnum,
 254                                                     newd->sys_name,
 255                                                     "state");
 256
 257                         newd->prev_state = read_dev_state(newd->state_fd);
 258                         newd->curr_state = newd->curr_state;
 259                 } else {
 260                         newd->state_fd = -1;
 261                 }
 262                 newd->next = new->info.devs;
 263                 new->info.devs = newd;
 264         }
 265         new->action_fd = sysfs_open(new->devnum, NULL, "sync_action");
 266         new->info.state_fd = sysfs_open(new->devnum, NULL, "array_state");
 267         new->resync_start_fd = sysfs_open(new->devnum, NULL, "resync_start");
 268         new->sync_pos_fd = sysfs_open(new->devnum, NULL, "sync_completed");
 269         new->sync_pos = 0;
 270
 271         sysfs_free(mdi);
 272         // finds and compares.
 273         if (container->ss->open_new(container, new, inst) < 0) {
 274                 // FIXME close all those files
 275                 new->container = NULL;
 276                 replace_array(container, victim, new);
 277                 return;
 278         }
 279         replace_array(container, victim, new);
 280         return;
 281 }
 282
 283 void manage(struct mdstat_ent *mdstat, struct supertype *container)
 284 {
 285         /* We have just read mdstat and need to compare it with
 286          * the known active arrays.
 287          * Arrays with the wrong metadata are ignored.
 288          */
 289
 290         for ( ; mdstat ; mdstat = mdstat->next) {
 291                 struct active_array *a;
 292                 if (mdstat->devnum == container->devnum) {
 293                         manage_container(mdstat, container);
 294                         continue;
 295                 }
 296                 if (mdstat->metadata_version == NULL ||
 297                     strncmp(mdstat->metadata_version, "external:/", 10) != 0 ||
 298                     strncmp(mdstat->metadata_version+10, container->devname,
 299                             strlen(container->devname)) != 0 ||
 300                     mdstat->metadata_version[10+strlen(container->devname)]
 301                       != '/')
 302                         /* Not for this array */
 303                         continue;
 304                 /* Looks like a member of this container */
 305                 for (a = container->arrays; a; a = a->next) {
 306                         if (mdstat->devnum == a->devnum) {
 307                                 if (a->container)
 308                                         manage_member(mdstat, a);
 309                                 break;
 310                         }
 311                 }
 312                 if (a == NULL || !a->container)
 313                         manage_new(mdstat, container, a);
 314         }
 315 }
 316
 317 static int handle_message(struct supertype *container, struct md_message *msg)
 318 {
 319         int err;
 320         struct md_generic_cmd *cmd = msg->buf;
 321
 322         if (!cmd)
 323                 return 0;
 324
 325         switch (cmd->action) {
 326         case md_action_remove_device:
 327
 328                 /* forward to the monitor */
 329                 active_cmd = cmd;
 330                 write(container->mgr_pipe[1], &err, 1);
 331                 read(container->mon_pipe[0], &err, 1);
 332                 return err;
 333
 334         default:
 335                 return -1;
 336         }
 337 }
 338
 339 void read_sock(struct supertype *container)
 340 {
 341         int fd;
 342         struct md_message msg;
 343         int terminate = 0;
 344         long fl;
 345         int tmo = 3; /* 3 second timeout before hanging up the socket */
 346
 347         fd = accept(container->sock, NULL, NULL);
 348         if (fd < 0)
 349                 return;
 350
 351         fl = fcntl(fd, F_GETFL, 0);
 352         fl |= O_NONBLOCK;
 353         fcntl(fd, F_SETFL, fl);
 354
 355         do {
 356                 int err;
 357
 358                 msg.buf = NULL;
 359
 360                 /* read and validate the message */
 361                 if (receive_message(fd, &msg, tmo) == 0) {
 362                         err = handle_message(container, &msg);
 363                         if (!err)
 364                                 ack(fd, msg.seq, tmo);
 365                         else
 366                                 nack(fd, err, tmo);
 367                 } else {
 368                         terminate = 1;
 369                         nack(fd, -1, tmo);
 370                 }
 371
 372                 if (msg.buf)
 373                         free(msg.buf);
 374         } while (!terminate);
 375
 376         close(fd);
 377 }
 378
 379 static int woke = 0;
 380 void wake_me(int sig)
 381 {
 382         woke = 1;
 383 }
 384
 385 void do_manager(struct supertype *container)
 386 {
 387         struct mdstat_ent *mdstat;
 388         sigset_t block, orig;
 389
 390         sigemptyset(&block);
 391         sigaddset(&block, SIGUSR1);
 392
 393         signal(SIGUSR1, wake_me);
 394
 395         do {
 396                 woke = 0;
 397
 398                 mdstat = mdstat_read(1, 0);
 399
 400                 manage(mdstat, container);
 401
 402                 read_sock(container);
 403
 404                 free_mdstat(mdstat);
 405
 406                 remove_old();
 407
 408                 sigprocmask(SIG_SETMASK, &block, &orig);
 409                 if (woke == 0)
 410                         mdstat_wait_fd(container->sock, &orig);
 411                 sigprocmask(SIG_SETMASK, &orig, NULL);
 412         } while(1);
 413 }