Monitor.c

   1 /*
   2  * mdadm - manage Linux "md" devices aka RAID arrays.
   3  *
   4  * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
   5  *
   6  *
   7  *    This program is free software; you can redistribute it and/or modify
   8  *    it under the terms of the GNU General Public License as published by
   9  *    the Free Software Foundation; either version 2 of the License, or
  10  *    (at your option) any later version.
  11  *
  12  *    This program is distributed in the hope that it will be useful,
  13  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *    GNU General Public License for more details.
  16  *
  17  *    You should have received a copy of the GNU General Public License
  18  *    along with this program; if not, write to the Free Software
  19  *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  *
  21  *    Author: Neil Brown
  22  *    Email: <neilb@suse.de>
  23  */
  24
  25 #include        "mdadm.h"
  26 #include        "md_p.h"
  27 #include        "md_u.h"
  28 #include        <sys/wait.h>
  29 #include        <signal.h>
  30 #include        <limits.h>
  31 #include        <syslog.h>
  32
  33 struct state {
  34         char *devname;
  35         char devnm[32]; /* to sync with mdstat info */
  36         unsigned int utime;
  37         int err;
  38         char *spare_group;
  39         int active, working, failed, spare, raid;
  40         int from_config;
  41         int from_auto;
  42         int expected_spares;
  43         int devstate[MAX_DISKS];
  44         dev_t devid[MAX_DISKS];
  45         int percent;
  46         char parent_devnm[32]; /* For subarray, devnm of parent.
  47                                 * For others, ""
  48                                 */
  49         struct supertype *metadata;
  50         struct state *subarray;/* for a container it is a link to first subarray
  51                                 * for a subarray it is a link to next subarray
  52                                 * in the same container */
  53         struct state *parent;  /* for a subarray it is a link to its container
  54                                 */
  55         struct state *next;
  56 };
  57
  58 struct alert_info {
  59         char *mailaddr;
  60         char *mailfrom;
  61         char *alert_cmd;
  62         int dosyslog;
  63 };
  64 static int make_daemon(char *pidfile);
  65 static int check_one_sharer(int scan);
  66 static void alert(char *event, char *dev, char *disc, struct alert_info *info);
  67 static int check_array(struct state *st, struct mdstat_ent *mdstat,
  68                        int test, struct alert_info *info,
  69                        int increments, char *prefer);
  70 static int add_new_arrays(struct mdstat_ent *mdstat, struct state **statelist,
  71                           int test, struct alert_info *info);
  72 static void try_spare_migration(struct state *statelist, struct alert_info *info);
  73 static void link_containers_with_subarrays(struct state *list);
  74
  75 int Monitor(struct mddev_dev *devlist,
  76             char *mailaddr, char *alert_cmd,
  77             struct context *c,
  78             int daemonise, int oneshot,
  79             int dosyslog, char *pidfile, int increments,
  80             int share)
  81 {
  82         /*
  83          * Every few seconds, scan every md device looking for changes
  84          * When a change is found, log it, possibly run the alert command,
  85          * and possibly send Email
  86          *
  87          * For each array, we record:
  88          *   Update time
  89          *   active/working/failed/spare drives
  90          *   State of each device.
  91          *   %rebuilt if rebuilding
  92          *
  93          * If the update time changes, check out all the data again
  94          * It is possible that we cannot get the state of each device
  95          * due to bugs in the md kernel module.
  96          * We also read /proc/mdstat to get rebuild percent,
  97          * and to get state on all active devices incase of kernel bug.
  98          *
  99          * Events are:
 100          *    Fail
 101          *      An active device had Faulty set or Active/Sync removed
 102          *    FailSpare
 103          *      A spare device had Faulty set
 104          *    SpareActive
 105          *      An active device had a reverse transition
 106          *    RebuildStarted
 107          *      percent went from -1 to +ve
 108          *    RebuildNN
 109          *      percent went from below to not-below NN%
 110          *    DeviceDisappeared
 111          *      Couldn't access a device which was previously visible
 112          *
 113          * if we detect an array with active<raid and spare==0
 114          * we look at other arrays that have same spare-group
 115          * If we find one with active==raid and spare>0,
 116          *  and if we can get_disk_info and find a name
 117          *  Then we hot-remove and hot-add to the other array
 118          *
 119          * If devlist is NULL, then we can monitor everything because --scan
 120          * was given.  We get an initial list from config file and add anything
 121          * that appears in /proc/mdstat
 122          */
 123
 124         struct state *statelist = NULL;
 125         struct state *st2;
 126         int finished = 0;
 127         struct mdstat_ent *mdstat = NULL;
 128         char *mailfrom = NULL;
 129         struct alert_info info;
 130         struct mddev_ident *mdlist;
 131
 132         if (!mailaddr) {
 133                 mailaddr = conf_get_mailaddr();
 134                 if (mailaddr && ! c->scan)
 135                         pr_err("Monitor using email address \"%s\" from config file\n",
 136                                mailaddr);
 137         }
 138         mailfrom = conf_get_mailfrom();
 139
 140         if (!alert_cmd) {
 141                 alert_cmd = conf_get_program();
 142                 if (alert_cmd && ! c->scan)
 143                         pr_err("Monitor using program \"%s\" from config file\n",
 144                                alert_cmd);
 145         }
 146         if (c->scan && !mailaddr && !alert_cmd && !dosyslog) {
 147                 pr_err("No mail address or alert command - not monitoring.\n");
 148                 return 1;
 149         }
 150         info.alert_cmd = alert_cmd;
 151         info.mailaddr = mailaddr;
 152         info.mailfrom = mailfrom;
 153         info.dosyslog = dosyslog;
 154
 155         if (daemonise) {
 156                 int rv = make_daemon(pidfile);
 157                 if (rv >= 0)
 158                         return rv;
 159         }
 160
 161         if (share)
 162                 if (check_one_sharer(c->scan))
 163                         return 1;
 164
 165         if (devlist == NULL) {
 166                 mdlist = conf_get_ident(NULL);
 167                 for (; mdlist; mdlist=mdlist->next) {
 168                         struct state *st;
 169                         if (mdlist->devname == NULL)
 170                                 continue;
 171                         if (strcasecmp(mdlist->devname, "<ignore>") == 0)
 172                                 continue;
 173                         st = xcalloc(1, sizeof *st);
 174                         if (mdlist->devname[0] == '/')
 175                                 st->devname = xstrdup(mdlist->devname);
 176                         else {
 177                                 st->devname = xmalloc(8+strlen(mdlist->devname)+1);
 178                                 strcpy(strcpy(st->devname, "/dev/md/"),
 179                                        mdlist->devname);
 180                         }
 181                         st->next = statelist;
 182                         st->devnm[0] = 0;
 183                         st->percent = RESYNC_UNKNOWN;
 184                         st->from_config = 1;
 185                         st->expected_spares = mdlist->spare_disks;
 186                         if (mdlist->spare_group)
 187                                 st->spare_group = xstrdup(mdlist->spare_group);
 188                         statelist = st;
 189                 }
 190         } else {
 191                 struct mddev_dev *dv;
 192                 for (dv=devlist ; dv; dv=dv->next) {
 193                         struct state *st = xcalloc(1, sizeof *st);
 194                         mdlist = conf_get_ident(dv->devname);
 195                         st->devname = xstrdup(dv->devname);
 196                         st->next = statelist;
 197                         st->devnm[0] = 0;
 198                         st->percent = RESYNC_UNKNOWN;
 199                         st->expected_spares = -1;
 200                         if (mdlist) {
 201                                 st->expected_spares = mdlist->spare_disks;
 202                                 if (mdlist->spare_group)
 203                                         st->spare_group = xstrdup(mdlist->spare_group);
 204                         }
 205                         statelist = st;
 206                 }
 207         }
 208
 209         while (! finished) {
 210                 int new_found = 0;
 211                 struct state *st, **stp;
 212                 int anydegraded = 0;
 213
 214                 if (mdstat)
 215                         free_mdstat(mdstat);
 216                 mdstat = mdstat_read(oneshot?0:1, 0);
 217                 if (!mdstat)
 218                         mdstat_close();
 219
 220                 for (st=statelist; st; st=st->next)
 221                         if (check_array(st, mdstat, c->test, &info,
 222                                         increments, c->prefer))
 223                                 anydegraded = 1;
 224
 225                 /* now check if there are any new devices found in mdstat */
 226                 if (c->scan)
 227                         new_found = add_new_arrays(mdstat, &statelist, c->test,
 228                                                    &info);
 229
 230                 /* If an array has active < raid && spare == 0 && spare_group != NULL
 231                  * Look for another array with spare > 0 and active == raid and same spare_group
 232                  *  if found, choose a device and hotremove/hotadd
 233                  */
 234                 if (share && anydegraded)
 235                         try_spare_migration(statelist, &info);
 236                 if (!new_found) {
 237                         if (oneshot)
 238                                 break;
 239                         else
 240                                 mdstat_wait(c->delay);
 241                 }
 242                 c->test = 0;
 243
 244                 for (stp = &statelist; (st = *stp) != NULL; ) {
 245                         if (st->from_auto && st->err > 5) {
 246                                 *stp = st->next;
 247                                 free(st->devname);
 248                                 free(st->spare_group);
 249                                 free(st);
 250                         } else
 251                                 stp = &st->next;
 252                 }
 253         }
 254         for (st2 = statelist; st2; st2 = statelist) {
 255                 statelist = st2->next;
 256                 free(st2);
 257         }
 258
 259         if (pidfile)
 260                 unlink(pidfile);
 261         return 0;
 262 }
 263
 264 static int make_daemon(char *pidfile)
 265 {
 266         /* Return:
 267          * -1 in the forked daemon
 268          *  0 in the parent
 269          *  1 on error
 270          * so a none-negative becomes the exit code.
 271          */
 272         int pid = fork();
 273         if (pid > 0) {
 274                 if (!pidfile)
 275                         printf("%d\n", pid);
 276                 else {
 277                         FILE *pid_file;
 278                         pid_file=fopen(pidfile, "w");
 279                         if (!pid_file)
 280                                 perror("cannot create pid file");
 281                         else {
 282                                 fprintf(pid_file,"%d\n", pid);
 283                                 fclose(pid_file);
 284                         }
 285                 }
 286                 return 0;
 287         }
 288         if (pid < 0) {
 289                 perror("daemonise");
 290                 return 1;
 291         }
 292         close(0);
 293         open("/dev/null", O_RDWR);
 294         dup2(0,1);
 295         dup2(0,2);
 296         setsid();
 297         return -1;
 298 }
 299
 300 static int check_one_sharer(int scan)
 301 {
 302         int pid, rv;
 303         FILE *fp;
 304         char dir[20];
 305         char path[100];
 306         struct stat buf;
 307         sprintf(path, "%s/autorebuild.pid", MDMON_DIR);
 308         fp = fopen(path, "r");
 309         if (fp) {
 310                 if (fscanf(fp, "%d", &pid) != 1)
 311                         pid = -1;
 312                 sprintf(dir, "/proc/%d", pid);
 313                 rv = stat(dir, &buf);
 314                 if (rv != -1) {
 315                         if (scan) {
 316                                 pr_err("Only one autorebuild process allowed in scan mode, aborting\n");
 317                                 fclose(fp);
 318                                 return 1;
 319                         } else {
 320                                 pr_err("Warning: One autorebuild process already running.\n");
 321                         }
 322                 }
 323                 fclose(fp);
 324         }
 325         if (scan) {
 326                 if (mkdir(MDMON_DIR, S_IRWXU) < 0 &&
 327                     errno != EEXIST) {
 328                         pr_err("Can't create autorebuild.pid file\n");
 329                 } else {
 330                         fp = fopen(path, "w");
 331                         if (!fp)
 332                                 pr_err("Cannot create autorebuild.pidfile\n");
 333                         else {
 334                                 pid = getpid();
 335                                 fprintf(fp, "%d\n", pid);
 336                                 fclose(fp);
 337                         }
 338                 }
 339         }
 340         return 0;
 341 }
 342
 343 static void alert(char *event, char *dev, char *disc, struct alert_info *info)
 344 {
 345         int priority;
 346
 347         if (!info->alert_cmd && !info->mailaddr && !info->dosyslog) {
 348                 time_t now = time(0);
 349
 350                 printf("%1.15s: %s on %s %s\n", ctime(&now)+4, event, dev, disc?disc:"unknown device");
 351         }
 352         if (info->alert_cmd) {
 353                 int pid = fork();
 354                 switch(pid) {
 355                 default:
 356                         waitpid(pid, NULL, 0);
 357                         break;
 358                 case -1:
 359                         break;
 360                 case 0:
 361                         execl(info->alert_cmd, info->alert_cmd,
 362                               event, dev, disc, NULL);
 363                         exit(2);
 364                 }
 365         }
 366         if (info->mailaddr &&
 367             (strncmp(event, "Fail", 4)==0 ||
 368              strncmp(event, "Test", 4)==0 ||
 369              strncmp(event, "Spares", 6)==0 ||
 370              strncmp(event, "Degrade", 7)==0)) {
 371                 FILE *mp = popen(Sendmail, "w");
 372                 if (mp) {
 373                         FILE *mdstat;
 374                         char hname[256];
 375                         gethostname(hname, sizeof(hname));
 376                         signal(SIGPIPE, SIG_IGN);
 377                         if (info->mailfrom)
 378                                 fprintf(mp, "From: %s\n", info->mailfrom);
 379                         else
 380                                 fprintf(mp, "From: %s monitoring <root>\n", Name);
 381                         fprintf(mp, "To: %s\n", info->mailaddr);
 382                         fprintf(mp, "Subject: %s event on %s:%s\n\n",
 383                                 event, dev, hname);
 384
 385                         fprintf(mp,
 386                                 "This is an automatically generated mail message from %s\n", Name);
 387                         fprintf(mp, "running on %s\n\n", hname);
 388
 389                         fprintf(mp,
 390                                 "A %s event had been detected on md device %s.\n\n", event, dev);
 391
 392                         if (disc && disc[0] != ' ')
 393                                 fprintf(mp,
 394                                         "It could be related to component device %s.\n\n", disc);
 395                         if (disc && disc[0] == ' ')
 396                                 fprintf(mp, "Extra information:%s.\n\n", disc);
 397
 398                         fprintf(mp, "Faithfully yours, etc.\n");
 399
 400                         mdstat = fopen("/proc/mdstat", "r");
 401                         if (mdstat) {
 402                                 char buf[8192];
 403                                 int n;
 404                                 fprintf(mp,
 405                                         "\nP.S. The /proc/mdstat file currently contains the following:\n\n");
 406                                 while ( (n=fread(buf, 1, sizeof(buf), mdstat)) > 0)
 407                                         n=fwrite(buf, 1, n, mp);
 408                                 fclose(mdstat);
 409                         }
 410                         pclose(mp);
 411                 }
 412         }
 413
 414         /* log the event to syslog maybe */
 415         if (info->dosyslog) {
 416                 /* Log at a different severity depending on the event.
 417                  *
 418                  * These are the critical events:  */
 419                 if (strncmp(event, "Fail", 4)==0 ||
 420                     strncmp(event, "Degrade", 7)==0 ||
 421                     strncmp(event, "DeviceDisappeared", 17)==0)
 422                         priority = LOG_CRIT;
 423                 /* Good to know about, but are not failures: */
 424                 else if (strncmp(event, "Rebuild", 7)==0 ||
 425                          strncmp(event, "MoveSpare", 9)==0 ||
 426                          strncmp(event, "Spares", 6) != 0)
 427                         priority = LOG_WARNING;
 428                 /* Everything else: */
 429                 else
 430                         priority = LOG_INFO;
 431
 432                 if (disc && disc[0] != ' ')
 433                         syslog(priority,
 434                                "%s event detected on md device %s, component device %s", event, dev, disc);
 435                 else if (disc)
 436                         syslog(priority,
 437                                "%s event detected on md device %s: %s",
 438                                event, dev, disc);
 439                 else
 440                         syslog(priority,
 441                                "%s event detected on md device %s",
 442                                event, dev);
 443         }
 444 }
 445
 446 static int check_array(struct state *st, struct mdstat_ent *mdstat,
 447                        int test, struct alert_info *ainfo,
 448                        int increments, char *prefer)
 449 {
 450         /* Update the state 'st' to reflect any changes shown in mdstat,
 451          * or found by directly examining the array, and return
 452          * '1' if the array is degraded, or '0' if it is optimal (or dead).
 453          */
 454         struct { int state, major, minor; } info[MAX_DISKS];
 455         struct mdinfo *sra = NULL;
 456         mdu_array_info_t array;
 457         struct mdstat_ent *mse = NULL, *mse2;
 458         char *dev = st->devname;
 459         int fd;
 460         int i;
 461         int remaining_disks;
 462         int last_disk;
 463         int new_array = 0;
 464         int retval;
 465
 466         if (test)
 467                 alert("TestMessage", dev, NULL, ainfo);
 468
 469         retval = 0;
 470
 471         fd = open(dev, O_RDONLY);
 472         if (fd < 0)
 473                 goto disappeared;
 474
 475         if (!md_array_active(fd))
 476                 goto disappeared;
 477
 478         fcntl(fd, F_SETFD, FD_CLOEXEC);
 479         if (md_get_array_info(fd, &array) < 0)
 480                 goto disappeared;
 481
 482         if (st->devnm[0] == 0)
 483                 strcpy(st->devnm, fd2devnm(fd));
 484
 485         sra = sysfs_read(-1, st->devnm, GET_LEVEL | GET_DISKS | GET_DEGRADED |
 486                          GET_MISMATCH | GET_DEVS | GET_STATE);
 487         if (!sra)
 488                 goto disappeared;
 489
 490         /* It's much easier to list what array levels can't
 491          * have a device disappear than all of them that can
 492          */
 493         if (sra->array.level == 0 || sra->array.level == -1) {
 494                 if (!st->err && !st->from_config)
 495                         alert("DeviceDisappeared", dev, " Wrong-Level", ainfo);
 496                 st->err++;
 497                 goto out;
 498         }
 499
 500         for (mse2 = mdstat ; mse2 ; mse2=mse2->next)
 501                 if (strcmp(mse2->devnm, st->devnm) == 0) {
 502                         mse2->devnm[0] = 0; /* flag it as "used" */
 503                         mse = mse2;
 504                 }
 505
 506         if (!mse) {
 507                 /* duplicated array in statelist
 508                  * or re-created after reading mdstat*/
 509                 st->err++;
 510                 close(fd);
 511                 goto out;
 512         }
 513         /* this array is in /proc/mdstat */
 514         if (array.utime == 0)
 515                 /* external arrays don't update utime, so
 516                  * just make sure it is always different. */
 517                 array.utime = st->utime + 1;;
 518
 519         if (st->err) {
 520                 /* New array appeared where previously had an error */
 521                 st->err = 0;
 522                 st->percent = RESYNC_NONE;
 523                 new_array = 1;
 524                 alert("NewArray", st->devname, NULL, ainfo);
 525         }
 526
 527         if (st->utime == array.utime && st->failed == sra->array.failed_disks &&
 528             st->working == sra->array.working_disks &&
 529             st->spare == sra->array.spare_disks &&
 530             (mse == NULL  || (mse->percent == st->percent))) {
 531                 if ((st->active < st->raid) && st->spare == 0)
 532                         retval = 1;
 533                 goto out;
 534         }
 535         if (st->utime == 0 && /* new array */
 536             mse->pattern && strchr(mse->pattern, '_') /* degraded */)
 537                 alert("DegradedArray", dev, NULL, ainfo);
 538
 539         if (st->utime == 0 && /* new array */ st->expected_spares > 0 &&
 540             sra->array.spare_disks < st->expected_spares)
 541                 alert("SparesMissing", dev, NULL, ainfo);
 542         if (st->percent < 0 && st->percent != RESYNC_UNKNOWN &&
 543             mse->percent >= 0)
 544                 alert("RebuildStarted", dev, NULL, ainfo);
 545         if (st->percent >= 0 && mse->percent >= 0 &&
 546             (mse->percent / increments) > (st->percent / increments)) {
 547                 char percentalert[15];
 548                 /*
 549                  * "RebuildNN" (10 chars) or "RebuildStarted" (15 chars)
 550                  */
 551
 552                 if((mse->percent / increments) == 0)
 553                         snprintf(percentalert, sizeof(percentalert),
 554                                  "RebuildStarted");
 555                 else
 556                         snprintf(percentalert, sizeof(percentalert),
 557                                  "Rebuild%02d", mse->percent);
 558
 559                 alert(percentalert, dev, NULL, ainfo);
 560         }
 561
 562         if (mse->percent == RESYNC_NONE && st->percent >= 0) {
 563                 /* Rebuild/sync/whatever just finished.
 564                  * If there is a number in /mismatch_cnt,
 565                  * we should report that.
 566                  */
 567                 if (sra && sra->mismatch_cnt > 0) {
 568                         char cnt[80];
 569                         snprintf(cnt, sizeof(cnt),
 570                                  " mismatches found: %d (on raid level %d)",
 571                                 sra->mismatch_cnt, sra->array.level);
 572                         alert("RebuildFinished", dev, cnt, ainfo);
 573                 } else
 574                         alert("RebuildFinished", dev, NULL, ainfo);
 575         }
 576         st->percent = mse->percent;
 577
 578         remaining_disks = sra->array.nr_disks;
 579         for (i = 0; i < MAX_DISKS && remaining_disks > 0; i++) {
 580                 mdu_disk_info_t disc;
 581                 disc.number = i;
 582                 if (md_get_disk_info(fd, &disc) >= 0) {
 583                         info[i].state = disc.state;
 584                         info[i].major = disc.major;
 585                         info[i].minor = disc.minor;
 586                         if (disc.major || disc.minor)
 587                                 remaining_disks --;
 588                 } else
 589                         info[i].major = info[i].minor = 0;
 590         }
 591         last_disk = i;
 592
 593         if (mse->metadata_version &&
 594             strncmp(mse->metadata_version, "external:", 9) == 0 &&
 595             is_subarray(mse->metadata_version+9)) {
 596                 char *sl;
 597                 strcpy(st->parent_devnm, mse->metadata_version+10);
 598                 sl = strchr(st->parent_devnm, '/');
 599                 if (sl)
 600                         *sl = 0;
 601         } else
 602                 st->parent_devnm[0] = 0;
 603         if (st->metadata == NULL && st->parent_devnm[0] == 0)
 604                 st->metadata = super_by_fd(fd, NULL);
 605
 606         for (i=0; i<MAX_DISKS; i++) {
 607                 mdu_disk_info_t disc = {0,0,0,0,0};
 608                 int newstate=0;
 609                 int change;
 610                 char *dv = NULL;
 611                 disc.number = i;
 612                 if (i < last_disk && (info[i].major || info[i].minor)) {
 613                         newstate = info[i].state;
 614                         dv = map_dev_preferred(info[i].major, info[i].minor, 1,
 615                                                prefer);
 616                         disc.state = newstate;
 617                         disc.major = info[i].major;
 618                         disc.minor = info[i].minor;
 619                 } else
 620                         newstate = (1 << MD_DISK_REMOVED);
 621
 622                 if (dv == NULL && st->devid[i])
 623                         dv = map_dev_preferred(major(st->devid[i]),
 624                                                minor(st->devid[i]), 1, prefer);
 625                 change = newstate ^ st->devstate[i];
 626                 if (st->utime && change && !st->err && !new_array) {
 627                         if ((st->devstate[i]&change) & (1 << MD_DISK_SYNC))
 628                                 alert("Fail", dev, dv, ainfo);
 629                         else if ((newstate & (1 << MD_DISK_FAULTY)) &&
 630                                  (disc.major || disc.minor) &&
 631                                  st->devid[i] == makedev(disc.major,
 632                                                          disc.minor))
 633                                 alert("FailSpare", dev, dv, ainfo);
 634                         else if ((newstate&change) & (1 << MD_DISK_SYNC))
 635                                 alert("SpareActive", dev, dv, ainfo);
 636                 }
 637                 st->devstate[i] = newstate;
 638                 st->devid[i] = makedev(disc.major, disc.minor);
 639         }
 640         st->active = sra->array.active_disks;
 641         st->working = sra->array.working_disks;
 642         st->spare = sra->array.spare_disks;
 643         st->failed = sra->array.failed_disks;
 644         st->utime = array.utime;
 645         st->raid = sra->array.raid_disks;
 646         st->err = 0;
 647         if ((st->active < st->raid) && st->spare == 0)
 648                 retval = 1;
 649
 650  out:
 651         if (sra)
 652                 sysfs_free(sra);
 653         if (fd > 0)
 654                 close(fd);
 655         return retval;
 656
 657  disappeared:
 658         if (!st->err)
 659                 alert("DeviceDisappeared", dev, NULL, ainfo);
 660         st->err++;
 661         goto out;
 662 }
 663
 664 static int add_new_arrays(struct mdstat_ent *mdstat, struct state **statelist,
 665                           int test, struct alert_info *info)
 666 {
 667         struct mdstat_ent *mse;
 668         int new_found = 0;
 669         char *name;
 670
 671         for (mse=mdstat; mse; mse=mse->next)
 672                 if (mse->devnm[0] &&
 673                     (!mse->level  || /* retrieve containers */
 674                      (strcmp(mse->level, "raid0") != 0 &&
 675                       strcmp(mse->level, "linear") != 0))
 676                         ) {
 677                         struct state *st = xcalloc(1, sizeof *st);
 678                         mdu_array_info_t array;
 679                         int fd;
 680
 681                         name = get_md_name(mse->devnm);
 682                         if (!name) {
 683                                 free(st);
 684                                 continue;
 685                         }
 686
 687                         st->devname = xstrdup(name);
 688                         if ((fd = open(st->devname, O_RDONLY)) < 0 ||
 689                             md_get_array_info(fd, &array) < 0) {
 690                                 /* no such array */
 691                                 if (fd >= 0)
 692                                         close(fd);
 693                                 put_md_name(st->devname);
 694                                 free(st->devname);
 695                                 if (st->metadata) {
 696                                         st->metadata->ss->free_super(st->metadata);
 697                                         free(st->metadata);
 698                                 }
 699                                 free(st);
 700                                 continue;
 701                         }
 702                         close(fd);
 703                         st->next = *statelist;
 704                         st->err = 1;
 705                         st->from_auto = 1;
 706                         strcpy(st->devnm, mse->devnm);
 707                         st->percent = RESYNC_UNKNOWN;
 708                         st->expected_spares = -1;
 709                         if (mse->metadata_version &&
 710                             strncmp(mse->metadata_version, "external:", 9) == 0 &&
 711                             is_subarray(mse->metadata_version+9)) {
 712                                 char *sl;
 713                                 strcpy(st->parent_devnm,
 714                                         mse->metadata_version+10);
 715                                 sl = strchr(st->parent_devnm, '/');
 716                                 *sl = 0;
 717                         } else
 718                                 st->parent_devnm[0] = 0;
 719                         *statelist = st;
 720                         if (test)
 721                                 alert("TestMessage", st->devname, NULL, info);
 722                         new_found = 1;
 723                 }
 724         return new_found;
 725 }
 726
 727 static int get_required_spare_criteria(struct state *st,
 728                                        struct spare_criteria *sc)
 729 {
 730         int fd;
 731
 732         if (!st->metadata ||
 733             !st->metadata->ss->get_spare_criteria) {
 734                 sc->min_size = 0;
 735                 sc->sector_size = 0;
 736                 return 0;
 737         }
 738
 739         fd = open(st->devname, O_RDONLY);
 740         if (fd < 0)
 741                 return 1;
 742         if (st->metadata->ss->external)
 743                 st->metadata->ss->load_container(st->metadata, fd, st->devname);
 744         else
 745                 st->metadata->ss->load_super(st->metadata, fd, st->devname);
 746         close(fd);
 747         if (!st->metadata->sb)
 748                 return 1;
 749
 750         st->metadata->ss->get_spare_criteria(st->metadata, sc);
 751         st->metadata->ss->free_super(st->metadata);
 752
 753         return 0;
 754 }
 755
 756 static int check_donor(struct state *from, struct state *to)
 757 {
 758         struct state *sub;
 759
 760         if (from == to)
 761                 return 0;
 762         if (from->parent)
 763                 /* Cannot move from a member */
 764                 return 0;
 765         if (from->err)
 766                 return 0;
 767         for (sub = from->subarray; sub; sub = sub->subarray)
 768                 /* If source array has degraded subarrays, don't
 769                  * remove anything
 770                  */
 771                 if (sub->active < sub->raid)
 772                         return 0;
 773         if (from->metadata->ss->external == 0)
 774                 if (from->active < from->raid)
 775                         return 0;
 776         if (from->spare <= 0)
 777                 return 0;
 778         return 1;
 779 }
 780
 781 static dev_t choose_spare(struct state *from, struct state *to,
 782                         struct domainlist *domlist, struct spare_criteria *sc)
 783 {
 784         int d;
 785         dev_t dev = 0;
 786
 787         for (d = from->raid; !dev && d < MAX_DISKS; d++) {
 788                 if (from->devid[d] > 0 &&
 789                     from->devstate[d] == 0) {
 790                         struct dev_policy *pol;
 791                         unsigned long long dev_size;
 792                         unsigned int dev_sector_size;
 793
 794                         if (to->metadata->ss->external &&
 795                             test_partition_from_id(from->devid[d]))
 796                                 continue;
 797
 798                         if (sc->min_size &&
 799                             dev_size_from_id(from->devid[d], &dev_size) &&
 800                             dev_size < sc->min_size)
 801                                 continue;
 802
 803                         if (sc->sector_size &&
 804                             dev_sector_size_from_id(from->devid[d],
 805                                                     &dev_sector_size) &&
 806                             sc->sector_size != dev_sector_size)
 807                                 continue;
 808
 809                         pol = devid_policy(from->devid[d]);
 810                         if (from->spare_group)
 811                                 pol_add(&pol, pol_domain,
 812                                         from->spare_group, NULL);
 813                         if (domain_test(domlist, pol, to->metadata->ss->name) == 1)
 814                             dev = from->devid[d];
 815                         dev_policy_free(pol);
 816                 }
 817         }
 818         return dev;
 819 }
 820
 821 static dev_t container_choose_spare(struct state *from, struct state *to,
 822                                     struct domainlist *domlist,
 823                                     struct spare_criteria *sc, int active)
 824 {
 825         /* This is similar to choose_spare, but we cannot trust devstate,
 826          * so we need to read the metadata instead
 827          */
 828         struct mdinfo *list;
 829         struct supertype *st = from->metadata;
 830         int fd = open(from->devname, O_RDONLY);
 831         int err;
 832         dev_t dev = 0;
 833
 834         if (fd < 0)
 835                 return 0;
 836         if (!st->ss->getinfo_super_disks) {
 837                 close(fd);
 838                 return 0;
 839         }
 840
 841         err = st->ss->load_container(st, fd, NULL);
 842         close(fd);
 843         if (err)
 844                 return 0;
 845
 846         if (from == to) {
 847                 /* We must check if number of active disks has not increased
 848                  * since ioctl in main loop. mdmon may have added spare
 849                  * to subarray. If so we do not need to look for more spares
 850                  * so return non zero value */
 851                 int active_cnt = 0;
 852                 struct mdinfo *dp;
 853                 list = st->ss->getinfo_super_disks(st);
 854                 if (!list) {
 855                         st->ss->free_super(st);
 856                         return 1;
 857                 }
 858                 dp = list->devs;
 859                 while (dp) {
 860                         if (dp->disk.state & (1<<MD_DISK_SYNC) &&
 861                             !(dp->disk.state & (1<<MD_DISK_FAULTY)))
 862                                 active_cnt++;
 863                         dp = dp->next;
 864                 }
 865                 sysfs_free(list);
 866                 if (active < active_cnt) {
 867                         /* Spare just activated.*/
 868                         st->ss->free_super(st);
 869                         return 1;
 870                 }
 871         }
 872
 873         /* We only need one spare so full list not needed */
 874         list = container_choose_spares(st, sc, domlist, from->spare_group,
 875                                        to->metadata->ss->name, 1);
 876         if (list) {
 877                 struct mdinfo *disks = list->devs;
 878                 if (disks)
 879                         dev = makedev(disks->disk.major, disks->disk.minor);
 880                 sysfs_free(list);
 881         }
 882         st->ss->free_super(st);
 883         return dev;
 884 }
 885
 886 static void try_spare_migration(struct state *statelist, struct alert_info *info)
 887 {
 888         struct state *from;
 889         struct state *st;
 890         struct spare_criteria sc;
 891
 892         link_containers_with_subarrays(statelist);
 893         for (st = statelist; st; st = st->next)
 894                 if (st->active < st->raid &&
 895                     st->spare == 0 && !st->err) {
 896                         struct domainlist *domlist = NULL;
 897                         int d;
 898                         struct state *to = st;
 899
 900                         if (to->parent_devnm[0] && !to->parent)
 901                                 /* subarray monitored without parent container
 902                                  * we can't move spares here */
 903                                 continue;
 904
 905                         if (to->parent)
 906                                 /* member of a container */
 907                                 to = to->parent;
 908
 909                         if (get_required_spare_criteria(to, &sc))
 910                                 continue;
 911                         if (to->metadata->ss->external) {
 912                                 /* We must make sure there is
 913                                  * no suitable spare in container already.
 914                                  * If there is we don't add more */
 915                                 dev_t devid = container_choose_spare(
 916                                         to, to, NULL, &sc, st->active);
 917                                 if (devid > 0)
 918                                         continue;
 919                         }
 920                         for (d = 0; d < MAX_DISKS; d++)
 921                                 if (to->devid[d])
 922                                         domainlist_add_dev(&domlist,
 923                                                            to->devid[d],
 924                                                            to->metadata->ss->name);
 925                         if (to->spare_group)
 926                                 domain_add(&domlist, to->spare_group);
 927                         /*
 928                          * No spare migration if the destination
 929                          * has no domain. Skip this array.
 930                          */
 931                         if (!domlist)
 932                                 continue;
 933                         for (from=statelist ; from ; from=from->next) {
 934                                 dev_t devid;
 935                                 if (!check_donor(from, to))
 936                                         continue;
 937                                 if (from->metadata->ss->external)
 938                                         devid = container_choose_spare(
 939                                                 from, to, domlist, &sc, 0);
 940                                 else
 941                                         devid = choose_spare(from, to, domlist,
 942                                                              &sc);
 943                                 if (devid > 0
 944                                     && move_spare(from->devname, to->devname, devid)) {
 945                                         alert("MoveSpare", to->devname, from->devname, info);
 946                                         break;
 947                                 }
 948                         }
 949                         domain_free(domlist);
 950                 }
 951 }
 952
 953 /* search the statelist to connect external
 954  * metadata subarrays with their containers
 955  * We always completely rebuild the tree from scratch as
 956  * that is safest considering the possibility of entries
 957  * disappearing or changing.
 958  */
 959 static void link_containers_with_subarrays(struct state *list)
 960 {
 961         struct state *st;
 962         struct state *cont;
 963         for (st = list; st; st = st->next) {
 964                 st->parent = NULL;
 965                 st->subarray = NULL;
 966         }
 967         for (st = list; st; st = st->next)
 968                 if (st->parent_devnm[0])
 969                         for (cont = list; cont; cont = cont->next)
 970                                 if (!cont->err &&
 971                                     cont->parent_devnm[0] == 0 &&
 972                                     strcmp(cont->devnm, st->parent_devnm) == 0) {
 973                                         st->parent = cont;
 974                                         st->subarray = cont->subarray;
 975                                         cont->subarray = st;
 976                                         break;
 977                                 }
 978 }
 979
 980 /* Not really Monitor but ... */
 981 int Wait(char *dev)
 982 {
 983         char devnm[32];
 984         int rv = 1;
 985         int frozen_remaining = 3;
 986
 987         if (!stat_is_blkdev(dev, NULL))
 988                 return 2;
 989         strcpy(devnm, dev);
 990
 991         while(1) {
 992                 struct mdstat_ent *ms = mdstat_read(1, 0);
 993                 struct mdstat_ent *e;
 994
 995                 for (e=ms ; e; e=e->next)
 996                         if (strcmp(e->devnm, devnm) == 0)
 997                                 break;
 998
 999                 if (e && e->percent == RESYNC_NONE) {
1000                         /* We could be in the brief pause before something
1001                          * starts. /proc/mdstat doesn't show that, but
1002                          * sync_action does.
1003                          */
1004                         struct mdinfo mdi;
1005                         char buf[21];
1006
1007                         if (sysfs_init(&mdi, -1, devnm))
1008                                 return 2;
1009                         if (sysfs_get_str(&mdi, NULL, "sync_action",
1010                                           buf, 20) > 0 &&
1011                             strcmp(buf,"idle\n") != 0) {
1012                                 e->percent = RESYNC_UNKNOWN;
1013                                 if (strcmp(buf, "frozen\n") == 0) {
1014                                         if (frozen_remaining == 0)
1015                                                 e->percent = RESYNC_NONE;
1016                                         else
1017                                                 frozen_remaining -= 1;
1018                                 }
1019                         }
1020                 }
1021                 if (!e || e->percent == RESYNC_NONE) {
1022                         if (e && e->metadata_version &&
1023                             strncmp(e->metadata_version, "external:", 9) == 0) {
1024                                 if (is_subarray(&e->metadata_version[9]))
1025                                         ping_monitor(&e->metadata_version[9]);
1026                                 else
1027                                         ping_monitor(devnm);
1028                         }
1029                         free_mdstat(ms);
1030                         return rv;
1031                 }
1032                 free_mdstat(ms);
1033                 rv = 0;
1034                 mdstat_wait(5);
1035         }
1036 }
1037
1038 static char *clean_states[] = {
1039         "clear", "inactive", "readonly", "read-auto", "clean", NULL };
1040
1041 int WaitClean(char *dev, int sock, int verbose)
1042 {
1043         int fd;
1044         struct mdinfo *mdi;
1045         int rv = 1;
1046         char devnm[32];
1047
1048         if (!stat_is_blkdev(dev, NULL))
1049                 return 2;
1050         fd = open(dev, O_RDONLY);
1051         if (fd < 0) {
1052                 if (verbose)
1053                         pr_err("Couldn't open %s: %s\n", dev, strerror(errno));
1054                 return 1;
1055         }
1056
1057         strcpy(devnm, fd2devnm(fd));
1058         mdi = sysfs_read(fd, devnm, GET_VERSION|GET_LEVEL|GET_SAFEMODE);
1059         if (!mdi) {
1060                 if (verbose)
1061                         pr_err("Failed to read sysfs attributes for %s\n", dev);
1062                 close(fd);
1063                 return 0;
1064         }
1065
1066         switch(mdi->array.level) {
1067         case LEVEL_LINEAR:
1068         case LEVEL_MULTIPATH:
1069         case 0:
1070                 /* safemode delay is irrelevant for these levels */
1071                 rv = 0;
1072         }
1073
1074         /* for internal metadata the kernel handles the final clean
1075          * transition, containers can never be dirty
1076          */
1077         if (!is_subarray(mdi->text_version))
1078                 rv = 0;
1079
1080         /* safemode disabled ? */
1081         if (mdi->safe_mode_delay == 0)
1082                 rv = 0;
1083
1084         if (rv) {
1085                 int state_fd = sysfs_open(fd2devnm(fd), NULL, "array_state");
1086                 char buf[20];
1087                 int delay = 5000;
1088
1089                 /* minimize the safe_mode_delay and prepare to wait up to 5s
1090                  * for writes to quiesce
1091                  */
1092                 sysfs_set_safemode(mdi, 1);
1093
1094                 /* wait for array_state to be clean */
1095                 while (1) {
1096                         rv = read(state_fd, buf, sizeof(buf));
1097                         if (rv < 0)
1098                                 break;
1099                         if (sysfs_match_word(buf, clean_states) <= 4)
1100                                 break;
1101                         rv = sysfs_wait(state_fd, &delay);
1102                         if (rv < 0 && errno != EINTR)
1103                                 break;
1104                         lseek(state_fd, 0, SEEK_SET);
1105                 }
1106                 if (rv < 0)
1107                         rv = 1;
1108                 else if (fping_monitor(sock) == 0 ||
1109                          ping_monitor(mdi->text_version) == 0) {
1110                         /* we need to ping to close the window between array
1111                          * state transitioning to clean and the metadata being
1112                          * marked clean
1113                          */
1114                         rv = 0;
1115                 } else
1116                         rv = 1;
1117                 if (rv && verbose)
1118                         pr_err("Error waiting for %s to be clean\n",
1119                                 dev);
1120
1121                 /* restore the original safe_mode_delay */
1122                 sysfs_set_safemode(mdi, mdi->safe_mode_delay);
1123                 close(state_fd);
1124         }
1125
1126         sysfs_free(mdi);
1127         close(fd);
1128
1129         return rv;
1130 }