Monitor.c

   1 /*
   2  * mdadm - manage Linux "md" devices aka RAID arrays.
   3  *
   4  * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
   5  *
   6  *
   7  *    This program is free software; you can redistribute it and/or modify
   8  *    it under the terms of the GNU General Public License as published by
   9  *    the Free Software Foundation; either version 2 of the License, or
  10  *    (at your option) any later version.
  11  *
  12  *    This program is distributed in the hope that it will be useful,
  13  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
  14  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  15  *    GNU General Public License for more details.
  16  *
  17  *    You should have received a copy of the GNU General Public License
  18  *    along with this program; if not, write to the Free Software
  19  *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  20  *
  21  *    Author: Neil Brown
  22  *    Email: <neilb@suse.de>
  23  */
  24
  25 #include        "mdadm.h"
  26 #include        "md_p.h"
  27 #include        "md_u.h"
  28 #include        <sys/wait.h>
  29 #include        <signal.h>
  30 #include        <limits.h>
  31 #include        <syslog.h>
  32
  33 struct state {
  34         char *devname;
  35         char devnm[32]; /* to sync with mdstat info */
  36         unsigned int utime;
  37         int err;
  38         char *spare_group;
  39         int active, working, failed, spare, raid;
  40         int from_config;
  41         int from_auto;
  42         int expected_spares;
  43         int devstate[MAX_DISKS];
  44         dev_t devid[MAX_DISKS];
  45         int percent;
  46         char parent_devnm[32]; /* For subarray, devnm of parent.
  47                                 * For others, ""
  48                                 */
  49         struct supertype *metadata;
  50         struct state *subarray;/* for a container it is a link to first subarray
  51                                 * for a subarray it is a link to next subarray
  52                                 * in the same container */
  53         struct state *parent;  /* for a subarray it is a link to its container
  54                                 */
  55         struct state *next;
  56 };
  57
  58 struct alert_info {
  59         char *mailaddr;
  60         char *mailfrom;
  61         char *alert_cmd;
  62         int dosyslog;
  63 };
  64 static int make_daemon(char *pidfile);
  65 static int check_one_sharer(int scan);
  66 static void write_autorebuild_pid(void);
  67 static void alert(char *event, char *dev, char *disc, struct alert_info *info);
  68 static int check_array(struct state *st, struct mdstat_ent *mdstat,
  69                        int test, struct alert_info *info,
  70                        int increments, char *prefer);
  71 static int add_new_arrays(struct mdstat_ent *mdstat, struct state **statelist,
  72                           int test, struct alert_info *info);
  73 static void try_spare_migration(struct state *statelist, struct alert_info *info);
  74 static void link_containers_with_subarrays(struct state *list);
  75
  76 int Monitor(struct mddev_dev *devlist,
  77             char *mailaddr, char *alert_cmd,
  78             struct context *c,
  79             int daemonise, int oneshot,
  80             int dosyslog, char *pidfile, int increments,
  81             int share)
  82 {
  83         /*
  84          * Every few seconds, scan every md device looking for changes
  85          * When a change is found, log it, possibly run the alert command,
  86          * and possibly send Email
  87          *
  88          * For each array, we record:
  89          *   Update time
  90          *   active/working/failed/spare drives
  91          *   State of each device.
  92          *   %rebuilt if rebuilding
  93          *
  94          * If the update time changes, check out all the data again
  95          * It is possible that we cannot get the state of each device
  96          * due to bugs in the md kernel module.
  97          * We also read /proc/mdstat to get rebuild percent,
  98          * and to get state on all active devices incase of kernel bug.
  99          *
 100          * Events are:
 101          *    Fail
 102          *      An active device had Faulty set or Active/Sync removed
 103          *    FailSpare
 104          *      A spare device had Faulty set
 105          *    SpareActive
 106          *      An active device had a reverse transition
 107          *    RebuildStarted
 108          *      percent went from -1 to +ve
 109          *    RebuildNN
 110          *      percent went from below to not-below NN%
 111          *    DeviceDisappeared
 112          *      Couldn't access a device which was previously visible
 113          *
 114          * if we detect an array with active<raid and spare==0
 115          * we look at other arrays that have same spare-group
 116          * If we find one with active==raid and spare>0,
 117          *  and if we can get_disk_info and find a name
 118          *  Then we hot-remove and hot-add to the other array
 119          *
 120          * If devlist is NULL, then we can monitor everything because --scan
 121          * was given.  We get an initial list from config file and add anything
 122          * that appears in /proc/mdstat
 123          */
 124
 125         struct state *statelist = NULL;
 126         struct state *st2;
 127         int finished = 0;
 128         struct mdstat_ent *mdstat = NULL;
 129         char *mailfrom;
 130         struct alert_info info;
 131         struct mddev_ident *mdlist;
 132         int delay_for_event = c->delay;
 133
 134         if (!mailaddr) {
 135                 mailaddr = conf_get_mailaddr();
 136                 if (mailaddr && ! c->scan)
 137                         pr_err("Monitor using email address \"%s\" from config file\n",
 138                                mailaddr);
 139         }
 140         mailfrom = conf_get_mailfrom();
 141
 142         if (!alert_cmd) {
 143                 alert_cmd = conf_get_program();
 144                 if (alert_cmd && !c->scan)
 145                         pr_err("Monitor using program \"%s\" from config file\n",
 146                                alert_cmd);
 147         }
 148         if (c->scan && !mailaddr && !alert_cmd && !dosyslog) {
 149                 pr_err("No mail address or alert command - not monitoring.\n");
 150                 return 1;
 151         }
 152         info.alert_cmd = alert_cmd;
 153         info.mailaddr = mailaddr;
 154         info.mailfrom = mailfrom;
 155         info.dosyslog = dosyslog;
 156
 157         if (share){
 158                 if (check_one_sharer(c->scan))
 159                         return 1;
 160         }
 161
 162         if (daemonise) {
 163                 int rv = make_daemon(pidfile);
 164                 if (rv >= 0)
 165                         return rv;
 166         }
 167
 168         if (share)
 169                 write_autorebuild_pid();
 170
 171         if (devlist == NULL) {
 172                 mdlist = conf_get_ident(NULL);
 173                 for (; mdlist; mdlist = mdlist->next) {
 174                         struct state *st;
 175
 176                         if (mdlist->devname == NULL)
 177                                 continue;
 178                         if (strcasecmp(mdlist->devname, "<ignore>") == 0)
 179                                 continue;
 180                         st = xcalloc(1, sizeof *st);
 181                         if (mdlist->devname[0] == '/')
 182                                 st->devname = xstrdup(mdlist->devname);
 183                         else {
 184                                 st->devname = xmalloc(8+strlen(mdlist->devname)+1);
 185                                 strcpy(strcpy(st->devname, "/dev/md/"),
 186                                        mdlist->devname);
 187                         }
 188                         st->next = statelist;
 189                         st->devnm[0] = 0;
 190                         st->percent = RESYNC_UNKNOWN;
 191                         st->from_config = 1;
 192                         st->expected_spares = mdlist->spare_disks;
 193                         if (mdlist->spare_group)
 194                                 st->spare_group = xstrdup(mdlist->spare_group);
 195                         statelist = st;
 196                 }
 197         } else {
 198                 struct mddev_dev *dv;
 199
 200                 for (dv = devlist; dv; dv = dv->next) {
 201                         struct state *st = xcalloc(1, sizeof *st);
 202                         mdlist = conf_get_ident(dv->devname);
 203                         st->devname = xstrdup(dv->devname);
 204                         st->next = statelist;
 205                         st->devnm[0] = 0;
 206                         st->percent = RESYNC_UNKNOWN;
 207                         st->expected_spares = -1;
 208                         if (mdlist) {
 209                                 st->expected_spares = mdlist->spare_disks;
 210                                 if (mdlist->spare_group)
 211                                         st->spare_group = xstrdup(mdlist->spare_group);
 212                         }
 213                         statelist = st;
 214                 }
 215         }
 216
 217         while (!finished) {
 218                 int new_found = 0;
 219                 struct state *st, **stp;
 220                 int anydegraded = 0;
 221                 int anyredundant = 0;
 222
 223                 if (mdstat)
 224                         free_mdstat(mdstat);
 225                 mdstat = mdstat_read(oneshot ? 0 : 1, 0);
 226
 227                 for (st = statelist; st; st = st->next) {
 228                         if (check_array(st, mdstat, c->test, &info,
 229                                         increments, c->prefer))
 230                                 anydegraded = 1;
 231                         /* for external arrays, metadata is filled for
 232                          * containers only
 233                          */
 234                         if (st->metadata && st->metadata->ss->external)
 235                                 continue;
 236                         if (st->err == 0 && !anyredundant)
 237                                 anyredundant = 1;
 238                 }
 239
 240                 /* now check if there are any new devices found in mdstat */
 241                 if (c->scan)
 242                         new_found = add_new_arrays(mdstat, &statelist, c->test,
 243                                                    &info);
 244
 245                 /* If an array has active < raid && spare == 0 && spare_group != NULL
 246                  * Look for another array with spare > 0 and active == raid and same spare_group
 247                  *  if found, choose a device and hotremove/hotadd
 248                  */
 249                 if (share && anydegraded)
 250                         try_spare_migration(statelist, &info);
 251                 if (!new_found) {
 252                         if (oneshot)
 253                                 break;
 254                         else if (!anyredundant) {
 255                                 break;
 256                         }
 257                         else {
 258                                 int wait_result = mdstat_wait(delay_for_event);
 259
 260                                 /*
 261                                  * If mdmonitor is awaken by event, set small delay once
 262                                  * to deal with udev and mdadm.
 263                                  */
 264                                 if (wait_result != 0) {
 265                                         if (c->delay > 5)
 266                                                 delay_for_event = 5;
 267                                 } else
 268                                         delay_for_event = c->delay;
 269
 270                                 mdstat_close();
 271                         }
 272                 }
 273                 c->test = 0;
 274
 275                 for (stp = &statelist; (st = *stp) != NULL; ) {
 276                         if (st->from_auto && st->err > 5) {
 277                                 *stp = st->next;
 278                                 free(st->devname);
 279                                 free(st->spare_group);
 280                                 free(st);
 281                         } else
 282                                 stp = &st->next;
 283                 }
 284         }
 285         for (st2 = statelist; st2; st2 = statelist) {
 286                 statelist = st2->next;
 287                 free(st2);
 288         }
 289
 290         if (pidfile)
 291                 unlink(pidfile);
 292         return 0;
 293 }
 294
 295 static int make_daemon(char *pidfile)
 296 {
 297         /* Return:
 298          * -1 in the forked daemon
 299          *  0 in the parent
 300          *  1 on error
 301          * so a none-negative becomes the exit code.
 302          */
 303         int pid = fork();
 304         if (pid > 0) {
 305                 if (!pidfile)
 306                         printf("%d\n", pid);
 307                 else {
 308                         FILE *pid_file = NULL;
 309                         int fd = open(pidfile, O_WRONLY | O_CREAT | O_TRUNC,
 310                                       0644);
 311                         if (fd >= 0)
 312                                 pid_file = fdopen(fd, "w");
 313                         if (!pid_file)
 314                                 perror("cannot create pid file");
 315                         else {
 316                                 fprintf(pid_file,"%d\n", pid);
 317                                 fclose(pid_file);
 318                         }
 319                 }
 320                 return 0;
 321         }
 322         if (pid < 0) {
 323                 perror("daemonise");
 324                 return 1;
 325         }
 326         manage_fork_fds(0);
 327         setsid();
 328         return -1;
 329 }
 330
 331 static int check_one_sharer(int scan)
 332 {
 333         int pid;
 334         FILE *comm_fp;
 335         FILE *fp;
 336         char comm_path[PATH_MAX];
 337         char path[PATH_MAX];
 338         char comm[20];
 339
 340         sprintf(path, "%s/autorebuild.pid", MDMON_DIR);
 341         fp = fopen(path, "r");
 342         if (fp) {
 343                 if (fscanf(fp, "%d", &pid) != 1)
 344                         pid = -1;
 345                 snprintf(comm_path, sizeof(comm_path),
 346                          "/proc/%d/comm", pid);
 347                 comm_fp = fopen(comm_path, "r");
 348                 if (comm_fp) {
 349                         if (fscanf(comm_fp, "%s", comm) &&
 350                             strncmp(basename(comm), Name, strlen(Name)) == 0) {
 351                                 if (scan) {
 352                                         pr_err("Only one autorebuild process allowed in scan mode, aborting\n");
 353                                         fclose(comm_fp);
 354                                         fclose(fp);
 355                                         return 1;
 356                                 } else {
 357                                         pr_err("Warning: One autorebuild process already running.\n");
 358                                 }
 359                         }
 360                         fclose(comm_fp);
 361                 }
 362                 fclose(fp);
 363         }
 364         return 0;
 365 }
 366
 367 static void write_autorebuild_pid()
 368 {
 369         char path[PATH_MAX];
 370         int pid;
 371         FILE *fp = NULL;
 372         sprintf(path, "%s/autorebuild.pid", MDMON_DIR);
 373
 374         if (mkdir(MDMON_DIR, 0700) < 0 && errno != EEXIST) {
 375                 pr_err("Can't create autorebuild.pid file\n");
 376         } else {
 377                 int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, 0700);
 378
 379                 if (fd >= 0)
 380                         fp = fdopen(fd, "w");
 381
 382                 if (!fp)
 383                         pr_err("Can't create autorebuild.pid file\n");
 384                 else {
 385                         pid = getpid();
 386                         fprintf(fp, "%d\n", pid);
 387                         fclose(fp);
 388                 }
 389         }
 390 }
 391
 392 static void alert(char *event, char *dev, char *disc, struct alert_info *info)
 393 {
 394         int priority;
 395
 396         if (!info->alert_cmd && !info->mailaddr && !info->dosyslog) {
 397                 time_t now = time(0);
 398
 399                 printf("%1.15s: %s on %s %s\n", ctime(&now) + 4,
 400                        event, dev, disc?disc:"unknown device");
 401         }
 402         if (info->alert_cmd) {
 403                 int pid = fork();
 404                 switch(pid) {
 405                 default:
 406                         waitpid(pid, NULL, 0);
 407                         break;
 408                 case -1:
 409                         break;
 410                 case 0:
 411                         execl(info->alert_cmd, info->alert_cmd,
 412                               event, dev, disc, NULL);
 413                         exit(2);
 414                 }
 415         }
 416         if (info->mailaddr && (strncmp(event, "Fail", 4) == 0 ||
 417                                strncmp(event, "Test", 4) == 0 ||
 418                                strncmp(event, "Spares", 6) == 0 ||
 419                                strncmp(event, "Degrade", 7) == 0)) {
 420                 FILE *mp = popen(Sendmail, "w");
 421                 if (mp) {
 422                         FILE *mdstat;
 423                         char hname[256];
 424                         gethostname(hname, sizeof(hname));
 425                         signal(SIGPIPE, SIG_IGN);
 426                         if (info->mailfrom)
 427                                 fprintf(mp, "From: %s\n", info->mailfrom);
 428                         else
 429                                 fprintf(mp, "From: %s monitoring <root>\n",
 430                                         Name);
 431                         fprintf(mp, "To: %s\n", info->mailaddr);
 432                         fprintf(mp, "Subject: %s event on %s:%s\n\n",
 433                                 event, dev, hname);
 434
 435                         fprintf(mp,
 436                                 "This is an automatically generated mail message from %s\n", Name);
 437                         fprintf(mp, "running on %s\n\n", hname);
 438
 439                         fprintf(mp,
 440                                 "A %s event had been detected on md device %s.\n\n", event, dev);
 441
 442                         if (disc && disc[0] != ' ')
 443                                 fprintf(mp,
 444                                         "It could be related to component device %s.\n\n", disc);
 445                         if (disc && disc[0] == ' ')
 446                                 fprintf(mp, "Extra information:%s.\n\n", disc);
 447
 448                         fprintf(mp, "Faithfully yours, etc.\n");
 449
 450                         mdstat = fopen("/proc/mdstat", "r");
 451                         if (mdstat) {
 452                                 char buf[8192];
 453                                 int n;
 454                                 fprintf(mp,
 455                                         "\nP.S. The /proc/mdstat file currently contains the following:\n\n");
 456                                 while ((n = fread(buf, 1, sizeof(buf),
 457                                                   mdstat)) > 0)
 458                                         n = fwrite(buf, 1, n, mp);
 459                                 fclose(mdstat);
 460                         }
 461                         pclose(mp);
 462                 }
 463         }
 464
 465         /* log the event to syslog maybe */
 466         if (info->dosyslog) {
 467                 /* Log at a different severity depending on the event.
 468                  *
 469                  * These are the critical events:  */
 470                 if (strncmp(event, "Fail", 4) == 0 ||
 471                     strncmp(event, "Degrade", 7) == 0 ||
 472                     strncmp(event, "DeviceDisappeared", 17) == 0)
 473                         priority = LOG_CRIT;
 474                 /* Good to know about, but are not failures: */
 475                 else if (strncmp(event, "Rebuild", 7) == 0 ||
 476                          strncmp(event, "MoveSpare", 9) == 0 ||
 477                          strncmp(event, "Spares", 6) != 0)
 478                         priority = LOG_WARNING;
 479                 /* Everything else: */
 480                 else
 481                         priority = LOG_INFO;
 482
 483                 if (disc && disc[0] != ' ')
 484                         syslog(priority,
 485                                "%s event detected on md device %s, component device %s", event, dev, disc);
 486                 else if (disc)
 487                         syslog(priority,
 488                                "%s event detected on md device %s: %s",
 489                                event, dev, disc);
 490                 else
 491                         syslog(priority,
 492                                "%s event detected on md device %s",
 493                                event, dev);
 494         }
 495 }
 496
 497 static int check_array(struct state *st, struct mdstat_ent *mdstat,
 498                        int test, struct alert_info *ainfo,
 499                        int increments, char *prefer)
 500 {
 501         /* Update the state 'st' to reflect any changes shown in mdstat,
 502          * or found by directly examining the array, and return
 503          * '1' if the array is degraded, or '0' if it is optimal (or dead).
 504          */
 505         struct { int state, major, minor; } info[MAX_DISKS];
 506         struct mdinfo *sra = NULL;
 507         mdu_array_info_t array;
 508         struct mdstat_ent *mse = NULL, *mse2;
 509         char *dev = st->devname;
 510         int fd;
 511         int i;
 512         int remaining_disks;
 513         int last_disk;
 514         int new_array = 0;
 515         int retval;
 516         int is_container = 0;
 517         unsigned long redundancy_only_flags = 0;
 518
 519         if (test)
 520                 alert("TestMessage", dev, NULL, ainfo);
 521
 522         retval = 0;
 523
 524         fd = open(dev, O_RDONLY);
 525         if (fd < 0)
 526                 goto disappeared;
 527
 528         if (st->devnm[0] == 0)
 529                 strcpy(st->devnm, fd2devnm(fd));
 530
 531         for (mse2 = mdstat; mse2; mse2 = mse2->next)
 532                 if (strcmp(mse2->devnm, st->devnm) == 0) {
 533                         mse2->devnm[0] = 0; /* flag it as "used" */
 534                         mse = mse2;
 535                 }
 536
 537         if (!mse) {
 538                 /* duplicated array in statelist
 539                  * or re-created after reading mdstat
 540                  */
 541                 st->err++;
 542                 goto out;
 543         }
 544
 545         if (mse->level == NULL)
 546                 is_container = 1;
 547
 548         if (!is_container && !md_array_active(fd))
 549                 goto disappeared;
 550
 551         fcntl(fd, F_SETFD, FD_CLOEXEC);
 552         if (md_get_array_info(fd, &array) < 0)
 553                 goto disappeared;
 554
 555         if (!is_container && map_name(pers, mse->level) > 0)
 556                 redundancy_only_flags |= GET_MISMATCH;
 557
 558         sra = sysfs_read(-1, st->devnm, GET_LEVEL | GET_DISKS | GET_DEVS |
 559                         GET_STATE | redundancy_only_flags);
 560
 561         if (!sra)
 562                 goto disappeared;
 563
 564         /* It's much easier to list what array levels can't
 565          * have a device disappear than all of them that can
 566          */
 567         if (sra->array.level == 0 || sra->array.level == -1) {
 568                 if (!st->err && !st->from_config)
 569                         alert("DeviceDisappeared", dev, " Wrong-Level", ainfo);
 570                 st->err++;
 571                 goto out;
 572         }
 573
 574         /* this array is in /proc/mdstat */
 575         if (array.utime == 0)
 576                 /* external arrays don't update utime, so
 577                  * just make sure it is always different. */
 578                 array.utime = st->utime + 1;;
 579
 580         if (st->err) {
 581                 /* New array appeared where previously had an error */
 582                 st->err = 0;
 583                 st->percent = RESYNC_NONE;
 584                 new_array = 1;
 585                 if (!is_container)
 586                         alert("NewArray", st->devname, NULL, ainfo);
 587         }
 588
 589         if (st->utime == array.utime && st->failed == sra->array.failed_disks &&
 590             st->working == sra->array.working_disks &&
 591             st->spare == sra->array.spare_disks &&
 592             (mse == NULL || (mse->percent == st->percent))) {
 593                 if ((st->active < st->raid) && st->spare == 0)
 594                         retval = 1;
 595                 goto out;
 596         }
 597         if (st->utime == 0 && /* new array */
 598             mse->pattern && strchr(mse->pattern, '_') /* degraded */)
 599                 alert("DegradedArray", dev, NULL, ainfo);
 600
 601         if (st->utime == 0 && /* new array */ st->expected_spares > 0 &&
 602             sra->array.spare_disks < st->expected_spares)
 603                 alert("SparesMissing", dev, NULL, ainfo);
 604         if (st->percent < 0 && st->percent != RESYNC_UNKNOWN &&
 605             mse->percent >= 0)
 606                 alert("RebuildStarted", dev, NULL, ainfo);
 607         if (st->percent >= 0 && mse->percent >= 0 &&
 608             (mse->percent / increments) > (st->percent / increments)) {
 609                 char percentalert[18];
 610                 /*
 611                  * "RebuildNN" (10 chars) or "RebuildStarted" (15 chars)
 612                  */
 613
 614                 if((mse->percent / increments) == 0)
 615                         snprintf(percentalert, sizeof(percentalert),
 616                                  "RebuildStarted");
 617                 else
 618                         snprintf(percentalert, sizeof(percentalert),
 619                                  "Rebuild%02d", mse->percent);
 620
 621                 alert(percentalert, dev, NULL, ainfo);
 622         }
 623
 624         if (mse->percent == RESYNC_NONE && st->percent >= 0) {
 625                 /* Rebuild/sync/whatever just finished.
 626                  * If there is a number in /mismatch_cnt,
 627                  * we should report that.
 628                  */
 629                 if (sra && sra->mismatch_cnt > 0) {
 630                         char cnt[80];
 631                         snprintf(cnt, sizeof(cnt),
 632                                  " mismatches found: %d (on raid level %d)",
 633                                  sra->mismatch_cnt, sra->array.level);
 634                         alert("RebuildFinished", dev, cnt, ainfo);
 635                 } else
 636                         alert("RebuildFinished", dev, NULL, ainfo);
 637         }
 638         st->percent = mse->percent;
 639
 640         remaining_disks = sra->array.nr_disks;
 641         for (i = 0; i < MAX_DISKS && remaining_disks > 0; i++) {
 642                 mdu_disk_info_t disc;
 643                 disc.number = i;
 644                 if (md_get_disk_info(fd, &disc) >= 0) {
 645                         info[i].state = disc.state;
 646                         info[i].major = disc.major;
 647                         info[i].minor = disc.minor;
 648                         if (disc.major || disc.minor)
 649                                 remaining_disks --;
 650                 } else
 651                         info[i].major = info[i].minor = 0;
 652         }
 653         last_disk = i;
 654
 655         if (mse->metadata_version &&
 656             strncmp(mse->metadata_version, "external:", 9) == 0 &&
 657             is_subarray(mse->metadata_version+9)) {
 658                 char *sl;
 659                 strcpy(st->parent_devnm, mse->metadata_version + 10);
 660                 sl = strchr(st->parent_devnm, '/');
 661                 if (sl)
 662                         *sl = 0;
 663         } else
 664                 st->parent_devnm[0] = 0;
 665         if (st->metadata == NULL && st->parent_devnm[0] == 0)
 666                 st->metadata = super_by_fd(fd, NULL);
 667
 668         for (i = 0; i < MAX_DISKS; i++) {
 669                 mdu_disk_info_t disc = {0, 0, 0, 0, 0};
 670                 int newstate = 0;
 671                 int change;
 672                 char *dv = NULL;
 673                 disc.number = i;
 674                 if (i < last_disk && (info[i].major || info[i].minor)) {
 675                         newstate = info[i].state;
 676                         dv = map_dev_preferred(info[i].major, info[i].minor, 1,
 677                                                prefer);
 678                         disc.state = newstate;
 679                         disc.major = info[i].major;
 680                         disc.minor = info[i].minor;
 681                 } else
 682                         newstate = (1 << MD_DISK_REMOVED);
 683
 684                 if (dv == NULL && st->devid[i])
 685                         dv = map_dev_preferred(major(st->devid[i]),
 686                                                minor(st->devid[i]), 1, prefer);
 687                 change = newstate ^ st->devstate[i];
 688                 if (st->utime && change && !st->err && !new_array) {
 689                         if ((st->devstate[i]&change) & (1 << MD_DISK_SYNC))
 690                                 alert("Fail", dev, dv, ainfo);
 691                         else if ((newstate & (1 << MD_DISK_FAULTY)) &&
 692                                  (disc.major || disc.minor) &&
 693                                  st->devid[i] == makedev(disc.major,
 694                                                          disc.minor))
 695                                 alert("FailSpare", dev, dv, ainfo);
 696                         else if ((newstate&change) & (1 << MD_DISK_SYNC))
 697                                 alert("SpareActive", dev, dv, ainfo);
 698                 }
 699                 st->devstate[i] = newstate;
 700                 st->devid[i] = makedev(disc.major, disc.minor);
 701         }
 702         st->active = sra->array.active_disks;
 703         st->working = sra->array.working_disks;
 704         st->spare = sra->array.spare_disks;
 705         st->failed = sra->array.failed_disks;
 706         st->utime = array.utime;
 707         st->raid = sra->array.raid_disks;
 708         st->err = 0;
 709         if ((st->active < st->raid) && st->spare == 0)
 710                 retval = 1;
 711
 712  out:
 713         if (sra)
 714                 sysfs_free(sra);
 715         if (fd >= 0)
 716                 close(fd);
 717         return retval;
 718
 719  disappeared:
 720         if (!st->err && !is_container)
 721                 alert("DeviceDisappeared", dev, NULL, ainfo);
 722         st->err++;
 723         goto out;
 724 }
 725
 726 static int add_new_arrays(struct mdstat_ent *mdstat, struct state **statelist,
 727                           int test, struct alert_info *info)
 728 {
 729         struct mdstat_ent *mse;
 730         int new_found = 0;
 731         char *name;
 732
 733         for (mse = mdstat; mse; mse = mse->next)
 734                 if (mse->devnm[0] && (!mse->level || /* retrieve containers */
 735                                       (strcmp(mse->level, "raid0") != 0 &&
 736                                        strcmp(mse->level, "linear") != 0))) {
 737                         struct state *st = xcalloc(1, sizeof *st);
 738                         mdu_array_info_t array;
 739                         int fd;
 740
 741                         name = get_md_name(mse->devnm);
 742                         if (!name) {
 743                                 free(st);
 744                                 continue;
 745                         }
 746
 747                         st->devname = xstrdup(name);
 748                         if ((fd = open(st->devname, O_RDONLY)) < 0 ||
 749                             md_get_array_info(fd, &array) < 0) {
 750                                 /* no such array */
 751                                 if (fd >= 0)
 752                                         close(fd);
 753                                 put_md_name(st->devname);
 754                                 free(st->devname);
 755                                 if (st->metadata) {
 756                                         st->metadata->ss->free_super(st->metadata);
 757                                         free(st->metadata);
 758                                 }
 759                                 free(st);
 760                                 continue;
 761                         }
 762                         close(fd);
 763                         st->next = *statelist;
 764                         st->err = 1;
 765                         st->from_auto = 1;
 766                         strcpy(st->devnm, mse->devnm);
 767                         st->percent = RESYNC_UNKNOWN;
 768                         st->expected_spares = -1;
 769                         if (mse->metadata_version &&
 770                             strncmp(mse->metadata_version,
 771                                     "external:", 9) == 0 &&
 772                             is_subarray(mse->metadata_version+9)) {
 773                                 char *sl;
 774                                 strcpy(st->parent_devnm,
 775                                         mse->metadata_version+10);
 776                                 sl = strchr(st->parent_devnm, '/');
 777                                 *sl = 0;
 778                         } else
 779                                 st->parent_devnm[0] = 0;
 780                         *statelist = st;
 781                         if (test)
 782                                 alert("TestMessage", st->devname, NULL, info);
 783                         new_found = 1;
 784                 }
 785         return new_found;
 786 }
 787
 788 static int get_required_spare_criteria(struct state *st,
 789                                        struct spare_criteria *sc)
 790 {
 791         int fd;
 792
 793         if (!st->metadata || !st->metadata->ss->get_spare_criteria) {
 794                 sc->min_size = 0;
 795                 sc->sector_size = 0;
 796                 return 0;
 797         }
 798
 799         fd = open(st->devname, O_RDONLY);
 800         if (fd < 0)
 801                 return 1;
 802         if (st->metadata->ss->external)
 803                 st->metadata->ss->load_container(st->metadata, fd, st->devname);
 804         else
 805                 st->metadata->ss->load_super(st->metadata, fd, st->devname);
 806         close(fd);
 807         if (!st->metadata->sb)
 808                 return 1;
 809
 810         st->metadata->ss->get_spare_criteria(st->metadata, sc);
 811         st->metadata->ss->free_super(st->metadata);
 812
 813         return 0;
 814 }
 815
 816 static int check_donor(struct state *from, struct state *to)
 817 {
 818         struct state *sub;
 819
 820         if (from == to)
 821                 return 0;
 822         if (from->parent)
 823                 /* Cannot move from a member */
 824                 return 0;
 825         if (from->err)
 826                 return 0;
 827         for (sub = from->subarray; sub; sub = sub->subarray)
 828                 /* If source array has degraded subarrays, don't
 829                  * remove anything
 830                  */
 831                 if (sub->active < sub->raid)
 832                         return 0;
 833         if (from->metadata->ss->external == 0)
 834                 if (from->active < from->raid)
 835                         return 0;
 836         if (from->spare <= 0)
 837                 return 0;
 838         return 1;
 839 }
 840
 841 static dev_t choose_spare(struct state *from, struct state *to,
 842                           struct domainlist *domlist, struct spare_criteria *sc)
 843 {
 844         int d;
 845         dev_t dev = 0;
 846
 847         for (d = from->raid; !dev && d < MAX_DISKS; d++) {
 848                 if (from->devid[d] > 0 && from->devstate[d] == 0) {
 849                         struct dev_policy *pol;
 850                         unsigned long long dev_size;
 851                         unsigned int dev_sector_size;
 852
 853                         if (to->metadata->ss->external &&
 854                             test_partition_from_id(from->devid[d]))
 855                                 continue;
 856
 857                         if (sc->min_size &&
 858                             dev_size_from_id(from->devid[d], &dev_size) &&
 859                             dev_size < sc->min_size)
 860                                 continue;
 861
 862                         if (sc->sector_size &&
 863                             dev_sector_size_from_id(from->devid[d],
 864                                                     &dev_sector_size) &&
 865                             sc->sector_size != dev_sector_size)
 866                                 continue;
 867
 868                         pol = devid_policy(from->devid[d]);
 869                         if (from->spare_group)
 870                                 pol_add(&pol, pol_domain,
 871                                         from->spare_group, NULL);
 872                         if (domain_test(domlist, pol,
 873                                         to->metadata->ss->name) == 1)
 874                             dev = from->devid[d];
 875                         dev_policy_free(pol);
 876                 }
 877         }
 878         return dev;
 879 }
 880
 881 static dev_t container_choose_spare(struct state *from, struct state *to,
 882                                     struct domainlist *domlist,
 883                                     struct spare_criteria *sc, int active)
 884 {
 885         /* This is similar to choose_spare, but we cannot trust devstate,
 886          * so we need to read the metadata instead
 887          */
 888         struct mdinfo *list;
 889         struct supertype *st = from->metadata;
 890         int fd = open(from->devname, O_RDONLY);
 891         int err;
 892         dev_t dev = 0;
 893
 894         if (fd < 0)
 895                 return 0;
 896         if (!st->ss->getinfo_super_disks) {
 897                 close(fd);
 898                 return 0;
 899         }
 900
 901         err = st->ss->load_container(st, fd, NULL);
 902         close(fd);
 903         if (err)
 904                 return 0;
 905
 906         if (from == to) {
 907                 /* We must check if number of active disks has not increased
 908                  * since ioctl in main loop. mdmon may have added spare
 909                  * to subarray. If so we do not need to look for more spares
 910                  * so return non zero value */
 911                 int active_cnt = 0;
 912                 struct mdinfo *dp;
 913                 list = st->ss->getinfo_super_disks(st);
 914                 if (!list) {
 915                         st->ss->free_super(st);
 916                         return 1;
 917                 }
 918                 dp = list->devs;
 919                 while (dp) {
 920                         if (dp->disk.state & (1 << MD_DISK_SYNC) &&
 921                             !(dp->disk.state & (1 << MD_DISK_FAULTY)))
 922                                 active_cnt++;
 923                         dp = dp->next;
 924                 }
 925                 sysfs_free(list);
 926                 if (active < active_cnt) {
 927                         /* Spare just activated.*/
 928                         st->ss->free_super(st);
 929                         return 1;
 930                 }
 931         }
 932
 933         /* We only need one spare so full list not needed */
 934         list = container_choose_spares(st, sc, domlist, from->spare_group,
 935                                        to->metadata->ss->name, 1);
 936         if (list) {
 937                 struct mdinfo *disks = list->devs;
 938                 if (disks)
 939                         dev = makedev(disks->disk.major, disks->disk.minor);
 940                 sysfs_free(list);
 941         }
 942         st->ss->free_super(st);
 943         return dev;
 944 }
 945
 946 static void try_spare_migration(struct state *statelist, struct alert_info *info)
 947 {
 948         struct state *from;
 949         struct state *st;
 950         struct spare_criteria sc;
 951
 952         link_containers_with_subarrays(statelist);
 953         for (st = statelist; st; st = st->next)
 954                 if (st->active < st->raid && st->spare == 0 && !st->err) {
 955                         struct domainlist *domlist = NULL;
 956                         int d;
 957                         struct state *to = st;
 958
 959                         if (to->parent_devnm[0] && !to->parent)
 960                                 /* subarray monitored without parent container
 961                                  * we can't move spares here */
 962                                 continue;
 963
 964                         if (to->parent)
 965                                 /* member of a container */
 966                                 to = to->parent;
 967
 968                         if (get_required_spare_criteria(to, &sc))
 969                                 continue;
 970                         if (to->metadata->ss->external) {
 971                                 /* We must make sure there is
 972                                  * no suitable spare in container already.
 973                                  * If there is we don't add more */
 974                                 dev_t devid = container_choose_spare(
 975                                         to, to, NULL, &sc, st->active);
 976                                 if (devid > 0)
 977                                         continue;
 978                         }
 979                         for (d = 0; d < MAX_DISKS; d++)
 980                                 if (to->devid[d])
 981                                         domainlist_add_dev(&domlist,
 982                                                            to->devid[d],
 983                                                            to->metadata->ss->name);
 984                         if (to->spare_group)
 985                                 domain_add(&domlist, to->spare_group);
 986                         /*
 987                          * No spare migration if the destination
 988                          * has no domain. Skip this array.
 989                          */
 990                         if (!domlist)
 991                                 continue;
 992                         for (from=statelist ; from ; from=from->next) {
 993                                 dev_t devid;
 994                                 if (!check_donor(from, to))
 995                                         continue;
 996                                 if (from->metadata->ss->external)
 997                                         devid = container_choose_spare(
 998                                                 from, to, domlist, &sc, 0);
 999                                 else
1000                                         devid = choose_spare(from, to, domlist,
1001                                                              &sc);
1002                                 if (devid > 0 &&
1003                                     move_spare(from->devname, to->devname,
1004                                                devid)) {
1005                                         alert("MoveSpare", to->devname,
1006                                               from->devname, info);
1007                                         break;
1008                                 }
1009                         }
1010                         domain_free(domlist);
1011                 }
1012 }
1013
1014 /* search the statelist to connect external
1015  * metadata subarrays with their containers
1016  * We always completely rebuild the tree from scratch as
1017  * that is safest considering the possibility of entries
1018  * disappearing or changing.
1019  */
1020 static void link_containers_with_subarrays(struct state *list)
1021 {
1022         struct state *st;
1023         struct state *cont;
1024         for (st = list; st; st = st->next) {
1025                 st->parent = NULL;
1026                 st->subarray = NULL;
1027         }
1028         for (st = list; st; st = st->next)
1029                 if (st->parent_devnm[0])
1030                         for (cont = list; cont; cont = cont->next)
1031                                 if (!cont->err && cont->parent_devnm[0] == 0 &&
1032                                     strcmp(cont->devnm, st->parent_devnm) == 0) {
1033                                         st->parent = cont;
1034                                         st->subarray = cont->subarray;
1035                                         cont->subarray = st;
1036                                         break;
1037                                 }
1038 }
1039
1040 /* Not really Monitor but ... */
1041 int Wait(char *dev)
1042 {
1043         char devnm[32];
1044         dev_t rdev;
1045         char *tmp;
1046         int rv = 1;
1047         int frozen_remaining = 3;
1048
1049         if (!stat_is_blkdev(dev, &rdev))
1050                 return 2;
1051
1052         tmp = devid2devnm(rdev);
1053         if (!tmp) {
1054                 pr_err("Cannot get md device name.\n");
1055                 return 2;
1056         }
1057
1058         strcpy(devnm, tmp);
1059
1060         while(1) {
1061                 struct mdstat_ent *ms = mdstat_read(1, 0);
1062                 struct mdstat_ent *e;
1063
1064                 for (e = ms; e; e = e->next)
1065                         if (strcmp(e->devnm, devnm) == 0)
1066                                 break;
1067
1068                 if (e && e->percent == RESYNC_NONE) {
1069                         /* We could be in the brief pause before something
1070                          * starts. /proc/mdstat doesn't show that, but
1071                          * sync_action does.
1072                          */
1073                         struct mdinfo mdi;
1074                         char buf[21];
1075
1076                         if (sysfs_init(&mdi, -1, devnm))
1077                                 return 2;
1078                         if (sysfs_get_str(&mdi, NULL, "sync_action",
1079                                           buf, 20) > 0 &&
1080                             strcmp(buf,"idle\n") != 0) {
1081                                 e->percent = RESYNC_UNKNOWN;
1082                                 if (strcmp(buf, "frozen\n") == 0) {
1083                                         if (frozen_remaining == 0)
1084                                                 e->percent = RESYNC_NONE;
1085                                         else
1086                                                 frozen_remaining -= 1;
1087                                 }
1088                         }
1089                 }
1090                 if (!e || e->percent == RESYNC_NONE) {
1091                         if (e && e->metadata_version &&
1092                             strncmp(e->metadata_version, "external:", 9) == 0) {
1093                                 if (is_subarray(&e->metadata_version[9]))
1094                                         ping_monitor(&e->metadata_version[9]);
1095                                 else
1096                                         ping_monitor(devnm);
1097                         }
1098                         free_mdstat(ms);
1099                         return rv;
1100                 }
1101                 free_mdstat(ms);
1102                 rv = 0;
1103                 mdstat_wait(5);
1104         }
1105 }
1106
1107 /* The state "broken" is used only for RAID0/LINEAR - it's the same as
1108  * "clean", but used in case the array has one or more members missing.
1109  */
1110 static char *clean_states[] = {
1111         "clear", "inactive", "readonly", "read-auto", "clean", "broken", NULL };
1112
1113 int WaitClean(char *dev, int verbose)
1114 {
1115         int fd;
1116         struct mdinfo *mdi;
1117         int rv = 1;
1118         char devnm[32];
1119
1120         if (!stat_is_blkdev(dev, NULL))
1121                 return 2;
1122         fd = open(dev, O_RDONLY);
1123         if (fd < 0) {
1124                 if (verbose)
1125                         pr_err("Couldn't open %s: %s\n", dev, strerror(errno));
1126                 return 1;
1127         }
1128
1129         strcpy(devnm, fd2devnm(fd));
1130         mdi = sysfs_read(fd, devnm, GET_VERSION|GET_LEVEL|GET_SAFEMODE);
1131         if (!mdi) {
1132                 if (verbose)
1133                         pr_err("Failed to read sysfs attributes for %s\n", dev);
1134                 close(fd);
1135                 return 0;
1136         }
1137
1138         switch(mdi->array.level) {
1139         case LEVEL_LINEAR:
1140         case LEVEL_MULTIPATH:
1141         case 0:
1142                 /* safemode delay is irrelevant for these levels */
1143                 rv = 0;
1144         }
1145
1146         /* for internal metadata the kernel handles the final clean
1147          * transition, containers can never be dirty
1148          */
1149         if (!is_subarray(mdi->text_version))
1150                 rv = 0;
1151
1152         /* safemode disabled ? */
1153         if (mdi->safe_mode_delay == 0)
1154                 rv = 0;
1155
1156         if (rv) {
1157                 int state_fd = sysfs_open(fd2devnm(fd), NULL, "array_state");
1158                 char buf[20];
1159                 int delay = 5000;
1160
1161                 /* minimize the safe_mode_delay and prepare to wait up to 5s
1162                  * for writes to quiesce
1163                  */
1164                 sysfs_set_safemode(mdi, 1);
1165
1166                 /* wait for array_state to be clean */
1167                 while (1) {
1168                         rv = read(state_fd, buf, sizeof(buf));
1169                         if (rv < 0)
1170                                 break;
1171                         if (sysfs_match_word(buf, clean_states) <
1172                             (int)ARRAY_SIZE(clean_states) - 1)
1173                                 break;
1174                         rv = sysfs_wait(state_fd, &delay);
1175                         if (rv < 0 && errno != EINTR)
1176                                 break;
1177                         lseek(state_fd, 0, SEEK_SET);
1178                 }
1179                 if (rv < 0)
1180                         rv = 1;
1181                 else if (ping_monitor(mdi->text_version) == 0) {
1182                         /* we need to ping to close the window between array
1183                          * state transitioning to clean and the metadata being
1184                          * marked clean
1185                          */
1186                         rv = 0;
1187                 } else {
1188                         rv = 1;
1189                         pr_err("Error connecting monitor with %s\n", dev);
1190                 }
1191                 if (rv && verbose)
1192                         pr_err("Error waiting for %s to be clean\n", dev);
1193
1194                 /* restore the original safe_mode_delay */
1195                 sysfs_set_safemode(mdi, mdi->safe_mode_delay);
1196                 close(state_fd);
1197         }
1198
1199         sysfs_free(mdi);
1200         close(fd);
1201
1202         return rv;
1203 }