Teach sysfs_add_disk() callers to use ->recovery_start versus 'insync' parameter
[thirdparty/mdadm.git] / Assemble.c
1 /*
2  * mdadm - manage Linux "md" devices aka RAID arrays.
3  *
4  * Copyright (C) 2001-2009 Neil Brown <neilb@suse.de>
5  *
6  *
7  *    This program is free software; you can redistribute it and/or modify
8  *    it under the terms of the GNU General Public License as published by
9  *    the Free Software Foundation; either version 2 of the License, or
10  *    (at your option) any later version.
11  *
12  *    This program is distributed in the hope that it will be useful,
13  *    but WITHOUT ANY WARRANTY; without even the implied warranty of
14  *    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  *    GNU General Public License for more details.
16  *
17  *    You should have received a copy of the GNU General Public License
18  *    along with this program; if not, write to the Free Software
19  *    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20  *
21  *    Author: Neil Brown
22  *    Email: <neilb@suse.de>
23  */
24
25 #include        "mdadm.h"
26 #include        <ctype.h>
27
28 static int name_matches(char *found, char *required, char *homehost)
29 {
30         /* See if the name found matches the required name, possibly
31          * prefixed with 'homehost'
32          */
33         char fnd[33];
34
35         strncpy(fnd, found, 32);
36         fnd[32] = 0;
37         if (strcmp(found, required)==0)
38                 return 1;
39         if (homehost) {
40                 int l = strlen(homehost);
41                 if (l < 32 && fnd[l] == ':' &&
42                     strcmp(fnd+l+1, required)==0)
43                         return 1;
44         }
45         return 0;
46 }
47
48 static int is_member_busy(char *metadata_version)
49 {
50         /* check if the given member array is active */
51         struct mdstat_ent *mdstat = mdstat_read(1, 0);
52         struct mdstat_ent *ent;
53         int busy = 0;
54
55         for (ent = mdstat; ent; ent = ent->next) {
56                 if (ent->metadata_version == NULL)
57                         continue;
58                 if (strncmp(ent->metadata_version, "external:", 9) != 0)
59                         continue;
60                 if (!is_subarray(&ent->metadata_version[9]))
61                         continue;
62                 /* Skip first char - it can be '/' or '-' */
63                 if (strcmp(&ent->metadata_version[10], metadata_version+1) == 0) {
64                         busy = 1;
65                         break;
66                 }
67         }
68         free_mdstat(mdstat);
69
70         return busy;
71 }
72
73 int Assemble(struct supertype *st, char *mddev,
74              mddev_ident_t ident,
75              mddev_dev_t devlist, char *backup_file,
76              int readonly, int runstop,
77              char *update, char *homehost, int require_homehost,
78              int verbose, int force)
79 {
80         /*
81          * The task of Assemble is to find a collection of
82          * devices that should (according to their superblocks)
83          * form an array, and to give this collection to the MD driver.
84          * In Linux-2.4 and later, this involves submitting a
85          * SET_ARRAY_INFO ioctl with no arg - to prepare
86          * the array - and then submit a number of
87          * ADD_NEW_DISK ioctls to add disks into
88          * the array.  Finally RUN_ARRAY might
89          * be submitted to start the array.
90          *
91          * Much of the work of Assemble is in finding and/or
92          * checking the disks to make sure they look right.
93          *
94          * If mddev is not set, then scan must be set and we
95          *  read through the config file for dev+uuid mapping
96          *  We recurse, setting mddev, for each device that
97          *    - isn't running
98          *    - has a valid uuid (or any uuid if !uuidset)
99          *
100          * If mddev is set, we try to determine state of md.
101          *   check version - must be at least 0.90.0
102          *   check kernel version.  must be at least 2.4.
103          *    If not, we can possibly fall back on START_ARRAY
104          *   Try to GET_ARRAY_INFO.
105          *     If possible, give up
106          *     If not, try to STOP_ARRAY just to make sure
107          *
108          * If !uuidset and scan, look in conf-file for uuid
109          *       If not found, give up
110          * If !devlist and scan and uuidset, get list of devs from conf-file
111          *
112          * For each device:
113          *   Check superblock - discard if bad
114          *   Check uuid (set if we don't have one) - discard if no match
115          *   Check superblock similarity if we have a superblock - discard if different
116          *   Record events, devicenum
117          * This should give us a list of devices for the array
118          * We should collect the most recent event number
119          *
120          * Count disks with recent enough event count
121          * While force && !enough disks
122          *    Choose newest rejected disks, update event count
123          *     mark clean and rewrite superblock
124          * If recent kernel:
125          *    SET_ARRAY_INFO
126          *    foreach device with recent events : ADD_NEW_DISK
127          *    if runstop == 1 || "enough" disks and runstop==0 -> RUN_ARRAY
128          * If old kernel:
129          *    Check the device numbers in superblock are right
130          *    update superblock if any changes
131          *    START_ARRAY
132          *
133          */
134         int mdfd;
135         int clean;
136         int auto_assem = (mddev == NULL && !ident->uuid_set &&
137                           ident->super_minor == UnSet && ident->name[0] == 0
138                           && (ident->container == NULL || ident->member == NULL));
139         int old_linux = 0;
140         int vers = vers; /* Keep gcc quite - it really is initialised */
141         struct {
142                 char *devname;
143                 int uptodate; /* set once we decide that this device is as
144                                * recent as everything else in the array.
145                                */
146                 struct mdinfo i;
147         } *devices;
148         int *best = NULL; /* indexed by raid_disk */
149         unsigned int bestcnt = 0;
150         int devcnt = 0;
151         unsigned int okcnt, sparecnt;
152         unsigned int req_cnt;
153         unsigned int i;
154         int most_recent = 0;
155         int chosen_drive;
156         int change = 0;
157         int inargv = 0;
158         int report_missmatch;
159         int bitmap_done;
160         int start_partial_ok = (runstop >= 0) && 
161                 (force || devlist==NULL || auto_assem);
162         unsigned int num_devs;
163         mddev_dev_t tmpdev;
164         struct mdinfo info;
165         struct mdinfo *content = NULL;
166         char *avail;
167         int nextspare = 0;
168         char *name = NULL;
169         int trustworthy;
170         char chosen_name[1024];
171
172         if (get_linux_version() < 2004000)
173                 old_linux = 1;
174
175         /*
176          * If any subdevs are listed, then any that don't
177          * match ident are discarded.  Remainder must all match and
178          * become the array.
179          * If no subdevs, then we scan all devices in the config file, but
180          * there must be something in the identity
181          */
182
183         if (!devlist &&
184             ident->uuid_set == 0 &&
185             ident->super_minor < 0 &&
186             ident->name[0] == 0 &&
187             (ident->container == NULL || ident->member == NULL) &&
188             ident->devices == NULL) {
189                 fprintf(stderr, Name ": No identity information available for %s - cannot assemble.\n",
190                         mddev ? mddev : "further assembly");
191                 return 1;
192         }
193
194         if (devlist == NULL)
195                 devlist = conf_get_devs();
196         else if (mddev)
197                 inargv = 1;
198
199         report_missmatch = ((inargv && verbose >= 0) || verbose > 0);
200  try_again:
201         /* We come back here when doing auto-assembly and attempting some
202          * set of devices failed.  Those are now marked as ->used==2 and
203          * we ignore them and try again
204          */
205
206         tmpdev = devlist; num_devs = 0;
207         while (tmpdev) {
208                 if (tmpdev->used)
209                         tmpdev->used = 2;
210                 else
211                         num_devs++;
212                 tmpdev = tmpdev->next;
213         }
214         devices = malloc(num_devs * sizeof(*devices));
215
216         if (!st && ident->st) st = ident->st;
217
218         if (verbose>0)
219             fprintf(stderr, Name ": looking for devices for %s\n",
220                     mddev ? mddev : "further assembly");
221
222         /* first walk the list of devices to find a consistent set
223          * that match the criterea, if that is possible.
224          * We flag the ones we like with 'used'.
225          */
226         for (tmpdev = devlist;
227              tmpdev;
228              tmpdev = tmpdev->next) {
229                 char *devname = tmpdev->devname;
230                 int dfd;
231                 struct stat stb;
232                 struct supertype *tst = dup_super(st);
233
234                 if (tmpdev->used > 1) continue;
235
236                 if (ident->devices &&
237                     !match_oneof(ident->devices, devname)) {
238                         if (report_missmatch)
239                                 fprintf(stderr, Name ": %s is not one of %s\n", devname, ident->devices);
240                         continue;
241                 }
242
243                 dfd = dev_open(devname, O_RDONLY|O_EXCL);
244                 if (dfd < 0) {
245                         if (report_missmatch)
246                                 fprintf(stderr, Name ": cannot open device %s: %s\n",
247                                         devname, strerror(errno));
248                         tmpdev->used = 2;
249                 } else if (fstat(dfd, &stb)< 0) {
250                         /* Impossible! */
251                         fprintf(stderr, Name ": fstat failed for %s: %s\n",
252                                 devname, strerror(errno));
253                         tmpdev->used = 2;
254                 } else if ((stb.st_mode & S_IFMT) != S_IFBLK) {
255                         fprintf(stderr, Name ": %s is not a block device.\n",
256                                 devname);
257                         tmpdev->used = 2;
258                 } else if (!tst && (tst = guess_super(dfd)) == NULL) {
259                         if (report_missmatch)
260                                 fprintf(stderr, Name ": no recogniseable superblock on %s\n",
261                                         devname);
262                         tmpdev->used = 2;
263                 } else if (auto_assem && st == NULL &&
264                            !conf_test_metadata(tst->ss->name)) {
265                         if (report_missmatch)
266                                 fprintf(stderr, Name ": %s has metadata type %s for which "
267                                         "auto-assembly is disabled\n",
268                                         devname, tst->ss->name);
269                         tmpdev->used = 2;
270                 } else if (tst->ss->load_super(tst,dfd, NULL)) {
271                         if (report_missmatch)
272                                 fprintf( stderr, Name ": no RAID superblock on %s\n",
273                                          devname);
274                 } else {
275                         content = &info;
276                         memset(content, 0, sizeof(*content));
277                         tst->ss->getinfo_super(tst, content);
278                 }
279                 if (dfd >= 0) close(dfd);
280
281                 if (tst && tst->sb && tst->ss->container_content
282                     && tst->loaded_container) {
283                         /* tmpdev is a container.  We need to be either
284                          * looking for a member, or auto-assembling
285                          */
286                         if (st) {
287                                 /* already found some components, this cannot
288                                  * be another one.
289                                  */
290                                 if (report_missmatch)
291                                         fprintf(stderr, Name ": %s is a container, but we are looking for components\n",
292                                                 devname);
293                                 goto loop;
294                         }
295
296                         if (ident->container) {
297                                 if (ident->container[0] == '/' &&
298                                     !same_dev(ident->container, devname)) {
299                                         if (report_missmatch)
300                                                 fprintf(stderr, Name ": %s is not the container required (%s)\n",
301                                                         devname, ident->container);
302                                         goto loop;
303                                 }
304                                 if (ident->container[0] != '/') {
305                                         /* we have a uuid */
306                                         int uuid[4];
307                                         if (!parse_uuid(ident->container, uuid) ||
308                                             !same_uuid(content->uuid, uuid, tst->ss->swapuuid)) {
309                                                 if (report_missmatch)
310                                                         fprintf(stderr, Name ": %s has wrong UUID to be required container\n",
311                                                                 devname);
312                                                 goto loop;
313                                         }
314                                 }
315                         }
316                         /* It is worth looking inside this container.
317                          */
318                         if (verbose > 0)
319                                 fprintf(stderr, Name ": looking in container %s\n",
320                                         devname);
321                 next_member:
322                         if (tmpdev->content)
323                                 content = tmpdev->content;
324                         else
325                                 content = tst->ss->container_content(tst);
326                         if (!content)
327                                 goto loop; /* empty container */
328
329                         tmpdev->content = content->next;
330                         if (tmpdev->content == NULL)
331                                 tmpdev->used = 2;
332
333                 } else if (ident->container || ident->member) {
334                         /* No chance of this matching if we don't have
335                          * a container */
336                         if (report_missmatch)
337                                 fprintf(stderr, Name "%s is not a container, and one is required.\n",
338                                         devname);
339                         goto loop;
340                 }
341
342                 if (ident->uuid_set && (!update || strcmp(update, "uuid")!= 0) &&
343                     (!tst || !tst->sb ||
344                      same_uuid(content->uuid, ident->uuid, tst->ss->swapuuid)==0)) {
345                         if (report_missmatch)
346                                 fprintf(stderr, Name ": %s has wrong uuid.\n",
347                                         devname);
348                         goto loop;
349                 }
350                 if (ident->name[0] && (!update || strcmp(update, "name")!= 0) &&
351                     (!tst || !tst->sb ||
352                      name_matches(content->name, ident->name, homehost)==0)) {
353                         if (report_missmatch)
354                                 fprintf(stderr, Name ": %s has wrong name.\n",
355                                         devname);
356                         goto loop;
357                 }
358                 if (ident->super_minor != UnSet &&
359                     (!tst || !tst->sb ||
360                      ident->super_minor != content->array.md_minor)) {
361                         if (report_missmatch)
362                                 fprintf(stderr, Name ": %s has wrong super-minor.\n",
363                                         devname);
364                         goto loop;
365                 }
366                 if (ident->level != UnSet &&
367                     (!tst || !tst->sb ||
368                      ident->level != content->array.level)) {
369                         if (report_missmatch)
370                                 fprintf(stderr, Name ": %s has wrong raid level.\n",
371                                         devname);
372                         goto loop;
373                 }
374                 if (ident->raid_disks != UnSet &&
375                     (!tst || !tst->sb ||
376                      ident->raid_disks!= content->array.raid_disks)) {
377                         if (report_missmatch)
378                                 fprintf(stderr, Name ": %s requires wrong number of drives.\n",
379                                         devname);
380                         goto loop;
381                 }
382                 if (auto_assem) {
383                         if (tst == NULL || tst->sb == NULL)
384                                 continue;
385                 }
386                 /* If we are this far, then we are nearly commited to this device.
387                  * If the super_block doesn't exist, or doesn't match others,
388                  * then we probably cannot continue
389                  * However if one of the arrays is for the homehost, and
390                  * the other isn't that can disambiguate.
391                  */
392
393                 if (!tst || !tst->sb) {
394                         fprintf(stderr, Name ": %s has no superblock - assembly aborted\n",
395                                 devname);
396                         if (st)
397                                 st->ss->free_super(st);
398                         return 1;
399                 }
400
401                 if (tst && tst->sb && tst->ss->container_content
402                     && tst->loaded_container) {
403                         /* we have the one container we need, don't keep
404                          * looking.  If the chosen member is active, skip.
405                          */
406                         if (is_member_busy(content->text_version)) {
407                                 if (report_missmatch)
408                                         fprintf(stderr, Name ": member %s in %s is already assembled\n",
409                                                 content->text_version,
410                                                 devname);
411                         skip:
412                                 if (tmpdev->content)
413                                         goto next_member;
414                                 tst->ss->free_super(tst);
415                                 tst = NULL;
416                                 content = NULL;
417                                 if (auto_assem)
418                                         goto loop;
419                                 return 1;
420                         }
421                         if (ident->member && ident->member[0]) {
422                                 char *s = strchr(content->text_version+1, '/');
423                                 if (s == NULL) {
424                                         fprintf(stderr, Name ": badly formatted version: %s\n",
425                                                 content->text_version);
426                                         goto skip;
427                                 }
428                                 if (strcmp(ident->member, s+1) != 0) {
429                                         if (report_missmatch)
430                                                 fprintf(stderr,
431                                                         Name ": skipping wrong member %s\n",
432                                                         content->text_version);
433                                         goto skip;
434                                 }
435                         }
436                         st = tst; tst = NULL;
437                         if (!auto_assem && tmpdev->next != NULL) {
438                                 fprintf(stderr, Name ": %s is a container, but is not "
439                                         "only device given: confused and aborting\n",
440                                         devname);
441                                 st->ss->free_super(st);
442                                 return 1;
443                         }
444                         if (verbose > 0)
445                                 fprintf(stderr, Name ": found match on member %s in %s\n",
446                                         content->text_version, devname);
447                         break;
448                 }
449                 if (st == NULL)
450                         st = dup_super(tst);
451                 if (st->minor_version == -1)
452                         st->minor_version = tst->minor_version;
453                 if (st->ss != tst->ss ||
454                     st->minor_version != tst->minor_version ||
455                     st->ss->compare_super(st, tst) != 0) {
456                         /* Some mismatch. If exactly one array matches this host,
457                          * we can resolve on that one.
458                          * Or, if we are auto assembling, we just ignore the second
459                          * for now.
460                          */
461                         if (auto_assem)
462                                 goto loop;
463                         if (homehost) {
464                                 int first = st->ss->match_home(st, homehost);
465                                 int last = tst->ss->match_home(tst, homehost);
466                                 if (first != last &&
467                                     (first == 1 || last == 1)) {
468                                         /* We can do something */
469                                         if (first) {/* just ignore this one */
470                                                 if (report_missmatch)
471                                                         fprintf(stderr, Name ": %s misses out due to wrong homehost\n",
472                                                                 devname);
473                                                 goto loop;
474                                         } else { /* reject all those sofar */
475                                                 mddev_dev_t td;
476                                                 if (report_missmatch)
477                                                         fprintf(stderr, Name ": %s overrides previous devices due to good homehost\n",
478                                                                 devname);
479                                                 for (td=devlist; td != tmpdev; td=td->next)
480                                                         if (td->used == 1)
481                                                                 td->used = 0;
482                                                 tmpdev->used = 1;
483                                                 goto loop;
484                                         }
485                                 }
486                         }
487                         fprintf(stderr, Name ": superblock on %s doesn't match others - assembly aborted\n",
488                                 devname);
489                         tst->ss->free_super(tst);
490                         st->ss->free_super(st);
491                         return 1;
492                 }
493
494                 tmpdev->used = 1;
495
496         loop:
497                 if (tmpdev->content)
498                         goto next_member;
499                 if (tst)
500                         tst->ss->free_super(tst);
501         }
502
503         if (!st || !st->sb || !content)
504                 return 2;
505
506         /* Now need to open the array device.  Use create_mddev */
507         if (content == &info)
508                 st->ss->getinfo_super(st, content);
509
510         trustworthy = FOREIGN;
511         name = content->name;
512         switch (st->ss->match_home(st, homehost)
513                 ?: st->ss->match_home(st, "any")) {
514         case 1:
515                 trustworthy = LOCAL;
516                 name = strchr(content->name, ':');
517                 if (name)
518                         name++;
519                 else
520                         name = content->name;
521                 break;
522         }
523         if (!auto_assem)
524                 /* If the array is listed in mdadm.conf or on
525                  * command line, then we trust the name
526                  * even if the array doesn't look local
527                  */
528                 trustworthy = LOCAL;
529
530         if (name[0] == 0 &&
531             content->array.level == LEVEL_CONTAINER) {
532                 name = content->text_version;
533                 trustworthy = METADATA;
534         }
535
536         if (name[0] && trustworthy != LOCAL &&
537             ! require_homehost &&
538             conf_name_is_free(name))
539                 trustworthy = LOCAL;
540
541         if (trustworthy == LOCAL &&
542             strchr(name, ':'))
543                 /* Ignore 'host:' prefix of name */
544                 name = strchr(name, ':')+1;
545
546         mdfd = create_mddev(mddev, name, ident->autof, trustworthy,
547                             chosen_name);
548         if (mdfd < 0) {
549                 st->ss->free_super(st);
550                 free(devices);
551                 if (auto_assem)
552                         goto try_again;
553                 return 1;
554         }
555         mddev = chosen_name;
556         vers = md_get_version(mdfd);
557         if (vers < 9000) {
558                 fprintf(stderr, Name ": Assemble requires driver version 0.90.0 or later.\n"
559                         "    Upgrade your kernel or try --build\n");
560                 close(mdfd);
561                 return 1;
562         }
563         if (mddev_busy(fd2devnum(mdfd))) {
564                 fprintf(stderr, Name ": %s already active, cannot restart it!\n",
565                         mddev);
566                 for (tmpdev = devlist ;
567                      tmpdev && tmpdev->used != 1;
568                      tmpdev = tmpdev->next)
569                         ;
570                 if (tmpdev && auto_assem)
571                         fprintf(stderr, Name ":   %s needed for %s...\n",
572                                 mddev, tmpdev->devname);
573                 close(mdfd);
574                 mdfd = -3;
575                 st->ss->free_super(st);
576                 free(devices);
577                 if (auto_assem)
578                         goto try_again;
579                 return 1;
580         }
581         ioctl(mdfd, STOP_ARRAY, NULL); /* just incase it was started but has no content */
582
583 #ifndef MDASSEMBLE
584         if (content != &info) {
585                 /* This is a member of a container.  Try starting the array. */
586                 return assemble_container_content(st, mdfd, content, runstop,
587                                            chosen_name, verbose);
588         }
589 #endif
590         /* Ok, no bad inconsistancy, we can try updating etc */
591         bitmap_done = 0;
592         content->update_private = NULL;
593         for (tmpdev = devlist; tmpdev; tmpdev=tmpdev->next) if (tmpdev->used == 1) {
594                 char *devname = tmpdev->devname;
595                 struct stat stb;
596                 /* looks like a good enough match to update the super block if needed */
597 #ifndef MDASSEMBLE
598                 if (update) {
599                         int dfd;
600                         /* prepare useful information in info structures */
601                         struct stat stb2;
602                         struct supertype *tst;
603                         fstat(mdfd, &stb2);
604
605                         if (strcmp(update, "uuid")==0 &&
606                             !ident->uuid_set) {
607                                 int rfd;
608                                 if ((rfd = open("/dev/urandom", O_RDONLY)) < 0 ||
609                                     read(rfd, ident->uuid, 16) != 16) {
610                                         *(__u32*)(ident->uuid) = random();
611                                         *(__u32*)(ident->uuid+1) = random();
612                                         *(__u32*)(ident->uuid+2) = random();
613                                         *(__u32*)(ident->uuid+3) = random();
614                                 }
615                                 if (rfd >= 0) close(rfd);
616                         }
617                         dfd = dev_open(devname, O_RDWR|O_EXCL);
618
619                         remove_partitions(dfd);
620
621                         tst = dup_super(st);
622                         tst->ss->load_super(tst, dfd, NULL);
623                         tst->ss->getinfo_super(tst, content);
624
625                         memcpy(content->uuid, ident->uuid, 16);
626                         strcpy(content->name, ident->name);
627                         content->array.md_minor = minor(stb2.st_rdev);
628
629                         tst->ss->update_super(tst, content, update,
630                                               devname, verbose,
631                                               ident->uuid_set, homehost);
632                         if (strcmp(update, "uuid")==0 &&
633                             !ident->uuid_set) {
634                                 ident->uuid_set = 1;
635                                 memcpy(ident->uuid, content->uuid, 16);
636                         }
637                         if (dfd < 0)
638                                 fprintf(stderr, Name ": Cannot open %s for superblock update\n",
639                                         devname);
640                         else if (tst->ss->store_super(tst, dfd))
641                                 fprintf(stderr, Name ": Could not re-write superblock on %s.\n",
642                                         devname);
643                         if (dfd >= 0)
644                                 close(dfd);
645
646                         if (strcmp(update, "uuid")==0 &&
647                             ident->bitmap_fd >= 0 && !bitmap_done) {
648                                 if (bitmap_update_uuid(ident->bitmap_fd,
649                                                        content->uuid,
650                                                        tst->ss->swapuuid) != 0)
651                                         fprintf(stderr, Name ": Could not update uuid on external bitmap.\n");
652                                 else
653                                         bitmap_done = 1;
654                         }
655                         tst->ss->free_super(tst);
656                 } else
657 #endif
658                 {
659                         struct supertype *tst = dup_super(st);
660                         int dfd;
661                         dfd = dev_open(devname, O_RDWR|O_EXCL);
662
663                         remove_partitions(dfd);
664
665                         tst->ss->load_super(tst, dfd, NULL);
666                         tst->ss->getinfo_super(tst, content);
667                         tst->ss->free_super(tst);
668                         close(dfd);
669                 }
670
671                 stat(devname, &stb);
672
673                 if (verbose > 0)
674                         fprintf(stderr, Name ": %s is identified as a member of %s, slot %d.\n",
675                                 devname, mddev, content->disk.raid_disk);
676                 devices[devcnt].devname = devname;
677                 devices[devcnt].uptodate = 0;
678                 devices[devcnt].i = *content;
679                 devices[devcnt].i.disk.major = major(stb.st_rdev);
680                 devices[devcnt].i.disk.minor = minor(stb.st_rdev);
681                 if (most_recent < devcnt) {
682                         if (devices[devcnt].i.events
683                             > devices[most_recent].i.events)
684                                 most_recent = devcnt;
685                 }
686                 if (content->array.level == LEVEL_MULTIPATH)
687                         /* with multipath, the raid_disk from the superblock is meaningless */
688                         i = devcnt;
689                 else
690                         i = devices[devcnt].i.disk.raid_disk;
691                 if (i+1 == 0) {
692                         if (nextspare < content->array.raid_disks)
693                                 nextspare = content->array.raid_disks;
694                         i = nextspare++;
695                 } else {
696                         if (i >= content->array.raid_disks &&
697                             i >= nextspare)
698                                 nextspare = i+1;
699                 }
700                 if (i < 10000) {
701                         if (i >= bestcnt) {
702                                 unsigned int newbestcnt = i+10;
703                                 int *newbest = malloc(sizeof(int)*newbestcnt);
704                                 unsigned int c;
705                                 for (c=0; c < newbestcnt; c++)
706                                         if (c < bestcnt)
707                                                 newbest[c] = best[c];
708                                         else
709                                                 newbest[c] = -1;
710                                 if (best)free(best);
711                                 best = newbest;
712                                 bestcnt = newbestcnt;
713                         }
714                         if (best[i] >=0 &&
715                             devices[best[i]].i.events
716                             == devices[devcnt].i.events
717                             && (devices[best[i]].i.disk.minor
718                                 != devices[devcnt].i.disk.minor)
719                             && st->ss == &super0
720                             && content->array.level != LEVEL_MULTIPATH) {
721                                 /* two different devices with identical superblock.
722                                  * Could be a mis-detection caused by overlapping
723                                  * partitions.  fail-safe.
724                                  */
725                                 fprintf(stderr, Name ": WARNING %s and %s appear"
726                                         " to have very similar superblocks.\n"
727                                         "      If they are really different, "
728                                         "please --zero the superblock on one\n"
729                                         "      If they are the same or overlap,"
730                                         " please remove one from %s.\n",
731                                         devices[best[i]].devname, devname,
732                                         inargv ? "the list" :
733                                            "the\n      DEVICE list in mdadm.conf"
734                                         );
735                                 close(mdfd);
736                                 return 1;
737                         }
738                         if (best[i] == -1
739                             || (devices[best[i]].i.events
740                                 < devices[devcnt].i.events))
741                                 best[i] = devcnt;
742                 }
743                 devcnt++;
744         }
745         free(content->update_private);
746         content->update_private = NULL;
747
748         if (devcnt == 0) {
749                 fprintf(stderr, Name ": no devices found for %s\n",
750                         mddev);
751                 if (st)
752                         st->ss->free_super(st);
753                 close(mdfd);
754                 return 1;
755         }
756
757         if (update && strcmp(update, "byteorder")==0)
758                 st->minor_version = 90;
759
760         st->ss->getinfo_super(st, content);
761         clean = content->array.state & 1;
762
763         /* now we have some devices that might be suitable.
764          * I wonder how many
765          */
766         avail = malloc(content->array.raid_disks);
767         memset(avail, 0, content->array.raid_disks);
768         okcnt = 0;
769         sparecnt=0;
770         for (i=0; i< bestcnt ;i++) {
771                 int j = best[i];
772                 int event_margin = 1; /* always allow a difference of '1'
773                                        * like the kernel does
774                                        */
775                 if (j < 0) continue;
776                 /* note: we ignore error flags in multipath arrays
777                  * as they don't make sense
778                  */
779                 if (content->array.level != LEVEL_MULTIPATH)
780                         if (!(devices[j].i.disk.state & (1<<MD_DISK_ACTIVE))) {
781                                 if (!(devices[j].i.disk.state
782                                       & (1<<MD_DISK_FAULTY)))
783                                         sparecnt++;
784                                 continue;
785                         }
786                 if (devices[j].i.events+event_margin >=
787                     devices[most_recent].i.events) {
788                         devices[j].uptodate = 1;
789                         if (i < content->array.raid_disks) {
790                                 okcnt++;
791                                 avail[i]=1;
792                         } else
793                                 sparecnt++;
794                 }
795         }
796         while (force && !enough(content->array.level, content->array.raid_disks,
797                                 content->array.layout, 1,
798                                 avail, okcnt)) {
799                 /* Choose the newest best drive which is
800                  * not up-to-date, update the superblock
801                  * and add it.
802                  */
803                 int fd;
804                 struct supertype *tst;
805                 long long current_events;
806                 chosen_drive = -1;
807                 for (i=0; i<content->array.raid_disks && i < bestcnt; i++) {
808                         int j = best[i];
809                         if (j>=0 &&
810                             !devices[j].uptodate &&
811                             (chosen_drive < 0 ||
812                              devices[j].i.events
813                              > devices[chosen_drive].i.events))
814                                 chosen_drive = j;
815                 }
816                 if (chosen_drive < 0)
817                         break;
818                 current_events = devices[chosen_drive].i.events;
819         add_another:
820                 if (verbose >= 0)
821                         fprintf(stderr, Name ": forcing event count in %s(%d) from %d upto %d\n",
822                                 devices[chosen_drive].devname,
823                                 devices[chosen_drive].i.disk.raid_disk,
824                                 (int)(devices[chosen_drive].i.events),
825                                 (int)(devices[most_recent].i.events));
826                 fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL);
827                 if (fd < 0) {
828                         fprintf(stderr, Name ": Couldn't open %s for write - not updating\n",
829                                 devices[chosen_drive].devname);
830                         devices[chosen_drive].i.events = 0;
831                         continue;
832                 }
833                 tst = dup_super(st);
834                 if (tst->ss->load_super(tst,fd, NULL)) {
835                         close(fd);
836                         fprintf(stderr, Name ": RAID superblock disappeared from %s - not updating.\n",
837                                 devices[chosen_drive].devname);
838                         devices[chosen_drive].i.events = 0;
839                         continue;
840                 }
841                 content->events = devices[most_recent].i.events;
842                 tst->ss->update_super(tst, content, "force-one",
843                                      devices[chosen_drive].devname, verbose,
844                                      0, NULL);
845
846                 if (tst->ss->store_super(tst, fd)) {
847                         close(fd);
848                         fprintf(stderr, Name ": Could not re-write superblock on %s\n",
849                                 devices[chosen_drive].devname);
850                         devices[chosen_drive].i.events = 0;
851                         tst->ss->free_super(tst);
852                         continue;
853                 }
854                 close(fd);
855                 devices[chosen_drive].i.events = devices[most_recent].i.events;
856                 devices[chosen_drive].uptodate = 1;
857                 avail[chosen_drive] = 1;
858                 okcnt++;
859                 tst->ss->free_super(tst);
860
861                 /* If there are any other drives of the same vintage,
862                  * add them in as well.  We can't lose and we might gain
863                  */
864                 for (i=0; i<content->array.raid_disks && i < bestcnt ; i++) {
865                         int j = best[i];
866                         if (j >= 0 &&
867                             !devices[j].uptodate &&
868                             devices[j].i.events == current_events) {
869                                 chosen_drive = j;
870                                 goto add_another;
871                         }
872                 }
873         }
874
875         /* Now we want to look at the superblock which the kernel will base things on
876          * and compare the devices that we think are working with the devices that the
877          * superblock thinks are working.
878          * If there are differences and --force is given, then update this chosen
879          * superblock.
880          */
881         chosen_drive = -1;
882         st->ss->free_super(st);
883         for (i=0; chosen_drive < 0 && i<bestcnt; i++) {
884                 int j = best[i];
885                 int fd;
886
887                 if (j<0)
888                         continue;
889                 if (!devices[j].uptodate)
890                         continue;
891                 chosen_drive = j;
892                 if ((fd=dev_open(devices[j].devname, O_RDONLY|O_EXCL))< 0) {
893                         fprintf(stderr, Name ": Cannot open %s: %s\n",
894                                 devices[j].devname, strerror(errno));
895                         close(mdfd);
896                         return 1;
897                 }
898                 if (st->ss->load_super(st,fd, NULL)) {
899                         close(fd);
900                         fprintf(stderr, Name ": RAID superblock has disappeared from %s\n",
901                                 devices[j].devname);
902                         close(mdfd);
903                         return 1;
904                 }
905                 close(fd);
906         }
907         if (st->sb == NULL) {
908                 fprintf(stderr, Name ": No suitable drives found for %s\n", mddev);
909                 close(mdfd);
910                 return 1;
911         }
912         st->ss->getinfo_super(st, content);
913 #ifndef MDASSEMBLE
914         sysfs_init(content, mdfd, 0);
915 #endif
916         for (i=0; i<bestcnt; i++) {
917                 int j = best[i];
918                 unsigned int desired_state;
919
920                 if (i < content->array.raid_disks)
921                         desired_state = (1<<MD_DISK_ACTIVE) | (1<<MD_DISK_SYNC);
922                 else
923                         desired_state = 0;
924
925                 if (j<0)
926                         continue;
927                 if (!devices[j].uptodate)
928                         continue;
929
930                 devices[j].i.disk.state = desired_state;
931                 if (!(devices[j].i.array.state & 1))
932                         clean = 0;
933
934                 if (st->ss->update_super(st, &devices[j].i, "assemble", NULL,
935                                          verbose, 0, NULL)) {
936                         if (force) {
937                                 if (verbose >= 0)
938                                         fprintf(stderr, Name ": "
939                                                 "clearing FAULTY flag for device %d in %s for %s\n",
940                                                 j, mddev, devices[j].devname);
941                                 change = 1;
942                         } else {
943                                 if (verbose >= -1)
944                                         fprintf(stderr, Name ": "
945                                                 "device %d in %s has wrong state in superblock, but %s seems ok\n",
946                                                 i, mddev, devices[j].devname);
947                         }
948                 }
949 #if 0
950                 if (!(super.disks[i].i.disk.state & (1 << MD_DISK_FAULTY))) {
951                         fprintf(stderr, Name ": devices %d of %s is not marked FAULTY in superblock, but cannot be found\n",
952                                 i, mddev);
953                 }
954 #endif
955         }
956         if (force && !clean &&
957             !enough(content->array.level, content->array.raid_disks,
958                     content->array.layout, clean,
959                     avail, okcnt)) {
960                 change += st->ss->update_super(st, content, "force-array",
961                                         devices[chosen_drive].devname, verbose,
962                                                0, NULL);
963                 clean = 1;
964         }
965
966         if (change) {
967                 int fd;
968                 fd = dev_open(devices[chosen_drive].devname, O_RDWR|O_EXCL);
969                 if (fd < 0) {
970                         fprintf(stderr, Name ": Could not open %s for write - cannot Assemble array.\n",
971                                 devices[chosen_drive].devname);
972                         close(mdfd);
973                         return 1;
974                 }
975                 if (st->ss->store_super(st, fd)) {
976                         close(fd);
977                         fprintf(stderr, Name ": Could not re-write superblock on %s\n",
978                                 devices[chosen_drive].devname);
979                         close(mdfd);
980                         return 1;
981                 }
982                 close(fd);
983         }
984
985         /* If we are in the middle of a reshape we may need to restore saved data
986          * that was moved aside due to the reshape overwriting live data
987          * The code of doing this lives in Grow.c
988          */
989 #ifndef MDASSEMBLE
990         if (content->reshape_active) {
991                 int err = 0;
992                 int *fdlist = malloc(sizeof(int)* bestcnt);
993                 if (verbose > 0)
994                         fprintf(stderr, Name ":%s has an active reshape - checking "
995                                 "if critical section needs to be restored\n",
996                                 chosen_name);
997                 for (i=0; i<bestcnt; i++) {
998                         int j = best[i];
999                         if (j >= 0) {
1000                                 fdlist[i] = dev_open(devices[j].devname, O_RDWR|O_EXCL);
1001                                 if (fdlist[i] < 0) {
1002                                         fprintf(stderr, Name ": Could not open %s for write - cannot Assemble array.\n",
1003                                                 devices[j].devname);
1004                                         err = 1;
1005                                         break;
1006                                 }
1007                         } else
1008                                 fdlist[i] = -1;
1009                 }
1010                 if (!err)
1011                         err = Grow_restart(st, content, fdlist, bestcnt, backup_file, verbose > 0);
1012                 while (i>0) {
1013                         i--;
1014                         if (fdlist[i]>=0) close(fdlist[i]);
1015                 }
1016                 if (err) {
1017                         fprintf(stderr, Name ": Failed to restore critical section for reshape, sorry.\n");
1018                         if (backup_file == NULL)
1019                                 fprintf(stderr,"      Possibly you needed to specify the --backup-file\n");
1020                         close(mdfd);
1021                         return err;
1022                 }
1023         }
1024 #endif
1025         /* count number of in-sync devices according to the superblock.
1026          * We must have this number to start the array without -s or -R
1027          */
1028         req_cnt = content->array.working_disks;
1029
1030         /* Almost ready to actually *do* something */
1031         if (!old_linux) {
1032                 int rv;
1033
1034                 /* First, fill in the map, so that udev can find our name
1035                  * as soon as we become active.
1036                  */
1037                 map_update(NULL, fd2devnum(mdfd), content->text_version,
1038                            content->uuid, chosen_name);
1039
1040                 rv = set_array_info(mdfd, st, content);
1041                 if (rv) {
1042                         fprintf(stderr, Name ": failed to set array info for %s: %s\n",
1043                                 mddev, strerror(errno));
1044                         close(mdfd);
1045                         return 1;
1046                 }
1047                 if (ident->bitmap_fd >= 0) {
1048                         if (ioctl(mdfd, SET_BITMAP_FILE, ident->bitmap_fd) != 0) {
1049                                 fprintf(stderr, Name ": SET_BITMAP_FILE failed.\n");
1050                                 close(mdfd);
1051                                 return 1;
1052                         }
1053                 } else if (ident->bitmap_file) {
1054                         /* From config file */
1055                         int bmfd = open(ident->bitmap_file, O_RDWR);
1056                         if (bmfd < 0) {
1057                                 fprintf(stderr, Name ": Could not open bitmap file %s\n",
1058                                         ident->bitmap_file);
1059                                 close(mdfd);
1060                                 return 1;
1061                         }
1062                         if (ioctl(mdfd, SET_BITMAP_FILE, bmfd) != 0) {
1063                                 fprintf(stderr, Name ": Failed to set bitmapfile for %s\n", mddev);
1064                                 close(bmfd);
1065                                 close(mdfd);
1066                                 return 1;
1067                         }
1068                         close(bmfd);
1069                 }
1070
1071                 /* First, add the raid disks, but add the chosen one last */
1072                 for (i=0; i<= bestcnt; i++) {
1073                         int j;
1074                         if (i < bestcnt) {
1075                                 j = best[i];
1076                                 if (j == chosen_drive)
1077                                         continue;
1078                         } else
1079                                 j = chosen_drive;
1080
1081                         if (j >= 0 /* && devices[j].uptodate */) {
1082                                 rv = add_disk(mdfd, st, content, &devices[j].i);
1083
1084                                 if (rv) {
1085                                         fprintf(stderr, Name ": failed to add "
1086                                                         "%s to %s: %s\n",
1087                                                 devices[j].devname,
1088                                                 mddev,
1089                                                 strerror(errno));
1090                                         if (i < content->array.raid_disks
1091                                             || i == bestcnt)
1092                                                 okcnt--;
1093                                         else
1094                                                 sparecnt--;
1095                                 } else if (verbose > 0)
1096                                         fprintf(stderr, Name ": added %s "
1097                                                         "to %s as %d\n",
1098                                                 devices[j].devname, mddev,
1099                                                 devices[j].i.disk.raid_disk);
1100                         } else if (verbose > 0 && i < content->array.raid_disks)
1101                                 fprintf(stderr, Name ": no uptodate device for "
1102                                                 "slot %d of %s\n",
1103                                         i, mddev);
1104                 }
1105
1106                 if (content->array.level == LEVEL_CONTAINER) {
1107                         if (verbose >= 0) {
1108                                 fprintf(stderr, Name ": Container %s has been "
1109                                         "assembled with %d drive%s",
1110                                         mddev, okcnt+sparecnt, okcnt+sparecnt==1?"":"s");
1111                                 if (okcnt < content->array.raid_disks)
1112                                         fprintf(stderr, " (out of %d)",
1113                                                 content->array.raid_disks);
1114                                 fprintf(stderr, "\n");
1115                         }
1116                         sysfs_uevent(content, "change");
1117                         wait_for(chosen_name, mdfd);
1118                         close(mdfd);
1119                         return 0;
1120                 }
1121
1122                 if (runstop == 1 ||
1123                     (runstop <= 0 &&
1124                      ( enough(content->array.level, content->array.raid_disks,
1125                               content->array.layout, clean, avail, okcnt) &&
1126                        (okcnt >= req_cnt || start_partial_ok)
1127                              ))) {
1128                         /* This array is good-to-go.
1129                          * If a reshape is in progress then we might need to
1130                          * continue monitoring it.  In that case we start
1131                          * it read-only and let the grow code make it writable.
1132                          */
1133                         int rv;
1134 #ifndef MDASSEMBLE
1135                         if (content->reshape_active &&
1136                             content->delta_disks <= 0)
1137                                 rv = Grow_continue(mdfd, st, content, backup_file);
1138                         else
1139 #endif
1140                                 rv = ioctl(mdfd, RUN_ARRAY, NULL);
1141                         if (rv == 0) {
1142                                 if (verbose >= 0) {
1143                                         fprintf(stderr, Name ": %s has been started with %d drive%s",
1144                                                 mddev, okcnt, okcnt==1?"":"s");
1145                                         if (okcnt < content->array.raid_disks)
1146                                                 fprintf(stderr, " (out of %d)", content->array.raid_disks);
1147                                         if (sparecnt)
1148                                                 fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s");
1149                                         fprintf(stderr, ".\n");
1150                                 }
1151                                 if (content->reshape_active &&
1152                                     content->array.level >= 4 &&
1153                                     content->array.level <= 6) {
1154                                         /* might need to increase the size
1155                                          * of the stripe cache - default is 256
1156                                          */
1157                                         if (256 < 4 * (content->array.chunk_size/4096)) {
1158                                                 struct mdinfo *sra = sysfs_read(mdfd, 0, 0);
1159                                                 if (sra)
1160                                                         sysfs_set_num(sra, NULL,
1161                                                                       "stripe_cache_size",
1162                                                                       (4 * content->array.chunk_size / 4096) + 1);
1163                                         }
1164                                 }
1165                                 wait_for(mddev, mdfd);
1166                                 close(mdfd);
1167                                 if (auto_assem) {
1168                                         int usecs = 1;
1169                                         /* There is a nasty race with 'mdadm --monitor'.
1170                                          * If it opens this device before we close it,
1171                                          * it gets an incomplete open on which IO
1172                                          * doesn't work and the capacity is
1173                                          * wrong.
1174                                          * If we reopen (to check for layered devices)
1175                                          * before --monitor closes, we loose.
1176                                          *
1177                                          * So: wait upto 1 second for there to be
1178                                          * a non-zero capacity.
1179                                          */
1180                                         while (usecs < 1000) {
1181                                                 mdfd = open(mddev, O_RDONLY);
1182                                                 if (mdfd >= 0) {
1183                                                         unsigned long long size;
1184                                                         if (get_dev_size(mdfd, NULL, &size) &&
1185                                                             size > 0)
1186                                                                 break;
1187                                                         close(mdfd);
1188                                                 }
1189                                                 usleep(usecs);
1190                                                 usecs <<= 1;
1191                                         }
1192                                 }
1193                                 return 0;
1194                         }
1195                         fprintf(stderr, Name ": failed to RUN_ARRAY %s: %s\n",
1196                                 mddev, strerror(errno));
1197
1198                         if (!enough(content->array.level, content->array.raid_disks,
1199                                     content->array.layout, 1, avail, okcnt))
1200                                 fprintf(stderr, Name ": Not enough devices to "
1201                                         "start the array.\n");
1202                         else if (!enough(content->array.level,
1203                                          content->array.raid_disks,
1204                                          content->array.layout, clean,
1205                                          avail, okcnt))
1206                                 fprintf(stderr, Name ": Not enough devices to "
1207                                         "start the array while not clean "
1208                                         "- consider --force.\n");
1209
1210                         if (auto_assem)
1211                                 ioctl(mdfd, STOP_ARRAY, NULL);
1212                         close(mdfd);
1213                         return 1;
1214                 }
1215                 if (runstop == -1) {
1216                         fprintf(stderr, Name ": %s assembled from %d drive%s",
1217                                 mddev, okcnt, okcnt==1?"":"s");
1218                         if (okcnt != content->array.raid_disks)
1219                                 fprintf(stderr, " (out of %d)", content->array.raid_disks);
1220                         fprintf(stderr, ", but not started.\n");
1221                         close(mdfd);
1222                         return 0;
1223                 }
1224                 if (verbose >= -1) {
1225                         fprintf(stderr, Name ": %s assembled from %d drive%s", mddev, okcnt, okcnt==1?"":"s");
1226                         if (sparecnt)
1227                                 fprintf(stderr, " and %d spare%s", sparecnt, sparecnt==1?"":"s");
1228                         if (!enough(content->array.level, content->array.raid_disks,
1229                                     content->array.layout, 1, avail, okcnt))
1230                                 fprintf(stderr, " - not enough to start the array.\n");
1231                         else if (!enough(content->array.level,
1232                                          content->array.raid_disks,
1233                                          content->array.layout, clean,
1234                                          avail, okcnt))
1235                                 fprintf(stderr, " - not enough to start the "
1236                                         "array while not clean - consider "
1237                                         "--force.\n");
1238                         else {
1239                                 if (req_cnt == content->array.raid_disks)
1240                                         fprintf(stderr, " - need all %d to start it", req_cnt);
1241                                 else
1242                                         fprintf(stderr, " - need %d of %d to start", req_cnt, content->array.raid_disks);
1243                                 fprintf(stderr, " (use --run to insist).\n");
1244                         }
1245                 }
1246                 if (auto_assem)
1247                         ioctl(mdfd, STOP_ARRAY, NULL);
1248                 close(mdfd);
1249                 return 1;
1250         } else {
1251                 /* The "chosen_drive" is a good choice, and if necessary, the superblock has
1252                  * been updated to point to the current locations of devices.
1253                  * so we can just start the array
1254                  */
1255                 unsigned long dev;
1256                 dev = makedev(devices[chosen_drive].i.disk.major,
1257                             devices[chosen_drive].i.disk.minor);
1258                 if (ioctl(mdfd, START_ARRAY, dev)) {
1259                     fprintf(stderr, Name ": Cannot start array: %s\n",
1260                             strerror(errno));
1261                 }
1262
1263         }
1264         close(mdfd);
1265         return 0;
1266 }
1267
1268 #ifndef MDASSEMBLE
1269 int assemble_container_content(struct supertype *st, int mdfd,
1270                                struct mdinfo *content, int runstop,
1271                                char *chosen_name, int verbose)
1272 {
1273         struct mdinfo *dev, *sra;
1274         int working = 0, preexist = 0;
1275         struct map_ent *map = NULL;
1276
1277         sysfs_init(content, mdfd, 0);
1278
1279         sra = sysfs_read(mdfd, 0, GET_VERSION);
1280         if (sra == NULL || strcmp(sra->text_version, content->text_version) != 0)
1281                 if (sysfs_set_array(content, md_get_version(mdfd)) != 0) {
1282                         close(mdfd);
1283                         return 1;
1284                 }
1285         if (sra)
1286                 sysfs_free(sra);
1287
1288         for (dev = content->devs; dev; dev = dev->next)
1289                 if (sysfs_add_disk(content, dev) == 0)
1290                         working++;
1291                 else if (errno == EEXIST)
1292                         preexist++;
1293         if (working == 0) {
1294                 close(mdfd);
1295                 return 1;/* Nothing new, don't try to start */
1296         }
1297         
1298         map_update(&map, fd2devnum(mdfd),
1299                    content->text_version,
1300                    content->uuid, chosen_name);
1301
1302         if (runstop > 0 ||
1303                  (working + preexist) >= content->array.working_disks) {
1304                 int err;
1305
1306                 switch(content->array.level) {
1307                 case LEVEL_LINEAR:
1308                 case LEVEL_MULTIPATH:
1309                 case 0:
1310                         err = sysfs_set_str(content, NULL, "array_state",
1311                                             "active");
1312                         break;
1313                 default:
1314                         err = sysfs_set_str(content, NULL, "array_state",
1315                                       "readonly");
1316                         /* start mdmon if needed. */
1317                         if (!err) {
1318                                 if (!mdmon_running(st->container_dev))
1319                                         start_mdmon(st->container_dev);
1320                                 ping_monitor(devnum2devname(st->container_dev));
1321                         }
1322                         break;
1323                 }
1324                 if (!err)
1325                         sysfs_set_safemode(content, content->safe_mode_delay);
1326                 if (verbose >= 0) {
1327                         if (err)
1328                                 fprintf(stderr, Name
1329                                         ": array %s now has %d devices",
1330                                         chosen_name, working + preexist);
1331                         else
1332                                 fprintf(stderr, Name
1333                                         ": Started %s with %d devices",
1334                                         chosen_name, working + preexist);
1335                         if (preexist)
1336                                 fprintf(stderr, " (%d new)", working);
1337                         fprintf(stderr, "\n");
1338                 }
1339                 if (!err)
1340                         wait_for(chosen_name, mdfd);
1341                 close(mdfd);
1342                 return 0;
1343                 /* FIXME should have an O_EXCL and wait for read-auto */
1344         } else {
1345                 if (verbose >= 0)
1346                         fprintf(stderr, Name
1347                                 ": %s assembled with %d devices but "
1348                                 "not started\n",
1349                                 chosen_name, working);
1350                 close(mdfd);
1351                 return 1;
1352         }
1353 }
1354 #endif
1355