]>
git.ipfire.org Git - thirdparty/mdadm.git/blob - Manage.c
2 * mdadm - manage Linux "md" devices aka RAID arrays.
4 * Copyright (C) 2001-2013 Neil Brown <neilb@suse.de>
7 * This program is free software; you can redistribute it and/or modify
8 * it under the terms of the GNU General Public License as published by
9 * the Free Software Foundation; either version 2 of the License, or
10 * (at your option) any later version.
12 * This program is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 * GNU General Public License for more details.
17 * You should have received a copy of the GNU General Public License
18 * along with this program; if not, write to the Free Software
19 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 * Email: <neilb@suse.de>
30 int Manage_ro(char *devname
, int fd
, int readonly
)
32 /* switch to readonly or rw
35 * first check that array is runing
36 * use RESTART_ARRAY_RW or STOP_ARRAY_RO
42 /* If this is an externally-managed array, we need to modify the
43 * metadata_version so that mdmon doesn't undo our change.
45 mdi
= sysfs_read(fd
, NULL
, GET_LEVEL
|GET_VERSION
);
47 mdi
->array
.major_version
== -1 &&
48 is_subarray(mdi
->text_version
)) {
50 strcpy(vers
, "external:");
51 strcat(vers
, mdi
->text_version
);
54 /* We set readonly ourselves. */
56 sysfs_set_str(mdi
, NULL
, "metadata_version", vers
);
59 rv
= sysfs_set_str(mdi
, NULL
, "array_state", "readonly");
62 pr_err("failed to set readonly for %s: %s\n",
63 devname
, strerror(errno
));
65 vers
[9] = mdi
->text_version
[0];
66 sysfs_set_str(mdi
, NULL
, "metadata_version", vers
);
72 /* We cannot set read/write - must signal mdmon */
74 sysfs_set_str(mdi
, NULL
, "metadata_version", vers
);
76 cp
= strchr(vers
+10, '/');
79 ping_monitor(vers
+10);
80 if (mdi
->array
.level
<= 0)
81 sysfs_set_str(mdi
, NULL
, "array_state", "active");
86 if (!md_array_active(fd
)) {
87 pr_err("%s does not appear to be active.\n", devname
);
93 if (ioctl(fd
, STOP_ARRAY_RO
, NULL
)) {
94 pr_err("failed to set readonly for %s: %s\n",
95 devname
, strerror(errno
));
99 } else if (readonly
< 0) {
100 if (ioctl(fd
, RESTART_ARRAY_RW
, NULL
)) {
101 pr_err("failed to set writable for %s: %s\n",
102 devname
, strerror(errno
));
112 static void remove_devices(char *devnm
, char *path
)
115 * Remove names at 'path' - possibly with
116 * partition suffixes - which link to the 'standard'
117 * name for devnm. These were probably created
118 * by mdadm when the array was assembled.
131 sprintf(base
, "/dev/%s", devnm
);
132 be
= base
+ strlen(base
);
134 path2
= xmalloc(strlen(path
)+20);
136 pe
= path2
+ strlen(path2
);
138 for (part
= 0; part
< 16; part
++) {
140 sprintf(be
, "p%d", part
);
143 sprintf(pe
, "p%d", part
);
145 sprintf(pe
, "%d", part
);
147 n
= readlink(path2
, link
, sizeof(link
));
148 if (n
> 0 && (int)strlen(base
) == n
&&
149 strncmp(link
, base
, n
) == 0)
155 int Manage_run(char *devname
, int fd
, struct context
*c
)
157 /* Run the array. Array must already be configured
164 pr_err("Cannot find %s in sysfs!!\n", devname
);
168 return IncrementalScan(c
, nm
);
171 int Manage_stop(char *devname
, int fd
, int verbose
, int will_retry
)
173 /* Stop the array. Array must already be configured
174 * 'will_retry' means that error messages are not wanted.
177 struct map_ent
*map
= NULL
;
184 unsigned long long rd1
, rd2
;
186 if (will_retry
&& verbose
== 0)
189 strcpy(devnm
, fd2devnm(fd
));
190 /* Get EXCL access first. If this fails, then attempting
191 * to stop is probably a bad idea.
193 mdi
= sysfs_read(fd
, NULL
, GET_LEVEL
|GET_COMPONENT
|GET_VERSION
);
194 if (mdi
&& is_subarray(mdi
->text_version
)) {
196 strncpy(container
, mdi
->text_version
+1, sizeof(container
));
197 container
[sizeof(container
)-1] = 0;
198 sl
= strchr(container
, '/');
205 while (((fd
= ((devname
[0] == '/')
206 ?open(devname
, O_RDONLY
|O_EXCL
)
207 :open_dev_flags(devnm
, O_RDONLY
|O_EXCL
))) < 0 ||
208 strcmp(fd2devnm(fd
), devnm
) != 0) && container
[0] &&
209 mdmon_running(container
) && count
) {
210 /* Can't open, so something might be wrong. However it
211 * is a container, so we might be racing with mdmon, so
216 flush_mdmon(container
);
219 if (fd
< 0 || strcmp(fd2devnm(fd
), devnm
) != 0) {
223 pr_err("Cannot get exclusive access to %s:Perhaps a running process, mounted filesystem or active volume group?\n",
227 /* If this is an mdmon managed array, just write 'inactive'
228 * to the array state and let mdmon clear up.
231 mdi
->array
.level
> 0 &&
232 is_subarray(mdi
->text_version
)) {
234 /* This is mdmon managed. */
237 /* As we had an O_EXCL open, any use of the device
238 * which blocks STOP_ARRAY is probably a transient use,
239 * so it is reasonable to retry for a while - 5 seconds.
243 (err
= sysfs_set_str(mdi
, NULL
,
252 pr_err("failed to stop array %s: %s\n",
253 devname
, strerror(errno
));
258 /* Give monitor a chance to act */
259 ping_monitor(mdi
->text_version
);
261 fd
= open_dev_excl(devnm
);
264 pr_err("failed to completely stop %s: Device is busy\n",
270 mdi
->array
.major_version
== -1 &&
271 mdi
->array
.minor_version
== -2 &&
272 !is_subarray(mdi
->text_version
)) {
273 struct mdstat_ent
*mds
, *m
;
274 /* container, possibly mdmon-managed.
275 * Make sure mdmon isn't opening it, which
276 * would interfere with the 'stop'
278 ping_monitor(mdi
->sys_name
);
280 /* now check that there are no existing arrays
281 * which are members of this array
283 mds
= mdstat_read(0, 0);
284 for (m
= mds
; m
; m
= m
->next
)
285 if (m
->metadata_version
&&
286 strncmp(m
->metadata_version
, "external:", 9)==0 &&
287 metadata_container_matches(m
->metadata_version
+9,
290 pr_err("Cannot stop container %s: member %s still active\n",
298 /* If the array is undergoing a reshape which changes the number
299 * of devices, then it would be nice to stop it at a point where
300 * it has completed a full number of stripes in both old and
301 * new layouts as this will allow the reshape to be reverted.
302 * So if 'sync_action' is "reshape" and 'raid_disks' shows two
303 * different numbers, then
305 * - set sync_max to next multiple of both data_disks and
306 * chunk sizes (or next but one)
308 * - wait on 'sync_completed' for that point to be reached.
310 if (mdi
&& (mdi
->array
.level
>= 4 && mdi
->array
.level
<= 6) &&
311 sysfs_attribute_available(mdi
, NULL
, "sync_action") &&
312 sysfs_attribute_available(mdi
, NULL
, "reshape_direction") &&
313 sysfs_get_str(mdi
, NULL
, "sync_action", buf
, 20) > 0 &&
314 strcmp(buf
, "reshape\n") == 0 &&
315 sysfs_get_two(mdi
, NULL
, "raid_disks", &rd1
, &rd2
) == 2) {
316 unsigned long long position
, curr
;
317 unsigned long long chunk1
, chunk2
;
318 unsigned long long rddiv
, chunkdiv
;
319 unsigned long long sectors
;
320 unsigned long long sync_max
, old_sync_max
;
321 unsigned long long completed
;
327 while (rd1
> rd2
&& delay
> 0 &&
328 sysfs_get_ll(mdi
, NULL
, "sync_max", &old_sync_max
) == 0) {
329 /* must be in the critical section - wait a bit */
334 if (sysfs_set_str(mdi
, NULL
, "sync_action", "frozen") != 0)
336 /* Array is frozen */
338 rd1
-= mdi
->array
.level
== 6 ? 2 : 1;
339 rd2
-= mdi
->array
.level
== 6 ? 2 : 1;
340 sysfs_get_str(mdi
, NULL
, "reshape_direction", buf
, sizeof(buf
));
341 if (strncmp(buf
, "back", 4) == 0)
343 if (sysfs_get_ll(mdi
, NULL
, "reshape_position", &position
) != 0) {
344 /* reshape must have finished now */
345 sysfs_set_str(mdi
, NULL
, "sync_action", "idle");
348 sysfs_get_two(mdi
, NULL
, "chunk_size", &chunk1
, &chunk2
);
351 rddiv
= GCD(rd1
, rd2
);
352 chunkdiv
= GCD(chunk1
, chunk2
);
353 sectors
= (chunk1
/chunkdiv
) * chunk2
* (rd1
/rddiv
) * rd2
;
356 /* Need to subtract 'reshape_position' from
357 * array size to get equivalent of sync_max.
358 * Size calculation based on raid5_size in kernel.
360 unsigned long long size
= mdi
->component_size
;
363 /* rd1 must be smaller */
364 /* Reshape may have progressed further backwards than
365 * recorded, so target even further back (hence "-1")
367 position
= (position
/ sectors
- 1) * sectors
;
368 /* rd1 is always the conversion factor between 'sync'
369 * position and 'reshape' position.
370 * We read 1 "new" stripe worth of data from where-ever,
371 * and when write out that full stripe.
373 sync_max
= size
- position
/rd1
;
375 /* Reshape will very likely be beyond position, and it may
376 * be too late to stop at '+1', so aim for '+2'
378 position
= (position
/ sectors
+ 2) * sectors
;
379 sync_max
= position
/rd1
;
381 if (sysfs_get_ll(mdi
, NULL
, "sync_max", &old_sync_max
) < 0)
382 old_sync_max
= mdi
->component_size
;
383 /* Must not advance sync_max as that could confuse
384 * the reshape monitor */
385 if (sync_max
< old_sync_max
)
386 sysfs_set_num(mdi
, NULL
, "sync_max", sync_max
);
387 sysfs_set_str(mdi
, NULL
, "sync_action", "idle");
389 /* That should have set things going again. Now we
390 * wait a little while (3 second max) for sync_completed
391 * to reach the target.
392 * The reshape process can block for 500msec if
393 * the sync speed limit is hit, so we need to wait
394 * a lot longer than that. 1 second is usually
398 scfd
= sysfs_open(mdi
->sys_name
, NULL
, "sync_completed");
399 while (scfd
>= 0 && delay
> 0 && old_sync_max
> 0) {
400 unsigned long long max_completed
;
401 sysfs_get_ll(mdi
, NULL
, "reshape_position", &curr
);
402 sysfs_fd_get_str(scfd
, buf
, sizeof(buf
));
403 if (strncmp(buf
, "none", 4) == 0) {
404 /* Either reshape has aborted, or hasn't
405 * quite started yet. Wait a bit and
406 * check 'sync_action' to see.
409 sysfs_get_str(mdi
, NULL
, "sync_action", buf
, sizeof(buf
));
410 if (strncmp(buf
, "reshape", 7) != 0)
414 if (sysfs_fd_get_two(scfd
, &completed
,
415 &max_completed
) == 2 &&
416 /* 'completed' sometimes reads as max-uulong */
417 completed
< max_completed
&&
418 (completed
> sync_max
||
419 (completed
== sync_max
&& curr
!= position
))) {
420 while (completed
> sync_max
) {
421 sync_max
+= sectors
/ rd1
;
427 if (sync_max
< old_sync_max
)
428 sysfs_set_num(mdi
, NULL
, "sync_max", sync_max
);
431 if (!backwards
&& curr
>= position
)
433 if (backwards
&& curr
<= position
)
435 sysfs_wait(scfd
, &delay
);
443 /* As we have an O_EXCL open, any use of the device
444 * which blocks STOP_ARRAY is probably a transient use,
445 * so it is reasonable to retry for a while - 5 seconds.
448 while (count
&& fd
>= 0 &&
449 (err
= ioctl(fd
, STOP_ARRAY
, NULL
)) < 0 && errno
== EBUSY
) {
453 if (fd
>= 0 && err
) {
455 pr_err("failed to stop array %s: %s\n",
456 devname
, strerror(errno
));
458 cont_err("Perhaps a running process, mounted filesystem or active volume group?\n");
464 if (get_linux_version() < 2006028) {
465 /* prior to 2.6.28, KOBJ_CHANGE was not sent when an md array
466 * was stopped, so We'll do it here just to be sure. Drop any
467 * partitions as well...
470 ioctl(fd
, BLKRRPART
, 0);
472 sysfs_uevent(mdi
, "change");
475 if (devnm
[0] && use_udev()) {
476 struct map_ent
*mp
= map_by_devnm(&map
, devnm
);
477 remove_devices(devnm
, mp
? mp
->path
: NULL
);
481 pr_err("stopped %s\n", devname
);
483 map_remove(&map
, devnm
);
491 static struct mddev_dev
*add_one(struct mddev_dev
*dv
, char *name
, char disp
)
493 struct mddev_dev
*new;
494 new = xmalloc(sizeof(*new));
495 memset(new, 0, sizeof(*new));
496 new->devname
= xstrdup(name
);
497 new->disposition
= disp
;
498 new->next
= dv
->next
;
503 static void add_faulty(struct mddev_dev
*dv
, int fd
, char disp
)
505 mdu_array_info_t array
;
506 mdu_disk_info_t disk
;
510 if (md_get_array_info(fd
, &array
) != 0)
513 remaining_disks
= array
.nr_disks
;
514 for (i
= 0; i
< MAX_DISKS
&& remaining_disks
> 0; i
++) {
517 if (md_get_disk_info(fd
, &disk
) != 0)
519 if (disk
.major
== 0 && disk
.minor
== 0)
522 if ((disk
.state
& 1) == 0) /* not faulty */
524 sprintf(buf
, "%d:%d", disk
.major
, disk
.minor
);
525 dv
= add_one(dv
, buf
, disp
);
529 static void add_detached(struct mddev_dev
*dv
, int fd
, char disp
)
531 mdu_array_info_t array
;
532 mdu_disk_info_t disk
;
536 if (md_get_array_info(fd
, &array
) != 0)
539 remaining_disks
= array
.nr_disks
;
540 for (i
= 0; i
< MAX_DISKS
&& remaining_disks
> 0; i
++) {
544 if (md_get_disk_info(fd
, &disk
) != 0)
546 if (disk
.major
== 0 && disk
.minor
== 0)
549 if (disp
== 'f' && (disk
.state
& 1) != 0) /* already faulty */
551 sprintf(buf
, "%d:%d", disk
.major
, disk
.minor
);
552 sfd
= dev_open(buf
, O_RDONLY
);
559 /* Probably not detached */
561 dv
= add_one(dv
, buf
, disp
);
565 static void add_set(struct mddev_dev
*dv
, int fd
, char set_char
)
567 mdu_array_info_t array
;
568 mdu_disk_info_t disk
;
573 if (md_get_array_info(fd
, &array
) != 0)
575 if (array
.level
!= 10)
577 copies
= ((array
.layout
& 0xff) *
578 ((array
.layout
>> 8) & 0xff));
579 if (array
.raid_disks
% copies
)
582 remaining_disks
= array
.nr_disks
;
583 for (i
= 0; i
< MAX_DISKS
&& remaining_disks
> 0; i
++) {
586 if (md_get_disk_info(fd
, &disk
) != 0)
588 if (disk
.major
== 0 && disk
.minor
== 0)
591 set
= disk
.raid_disk
% copies
;
592 if (set_char
!= set
+ 'A')
594 sprintf(buf
, "%d:%d", disk
.major
, disk
.minor
);
595 dv
= add_one(dv
, buf
, dv
->disposition
);
599 int attempt_re_add(int fd
, int tfd
, struct mddev_dev
*dv
,
600 struct supertype
*dev_st
, struct supertype
*tst
,
602 char *update
, char *devname
, int verbose
,
603 mdu_array_info_t
*array
)
609 dev_st
->ss
->getinfo_super(dev_st
, &mdi
, NULL
);
610 dev_st
->ss
->uuid_from_super(dev_st
, ouuid
);
612 tst
->ss
->uuid_from_super(tst
, duuid
);
614 /* Assume uuid matches: kernel will check */
615 memcpy(duuid
, ouuid
, sizeof(ouuid
));
616 if ((mdi
.disk
.state
& (1<<MD_DISK_ACTIVE
)) &&
617 !(mdi
.disk
.state
& (1<<MD_DISK_FAULTY
)) &&
618 memcmp(duuid
, ouuid
, sizeof(ouuid
))==0) {
619 /* Looks like it is worth a
620 * try. Need to make sure
621 * kernel will accept it
624 mdu_disk_info_t disc
;
625 /* re-add doesn't work for version-1 superblocks
628 if (array
->major_version
== 1 &&
629 get_linux_version() <= 2006018)
631 disc
.number
= mdi
.disk
.number
;
632 if (md_get_disk_info(fd
, &disc
) != 0 ||
633 disc
.major
!= 0 || disc
.minor
!= 0)
635 disc
.major
= major(rdev
);
636 disc
.minor
= minor(rdev
);
637 disc
.number
= mdi
.disk
.number
;
638 disc
.raid_disk
= mdi
.disk
.raid_disk
;
639 disc
.state
= mdi
.disk
.state
;
640 if (array
->state
& (1 << MD_SB_CLUSTERED
)) {
641 /* extra flags are needed when adding to a cluster as
642 * there are two cases to distinguish
644 if (dv
->disposition
== 'c')
645 disc
.state
|= (1 << MD_DISK_CANDIDATE
);
647 disc
.state
|= (1 << MD_DISK_CLUSTER_ADD
);
649 if (dv
->writemostly
== FlagSet
)
650 disc
.state
|= 1 << MD_DISK_WRITEMOSTLY
;
651 if (dv
->writemostly
== FlagClear
)
652 disc
.state
&= ~(1 << MD_DISK_WRITEMOSTLY
);
653 if (dv
->failfast
== FlagSet
)
654 disc
.state
|= 1 << MD_DISK_FAILFAST
;
655 if (dv
->failfast
== FlagClear
)
656 disc
.state
&= ~(1 << MD_DISK_FAILFAST
);
657 remove_partitions(tfd
);
658 if (update
|| dv
->writemostly
!= FlagDefault
||
659 dv
->failfast
!= FlagDefault
) {
661 tfd
= dev_open(dv
->devname
, O_RDWR
);
663 pr_err("failed to open %s for superblock update during re-add\n", dv
->devname
);
667 if (dv
->writemostly
== FlagSet
)
668 rv
= dev_st
->ss
->update_super(
669 dev_st
, NULL
, "writemostly",
670 devname
, verbose
, 0, NULL
);
671 if (dv
->writemostly
== FlagClear
)
672 rv
= dev_st
->ss
->update_super(
673 dev_st
, NULL
, "readwrite",
674 devname
, verbose
, 0, NULL
);
675 if (dv
->failfast
== FlagSet
)
676 rv
= dev_st
->ss
->update_super(
677 dev_st
, NULL
, "failfast",
678 devname
, verbose
, 0, NULL
);
679 if (dv
->failfast
== FlagClear
)
680 rv
= dev_st
->ss
->update_super(
681 dev_st
, NULL
, "nofailfast",
682 devname
, verbose
, 0, NULL
);
684 rv
= dev_st
->ss
->update_super(
685 dev_st
, NULL
, update
,
686 devname
, verbose
, 0, NULL
);
688 rv
= dev_st
->ss
->store_super(dev_st
, tfd
);
691 pr_err("failed to update superblock during re-add\n");
695 /* don't even try if disk is marked as faulty */
697 if (ioctl(fd
, ADD_NEW_DISK
, &disc
) == 0) {
699 pr_err("re-added %s\n", dv
->devname
);
702 if (errno
== ENOMEM
|| errno
== EROFS
) {
703 pr_err("add new device failed for %s: %s\n",
704 dv
->devname
, strerror(errno
));
705 if (dv
->disposition
== 'M')
714 int Manage_add(int fd
, int tfd
, struct mddev_dev
*dv
,
715 struct supertype
*tst
, mdu_array_info_t
*array
,
716 int force
, int verbose
, char *devname
,
717 char *update
, unsigned long rdev
, unsigned long long array_size
,
720 unsigned long long ldsize
;
721 struct supertype
*dev_st
;
723 mdu_disk_info_t disc
;
725 if (!get_dev_size(tfd
, dv
->devname
, &ldsize
)) {
726 if (dv
->disposition
== 'M')
732 if (tst
->ss
== &super0
&& ldsize
> 4ULL*1024*1024*1024*1024) {
733 /* More than 4TB is wasted on v0.90 */
735 pr_err("%s is larger than %s can effectively use.\n"
736 " Add --force is you really want to add this device.\n",
737 dv
->devname
, devname
);
740 pr_err("%s is larger than %s can effectively use.\n"
741 " Adding anyway as --force was given.\n",
742 dv
->devname
, devname
);
744 if (!tst
->ss
->external
&& array
->major_version
== 0) {
745 if (ioctl(fd
, HOT_ADD_DISK
, rdev
)==0) {
747 pr_err("hot added %s\n",
752 pr_err("hot add failed for %s: %s\n",
753 dv
->devname
, strerror(errno
));
757 if (array
->not_persistent
== 0 || tst
->ss
->external
) {
759 /* need to find a sample superblock to copy, and
760 * a spare slot to use.
761 * For 'external' array (well, container based),
762 * We can just load the metadata for the array->
766 /* already loaded */;
767 else if (tst
->ss
->external
) {
768 tst
->ss
->load_container(tst
, fd
, NULL
);
769 } else for (j
= 0; j
< tst
->max_devs
; j
++) {
773 if (md_get_disk_info(fd
, &disc
))
775 if (disc
.major
==0 && disc
.minor
==0)
777 if ((disc
.state
& 4)==0) /* sync */
779 /* Looks like a good device to try */
780 dev
= map_dev(disc
.major
, disc
.minor
, 1);
783 dfd
= dev_open(dev
, O_RDONLY
);
786 if (tst
->ss
->load_super(tst
, dfd
,
794 /* FIXME this is a bad test to be using */
795 if (!tst
->sb
&& (dv
->disposition
!= 'a' &&
796 dv
->disposition
!= 'S')) {
797 /* we are re-adding a device to a
798 * completely dead array - have to depend
801 } else if (!tst
->sb
) {
802 pr_err("cannot load array metadata from %s\n", devname
);
806 /* Make sure device is large enough */
807 if (dv
->disposition
!= 'j' && /* skip size check for Journal */
809 tst
->ss
->avail_size(tst
, ldsize
/512, INVALID_SECTORS
) <
811 if (dv
->disposition
== 'M')
813 pr_err("%s not large enough to join array\n",
818 /* Possibly this device was recently part of
819 * the array and was temporarily removed, and
820 * is now being re-added. If so, we can
824 if (array
->not_persistent
== 0) {
825 dev_st
= dup_super(tst
);
826 dev_st
->ss
->load_super(dev_st
, tfd
, NULL
);
827 if (dev_st
->sb
&& dv
->disposition
!= 'S') {
830 rv
= attempt_re_add(fd
, tfd
, dv
, dev_st
, tst
,
831 rdev
, update
, devname
,
833 dev_st
->ss
->free_super(dev_st
);
838 if (dv
->disposition
== 'M') {
840 pr_err("--re-add for %s to %s is not possible\n",
841 dv
->devname
, devname
);
844 if (dv
->disposition
== 'A') {
845 pr_err("--re-add for %s to %s is not possible\n",
846 dv
->devname
, devname
);
849 if (array
->active_disks
< array
->raid_disks
) {
850 char *avail
= xcalloc(array
->raid_disks
, 1);
854 for (d
= 0; d
< MAX_DISKS
&& found
< array
->nr_disks
; d
++) {
856 if (md_get_disk_info(fd
, &disc
))
858 if (disc
.major
== 0 && disc
.minor
== 0)
860 if (!(disc
.state
& (1<<MD_DISK_SYNC
)))
862 avail
[disc
.raid_disk
] = 1;
865 array_failed
= !enough(array
->level
, array
->raid_disks
,
866 array
->layout
, 1, avail
);
871 pr_err("%s has failed so using --add cannot work and might destroy\n",
873 pr_err("data on %s. You should stop the array and re-assemble it.\n",
878 /* non-persistent. Must ensure that new drive
879 * is at least array->size big.
881 if (ldsize
/512 < array_size
) {
882 pr_err("%s not large enough to join array\n",
887 /* committed to really trying this device now*/
888 remove_partitions(tfd
);
890 /* in 2.6.17 and earlier, version-1 superblocks won't
891 * use the number we write, but will choose a free number.
892 * we must choose the same free number, which requires
893 * starting at 'raid_disks' and counting up
895 for (j
= array
->raid_disks
; j
< tst
->max_devs
; j
++) {
897 if (md_get_disk_info(fd
, &disc
))
899 if (disc
.major
==0 && disc
.minor
==0)
901 if (disc
.state
& 8) /* removed */
904 disc
.major
= major(rdev
);
905 disc
.minor
= minor(rdev
);
909 disc
.number
= raid_slot
;
912 /* only add journal to array that supports journaling */
913 if (dv
->disposition
== 'j') {
917 mdp
= sysfs_read(fd
, NULL
, GET_ARRAY_STATE
);
919 pr_err("%s unable to read array state.\n", devname
);
923 if (mdp
->array_state
!= ARRAY_READONLY
) {
925 pr_err("%s is not readonly, cannot add journal.\n", devname
);
931 tst
->ss
->getinfo_super(tst
, &mdi
, NULL
);
932 if (mdi
.journal_device_required
== 0) {
933 pr_err("%s does not support journal device.\n", devname
);
939 if (array
->not_persistent
==0) {
941 if (dv
->disposition
== 'j')
942 disc
.state
|= (1 << MD_DISK_JOURNAL
) | (1 << MD_DISK_SYNC
);
943 if (dv
->writemostly
== FlagSet
)
944 disc
.state
|= 1 << MD_DISK_WRITEMOSTLY
;
945 if (dv
->failfast
== FlagSet
)
946 disc
.state
|= 1 << MD_DISK_FAILFAST
;
947 dfd
= dev_open(dv
->devname
, O_RDWR
| O_EXCL
|O_DIRECT
);
948 if (tst
->ss
->add_to_super(tst
, &disc
, dfd
,
949 dv
->devname
, INVALID_SECTORS
))
951 if (tst
->ss
->write_init_super(tst
))
953 } else if (dv
->disposition
== 'A') {
954 /* this had better be raid1.
955 * As we are "--re-add"ing we must find a spare slot
958 char *used
= xcalloc(array
->raid_disks
, 1);
959 for (j
= 0; j
< tst
->max_devs
; j
++) {
960 mdu_disk_info_t disc2
;
962 if (md_get_disk_info(fd
, &disc2
))
964 if (disc2
.major
==0 && disc2
.minor
==0)
966 if (disc2
.state
& 8) /* removed */
968 if (disc2
.raid_disk
< 0)
970 if (disc2
.raid_disk
> array
->raid_disks
)
972 used
[disc2
.raid_disk
] = 1;
974 for (j
= 0 ; j
< array
->raid_disks
; j
++)
977 disc
.state
|= (1<<MD_DISK_SYNC
);
983 if (array
->state
& (1 << MD_SB_CLUSTERED
)) {
984 if (dv
->disposition
== 'c')
985 disc
.state
|= (1 << MD_DISK_CANDIDATE
);
987 disc
.state
|= (1 << MD_DISK_CLUSTER_ADD
);
990 if (dv
->writemostly
== FlagSet
)
991 disc
.state
|= (1 << MD_DISK_WRITEMOSTLY
);
992 if (dv
->failfast
== FlagSet
)
993 disc
.state
|= (1 << MD_DISK_FAILFAST
);
994 if (tst
->ss
->external
) {
996 * to an external metadata container */
997 struct mdinfo new_mdi
;
1003 strcpy(devnm
, fd2devnm(fd
));
1005 container_fd
= open_dev_excl(devnm
);
1006 if (container_fd
< 0) {
1007 pr_err("add failed for %s: could not get exclusive access to container\n",
1009 tst
->ss
->free_super(tst
);
1013 Kill(dv
->devname
, NULL
, 0, -1, 0);
1014 dfd
= dev_open(dv
->devname
, O_RDWR
| O_EXCL
|O_DIRECT
);
1015 if (mdmon_running(tst
->container_devnm
))
1016 tst
->update_tail
= &tst
->updates
;
1017 if (tst
->ss
->add_to_super(tst
, &disc
, dfd
,
1018 dv
->devname
, INVALID_SECTORS
)) {
1020 close(container_fd
);
1023 if (tst
->update_tail
)
1024 flush_metadata_updates(tst
);
1026 tst
->ss
->sync_metadata(tst
);
1028 sra
= sysfs_read(container_fd
, NULL
, 0);
1030 pr_err("add failed for %s: sysfs_read failed\n",
1032 close(container_fd
);
1033 tst
->ss
->free_super(tst
);
1036 sra
->array
.level
= LEVEL_CONTAINER
;
1037 /* Need to set data_offset and component_size */
1038 tst
->ss
->getinfo_super(tst
, &new_mdi
, NULL
);
1039 new_mdi
.disk
.major
= disc
.major
;
1040 new_mdi
.disk
.minor
= disc
.minor
;
1041 new_mdi
.recovery_start
= 0;
1042 /* Make sure fds are closed as they are O_EXCL which
1043 * would block add_disk */
1044 tst
->ss
->free_super(tst
);
1045 if (sysfs_add_disk(sra
, &new_mdi
, 0) != 0) {
1046 pr_err("add new device to external metadata failed for %s\n", dv
->devname
);
1047 close(container_fd
);
1051 ping_monitor(devnm
);
1053 close(container_fd
);
1055 tst
->ss
->free_super(tst
);
1056 if (ioctl(fd
, ADD_NEW_DISK
, &disc
)) {
1057 if (dv
->disposition
== 'j')
1058 pr_err("Failed to hot add %s as journal, "
1059 "please try restart %s.\n", dv
->devname
, devname
);
1061 pr_err("add new device failed for %s as %d: %s\n",
1062 dv
->devname
, j
, strerror(errno
));
1065 if (dv
->disposition
== 'j') {
1066 pr_err("Journal added successfully, making %s read-write\n", devname
);
1067 if (Manage_ro(devname
, fd
, -1))
1068 pr_err("Failed to make %s read-write\n", devname
);
1073 pr_err("added %s\n", dv
->devname
);
1077 int Manage_remove(struct supertype
*tst
, int fd
, struct mddev_dev
*dv
,
1078 int sysfd
, unsigned long rdev
, int force
, int verbose
, char *devname
)
1083 if (tst
->ss
->external
) {
1084 /* To remove a device from a container, we must
1085 * check that it isn't in use in an array.
1086 * This involves looking in the 'holders'
1087 * directory - there must be just one entry,
1089 * To ensure that it doesn't get used as a
1090 * hot spare while we are checking, we
1091 * get an O_EXCL open on the container
1095 strcpy(devnm
, fd2devnm(fd
));
1096 lfd
= open_dev_excl(devnm
);
1098 pr_err("Cannot get exclusive access to container - odd\n");
1101 /* We may not be able to check on holders in
1102 * sysfs, either because we don't have the dev num
1103 * (rdev == 0) or because the device has been detached
1104 * and the 'holders' directory no longer exists
1105 * (ret == -1). In that case, assume it is OK to
1112 * The drive has already been set to 'faulty', however
1113 * monitor might not have had time to process it and the
1114 * drive might still have an entry in the 'holders'
1115 * directory. Try a few times to avoid a false error
1120 ret
= sysfs_unique_holder(devnm
, rdev
);
1123 usleep(100 * 1000); /* 100ms */
1124 } while (--count
> 0);
1127 pr_err("%s is not a member, cannot remove.\n",
1133 pr_err("%s is still in use, cannot remove.\n",
1140 /* FIXME check that it is a current member */
1142 /* device has been removed and we don't know
1143 * the major:minor number
1145 err
= sys_hot_remove_disk(sysfd
, force
);
1147 err
= hot_remove_disk(fd
, rdev
, force
);
1148 if (err
&& errno
== ENODEV
) {
1149 /* Old kernels rejected this if no personality
1151 struct mdinfo
*sra
= sysfs_read(fd
, NULL
, GET_DEVS
);
1152 struct mdinfo
*dv
= NULL
;
1155 for ( ; dv
; dv
=dv
->next
)
1156 if (dv
->disk
.major
== (int)major(rdev
) &&
1157 dv
->disk
.minor
== (int)minor(rdev
))
1160 err
= sysfs_set_str(sra
, dv
,
1168 pr_err("hot remove failed for %s: %s\n", dv
->devname
,
1174 if (tst
->ss
->external
) {
1176 * Before dropping our exclusive open we make an
1177 * attempt at preventing mdmon from seeing an
1178 * 'add' event before reconciling this 'remove'
1181 char *devnm
= fd2devnm(fd
);
1184 pr_err("unable to get container name\n");
1188 ping_manager(devnm
);
1193 pr_err("hot removed %s from %s\n",
1194 dv
->devname
, devname
);
1198 int Manage_replace(struct supertype
*tst
, int fd
, struct mddev_dev
*dv
,
1199 unsigned long rdev
, int verbose
, char *devname
)
1201 struct mdinfo
*mdi
, *di
;
1202 if (tst
->ss
->external
) {
1203 pr_err("--replace only supported for native metadata (0.90 or 1.x)\n");
1206 /* Need to find the device in sysfs and add 'want_replacement' to the
1209 mdi
= sysfs_read(fd
, NULL
, GET_DEVS
);
1210 if (!mdi
|| !mdi
->devs
) {
1211 pr_err("Cannot find status of %s to enable replacement - strange\n",
1215 for (di
= mdi
->devs
; di
; di
= di
->next
)
1216 if (di
->disk
.major
== (int)major(rdev
) &&
1217 di
->disk
.minor
== (int)minor(rdev
))
1221 if (di
->disk
.raid_disk
< 0) {
1222 pr_err("%s is not active and so cannot be replaced.\n",
1227 rv
= sysfs_set_str(mdi
, di
,
1228 "state", "want_replacement");
1231 pr_err("Failed to request replacement for %s\n",
1236 pr_err("Marked %s (device %d in %s) for replacement\n",
1237 dv
->devname
, di
->disk
.raid_disk
, devname
);
1238 /* If there is a matching 'with', we need to tell it which
1241 while (dv
&& dv
->disposition
!= 'W')
1244 dv
->disposition
= 'w';
1245 dv
->used
= di
->disk
.raid_disk
;
1250 pr_err("%s not found in %s so cannot --replace it\n",
1251 dv
->devname
, devname
);
1255 int Manage_with(struct supertype
*tst
, int fd
, struct mddev_dev
*dv
,
1256 unsigned long rdev
, int verbose
, char *devname
)
1258 struct mdinfo
*mdi
, *di
;
1259 /* try to set 'slot' for 'rdev' in 'fd' to 'dv->used' */
1260 mdi
= sysfs_read(fd
, NULL
, GET_DEVS
|GET_STATE
);
1261 if (!mdi
|| !mdi
->devs
) {
1262 pr_err("Cannot find status of %s to enable replacement - strange\n",
1266 for (di
= mdi
->devs
; di
; di
= di
->next
)
1267 if (di
->disk
.major
== (int)major(rdev
) &&
1268 di
->disk
.minor
== (int)minor(rdev
))
1272 if (di
->disk
.state
& (1<<MD_DISK_FAULTY
)) {
1273 pr_err("%s is faulty and cannot be a replacement\n",
1278 if (di
->disk
.raid_disk
>= 0) {
1279 pr_err("%s is active and cannot be a replacement\n",
1284 rv
= sysfs_set_num(mdi
, di
,
1288 pr_err("Failed to set %s as preferred replacement.\n",
1293 pr_err("Marked %s in %s as replacement for device %d\n",
1294 dv
->devname
, devname
, dv
->used
);
1298 pr_err("%s not found in %s so cannot make it preferred replacement\n",
1299 dv
->devname
, devname
);
1303 int Manage_subdevs(char *devname
, int fd
,
1304 struct mddev_dev
*devlist
, int verbose
, int test
,
1305 char *update
, int force
)
1307 /* Do something to each dev.
1309 * 'a' - add the device
1311 * If that fails EINVAL, try ADD_NEW_DISK
1312 * 'S' - add the device as a spare - don't try re-add
1313 * 'j' - add the device as a journal device
1314 * 'A' - re-add the device
1315 * 'r' - remove the device: HOT_REMOVE_DISK
1316 * device can be 'faulty' or 'detached' in which case all
1317 * matching devices are removed.
1318 * 'f' - set the device faulty SET_DISK_FAULTY
1319 * device can be 'detached' in which case any device that
1320 * is inaccessible will be marked faulty.
1321 * 'R' - mark this device as wanting replacement.
1322 * 'W' - this device is added if necessary and activated as
1323 * a replacement for a previous 'R' device.
1325 * 'w' - 'W' will be changed to 'w' when it is paired with
1326 * a 'R' device. If a 'W' is found while walking the list
1327 * it must be unpaired, and is an error.
1328 * 'M' - this is created by a 'missing' target. It is a slight
1330 * 'F' - Another variant of 'A', where the device was faulty
1331 * so must be removed from the array first.
1332 * 'c' - confirm the device as found (for clustered environments)
1334 * For 'f' and 'r', the device can also be a kernel-internal
1335 * name such as 'sdb'.
1337 mdu_array_info_t array
;
1338 unsigned long long array_size
;
1339 struct mddev_dev
*dv
;
1341 struct supertype
*tst
;
1342 char *subarray
= NULL
;
1344 int count
= 0; /* number of actions taken */
1346 struct mdinfo devinfo
;
1351 if (sysfs_init(&info
, fd
, NULL
)) {
1352 pr_err("sysfs not availabile for %s\n", devname
);
1356 if (md_get_array_info(fd
, &array
)) {
1357 pr_err("Cannot get array info for %s\n", devname
);
1360 /* array.size is only 32 bits and may be truncated.
1361 * So read from sysfs if possible, and record number of sectors
1364 array_size
= get_component_size(fd
);
1365 if (array_size
<= 0)
1366 array_size
= array
.size
* 2;
1368 tst
= super_by_fd(fd
, &subarray
);
1370 pr_err("unsupport array - version %d.%d\n",
1371 array
.major_version
, array
.minor_version
);
1375 for (dv
= devlist
; dv
; dv
= dv
->next
) {
1376 unsigned long rdev
= 0; /* device to add/remove etc */
1381 if (dv
->disposition
== 'c') {
1382 rv
= parse_cluster_confirm_arg(dv
->devname
,
1386 pr_err("Could not get the devname of cluster\n");
1391 if (strcmp(dv
->devname
, "failed") == 0 ||
1392 strcmp(dv
->devname
, "faulty") == 0) {
1393 if (dv
->disposition
!= 'A' && dv
->disposition
!= 'r') {
1394 pr_err("%s only meaningful with -r or --re-add, not -%c\n",
1395 dv
->devname
, dv
->disposition
);
1398 add_faulty(dv
, fd
, (dv
->disposition
== 'A'
1402 if (strcmp(dv
->devname
, "detached") == 0) {
1403 if (dv
->disposition
!= 'r' && dv
->disposition
!= 'f') {
1404 pr_err("%s only meaningful with -r of -f, not -%c\n",
1405 dv
->devname
, dv
->disposition
);
1408 add_detached(dv
, fd
, dv
->disposition
);
1412 if (strcmp(dv
->devname
, "missing") == 0) {
1413 struct mddev_dev
*add_devlist
;
1414 struct mddev_dev
**dp
;
1415 if (dv
->disposition
== 'c') {
1416 rv
= ioctl(fd
, CLUSTERED_DISK_NACK
, NULL
);
1420 if (dv
->disposition
!= 'A') {
1421 pr_err("'missing' only meaningful with --re-add\n");
1424 add_devlist
= conf_get_devs();
1425 if (add_devlist
== NULL
) {
1426 pr_err("no devices to scan for missing members.");
1429 for (dp
= &add_devlist
; *dp
; dp
= & (*dp
)->next
)
1430 /* 'M' (for 'missing') is like 'A' without errors */
1431 (*dp
)->disposition
= 'M';
1433 dv
->next
= add_devlist
;
1437 if (strncmp(dv
->devname
, "set-", 4) == 0 &&
1438 strlen(dv
->devname
) == 5) {
1441 if (dv
->disposition
!= 'r' &&
1442 dv
->disposition
!= 'f') {
1443 pr_err("'%s' only meaningful with -r or -f\n",
1447 if (array
.level
!= 10) {
1448 pr_err("'%s' only meaningful with RAID10 arrays\n",
1452 copies
= ((array
.layout
& 0xff) *
1453 ((array
.layout
>> 8) & 0xff));
1454 if (array
.raid_disks
% copies
!= 0 ||
1455 dv
->devname
[4] < 'A' ||
1456 dv
->devname
[4] >= 'A' + copies
||
1458 pr_err("'%s' not meaningful with this array\n",
1462 add_set(dv
, fd
, dv
->devname
[4]);
1466 if (strchr(dv
->devname
, '/') == NULL
&&
1467 strchr(dv
->devname
, ':') == NULL
&&
1468 strlen(dv
->devname
) < 50) {
1469 /* Assume this is a kernel-internal name like 'sda1' */
1472 if (dv
->disposition
!= 'r' && dv
->disposition
!= 'f') {
1473 pr_err("%s only meaningful with -r or -f, not -%c\n",
1474 dv
->devname
, dv
->disposition
);
1478 sprintf(dname
, "dev-%s", dv
->devname
);
1479 sysfd
= sysfs_open(fd2devnm(fd
), dname
, "block/dev");
1482 if (sysfs_fd_get_str(sysfd
, dn
, 20) > 0 &&
1483 sscanf(dn
, "%d:%d", &mj
,&mn
) == 2) {
1484 rdev
= makedev(mj
,mn
);
1491 sysfd
= sysfs_open(fd2devnm(fd
), dname
, "state");
1493 pr_err("%s does not appear to be a component of %s\n",
1494 dv
->devname
, devname
);
1498 } else if ((dv
->disposition
== 'r' ||
1499 dv
->disposition
== 'f') &&
1500 get_maj_min(dv
->devname
, &mj
, &mn
)) {
1501 /* for 'fail' and 'remove', the device might
1504 rdev
= makedev(mj
, mn
);
1506 tfd
= dev_open(dv
->devname
, O_RDONLY
);
1508 fstat_is_blkdev(tfd
, dv
->devname
, &rdev
);
1511 int open_err
= errno
;
1512 if (!stat_is_blkdev(dv
->devname
, &rdev
)) {
1513 if (dv
->disposition
== 'M')
1514 /* non-fatal. Also improbable */
1518 if (dv
->disposition
== 'r')
1519 /* Be happy, the stat worked, that is
1520 * enough for --remove
1524 if (dv
->disposition
== 'M')
1527 pr_err("Cannot open %s: %s\n",
1528 dv
->devname
, strerror(open_err
));
1533 switch(dv
->disposition
){
1535 pr_err("internal error - devmode[%s]=%d\n",
1536 dv
->devname
, dv
->disposition
);
1539 case 'S': /* --add-spare */
1540 case 'j': /* --add-journal */
1542 case 'M': /* --re-add missing */
1543 case 'F': /* --re-add faulty */
1544 case 'c': /* --cluster-confirm */
1545 /* add the device */
1547 pr_err("Cannot add disks to a \'member\' array, perform this operation on the parent container\n");
1551 /* Let's first try to write re-add to sysfs */
1553 (dv
->disposition
== 'A' || dv
->disposition
== 'F')) {
1554 sysfs_init_dev(&devinfo
, rdev
);
1555 if (sysfs_set_str(&info
, &devinfo
, "state", "re-add") == 0) {
1556 pr_err("re-add %s to %s succeed\n",
1557 dv
->devname
, info
.sys_name
);
1562 if (dv
->disposition
== 'F')
1563 /* Need to remove first */
1564 hot_remove_disk(fd
, rdev
, force
);
1565 /* Make sure it isn't in use (in 2.6 or later) */
1566 tfd
= dev_open(dv
->devname
, O_RDONLY
|O_EXCL
);
1568 /* We know no-one else is using it. We'll
1569 * need non-exclusive access to add it, so
1573 tfd
= dev_open(dv
->devname
, O_RDONLY
);
1576 if (dv
->disposition
== 'M')
1578 pr_err("Cannot open %s: %s\n",
1579 dv
->devname
, strerror(errno
));
1583 if (sysfs_freeze_array(&info
) == 1)
1588 rv
= Manage_add(fd
, tfd
, dv
, tst
, &array
,
1589 force
, verbose
, devname
, update
,
1590 rdev
, array_size
, raid_slot
);
1602 pr_err("Cannot remove disks from a \'member\' array, perform this operation on the parent container\n");
1605 rv
= Manage_remove(tst
, fd
, dv
, sysfd
,
1606 rdev
, verbose
, force
,
1617 case 'f': /* set faulty */
1618 /* FIXME check current member */
1619 if ((sysfd
>= 0 && write(sysfd
, "faulty", 6) != 6) ||
1620 (sysfd
< 0 && ioctl(fd
, SET_DISK_FAULTY
,
1624 pr_err("set device faulty failed for %s: %s\n",
1625 dv
->devname
, strerror(errno
));
1635 pr_err("set %s faulty in %s\n",
1636 dv
->devname
, devname
);
1638 case 'R': /* Mark as replaceable */
1640 pr_err("Cannot replace disks in a \'member\' array, perform this operation on the parent container\n");
1644 if (sysfs_freeze_array(&info
) == 1)
1649 rv
= Manage_replace(tst
, fd
, dv
,
1658 case 'W': /* --with device that doesn't match */
1659 pr_err("No matching --replace device for --with %s\n",
1662 case 'w': /* --with device which was matched */
1663 rv
= Manage_with(tst
, fd
, dv
,
1664 rdev
, verbose
, devname
);
1671 sysfs_set_str(&info
, NULL
, "sync_action","idle");
1672 if (test
&& count
== 0)
1678 sysfs_set_str(&info
, NULL
, "sync_action","idle");
1679 return !test
&& busy
? 2 : 1;
1682 int autodetect(void)
1684 /* Open any md device, and issue the RAID_AUTORUN ioctl */
1686 int fd
= dev_open("9:0", O_RDONLY
);
1688 if (ioctl(fd
, RAID_AUTORUN
, 0) == 0)
1695 int Update_subarray(char *dev
, char *subarray
, char *update
, struct mddev_ident
*ident
, int verbose
)
1697 struct supertype supertype
, *st
= &supertype
;
1700 memset(st
, 0, sizeof(*st
));
1702 fd
= open_subarray(dev
, subarray
, st
, verbose
< 0);
1706 if (!st
->ss
->update_subarray
) {
1708 pr_err("Operation not supported for %s metadata\n",
1713 if (mdmon_running(st
->devnm
))
1714 st
->update_tail
= &st
->updates
;
1716 rv
= st
->ss
->update_subarray(st
, subarray
, update
, ident
);
1720 pr_err("Failed to update %s of subarray-%s in %s\n",
1721 update
, subarray
, dev
);
1722 } else if (st
->update_tail
)
1723 flush_metadata_updates(st
);
1725 st
->ss
->sync_metadata(st
);
1727 if (rv
== 0 && strcmp(update
, "name") == 0 && verbose
>= 0)
1728 pr_err("Updated subarray-%s name from %s, UUIDs may have changed\n",
1732 st
->ss
->free_super(st
);
1738 /* Move spare from one array to another If adding to destination array fails
1739 * add back to original array.
1740 * Returns 1 on success, 0 on failure */
1741 int move_spare(char *from_devname
, char *to_devname
, dev_t devid
)
1743 struct mddev_dev devlist
;
1746 /* try to remove and add */
1747 int fd1
= open(to_devname
, O_RDONLY
);
1748 int fd2
= open(from_devname
, O_RDONLY
);
1750 if (fd1
< 0 || fd2
< 0) {
1751 if (fd1
>=0) close(fd1
);
1752 if (fd2
>=0) close(fd2
);
1756 devlist
.next
= NULL
;
1758 devlist
.writemostly
= FlagDefault
;
1759 devlist
.failfast
= FlagDefault
;
1760 devlist
.devname
= devname
;
1761 sprintf(devname
, "%d:%d", major(devid
), minor(devid
));
1763 devlist
.disposition
= 'r';
1764 if (Manage_subdevs(from_devname
, fd2
, &devlist
, -1, 0, NULL
, 0) == 0) {
1765 devlist
.disposition
= 'a';
1766 if (Manage_subdevs(to_devname
, fd1
, &devlist
, -1, 0, NULL
, 0) == 0) {
1767 /* make sure manager is aware of changes */
1768 ping_manager(to_devname
);
1769 ping_manager(from_devname
);
1774 else Manage_subdevs(from_devname
, fd2
, &devlist
, -1, 0, NULL
, 0);