]>
Commit | Line | Data |
---|---|---|
64c4757e | 1 | /* |
9a9dab36 | 2 | * mdadm - manage Linux "md" devices aka RAID arrays. |
64c4757e | 3 | * |
4f589ad0 | 4 | * Copyright (C) 2001-2006 Neil Brown <neilb@suse.de> |
64c4757e NB |
5 | * |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License as published by | |
9 | * the Free Software Foundation; either version 2 of the License, or | |
10 | * (at your option) any later version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program; if not, write to the Free Software | |
19 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA | |
20 | * | |
21 | * Author: Neil Brown | |
22 | * Email: <neilb@cse.unsw.edu.au> | |
23 | * Paper: Neil Brown | |
24 | * School of Computer Science and Engineering | |
25 | * The University of New South Wales | |
26 | * Sydney, 2052 | |
27 | * Australia | |
28 | */ | |
29 | ||
4ccdb956 | 30 | #define _GNU_SOURCE |
c2c9bb6f | 31 | #define _FILE_OFFSET_BITS 64 |
64c4757e | 32 | #include <unistd.h> |
e39b673e | 33 | #if !defined(__dietlibc__) && !defined(__KLIBC__) |
64c4757e | 34 | extern __off64_t lseek64 __P ((int __fd, __off64_t __offset, int __whence)); |
f783ca4f | 35 | #else |
fffdbe5e | 36 | # if defined(__NO_STAT64) || __WORDSIZE != 32 |
f783ca4f NB |
37 | # define lseek64 lseek |
38 | # endif | |
98c6faba | 39 | #endif |
64c4757e NB |
40 | |
41 | #include <sys/types.h> | |
42 | #include <sys/stat.h> | |
43 | #include <stdlib.h> | |
44 | #include <time.h> | |
11018a45 | 45 | #include <sys/time.h> |
64c4757e NB |
46 | #include <getopt.h> |
47 | #include <fcntl.h> | |
48 | #include <stdio.h> | |
49 | #include <errno.h> | |
50 | #include <string.h> | |
773135f5 | 51 | #include <syslog.h> |
b56c3630 NB |
52 | #ifdef __dietlibc__ |
53 | #include <strings.h> | |
280a927d NB |
54 | /* dietlibc has deprecated random and srandom!! */ |
55 | #define random rand | |
56 | #define srandom srand | |
98c6faba NB |
57 | #endif |
58 | ||
64c4757e NB |
59 | |
60 | #include <linux/kdev_t.h> | |
e0d19036 NB |
61 | /*#include <linux/fs.h> */ |
62 | #include <sys/mount.h> | |
63 | #include <asm/types.h> | |
64c4757e NB |
64 | #include <sys/ioctl.h> |
65 | #define MD_MAJOR 9 | |
dd0781e5 | 66 | #define MdpMinorShift 6 |
64c4757e | 67 | |
e0d19036 | 68 | #ifndef BLKGETSIZE64 |
98c6faba | 69 | #define BLKGETSIZE64 _IOR(0x12,114,size_t) /* return device size in bytes (u64 *arg) */ |
e0d19036 | 70 | #endif |
56eb10c0 | 71 | |
c82f047c NB |
72 | #define DEFAULT_BITMAP_CHUNK 4096 |
73 | #define DEFAULT_BITMAP_DELAY 5 | |
dfd4d8ee | 74 | #define DEFAULT_MAX_WRITE_BEHIND 256 |
64c4757e NB |
75 | |
76 | #include "md_u.h" | |
e0d19036 | 77 | #include "md_p.h" |
c82f047c | 78 | #include "bitmap.h" |
f7dd881f | 79 | #include "msg.h" |
64c4757e | 80 | |
1e0d770c | 81 | #include <endian.h> |
efd441d1 NB |
82 | /* Redhat don't like to #include <asm/byteorder.h>, and |
83 | * some time include <linux/byteorder/xxx_endian.h> isn't enough, | |
84 | * and there is no standard conversion function so... */ | |
0ae03b8a NB |
85 | /* And dietlibc doesn't think byteswap is ok, so.. */ |
86 | /* #include <byteswap.h> */ | |
87 | #define bswap_16(x) (((x) & 0x00ffU) << 8 | \ | |
88 | ((x) & 0xff00U) >> 8) | |
89 | #define bswap_32(x) (((x) & 0x000000ffU) << 24 | \ | |
90 | ((x) & 0xff000000U) >> 24 | \ | |
91 | ((x) & 0x0000ff00U) << 8 | \ | |
92 | ((x) & 0x00ff0000U) >> 8) | |
93 | #define bswap_64(x) (((x) & 0x00000000000000ffULL) << 56 | \ | |
94 | ((x) & 0xff00000000000000ULL) >> 56 | \ | |
95 | ((x) & 0x000000000000ff00ULL) << 40 | \ | |
96 | ((x) & 0x00ff000000000000ULL) >> 40 | \ | |
97 | ((x) & 0x0000000000ff0000ULL) << 24 | \ | |
98 | ((x) & 0x0000ff0000000000ULL) >> 24 | \ | |
99 | ((x) & 0x00000000ff000000ULL) << 8 | \ | |
100 | ((x) & 0x000000ff00000000ULL) >> 8) | |
101 | ||
e39b673e | 102 | #if !defined(__KLIBC__) |
efd441d1 NB |
103 | #if BYTE_ORDER == LITTLE_ENDIAN |
104 | #define __cpu_to_le16(_x) (_x) | |
105 | #define __cpu_to_le32(_x) (_x) | |
106 | #define __cpu_to_le64(_x) (_x) | |
107 | #define __le16_to_cpu(_x) (_x) | |
108 | #define __le32_to_cpu(_x) (_x) | |
109 | #define __le64_to_cpu(_x) (_x) | |
974e620d NB |
110 | |
111 | #define __cpu_to_be16(_x) bswap_16(_x) | |
112 | #define __cpu_to_be32(_x) bswap_32(_x) | |
113 | #define __cpu_to_be64(_x) bswap_64(_x) | |
114 | #define __be16_to_cpu(_x) bswap_16(_x) | |
115 | #define __be32_to_cpu(_x) bswap_32(_x) | |
116 | #define __be64_to_cpu(_x) bswap_64(_x) | |
efd441d1 NB |
117 | #elif BYTE_ORDER == BIG_ENDIAN |
118 | #define __cpu_to_le16(_x) bswap_16(_x) | |
119 | #define __cpu_to_le32(_x) bswap_32(_x) | |
120 | #define __cpu_to_le64(_x) bswap_64(_x) | |
121 | #define __le16_to_cpu(_x) bswap_16(_x) | |
122 | #define __le32_to_cpu(_x) bswap_32(_x) | |
123 | #define __le64_to_cpu(_x) bswap_64(_x) | |
974e620d NB |
124 | |
125 | #define __cpu_to_be16(_x) (_x) | |
126 | #define __cpu_to_be32(_x) (_x) | |
127 | #define __cpu_to_be64(_x) (_x) | |
128 | #define __be16_to_cpu(_x) (_x) | |
129 | #define __be32_to_cpu(_x) (_x) | |
130 | #define __be64_to_cpu(_x) (_x) | |
1e0d770c NB |
131 | #else |
132 | # error "unknown endianness." | |
133 | #endif | |
e39b673e | 134 | #endif /* __KLIBC__ */ |
1e0d770c NB |
135 | |
136 | ||
137 | ||
4b1ac34b NB |
138 | /* general information that might be extracted from a superblock */ |
139 | struct mdinfo { | |
140 | mdu_array_info_t array; | |
141 | mdu_disk_info_t disk; | |
142 | __u64 events; | |
fbf8a0b7 | 143 | int uuid[4]; |
31317663 | 144 | char name[33]; |
353632d9 | 145 | unsigned long long data_offset; |
598f0d58 NB |
146 | unsigned long long component_size; /* same as array.size, except in |
147 | * sectors and up to 64bits. | |
148 | */ | |
353632d9 NB |
149 | int reshape_active; |
150 | unsigned long long reshape_progress; | |
0fd5c350 | 151 | unsigned long long resync_start; |
a67dd8cc | 152 | unsigned long safe_mode_delay; /* ms delay to mark clean */ |
353632d9 | 153 | int new_level, delta_disks, new_layout, new_chunk; |
06c7f68e | 154 | int errors; |
7e0f6979 NB |
155 | int cache_size; /* size of raid456 stripe cache*/ |
156 | int mismatch_cnt; | |
157 | char text_version[50]; | |
dd15dc4a NB |
158 | |
159 | int container_member; /* for assembling external-metatdata arrays | |
160 | * This is to be used internally by metadata | |
161 | * handler only */ | |
06c7f68e NB |
162 | |
163 | char sys_name[20]; | |
7e0f6979 | 164 | struct mdinfo *devs; |
06c7f68e | 165 | struct mdinfo *next; |
549e9569 NB |
166 | |
167 | /* Device info for mdmon: */ | |
168 | int state_fd; | |
8d45d196 DW |
169 | #define DS_FAULTY 1 |
170 | #define DS_INSYNC 2 | |
171 | #define DS_WRITE_MOSTLY 4 | |
172 | #define DS_SPARE 8 | |
173 | #define DS_BLOCKED 16 | |
174 | #define DS_REMOVE 1024 | |
92967543 | 175 | #define DS_UNBLOCK 2048 |
549e9569 NB |
176 | int prev_state, curr_state, next_state; |
177 | ||
4b1ac34b NB |
178 | }; |
179 | ||
5bbb4842 NB |
180 | struct createinfo { |
181 | int uid; | |
182 | int gid; | |
183 | int autof; | |
184 | int mode; | |
38098016 | 185 | int symlinks; |
058574b1 | 186 | struct supertype *supertype; |
5bbb4842 NB |
187 | }; |
188 | ||
9a9dab36 | 189 | #define Name "mdadm" |
682c7051 | 190 | |
e0d19036 NB |
191 | enum mode { |
192 | ASSEMBLE=1, | |
193 | BUILD, | |
194 | CREATE, | |
195 | MANAGE, | |
196 | MISC, | |
197 | MONITOR, | |
dd0781e5 | 198 | GROW, |
8382f19b | 199 | INCREMENTAL, |
1f48664b | 200 | AUTODETECT, |
e0d19036 NB |
201 | }; |
202 | ||
64c4757e | 203 | extern char short_options[]; |
024768c4 | 204 | extern char short_bitmap_options[]; |
c06487ce | 205 | extern char short_bitmap_auto_options[]; |
64c4757e | 206 | extern struct option long_options[]; |
56eedc1a | 207 | extern char Version[], Usage[], Help[], OptionHelp[], |
dd0781e5 | 208 | Help_create[], Help_build[], Help_assemble[], Help_grow[], |
8382f19b | 209 | Help_incr[], |
e0d19036 | 210 | Help_manage[], Help_misc[], Help_monitor[], Help_config[]; |
64c4757e | 211 | |
997aed5d NB |
212 | /* for option that don't have short equivilents, we assign arbitrary |
213 | * small numbers. '1' means an undecorated option, so we start at '2'. | |
214 | */ | |
215 | enum special_options { | |
216 | AssumeClean = 2, | |
217 | BitmapChunk, | |
218 | WriteBehind, | |
219 | ReAdd, | |
220 | NoDegraded, | |
221 | Sparc22, | |
222 | BackupFile, | |
223 | HomeHost, | |
589395d6 | 224 | AutoHomeHost, |
38098016 | 225 | Symlinks, |
1f48664b | 226 | AutoDetect, |
1770662b | 227 | Waitclean, |
4cce4069 | 228 | DetailPlatform, |
997aed5d NB |
229 | }; |
230 | ||
64c4757e | 231 | /* structures read from config file */ |
52826846 NB |
232 | /* List of mddevice names and identifiers |
233 | * Identifiers can be: | |
234 | * uuid=128-hex-uuid | |
235 | * super-minor=decimal-minor-number-from-superblock | |
236 | * devices=comma,separated,list,of,device,names,with,wildcards | |
237 | * | |
238 | * If multiple fields are present, the intersection of all matching | |
239 | * devices is considered | |
240 | */ | |
98c6faba | 241 | #define UnSet (0xfffe) |
52826846 | 242 | typedef struct mddev_ident_s { |
dd0781e5 | 243 | char *devname; |
aba69144 | 244 | |
dd0781e5 | 245 | int uuid_set; |
3fa06e9d | 246 | int uuid[4]; |
947fd4dd | 247 | char name[33]; |
52826846 | 248 | |
98c6faba | 249 | unsigned int super_minor; |
52826846 | 250 | |
dd0781e5 | 251 | char *devices; /* comma separated list of device |
52826846 NB |
252 | * names with wild cards |
253 | */ | |
dd0781e5 | 254 | int level; |
98c6faba NB |
255 | unsigned int raid_disks; |
256 | unsigned int spare_disks; | |
82d9eba6 | 257 | struct supertype *st; |
dd0781e5 NB |
258 | int autof; /* 1 for normal, 2 for partitioned */ |
259 | char *spare_group; | |
7ef02d01 | 260 | char *bitmap_file; |
c82f047c | 261 | int bitmap_fd; |
dd0781e5 | 262 | |
1771a6e2 N |
263 | char *container; /* /dev/whatever name of container, or |
264 | * uuid of container. You would expect | |
265 | * this to be the 'devname' or UUID | |
dbb44303 N |
266 | * of some other entry. |
267 | */ | |
268 | char *member; /* subarray within a container */ | |
269 | ||
52826846 NB |
270 | struct mddev_ident_s *next; |
271 | } *mddev_ident_t; | |
64c4757e NB |
272 | |
273 | /* List of device names - wildcards expanded */ | |
274 | typedef struct mddev_dev_s { | |
275 | char *devname; | |
cd29a5c8 NB |
276 | char disposition; /* 'a' for add, 'r' for remove, 'f' for fail. |
277 | * Not set for names read from .config | |
278 | */ | |
b3d31955 | 279 | char writemostly; /* 1 for 'set writemostly', 2 for 'clear writemostly' */ |
fe80f49b | 280 | char re_add; |
811e6cbe | 281 | char used; /* set when used */ |
9008ed1c N |
282 | struct mdinfo *content; /* If devname is a container, this might list |
283 | * the remaining member arrays. */ | |
64c4757e NB |
284 | struct mddev_dev_s *next; |
285 | } *mddev_dev_t; | |
286 | ||
682c7051 NB |
287 | typedef struct mapping { |
288 | char *name; | |
289 | int num; | |
290 | } mapping_t; | |
291 | ||
e0d19036 NB |
292 | |
293 | struct mdstat_ent { | |
294 | char *dev; | |
295 | int devnum; | |
296 | int active; | |
297 | char *level; | |
298 | char *pattern; /* U or up, _ for down */ | |
299 | int percent; /* -1 if no resync */ | |
e5329c37 | 300 | int resync; /* 1 if resync, 0 if recovery */ |
549e9569 NB |
301 | int devcnt; |
302 | int raid_disks; | |
303 | int chunk_size; | |
304 | char * metadata_version; | |
e0d19036 NB |
305 | struct mdstat_ent *next; |
306 | }; | |
307 | ||
22a88995 | 308 | extern struct mdstat_ent *mdstat_read(int hold, int start); |
e0d19036 | 309 | extern void free_mdstat(struct mdstat_ent *ms); |
dd0781e5 | 310 | extern void mdstat_wait(int seconds); |
1ed3f387 | 311 | extern void mdstat_wait_fd(int fd, const sigset_t *sigmask); |
8382f19b NB |
312 | extern int mddev_busy(int devnum); |
313 | ||
314 | struct map_ent { | |
315 | struct map_ent *next; | |
316 | int devnum; | |
1522c538 | 317 | char metadata[20]; |
8382f19b | 318 | int uuid[4]; |
195254b8 | 319 | int bad; |
8382f19b NB |
320 | char *path; |
321 | }; | |
1522c538 | 322 | extern int map_update(struct map_ent **mpp, int devnum, char *metadata, |
8382f19b NB |
323 | int uuid[4], char *path); |
324 | extern struct map_ent *map_by_uuid(struct map_ent **map, int uuid[4]); | |
f2e55ecc N |
325 | extern struct map_ent *map_by_devnum(struct map_ent **map, int devnum); |
326 | extern struct map_ent *map_by_name(struct map_ent **map, char *name); | |
8382f19b NB |
327 | extern void map_read(struct map_ent **melp); |
328 | extern int map_write(struct map_ent *mel); | |
329 | extern void map_delete(struct map_ent **mapp, int devnum); | |
330 | extern void map_free(struct map_ent *map); | |
331 | extern void map_add(struct map_ent **melp, | |
1522c538 | 332 | int devnum, char *metadata, int uuid[4], char *path); |
ad5bc697 N |
333 | extern int map_lock(struct map_ent **melp); |
334 | extern void map_unlock(struct map_ent **melp); | |
e0d19036 | 335 | |
e86c9dd6 NB |
336 | /* various details can be requested */ |
337 | #define GET_LEVEL 1 | |
338 | #define GET_LAYOUT 2 | |
339 | #define GET_COMPONENT 4 | |
340 | #define GET_CHUNK 8 | |
758d3a8e | 341 | #define GET_CACHE 16 |
37dfc3d6 | 342 | #define GET_MISMATCH 32 |
8382f19b | 343 | #define GET_VERSION 64 |
549e9569 | 344 | #define GET_DISKS 128 |
f1d26766 | 345 | #define GET_DEGRADED 256 |
1770662b | 346 | #define GET_SAFEMODE 512 |
e86c9dd6 NB |
347 | |
348 | #define GET_DEVS 1024 /* gets role, major, minor */ | |
349 | #define GET_OFFSET 2048 | |
350 | #define GET_SIZE 4096 | |
351 | #define GET_STATE 8192 | |
352 | #define GET_ERROR 16384 | |
353 | ||
354 | /* If fd >= 0, get the array it is open on, | |
355 | * else use devnum. >=0 -> major9. <0..... | |
356 | */ | |
549e9569 | 357 | extern int sysfs_open(int devnum, char *devname, char *attr); |
f35f2525 | 358 | extern void sysfs_init(struct mdinfo *mdi, int fd, int devnum); |
7e0f6979 NB |
359 | extern void sysfs_free(struct mdinfo *sra); |
360 | extern struct mdinfo *sysfs_read(int fd, int devnum, unsigned long options); | |
1770662b DW |
361 | extern int sysfs_attr_match(const char *attr, const char *str); |
362 | extern int sysfs_match_word(const char *word, char **list); | |
7e0f6979 | 363 | extern int sysfs_set_str(struct mdinfo *sra, struct mdinfo *dev, |
e86c9dd6 | 364 | char *name, char *val); |
7e0f6979 | 365 | extern int sysfs_set_num(struct mdinfo *sra, struct mdinfo *dev, |
e86c9dd6 | 366 | char *name, unsigned long long val); |
97590376 | 367 | extern int sysfs_uevent(struct mdinfo *sra, char *event); |
7e0f6979 | 368 | extern int sysfs_get_ll(struct mdinfo *sra, struct mdinfo *dev, |
e86c9dd6 | 369 | char *name, unsigned long long *val); |
8ed3e5e1 | 370 | extern int sysfs_set_safemode(struct mdinfo *sra, unsigned long ms); |
f35f2525 | 371 | extern int sysfs_set_array(struct mdinfo *info, int vers); |
2318b9f0 | 372 | extern int sysfs_add_disk(struct mdinfo *sra, struct mdinfo *sd); |
f1665f72 | 373 | extern int sysfs_disk_to_scsi_id(int fd, __u32 *id); |
f94d52f4 | 374 | extern int sysfs_unique_holder(int devnum, long rdev); |
c69b251b | 375 | extern int load_sys(char *path, char *buf); |
e86c9dd6 NB |
376 | |
377 | ||
378 | extern int save_stripes(int *source, unsigned long long *offsets, | |
379 | int raid_disks, int chunk_size, int level, int layout, | |
380 | int nwrites, int *dest, | |
381 | unsigned long long start, unsigned long long length); | |
353632d9 NB |
382 | extern int restore_stripes(int *dest, unsigned long long *offsets, |
383 | int raid_disks, int chunk_size, int level, int layout, | |
384 | int source, unsigned long long read_offset, | |
385 | unsigned long long start, unsigned long long length); | |
e86c9dd6 | 386 | |
52826846 NB |
387 | #ifndef Sendmail |
388 | #define Sendmail "/usr/lib/sendmail -t" | |
389 | #endif | |
390 | ||
773135f5 NB |
391 | #define SYSLOG_FACILITY LOG_DAEMON |
392 | ||
682c7051 NB |
393 | extern char *map_num(mapping_t *map, int num); |
394 | extern int map_name(mapping_t *map, char *name); | |
b5e64645 | 395 | extern mapping_t r5layout[], pers[], modes[], faultylayout[]; |
682c7051 | 396 | |
16c6fa80 | 397 | extern char *map_dev(int major, int minor, int create); |
64c4757e | 398 | |
549e9569 | 399 | struct active_array; |
2e735d19 | 400 | struct metadata_update; |
64c4757e | 401 | |
6adfd3af NB |
402 | /* A superswitch provides entry point the a metadata handler. |
403 | * | |
404 | * The super_switch primarily operates on some "metadata" that | |
405 | * is accessed via the 'supertype'. | |
406 | * This metadata has one of three possible sources. | |
407 | * 1/ It is read from a single device. In this case it may not completely | |
408 | * describe the array or arrays as some information might be on other | |
409 | * devices. | |
410 | * 2/ It is read from all devices in a container. In this case all | |
411 | * information is present. | |
412 | * 3/ It is created by ->init_super / ->add_to_super. In this case it will | |
413 | * be complete once enough ->add_to_super calls have completed. | |
414 | * | |
415 | * When creating an array inside a container, the metadata will be | |
416 | * formed by a combination of 2 and 3. The metadata or the array is read, | |
417 | * then new information is added. | |
418 | * | |
419 | * The metadata must sometimes have a concept of a 'current' array | |
420 | * and a 'current' device. | |
421 | * The 'current' array is set by init_super to be the newly created array, | |
422 | * or is set by super_by_fd when it finds it is looking at an array inside | |
423 | * a container. | |
424 | * | |
425 | * The 'current' device is either the device that the metadata was read from | |
426 | * in case 1, or the last device added by add_to_super in case 3. | |
427 | * Case 2 does not identify a 'current' device. | |
428 | */ | |
f9ce90ba | 429 | extern struct superswitch { |
6adfd3af NB |
430 | |
431 | /* Used to report details of metadata read from a component | |
432 | * device. ->load_super has been called. | |
433 | */ | |
3da92f27 NB |
434 | void (*examine_super)(struct supertype *st, char *homehost); |
435 | void (*brief_examine_super)(struct supertype *st); | |
0d726f17 | 436 | void (*export_examine_super)(struct supertype *st); |
6adfd3af NB |
437 | |
438 | /* Used to report details of an active array. | |
439 | * ->load_super was possibly given a 'component' string. | |
440 | */ | |
3da92f27 | 441 | void (*detail_super)(struct supertype *st, char *homehost); |
3da92f27 | 442 | void (*brief_detail_super)(struct supertype *st); |
0d726f17 | 443 | void (*export_detail_super)(struct supertype *st); |
6adfd3af | 444 | |
4cce4069 DW |
445 | /* Optional: platform hardware / firmware details */ |
446 | int (*detail_platform)(int verbose); | |
447 | ||
6adfd3af NB |
448 | /* Used: |
449 | * to get uuid to storing in bitmap metadata | |
450 | * and 'reshape' backup-data metadata | |
451 | * To see if a device is being re-added to an array it was part of. | |
452 | */ | |
3da92f27 | 453 | void (*uuid_from_super)(struct supertype *st, int uuid[4]); |
6adfd3af NB |
454 | |
455 | /* Extra generic details from metadata. This could be details about | |
456 | * the container, or about an individual array within the container. | |
457 | * The determination is made either by: | |
458 | * load_super being given a 'component' string. | |
459 | * validate_geometry determining what to create. | |
d2ca6449 NB |
460 | * The info includes both array information and device information. |
461 | * The particular device should be: | |
462 | * The last device added by add_to_super | |
463 | * The device the metadata was loaded from by load_super | |
6adfd3af | 464 | */ |
3da92f27 | 465 | void (*getinfo_super)(struct supertype *st, struct mdinfo *info); |
6adfd3af NB |
466 | |
467 | /* Check if the given metadata is flagged as belonging to "this" | |
9362c1c8 | 468 | * host. 0 for 'no', 1 for 'yes', -1 for "Don't record homehost" |
6adfd3af | 469 | */ |
3da92f27 | 470 | int (*match_home)(struct supertype *st, char *homehost); |
6adfd3af NB |
471 | |
472 | /* Make one of several generic modifications to metadata | |
473 | * prior to assembly (or other times). | |
474 | * sparc2.2 - first bug in early 0.90 metadata | |
475 | * super-minor - change name of 0.90 metadata | |
476 | * summaries - 'correct' any redundant data | |
477 | * resync - mark array as dirty to trigger a resync. | |
478 | * uuid - set new uuid - only 0.90 or 1.x | |
479 | * name - change the name of the array (where supported) | |
480 | * homehost - change which host this array is tied to. | |
481 | * devicesize - If metadata is at start of device, change recorded | |
482 | * device size to match actual device size | |
483 | * byteorder - swap bytes for 0.90 metadata | |
484 | * | |
485 | * force-one - mark that device as uptodate, not old or failed. | |
486 | * force-array - mark array as clean if it would not otherwise | |
487 | * assemble | |
488 | * assemble - not sure how this is different from force-one... | |
489 | * linear-grow-new - add a new device to a linear array, but don't | |
490 | * change the size: so superblock still matches | |
491 | * linear-grow-update - now change the size of the array. | |
492 | */ | |
68c7d6d7 | 493 | int (*update_super)(struct supertype *st, struct mdinfo *info, |
3da92f27 | 494 | char *update, |
e5eac01f NB |
495 | char *devname, int verbose, |
496 | int uuid_set, char *homehost); | |
6adfd3af NB |
497 | |
498 | /* Create new metadata for new array as described. This could | |
499 | * be a new container, or an array in a pre-existing container. | |
500 | * Also used to zero metadata prior to writing it to invalidate old | |
501 | * metadata. | |
502 | */ | |
3da92f27 NB |
503 | int (*init_super)(struct supertype *st, mdu_array_info_t *info, |
504 | unsigned long long size, char *name, | |
505 | char *homehost, int *uuid); | |
6adfd3af NB |
506 | |
507 | /* update the metadata to include new device, either at create or | |
508 | * when hot-adding a spare. | |
509 | */ | |
f20c3968 | 510 | int (*add_to_super)(struct supertype *st, mdu_disk_info_t *dinfo, |
111d01fc | 511 | int fd, char *devname); |
6adfd3af NB |
512 | |
513 | /* Write metadata to one device when fixing problems or adding | |
514 | * a new device. | |
515 | */ | |
3da92f27 | 516 | int (*store_super)(struct supertype *st, int fd); |
6adfd3af NB |
517 | |
518 | /* Write all metadata for this array. | |
519 | */ | |
111d01fc | 520 | int (*write_init_super)(struct supertype *st); |
64557c33 | 521 | int (*compare_super)(struct supertype *st, struct supertype *tst); |
3da92f27 | 522 | int (*load_super)(struct supertype *st, int fd, char *devname); |
82d9eba6 | 523 | struct supertype * (*match_metadata_desc)(char *arg); |
1bf4e2d9 | 524 | __u64 (*avail_size)(struct supertype *st, __u64 size); |
3da92f27 | 525 | int (*add_internal_bitmap)(struct supertype *st, int *chunkp, |
199171a2 | 526 | int delay, int write_behind, |
f9c25f1d | 527 | unsigned long long size, int may_change, int major); |
3da92f27 NB |
528 | void (*locate_bitmap)(struct supertype *st, int fd); |
529 | int (*write_bitmap)(struct supertype *st, int fd); | |
530 | void (*free_super)(struct supertype *st); | |
78e44928 NB |
531 | |
532 | /* validate_geometry is called with an st returned by | |
533 | * match_metadata_desc. | |
534 | * It should check that the geometry described in compatible with | |
535 | * the metadata type. It will be called repeatedly as devices | |
536 | * added to validate changing size and new devices. If there are | |
537 | * inter-device dependencies, it should record sufficient details | |
538 | * so these can be validated. | |
539 | */ | |
17f25ca6 NB |
540 | int (*validate_geometry)(struct supertype *st, int level, int layout, |
541 | int raiddisks, | |
542 | int chunk, unsigned long long size, | |
2c514b71 NB |
543 | char *subdev, unsigned long long *freesize, |
544 | int verbose); | |
598f0d58 NB |
545 | |
546 | struct mdinfo *(*container_content)(struct supertype *st); | |
547 | ||
549e9569 | 548 | /* for mdmon */ |
cba0191b NB |
549 | int (*open_new)(struct supertype *c, struct active_array *a, |
550 | char *inst); | |
ed9d66aa NB |
551 | |
552 | /* Tell the metadata handler the current state of the array. | |
553 | * This covers whether it is known to be consistent (no pending writes) | |
01f157d7 | 554 | * and how far along a resync is known to have progressed |
ed9d66aa NB |
555 | * (in a->resync_start). |
556 | * resync status is really irrelevant if the array is not consistent, | |
557 | * but some metadata (DDF!) have a place to record the distinction. | |
01f157d7 N |
558 | * If 'consistent' is '2', then the array can mark it dirty if a |
559 | * resync/recovery/whatever is required, or leave it clean if not. | |
560 | * Return value is 0 dirty (not consistent) and 1 if clean. | |
561 | * it is only really important if consistent is passed in as '2'. | |
4e5528c6 | 562 | */ |
01f157d7 | 563 | int (*set_array_state)(struct active_array *a, int consistent); |
7a7cc504 NB |
564 | |
565 | /* When the state of a device might have changed, we call set_disk to | |
566 | * tell the metadata what the current state is. | |
567 | * Typically this happens on spare->in_sync and (spare|in_sync)->faulty | |
568 | * transitions. | |
569 | * set_disk might be called when the state of the particular disk has | |
570 | * not in fact changed. | |
571 | */ | |
8d45d196 | 572 | void (*set_disk)(struct active_array *a, int n, int state); |
2e735d19 NB |
573 | void (*sync_metadata)(struct supertype *st); |
574 | void (*process_update)(struct supertype *st, | |
575 | struct metadata_update *update); | |
edd8d13c NB |
576 | void (*prepare_update)(struct supertype *st, |
577 | struct metadata_update *update); | |
549e9569 | 578 | |
6c3fb95c NB |
579 | /* activate_spare will check if the array is degraded and, if it |
580 | * is, try to find some spare space in the container. | |
581 | * On success, it add appropriate updates (For process_update) to | |
582 | * to the 'updates' list and returns a list of 'mdinfo' identifying | |
583 | * the device, or devices as there might be multiple missing | |
584 | * devices and multiple spares available. | |
585 | */ | |
586 | struct mdinfo *(*activate_spare)(struct active_array *a, | |
587 | struct metadata_update **updates); | |
549e9569 | 588 | |
f277ce36 | 589 | int swapuuid; /* true if uuid is bigending rather than hostendian */ |
d03373f1 | 590 | int external; |
4cce4069 | 591 | const char *name; /* canonical metadata name */ |
0063ecba | 592 | } super0, super1, super_ddf, *superlist[]; |
82d9eba6 | 593 | |
0063ecba | 594 | extern struct superswitch super_imsm; |
cdddbdbc | 595 | |
edd8d13c NB |
596 | struct metadata_update { |
597 | int len; | |
598 | char *buf; | |
599 | void *space; /* allocated space that monitor will use */ | |
600 | struct metadata_update *next; | |
601 | }; | |
602 | ||
6adfd3af NB |
603 | /* A supertype holds a particular collection of metadata. |
604 | * It identifies the metadata type by the superswitch, and the particular | |
605 | * sub-version of that metadata type. | |
606 | * metadata read in or created is stored in 'sb' and 'info'. | |
607 | * There are also fields used by mdmon to track containers. | |
608 | * | |
352452c3 N |
609 | * A supertype may refer to: |
610 | * Just an array, possibly in a container | |
611 | * A container, not identifying any particular array | |
612 | * Info read from just one device, not yet fully describing the array/container. | |
613 | * | |
614 | * | |
6adfd3af NB |
615 | * A supertype is created by: |
616 | * super_by_fd | |
617 | * guess_super | |
618 | * dup_super | |
619 | */ | |
82d9eba6 NB |
620 | struct supertype { |
621 | struct superswitch *ss; | |
622 | int minor_version; | |
ea329559 | 623 | int max_devs; |
2f6079dc | 624 | int container_dev; /* devnum of container */ |
f7e7067b | 625 | char subarray[32]; /* name of array inside container */ |
64557c33 | 626 | void *sb; |
111d01fc | 627 | void *info; |
352452c3 N |
628 | int loaded_container; /* Set if load_super found a container, |
629 | * not just one device */ | |
549e9569 | 630 | |
edd8d13c NB |
631 | struct metadata_update *updates; |
632 | struct metadata_update **update_tail; | |
633 | ||
549e9569 NB |
634 | /* extra stuff used by mdmon */ |
635 | struct active_array *arrays; | |
549e9569 | 636 | int sock; /* listen to external programs */ |
549e9569 NB |
637 | int devnum; |
638 | char *devname; /* e.g. md0. This appears in metadata_verison: | |
639 | * external:/md0/12 | |
640 | */ | |
641 | int devcnt; | |
642 | ||
643 | struct mdinfo *devs; | |
644 | ||
82d9eba6 | 645 | }; |
f9ce90ba | 646 | |
1686dc25 | 647 | extern struct supertype *super_by_fd(int fd); |
82d9eba6 | 648 | extern struct supertype *guess_super(int fd); |
3da92f27 | 649 | extern struct supertype *dup_super(struct supertype *st); |
beae1dfe | 650 | extern int get_dev_size(int fd, char *dname, unsigned long long *sizep); |
8382f19b NB |
651 | extern void get_one_disk(int mdfd, mdu_array_info_t *ainf, |
652 | mdu_disk_info_t *disk); | |
a714580e | 653 | void wait_for(char *dev); |
f9ce90ba | 654 | |
838acbc2 NB |
655 | #if __GNUC__ < 3 |
656 | struct stat64; | |
657 | #endif | |
658 | ||
45e878bb NB |
659 | #define HAVE_NFTW we assume |
660 | #define HAVE_FTW | |
661 | ||
173fc515 | 662 | #ifdef UCLIBC |
45e878bb NB |
663 | # include <features.h> |
664 | # ifndef __UCLIBC_HAS_FTW__ | |
665 | # undef HAVE_FTW | |
666 | # undef HAVE_NFTW | |
667 | # endif | |
668 | #endif | |
669 | ||
670 | #ifdef __dietlibc__ | |
671 | # undef HAVE_NFTW | |
672 | #endif | |
673 | ||
e39b673e | 674 | #if defined(__KLIBC__) |
675 | # undef HAVE_NFTW | |
676 | # undef HAVE_FTW | |
677 | #endif | |
678 | ||
45e878bb | 679 | #ifndef HAVE_NFTW |
173fc515 | 680 | # define FTW_PHYS 1 |
45e878bb NB |
681 | # ifndef HAVE_FTW |
682 | struct FTW {}; | |
173fc515 NB |
683 | # endif |
684 | #endif | |
685 | ||
45e878bb NB |
686 | #ifdef HAVE_FTW |
687 | # include <ftw.h> | |
688 | #endif | |
689 | ||
173fc515 NB |
690 | extern int add_dev(const char *name, const struct stat *stb, int flag, struct FTW *s); |
691 | ||
f9ce90ba | 692 | |
64c4757e | 693 | extern int Manage_ro(char *devname, int fd, int readonly); |
91f068bf | 694 | extern int Manage_runstop(char *devname, int fd, int runstop, int quiet); |
dd0781e5 | 695 | extern int Manage_resize(char *devname, int fd, long long size, int raid_disks); |
b5e64645 | 696 | extern int Manage_reconfig(char *devname, int fd, int layout); |
64c4757e | 697 | extern int Manage_subdevs(char *devname, int fd, |
dab6685f | 698 | mddev_dev_t devlist, int verbose); |
1f48664b | 699 | extern int autodetect(void); |
e5329c37 | 700 | extern int Grow_Add_device(char *devname, int fd, char *newdev); |
8fac0577 | 701 | extern int Grow_addbitmap(char *devname, int fd, char *file, int chunk, int delay, int write_behind, int force); |
06b0d786 | 702 | extern int Grow_reshape(char *devname, int fd, int quiet, char *backup_file, |
e86c9dd6 NB |
703 | long long size, |
704 | int level, int layout, int chunksize, int raid_disks); | |
06b0d786 NB |
705 | extern int Grow_restart(struct supertype *st, struct mdinfo *info, |
706 | int *fdlist, int cnt, char *backup_file); | |
64c4757e NB |
707 | |
708 | ||
7f91af49 | 709 | extern int Assemble(struct supertype *st, char *mddev, |
52826846 | 710 | mddev_ident_t ident, |
06b0d786 | 711 | mddev_dev_t devlist, char *backup_file, |
64c4757e | 712 | int readonly, int runstop, |
e5eac01f | 713 | char *update, char *homehost, |
64c4757e NB |
714 | int verbose, int force); |
715 | ||
7f91af49 N |
716 | extern int Build(char *mddev, int chunk, int level, int layout, |
717 | int raiddisks, mddev_dev_t devlist, int assume_clean, | |
718 | char *bitmap_file, int bitmap_chunk, int write_behind, | |
719 | int delay, int verbose, int autof); | |
64c4757e NB |
720 | |
721 | ||
7f91af49 | 722 | extern int Create(struct supertype *st, char *mddev, |
5dd497ee | 723 | int chunk, int level, int layout, unsigned long long size, int raiddisks, int sparedisks, |
3d3dd91e | 724 | char *name, char *homehost, int *uuid, |
cd29a5c8 | 725 | int subdevs, mddev_dev_t devlist, |
47d79ef8 | 726 | int runstop, int verbose, int force, int assume_clean, |
7f91af49 | 727 | char *bitmap_file, int bitmap_chunk, int write_behind, int delay, int autof); |
64c4757e | 728 | |
54bad364 | 729 | extern int Detail(char *dev, int brief, int export, int test, char *homehost); |
4cce4069 | 730 | extern int Detail_Platform(struct superswitch *ss, int scan, int verbose); |
e0d19036 | 731 | extern int Query(char *dev); |
0d726f17 KS |
732 | extern int Examine(mddev_dev_t devlist, int brief, int export, int scan, |
733 | int SparcAdjust, struct supertype *forcest, char *homehost); | |
cd29a5c8 | 734 | extern int Monitor(mddev_dev_t devlist, |
e0d19036 | 735 | char *mailaddr, char *alert_cmd, |
aa88f531 | 736 | int period, int daemonise, int scan, int oneshot, |
8aec876d | 737 | int dosyslog, int test, char *pidfile); |
64c4757e | 738 | |
111d01fc | 739 | extern int Kill(char *dev, int force, int quiet, int noexcl); |
b90c0e9a | 740 | extern int Wait(char *dev); |
27dec8fa | 741 | extern int WaitClean(char *dev, int verbose); |
9a9dab36 | 742 | |
8382f19b NB |
743 | extern int Incremental(char *devname, int verbose, int runstop, |
744 | struct supertype *st, char *homehost, int autof); | |
598f0d58 | 745 | extern int Incremental_container(struct supertype *st, char *devname, |
215bb3f7 N |
746 | int verbose, int runstop, int autof, |
747 | int trustworthy); | |
8382f19b NB |
748 | extern void RebuildMap(void); |
749 | extern int IncrementalScan(int verbose); | |
750 | ||
c82f047c NB |
751 | extern int CreateBitmap(char *filename, int force, char uuid[16], |
752 | unsigned long chunksize, unsigned long daemon_sleep, | |
dfd4d8ee | 753 | unsigned long write_behind, |
dcec9ee5 NB |
754 | unsigned long long array_size, |
755 | int major); | |
55935d51 | 756 | extern int ExamineBitmap(char *filename, int brief, struct supertype *st); |
bf4fb153 | 757 | extern int bitmap_update_uuid(int fd, int *uuid, int swap); |
2a528478 | 758 | extern unsigned long bitmap_sectors(struct bitmap_super_s *bsb); |
c82f047c | 759 | |
64c4757e | 760 | extern int md_get_version(int fd); |
e0d19036 | 761 | extern int get_linux_version(void); |
64c4757e | 762 | extern int parse_uuid(char *str, int uuid[4]); |
682c7051 NB |
763 | extern int check_ext2(int fd, char *name); |
764 | extern int check_reiser(int fd, char *name); | |
765 | extern int check_raid(int fd, char *name); | |
64c4757e | 766 | |
dd0781e5 | 767 | extern int get_mdp_major(void); |
8b0dabea | 768 | extern int dev_open(char *dev, int flags); |
e8a70c89 | 769 | extern int open_dev(int devnum); |
a931db9e | 770 | extern int open_dev_excl(int devnum); |
8d80900b | 771 | extern int is_standard(char *dev, int *nump); |
9008ed1c | 772 | extern int same_dev(char *one, char *two); |
dd0781e5 | 773 | |
f1ae21c4 | 774 | extern int parse_auto(char *str, char *msg, int config); |
8aec876d NB |
775 | extern mddev_ident_t conf_get_ident(char *dev); |
776 | extern mddev_dev_t conf_get_devs(void); | |
8382f19b | 777 | extern int conf_test_dev(char *devname); |
8aec876d NB |
778 | extern struct createinfo *conf_get_create_info(void); |
779 | extern void set_conffile(char *file); | |
780 | extern char *conf_get_mailaddr(void); | |
781 | extern char *conf_get_mailfrom(void); | |
782 | extern char *conf_get_program(void); | |
783 | extern char *conf_get_homehost(void); | |
e0d19036 | 784 | extern char *conf_line(FILE *file); |
98c6faba | 785 | extern char *conf_word(FILE *file, int allow_key); |
e0d19036 NB |
786 | extern void free_line(char *line); |
787 | extern int match_oneof(char *devices, char *devname); | |
e0d19036 | 788 | extern void uuid_from_super(int uuid[4], mdp_super_t *super); |
36ba7d48 | 789 | extern const int uuid_match_any[4]; |
f277ce36 | 790 | extern int same_uuid(int a[4], int b[4], int swapuuid); |
350f29f9 | 791 | extern void copy_uuid(void *a, int b[4], int swapuuid); |
d7288ddc | 792 | extern char *fname_from_uuid(struct supertype *st, |
ff54de6e | 793 | struct mdinfo *info, char *buf, char sep); |
4b1ac34b | 794 | extern unsigned long calc_csum(void *super, int bytes); |
583315d9 | 795 | extern int enough(int level, int raid_disks, int layout, int clean, |
265e0f17 | 796 | char *avail, int avail_disks); |
e0d19036 | 797 | extern int ask(char *mesg); |
8fac0577 | 798 | extern unsigned long long get_component_size(int fd); |
0430ed48 | 799 | extern void remove_partitions(int fd); |
5f8097be NB |
800 | extern unsigned long long calc_array_size(int level, int raid_disks, int layout, |
801 | int chunksize, unsigned long long devsize); | |
edd8d13c NB |
802 | extern int flush_metadata_updates(struct supertype *st); |
803 | extern void append_metadata_update(struct supertype *st, void *buf, int len); | |
6234c63c DW |
804 | extern int assemble_container_content(struct supertype *st, int mdfd, |
805 | struct mdinfo *content, int runstop, | |
806 | char *chosen_name, int verbose); | |
e0d19036 | 807 | |
7801ac20 N |
808 | extern int add_disk(int mdfd, struct supertype *st, |
809 | struct mdinfo *sra, struct mdinfo *info); | |
f35f2525 | 810 | extern int set_array_info(int mdfd, struct supertype *st, struct mdinfo *info); |
cd29a5c8 | 811 | |
56eb10c0 | 812 | extern char *human_size(long long bytes); |
e4965ef8 N |
813 | extern char *human_size_brief(long long bytes); |
814 | extern void print_r10_layout(int layout); | |
e0d19036 | 815 | |
63152c1b NB |
816 | #define NoMdDev (1<<23) |
817 | extern int find_free_devnum(int use_partitions); | |
818 | ||
e0d19036 NB |
819 | extern void put_md_name(char *name); |
820 | extern char *get_md_name(int dev); | |
821 | ||
c913b90e | 822 | extern char DefaultConfFile[]; |
b5e64645 | 823 | |
69207ff6 N |
824 | extern int create_mddev(char *dev, char *name, int autof, int trustworthy, |
825 | char *chosen); | |
826 | /* values for 'trustworthy' */ | |
827 | #define LOCAL 1 | |
828 | #define FOREIGN 2 | |
829 | #define METADATA 3 | |
6be1d39d | 830 | extern int open_mddev(char *dev, int report_errors); |
a322f70c | 831 | extern int open_container(int fd); |
b5e64645 | 832 | |
a931db9e NB |
833 | extern int mdmon_running(int devnum); |
834 | extern int signal_mdmon(int devnum); | |
40ebbb9c | 835 | extern int check_env(char *name); |
8850ee3e | 836 | extern int start_mdmon(int devnum); |
a931db9e | 837 | |
2f6079dc | 838 | extern char *devnum2devname(int num); |
77472ff8 | 839 | extern int devname2devnum(char *name); |
c94709e8 | 840 | extern int stat2devnum(struct stat *st); |
2f6079dc | 841 | extern int fd2devnum(int fd); |
b5e64645 | 842 | |
a931db9e NB |
843 | static inline int dev2major(int d) |
844 | { | |
845 | if (d >= 0) | |
846 | return MD_MAJOR; | |
847 | else | |
848 | return get_mdp_major(); | |
849 | } | |
850 | ||
851 | static inline int dev2minor(int d) | |
852 | { | |
853 | if (d >= 0) | |
854 | return d; | |
855 | return (-1-d) << MdpMinorShift; | |
856 | } | |
857 | ||
6416d527 NB |
858 | static inline int ROUND_UP(int a, int base) |
859 | { | |
860 | return ((a+base-1)/base)*base; | |
861 | } | |
862 | ||
3c558363 N |
863 | static inline int is_subarray(char *vers) |
864 | { | |
e9dd1598 N |
865 | /* The version string for a 'subarray' (an array in a container) |
866 | * is | |
867 | * /containername/componentname for normal read-write arrays | |
868 | * -containername/componentname for read-only arrays. | |
869 | * containername is e.g. md0, md_d1 | |
870 | * componentname is dependant on the metadata. e.g. '1' 'S1' ... | |
871 | */ | |
872 | return (*vers == '/' || *vers == '-'); | |
3c558363 N |
873 | } |
874 | ||
2a24d7b6 DW |
875 | #ifdef DEBUG |
876 | #define dprintf(fmt, arg...) \ | |
877 | fprintf(stderr, fmt, ##arg) | |
878 | #else | |
879 | #define dprintf(fmt, arg...) \ | |
880 | ({ if (0) fprintf(stderr, fmt, ##arg); 0; }) | |
881 | #endif | |
882 | ||
b5e64645 NB |
883 | #define LEVEL_MULTIPATH (-4) |
884 | #define LEVEL_LINEAR (-1) | |
885 | #define LEVEL_FAULTY (-5) | |
886 | ||
17f25ca6 NB |
887 | /* kernel module doesn't know about these */ |
888 | #define LEVEL_CONTAINER (-100) | |
a322f70c | 889 | #define LEVEL_UNSUPPORTED (-200) |
17f25ca6 | 890 | |
b5e64645 NB |
891 | |
892 | /* faulty stuff */ | |
893 | ||
894 | #define WriteTransient 0 | |
895 | #define ReadTransient 1 | |
896 | #define WritePersistent 2 | |
897 | #define ReadPersistent 3 | |
898 | #define WriteAll 4 /* doesn't go to device */ | |
899 | #define ReadFixable 5 | |
900 | #define Modes 6 | |
901 | ||
902 | #define ClearErrors 31 | |
903 | #define ClearFaults 30 | |
904 | ||
905 | #define AllPersist 100 /* internal use only */ | |
906 | #define NoPersist 101 | |
907 | ||
908 | #define ModeMask 0x1f | |
909 | #define ModeShift 5 | |
fe6729fa NB |
910 | |
911 | ||
912 | #ifdef __TINYC__ | |
913 | #undef minor | |
914 | #undef major | |
915 | #undef makedev | |
916 | #define minor(x) ((x)&0xff) | |
917 | #define major(x) (((x)>>8)&0xff) | |
918 | #define makedev(M,m) (((M)<<8) | (m)) | |
919 | #endif | |
920 | ||
e86c9dd6 NB |
921 | /* for raid5 */ |
922 | #define ALGORITHM_LEFT_ASYMMETRIC 0 | |
923 | #define ALGORITHM_RIGHT_ASYMMETRIC 1 | |
924 | #define ALGORITHM_LEFT_SYMMETRIC 2 | |
925 | #define ALGORITHM_RIGHT_SYMMETRIC 3 |