2 * mk_hugefiles.c -- create huge files
5 #define _XOPEN_SOURCE 600 /* for inclusion of PATH_MAX in Solaris */
6 #define _BSD_SOURCE /* for makedev() and major() */
7 #define _DEFAULT_SOURCE /* since glibc 2.20 _BSD_SOURCE is deprecated */
18 #include <sys/utsname.h>
35 #ifdef HAVE_SYS_IOCTL_H
36 #include <sys/ioctl.h>
38 #include <sys/types.h>
40 #ifdef HAVE_SYS_SYSMACROS_H
41 #include <sys/sysmacros.h>
45 #include <blkid/blkid.h>
47 #include "ext2fs/ext2_fs.h"
48 #include "ext2fs/ext2fsP.h"
49 #include "et/com_err.h"
50 #include "uuid/uuid.h"
52 #include "ext2fs/ext2fs.h"
54 #include "support/profile.h"
55 #include "support/prof_err.h"
56 #include "support/nls-enable.h"
61 static blk64_t num_blocks
;
62 static blk64_t num_slack
;
63 static unsigned long num_files
;
65 static char *fn_prefix
;
66 static int idx_digits
;
68 static char *fn_numbuf
;
69 int zero_hugefile
= 1;
71 #define SYSFS_PATH_LEN 300
72 typedef char sysfs_path_t
[SYSFS_PATH_LEN
];
76 * We are very careful to avoid needing to worry about buffer
77 * overflows, so we don't really need to use snprintf() except as an
78 * additional safety check. So if snprintf() is not present, it's
79 * safe to fall back to vsprintf(). This provides portability since
80 * vsprintf() is guaranteed by C89, while snprintf() is only
81 * guaranteed by C99 --- which for example, Microsoft Visual Studio
82 * has *still* not bothered to implement. :-/ (Not that I expect
83 * mke2fs to be ported to MS Visual Studio any time soon, but
84 * libext2fs *does* get built on Microsoft platforms, and we might
85 * want to move this into libext2fs some day.)
87 static int my_snprintf(char *str
, size_t size
, const char *format
, ...)
93 ret
= vsprintf(str
, format
, ap
);
98 #define snprintf my_snprintf
102 * Fall back to Linux's definitions of makedev and major are needed.
103 * The search_sysfs_block() function is highly unlikely to work on
104 * non-Linux systems anyway.
107 #define makedev(maj, min) (((maj) << 8) + (min))
110 static char *search_sysfs_block(dev_t devno
, sysfs_path_t ret_path
)
112 struct dirent
*de
, *p_de
;
113 DIR *dir
= NULL
, *p_dir
= NULL
;
115 sysfs_path_t path
, p_path
;
116 unsigned int major
, minor
;
117 char *ret
= ret_path
;
120 if ((dir
= opendir("/sys/block")) == NULL
)
122 while ((de
= readdir(dir
)) != NULL
) {
123 if (!strcmp(de
->d_name
, ".") || !strcmp(de
->d_name
, "..") ||
124 strlen(de
->d_name
) > sizeof(path
)-32)
126 snprintf(path
, SYSFS_PATH_LEN
,
127 "/sys/block/%s/dev", de
->d_name
);
128 f
= fopen(path
, "r");
130 (fscanf(f
, "%u:%u", &major
, &minor
) == 2)) {
132 if (makedev(major
, minor
) == devno
) {
133 snprintf(ret_path
, SYSFS_PATH_LEN
,
134 "/sys/block/%s", de
->d_name
);
138 if (major(devno
) != major
)
145 snprintf(path
, SYSFS_PATH_LEN
, "/sys/block/%s", de
->d_name
);
149 if ((p_dir
= opendir(path
)) == NULL
)
151 while ((p_de
= readdir(p_dir
)) != NULL
) {
152 if (!strcmp(p_de
->d_name
, ".") ||
153 !strcmp(p_de
->d_name
, "..") ||
154 (strlen(p_de
->d_name
) >
155 SYSFS_PATH_LEN
- strlen(path
) - 32))
157 snprintf(p_path
, SYSFS_PATH_LEN
, "%s/%s/dev",
160 f
= fopen(p_path
, "r");
162 (fscanf(f
, "%u:%u", &major
, &minor
) == 2) &&
163 (((major
<< 8) + minor
) == devno
)) {
165 snprintf(ret_path
, SYSFS_PATH_LEN
, "%s/%s",
182 static blk64_t
get_partition_start(const char *device_name
)
184 unsigned long long start
;
191 if ((stat(device_name
, &st
) < 0) || !S_ISBLK(st
.st_mode
))
194 cp
= search_sysfs_block(st
.st_rdev
, path
);
197 if (strlen(path
) > SYSFS_PATH_LEN
- sizeof("/start"))
199 strcat(path
, "/start");
200 f
= fopen(path
, "r");
203 n
= fscanf(f
, "%llu", &start
);
205 return (n
== 1) ? start
: 0;
208 static errcode_t
create_directory(ext2_filsys fs
, char *dir
,
212 struct ext2_inode inode
;
213 ext2_ino_t ino
= EXT2_ROOT_INO
;
215 errcode_t retval
= 0;
216 char *fn
, *cp
, *next
;
218 fn
= malloc(strlen(dir
) + 1);
225 next
= strchr(cp
, '/');
229 retval
= ext2fs_new_inode(fs
, ino
, LINUX_S_IFDIR
,
234 retval
= ext2fs_mkdir(fs
, ino
, newdir
, cp
);
239 retval
= ext2fs_read_inode(fs
, ino
, &inode
);
243 inode
.i_uid
= uid
& 0xFFFF;
244 ext2fs_set_i_uid_high(inode
, (uid
>> 16) & 0xffff);
245 inode
.i_gid
= gid
& 0xFFFF;
246 ext2fs_set_i_gid_high(inode
, (gid
>> 16) & 0xffff);
247 retval
= ext2fs_write_inode(fs
, ino
, &inode
);
251 if (next
== NULL
|| *next
== '\0')
262 static errcode_t
mk_hugefile(ext2_filsys fs
, blk64_t num
,
263 ext2_ino_t dir
, unsigned long idx
, ext2_ino_t
*ino
)
267 blk64_t lblk
, bend
= 0;
271 struct ext2_inode inode
;
272 ext2_extent_handle_t handle
;
274 retval
= ext2fs_new_inode(fs
, 0, LINUX_S_IFREG
, NULL
, ino
);
278 memset(&inode
, 0, sizeof(struct ext2_inode
));
279 inode
.i_mode
= LINUX_S_IFREG
| (0666 & ~fs
->umask
);
280 inode
.i_links_count
= 1;
281 inode
.i_uid
= uid
& 0xFFFF;
282 ext2fs_set_i_uid_high(inode
, (uid
>> 16) & 0xffff);
283 inode
.i_gid
= gid
& 0xFFFF;
284 ext2fs_set_i_gid_high(inode
, (gid
>> 16) & 0xffff);
286 retval
= ext2fs_write_new_inode(fs
, *ino
, &inode
);
290 ext2fs_inode_alloc_stats2(fs
, *ino
, +1, 0);
292 retval
= ext2fs_extent_open2(fs
, *ino
, &inode
, &handle
);
297 * We don't use ext2fs_fallocate() here because hugefiles are
298 * designed to be physically contiguous (if the block group
299 * descriptors are configured to be in a single block at the
300 * beginning of the file system, by using the
301 * packed_meta_blocks layout), with the extent tree blocks
302 * allocated near the beginning of the file system.
305 left
= num
? num
: 1;
310 retval
= ext2fs_find_first_zero_block_bitmap2(fs
->block_map
,
311 goal
, ext2fs_blocks_count(fs
->super
) - 1, &end
);
316 retval
= ext2fs_find_first_set_block_bitmap2(fs
->block_map
, goal
,
317 ext2fs_blocks_count(fs
->super
) - 1, &bend
);
318 if (retval
== ENOENT
) {
319 bend
= ext2fs_blocks_count(fs
->super
);
323 if (!num
|| bend
- goal
< left
)
330 ext2fs_block_alloc_stats_range(fs
, pblk
, n
, +1);
334 retval
= ext2fs_zero_blocks2(fs
, pblk
, n
,
338 com_err(program_name
, retval
,
339 _("while zeroing block %llu "
340 "for hugefile"), ret_blk
);
345 struct ext2fs_extent newextent
;
347 if (l
> EXT_INIT_MAX_LEN
)
348 l
= EXT_INIT_MAX_LEN
;
351 newextent
.e_pblk
= pblk
;
352 newextent
.e_lblk
= lblk
;
353 newextent
.e_flags
= 0;
355 retval
= ext2fs_extent_insert(handle
,
356 EXT2_EXTENT_INSERT_AFTER
, &newextent
);
365 retval
= ext2fs_read_inode(fs
, *ino
, &inode
);
369 retval
= ext2fs_iblk_add_blocks(fs
, &inode
,
370 count
/ EXT2FS_CLUSTER_RATIO(fs
));
373 size
= (__u64
) count
* fs
->blocksize
;
374 retval
= ext2fs_inode_size_set(fs
, &inode
, size
);
378 retval
= ext2fs_write_new_inode(fs
, *ino
, &inode
);
383 sprintf(fn_numbuf
, "%0*lu", idx_digits
, idx
);
384 else if (num_files
> 1)
385 sprintf(fn_numbuf
, "%lu", idx
);
388 retval
= ext2fs_link(fs
, dir
, fn_buf
, *ino
, EXT2_FT_REG_FILE
);
389 if (retval
== EXT2_ET_DIR_NO_SPACE
) {
390 retval
= ext2fs_expand_dir(fs
, dir
);
401 ext2fs_extent_free(handle
);
406 static blk64_t
calc_overhead(ext2_filsys fs
, blk64_t num
)
408 blk64_t e_blocks
, e_blocks2
, e_blocks3
, e_blocks4
;
409 int extents_per_block
;
410 int extents
= (num
+ EXT_INIT_MAX_LEN
- 1) / EXT_INIT_MAX_LEN
;
416 * This calculation is due to the fact that we are inefficient
417 * in how handle extent splits when appending to the end of
418 * the extent tree. Sigh. We should fix this so that we can
419 * actually store 340 extents per 4k block, instead of only 170.
421 extents_per_block
= ((fs
->blocksize
-
422 sizeof(struct ext3_extent_header
)) /
423 sizeof(struct ext3_extent
));
424 extents_per_block
= (extents_per_block
/ 2) - 1;
426 e_blocks
= (extents
+ extents_per_block
- 1) / extents_per_block
;
427 e_blocks2
= (e_blocks
+ extents_per_block
- 1) / extents_per_block
;
428 e_blocks3
= (e_blocks2
+ extents_per_block
- 1) / extents_per_block
;
429 e_blocks4
= (e_blocks3
+ extents_per_block
- 1) / extents_per_block
;
430 return (e_blocks
+ e_blocks2
+ e_blocks3
+ e_blocks4
) *
431 EXT2FS_CLUSTER_RATIO(fs
);
435 * Find the place where we should start allocating blocks for the huge
436 * files. Leave <slack> free blocks at the beginning of the file
437 * system for things like metadata blocks.
439 static blk64_t
get_start_block(ext2_filsys fs
, blk64_t slack
)
442 blk64_t blk
= fs
->super
->s_first_data_block
, next
;
443 blk64_t last_blk
= ext2fs_blocks_count(fs
->super
) - 1;
446 retval
= ext2fs_find_first_zero_block_bitmap2(fs
->block_map
,
447 blk
, last_blk
, &blk
);
451 retval
= ext2fs_find_first_set_block_bitmap2(fs
->block_map
,
452 blk
, last_blk
, &next
);
456 if (next
- blk
> slack
) {
461 slack
-= (next
- blk
);
467 static blk64_t
round_up_align(blk64_t b
, unsigned long align
,
474 part_offset
= part_offset
% align
;
475 m
= (b
+ part_offset
) % align
;
481 errcode_t
mk_hugefiles(ext2_filsys fs
, const char *device_name
)
486 blk64_t fs_blocks
, part_offset
= 0;
491 if (!get_bool_from_profile(fs_types
, "make_hugefiles", 0))
494 if (!ext2fs_has_feature_extents(fs
->super
))
495 return EXT2_ET_EXTENT_NOT_SUPPORTED
;
497 uid
= get_int_from_profile(fs_types
, "hugefiles_uid", 0);
498 gid
= get_int_from_profile(fs_types
, "hugefiles_gid", 0);
499 fs
->umask
= get_int_from_profile(fs_types
, "hugefiles_umask", 077);
500 num_files
= get_int_from_profile(fs_types
, "num_hugefiles", 0);
501 t
= get_string_from_profile(fs_types
, "hugefiles_slack", "1M");
502 num_slack
= parse_num_blocks2(t
, fs
->super
->s_log_block_size
);
504 t
= get_string_from_profile(fs_types
, "hugefiles_size", "0");
505 num_blocks
= parse_num_blocks2(t
, fs
->super
->s_log_block_size
);
507 t
= get_string_from_profile(fs_types
, "hugefiles_align", "0");
508 align
= parse_num_blocks2(t
, fs
->super
->s_log_block_size
);
510 if (get_bool_from_profile(fs_types
, "hugefiles_align_disk", 0)) {
511 part_offset
= get_partition_start(device_name
) /
512 (fs
->blocksize
/ 512);
513 if (part_offset
% EXT2FS_CLUSTER_RATIO(fs
)) {
515 _("Partition offset of %llu (%uk) blocks "
516 "not compatible with cluster size %u.\n"),
517 part_offset
, fs
->blocksize
,
518 EXT2_CLUSTER_SIZE(fs
->super
));
522 num_blocks
= round_up_align(num_blocks
, align
, 0);
523 zero_hugefile
= get_bool_from_profile(fs_types
, "zero_hugefiles",
526 t
= get_string_from_profile(fs_types
, "hugefiles_dir", "/");
527 retval
= create_directory(fs
, t
, &dir
);
532 fn_prefix
= get_string_from_profile(fs_types
, "hugefiles_name",
534 idx_digits
= get_int_from_profile(fs_types
, "hugefiles_digits", 5);
535 d
= int_log10(num_files
) + 1;
538 dsize
= strlen(fn_prefix
) + d
+ 16;
539 fn_buf
= malloc(dsize
);
544 strcpy(fn_buf
, fn_prefix
);
545 fn_numbuf
= fn_buf
+ strlen(fn_prefix
);
548 fs_blocks
= ext2fs_free_blocks_count(fs
->super
);
549 if (fs_blocks
< num_slack
+ align
)
551 fs_blocks
-= num_slack
+ align
;
552 if (num_blocks
&& num_blocks
> fs_blocks
)
554 if (num_blocks
== 0 && num_files
== 0)
557 if (num_files
== 0 && num_blocks
) {
558 num_files
= fs_blocks
/ num_blocks
;
559 fs_blocks
-= (num_files
/ 16) + 1;
560 fs_blocks
-= calc_overhead(fs
, num_blocks
) * num_files
;
561 num_files
= fs_blocks
/ num_blocks
;
564 if (num_blocks
== 0 && num_files
> 1) {
565 num_blocks
= fs_blocks
/ num_files
;
566 fs_blocks
-= (num_files
/ 16) + 1;
567 fs_blocks
-= calc_overhead(fs
, num_blocks
) * num_files
;
568 num_blocks
= fs_blocks
/ num_files
;
571 num_slack
+= (calc_overhead(fs
, num_blocks
? num_blocks
: fs_blocks
) *
573 num_slack
+= (num_files
/ 16) + 1; /* space for dir entries */
574 goal
= get_start_block(fs
, num_slack
);
575 goal
= round_up_align(goal
, align
, part_offset
);
577 if ((num_blocks
? num_blocks
: fs_blocks
) >
578 (0x80000000UL
/ fs
->blocksize
))
579 ext2fs_set_feature_large_file(fs
->super
);
582 if (zero_hugefile
&& verbose
)
583 printf("%s", _("Huge files will be zero'ed\n"));
584 printf(_("Creating %lu huge file(s) "), num_files
);
586 printf(_("with %llu blocks each"), num_blocks
);
589 for (i
=0; i
< num_files
; i
++) {
592 retval
= mk_hugefile(fs
, num_blocks
, dir
, i
, &ino
);
594 com_err(program_name
, retval
,
595 _("while creating huge file %lu"), i
);
600 fputs(_("done\n"), stdout
);