2 * mkcramfs - make a cramfs file system
4 * Copyright (C) 1999-2001 Transmeta Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 * Old version would die on largish filesystems. Change to mmap the
23 * files one by one instaed of all simultaneously. - aeb, 2002-11-01
26 #include <sys/types.h>
31 #include <sys/fcntl.h>
44 #define PAD_SIZE 512 /* only 0 and 512 supported by kernel */
46 static const char *progname
= "mkcramfs";
47 static int verbose
= 0;
50 #define PAGE_CACHE_SIZE (16384)
51 #elif defined __alpha__
52 #define PAGE_CACHE_SIZE (8192)
54 #define PAGE_CACHE_SIZE (4096)
57 /* The kernel assumes PAGE_CACHE_SIZE as block size. */
58 static unsigned int blksize
= PAGE_CACHE_SIZE
; /* settable via -b option */
59 static long total_blocks
= 0, total_nodes
= 1; /* pre-count the root node */
60 static int image_length
= 0;
63 * If opt_holes is set, then mkcramfs can create explicit holes in the
64 * data, which saves 26 bytes per hole (which is a lot smaller a
65 * saving than for most filesystems).
67 * Note that kernels up to at least 2.3.39 don't support cramfs holes,
68 * which is why this is turned off by default.
70 static int opt_edition
= 0;
71 static int opt_errors
= 0;
72 static int opt_holes
= 0;
73 static int opt_pad
= 0;
74 static char *opt_image
= NULL
;
75 static char *opt_name
= NULL
;
77 static int warn_dev
= 0;
78 static int warn_gid
= 0;
79 static int warn_namelen
= 0;
80 static int warn_skip
= 0;
81 static int warn_size
= 0;
82 static int warn_uid
= 0;
85 # define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
88 /* In-core version of inode / directory entry. */
92 unsigned int mode
, size
, uid
, gid
;
93 unsigned char md5sum
[16];
100 struct entry
*same
; /* points to other identical file */
101 unsigned int offset
; /* pointer to compressed data in archive */
102 unsigned int dir_offset
; /* offset of directory entry in archive */
105 struct entry
*child
; /* NULL for non-directory and empty dir */
110 * Width of various bitfields in struct cramfs_inode.
111 * Used only to generate warnings.
113 #define CRAMFS_SIZE_WIDTH 24
114 #define CRAMFS_UID_WIDTH 16
115 #define CRAMFS_GID_WIDTH 8
116 #define CRAMFS_OFFSET_WIDTH 26
118 /* Input status of 0 to print help and exit without an error. */
121 FILE *stream
= status
? stderr
: stdout
;
124 _("usage: %s [-v] [-b blksz] [-e edition] [-i file] [-n name] "
126 " -h print this help\n"
128 " -E make all warnings errors "
129 "(non-zero exit status)\n"
130 " -b blksz use this blocksize, must equal page size\n"
131 " -e edition set edition number (part of fsid)\n"
132 " -i file insert a file image into the filesystem "
133 "(requires >= 2.4.0)\n"
134 " -n name set name of cramfs filesystem\n"
135 " -p pad by %d bytes for boot code\n"
136 " -s sort directory entries (old option, ignored)\n"
137 " -z make explicit holes (requires >= 2.3.39)\n"
138 " dirname root of the filesystem to be compressed\n"
139 " outfile output file\n"),
147 xmalloc (size_t size
) {
148 void *t
= malloc(size
);
151 exit(8); /* out of memory */
157 do_mmap(char *path
, unsigned int size
, unsigned int mode
){
165 start
= xmalloc(size
);
166 if (readlink(path
, start
, size
) < 0) {
174 fd
= open(path
, O_RDONLY
);
181 start
= mmap(NULL
, size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
182 if (-1 == (int) (long) start
) {
192 do_munmap(char *start
, unsigned int size
, unsigned int mode
){
199 /* compute md5sums, so that we do not have to compare every pair of files */
201 mdfile(struct entry
*e
) {
205 start
= do_mmap(e
->path
, e
->size
, e
->mode
);
210 MD5Update(&ctx
, start
, e
->size
);
211 MD5Final(e
->md5sum
, &ctx
);
213 do_munmap(start
, e
->size
, e
->mode
);
215 e
->flags
|= HAVE_MD5
;
219 /* md5 digests are equal; files are almost certainly the same,
220 but just to be sure, do the comparison */
222 identical_file(struct entry
*e1
, struct entry
*e2
){
223 char *start1
, *start2
;
226 start1
= do_mmap(e1
->path
, e1
->size
, e1
->mode
);
229 start2
= do_mmap(e2
->path
, e2
->size
, e2
->mode
);
232 equal
= !memcmp(start1
, start2
, e1
->size
);
233 do_munmap(start1
, e1
->size
, e1
->mode
);
234 do_munmap(start2
, e2
->size
, e2
->mode
);
239 * The longest file name component to allow for in the input directory tree.
240 * Ext2fs (and many others) allow up to 255 bytes. A couple of filesystems
241 * allow longer (e.g. smbfs 1024), but there isn't much use in supporting
242 * >255-byte names in the input directory tree given that such names get
243 * truncated to 255 bytes when written to cramfs.
245 #define MAX_INPUT_NAMELEN 255
247 static int find_identical_file(struct entry
*orig
, struct entry
*new)
253 if (orig
->size
== new->size
&& orig
->path
) {
259 if ((orig
->flags
& HAVE_MD5
) && (new->flags
& HAVE_MD5
) &&
260 !memcmp(orig
->md5sum
, new->md5sum
, 16) &&
261 identical_file(orig
, new)) {
266 return find_identical_file(orig
->child
, new) ||
267 find_identical_file(orig
->next
, new);
270 static void eliminate_doubles(struct entry
*root
, struct entry
*orig
) {
272 if (orig
->size
&& orig
->path
)
273 find_identical_file(root
,orig
);
274 eliminate_doubles(root
,orig
->child
);
275 eliminate_doubles(root
,orig
->next
);
280 * We define our own sorting function instead of using alphasort which
281 * uses strcoll and changes ordering based on locale information.
283 static int cramsort (const void *a
, const void *b
)
285 return strcmp ((*(const struct dirent
**) a
)->d_name
,
286 (*(const struct dirent
**) b
)->d_name
);
289 static unsigned int parse_directory(struct entry
*root_entry
, const char *name
, struct entry
**prev
, loff_t
*fslen_ub
)
291 struct dirent
**dirlist
;
292 int totalsize
= 0, dircount
, dirindex
;
293 char *path
, *endpath
;
294 size_t len
= strlen(name
);
296 /* Set up the path. */
297 /* TODO: Reuse the parent's buffer to save memcpy'ing and duplication. */
298 path
= xmalloc(len
+ 1 + MAX_INPUT_NAMELEN
+ 1);
299 memcpy(path
, name
, len
);
300 endpath
= path
+ len
;
304 /* read in the directory and sort */
305 dircount
= scandir(name
, &dirlist
, 0, cramsort
);
312 /* process directory */
313 for (dirindex
= 0; dirindex
< dircount
; dirindex
++) {
314 struct dirent
*dirent
;
320 dirent
= dirlist
[dirindex
];
322 /* Ignore "." and ".." - we won't be adding them
324 if (dirent
->d_name
[0] == '.') {
325 if (dirent
->d_name
[1] == '\0')
327 if (dirent
->d_name
[1] == '.') {
328 if (dirent
->d_name
[2] == '\0')
332 namelen
= strlen(dirent
->d_name
);
333 if (namelen
> MAX_INPUT_NAMELEN
) {
335 _("Very long (%u bytes) filename `%s' found.\n"
336 " Please increase MAX_INPUT_NAMELEN in "
337 "mkcramfs.c and recompile. Exiting.\n"),
338 namelen
, dirent
->d_name
);
341 memcpy(endpath
, dirent
->d_name
, namelen
+ 1);
343 if (lstat(path
, &st
) < 0) {
348 entry
= calloc(1, sizeof(struct entry
));
353 entry
->name
= strdup(dirent
->d_name
);
359 /* Can't happen when reading from ext2fs. */
361 /* TODO: we ought to avoid chopping in half
362 multi-byte UTF8 characters. */
363 entry
->name
[namelen
= 255] = '\0';
366 entry
->mode
= st
.st_mode
;
367 entry
->size
= st
.st_size
;
368 entry
->uid
= st
.st_uid
;
369 if (entry
->uid
>= 1 << CRAMFS_UID_WIDTH
)
371 entry
->gid
= st
.st_gid
;
372 if (entry
->gid
>= 1 << CRAMFS_GID_WIDTH
)
373 /* TODO: We ought to replace with a default
374 gid instead of truncating; otherwise there
375 are security problems. Maybe mode should
376 be &= ~070. Same goes for uid once Linux
377 supports >16-bit uids. */
379 size
= sizeof(struct cramfs_inode
) + ((namelen
+ 3) & ~3);
381 if (S_ISDIR(st
.st_mode
)) {
382 entry
->size
= parse_directory(root_entry
, path
, &entry
->child
, fslen_ub
);
383 } else if (S_ISREG(st
.st_mode
)) {
384 entry
->path
= strdup(path
);
386 if (entry
->size
>= (1 << CRAMFS_SIZE_WIDTH
)) {
388 entry
->size
= (1 << CRAMFS_SIZE_WIDTH
) - 1;
391 } else if (S_ISLNK(st
.st_mode
)) {
392 entry
->path
= strdup(path
);
393 } else if (S_ISFIFO(st
.st_mode
) || S_ISSOCK(st
.st_mode
)) {
394 /* maybe we should skip sockets */
397 entry
->size
= st
.st_rdev
;
398 if (entry
->size
& -(1<<CRAMFS_SIZE_WIDTH
))
402 if (S_ISREG(st
.st_mode
) || S_ISLNK(st
.st_mode
)) {
403 int blocks
= ((entry
->size
- 1) / blksize
+ 1);
405 /* block pointers & data expansion allowance + data */
407 *fslen_ub
+= (4+26)*blocks
+ entry
->size
+ 3;
410 /* Link it into the list */
416 free(dirlist
); /* allocated by scandir() with malloc() */
420 /* Returns sizeof(struct cramfs_super), which includes the root inode. */
421 static unsigned int write_superblock(struct entry
*root
, char *base
, int size
)
423 struct cramfs_super
*super
= (struct cramfs_super
*) base
;
424 unsigned int offset
= sizeof(struct cramfs_super
) + image_length
;
430 super
->magic
= CRAMFS_MAGIC
;
431 super
->flags
= CRAMFS_FLAG_FSID_VERSION_2
| CRAMFS_FLAG_SORTED_DIRS
;
433 super
->flags
|= CRAMFS_FLAG_HOLES
;
434 if (image_length
> 0)
435 super
->flags
|= CRAMFS_FLAG_SHIFTED_ROOT_OFFSET
;
437 memcpy(super
->signature
, CRAMFS_SIGNATURE
, sizeof(super
->signature
));
439 super
->fsid
.crc
= crc32(0L, Z_NULL
, 0);
440 super
->fsid
.edition
= opt_edition
;
441 super
->fsid
.blocks
= total_blocks
;
442 super
->fsid
.files
= total_nodes
;
444 memset(super
->name
, 0x00, sizeof(super
->name
));
446 strncpy(super
->name
, opt_name
, sizeof(super
->name
));
448 strncpy(super
->name
, "Compressed", sizeof(super
->name
));
450 super
->root
.mode
= root
->mode
;
451 super
->root
.uid
= root
->uid
;
452 super
->root
.gid
= root
->gid
;
453 super
->root
.size
= root
->size
;
454 super
->root
.offset
= offset
>> 2;
459 static void set_data_offset(struct entry
*entry
, char *base
, unsigned long offset
)
461 struct cramfs_inode
*inode
= (struct cramfs_inode
*) (base
+ entry
->dir_offset
);
462 if (offset
>= (1 << (2 + CRAMFS_OFFSET_WIDTH
))) {
463 fprintf(stderr
, _("filesystem too big. Exiting.\n"));
466 inode
->offset
= (offset
>> 2);
471 * We do a width-first printout of the directory
472 * entries, using a stack to remember the directories
475 #define MAXENTRIES (100)
476 static unsigned int write_directory_structure(struct entry
*entry
, char *base
, unsigned int offset
)
478 int stack_entries
= 0;
479 struct entry
*entry_stack
[MAXENTRIES
];
482 int dir_start
= stack_entries
;
484 struct cramfs_inode
*inode
=
485 (struct cramfs_inode
*) (base
+ offset
);
486 size_t len
= strlen(entry
->name
);
488 entry
->dir_offset
= offset
;
490 inode
->mode
= entry
->mode
;
491 inode
->uid
= entry
->uid
;
492 inode
->gid
= entry
->gid
;
493 inode
->size
= entry
->size
;
495 /* Non-empty directories, regfiles and symlinks will
496 write over inode->offset later. */
498 offset
+= sizeof(struct cramfs_inode
);
499 total_nodes
++; /* another node */
500 memcpy(base
+ offset
, entry
->name
, len
);
501 /* Pad up the name to a 4-byte boundary */
503 *(base
+ offset
+ len
) = '\0';
506 inode
->namelen
= len
>> 2;
510 printf(" %s\n", entry
->name
);
512 if (stack_entries
>= MAXENTRIES
) {
514 _("Exceeded MAXENTRIES. Raise"
515 " this value in mkcramfs.c "
516 "and recompile. Exiting.\n")
520 entry_stack
[stack_entries
] = entry
;
527 * Reverse the order the stack entries pushed during
528 * this directory, for a small optimization of disk
529 * access in the created fs. This change makes things
533 struct entry
**lo
= entry_stack
+ dir_start
;
534 struct entry
**hi
= entry_stack
+ stack_entries
;
544 /* Pop a subdirectory entry from the stack, and recurse. */
548 entry
= entry_stack
[stack_entries
];
550 set_data_offset(entry
, base
, offset
);
552 printf("'%s':\n", entry
->name
);
553 entry
= entry
->child
;
558 static int is_zero(char const *begin
, unsigned len
)
561 /* Returns non-zero iff the first LEN bytes from BEGIN are
563 return (len
-- == 0 ||
571 memcmp(begin
, begin
+ 4, len
) == 0))))))));
573 /* Never create holes. */
578 * One 4-byte pointer per block and then the actual blocked
579 * output. The first block does not need an offset pointer,
580 * as it will start immediately after the pointer block;
581 * so the i'th pointer points to the end of the i'th block
582 * (i.e. the start of the (i+1)'th block or past EOF).
584 * Note that size > 0, as a zero-sized file wouldn't ever
585 * have gotten here in the first place.
588 do_compress(char *base
, unsigned int offset
, char const *name
,
589 char *path
, unsigned int size
, unsigned int mode
)
591 unsigned long original_size
, original_offset
, new_size
, blocks
, curr
;
595 /* get uncompressed data */
596 start
= do_mmap(path
, size
, mode
);
601 original_size
= size
;
602 original_offset
= offset
;
603 blocks
= (size
- 1) / blksize
+ 1;
604 curr
= offset
+ 4 * blocks
;
606 total_blocks
+= blocks
;
609 unsigned long len
= 2 * blksize
;
610 unsigned int input
= size
;
614 if (!is_zero (p
, input
)) {
615 compress(base
+ curr
, &len
, p
, input
);
620 if (len
> blksize
*2) {
621 /* (I don't think this can happen with zlib.) */
622 printf(_("AIEEE: block \"compressed\" to > "
623 "2*blocklength (%ld)\n"),
628 *(u32
*) (base
+ offset
) = curr
;
632 do_munmap(start
, original_size
, mode
);
634 curr
= (curr
+ 3) & ~3;
635 new_size
= curr
- original_offset
;
636 /* TODO: Arguably, original_size in these 2 lines should be
637 st_blocks * 512. But if you say that, then perhaps
638 administrative data should also be included in both. */
639 change
= new_size
- original_size
;
641 printf(_("%6.2f%% (%+d bytes)\t%s\n"),
642 (change
* 100) / (double) original_size
, change
, name
);
649 * Traverse the entry tree, writing data for every item that has
650 * non-null entry->path (i.e. every symlink and non-empty
654 write_data(struct entry
*entry
, char *base
, unsigned int offset
) {
657 for (e
= entry
; e
; e
= e
->next
) {
660 set_data_offset(e
, base
, e
->same
->offset
);
661 e
->offset
= e
->same
->offset
;
663 set_data_offset(e
, base
, offset
);
665 offset
= do_compress(base
, offset
, e
->name
,
666 e
->path
, e
->size
,e
->mode
);
669 offset
= write_data(e
->child
, base
, offset
);
674 static unsigned int write_file(char *file
, char *base
, unsigned int offset
)
679 fd
= open(file
, O_RDONLY
);
684 buf
= mmap(NULL
, image_length
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
685 memcpy(base
+ offset
, buf
, image_length
);
686 munmap(buf
, image_length
);
688 /* Pad up the image_length to a 4-byte boundary */
689 while (image_length
& 3) {
690 *(base
+ offset
+ image_length
) = '\0';
693 return (offset
+ image_length
);
697 * Maximum size fs you can create is roughly 256MB. (The last file's
698 * data must begin within 256MB boundary but can extend beyond that.)
700 * Note that if you want it to fit in a ROM then you're limited to what the
701 * hardware and kernel can support (64MB?).
705 return (((1 << CRAMFS_OFFSET_WIDTH
) - 1) << 2) /* offset */
706 + (1 << CRAMFS_SIZE_WIDTH
) - 1 /* filesize */
707 + (1 << CRAMFS_SIZE_WIDTH
) * 4 / blksize
; /* block pointers */
713 * mkcramfs directory-name outfile
715 * where "directory-name" is simply the root of the directory
716 * tree that we want to generate a compressed filesystem out
719 int main(int argc
, char **argv
)
721 struct stat st
; /* used twice... */
722 struct entry
*root_entry
;
724 ssize_t offset
, written
;
726 /* initial guess (upper-bound) of required filesystem size */
727 loff_t fslen_ub
= sizeof(struct cramfs_super
);
728 unsigned int fslen_max
;
729 char const *dirname
, *outfile
;
730 u32 crc
= crc32(0L, Z_NULL
, 0);
738 if ((p
= strrchr(progname
, '/')) != NULL
)
742 /* command line options */
743 while ((c
= getopt(argc
, argv
, "hb:Ee:i:n:psVvz")) != EOF
) {
748 blksize
= atoi(optarg
);
756 opt_edition
= atoi(optarg
);
760 if (lstat(opt_image
, &st
) < 0) {
764 image_length
= st
.st_size
; /* may be padded later */
765 fslen_ub
+= (image_length
+ 3); /* 3 is for padding */
772 fslen_ub
+= PAD_SIZE
;
775 /* old option, ignored */
778 printf(_("%s from %s\n"),
779 progname
, util_linux_version
);
790 if ((argc
- optind
) != 2)
792 dirname
= argv
[optind
];
793 outfile
= argv
[optind
+ 1];
795 if (stat(dirname
, &st
) < 0) {
799 fd
= open(outfile
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
801 root_entry
= calloc(1, sizeof(struct entry
));
806 root_entry
->mode
= st
.st_mode
;
807 root_entry
->uid
= st
.st_uid
;
808 root_entry
->gid
= st
.st_gid
;
810 root_entry
->size
= parse_directory(root_entry
, dirname
, &root_entry
->child
, &fslen_ub
);
812 /* always allocate a multiple of blksize bytes because that's
813 what we're going to write later on */
814 fslen_ub
= ((fslen_ub
- 1) | (blksize
- 1)) + 1;
815 fslen_max
= maxfslen();
817 if (fslen_ub
> fslen_max
) {
819 _("warning: guestimate of required size (upper bound) "
820 "is %LdMB, but maximum image size is %uMB. "
821 "We might die prematurely.\n"),
824 fslen_ub
= fslen_max
;
827 /* find duplicate files */
828 eliminate_doubles(root_entry
,root_entry
);
830 /* TODO: Why do we use a private/anonymous mapping here
831 followed by a write below, instead of just a shared mapping
832 and a couple of ftruncate calls? Is it just to save us
833 having to deal with removing the file afterwards? If we
834 really need this huge anonymous mapping, we ought to mmap
835 in smaller chunks, so that the user doesn't need nn MB of
836 RAM free. If the reason is to be able to write to
837 un-mmappable block devices, then we could try shared mmap
838 and revert to anonymous mmap if the shared mmap fails. */
839 rom_image
= mmap(NULL
,
841 PROT_READ
| PROT_WRITE
,
842 MAP_PRIVATE
| MAP_ANONYMOUS
,
845 if (-1 == (int) (long) rom_image
) {
846 perror("ROM image map");
850 /* Skip the first opt_pad bytes for boot loader code */
852 memset(rom_image
, 0x00, opt_pad
);
854 /* Skip the superblock and come back to write it later. */
855 offset
+= sizeof(struct cramfs_super
);
857 /* Insert a file image. */
860 printf(_("Including: %s\n"), opt_image
);
861 offset
= write_file(opt_image
, rom_image
, offset
);
864 offset
= write_directory_structure(root_entry
->child
, rom_image
, offset
);
866 printf(_("Directory data: %d bytes\n"), offset
);
868 offset
= write_data(root_entry
, rom_image
, offset
);
870 /* We always write a multiple of blksize bytes, so that
872 offset
= ((offset
- 1) | (blksize
- 1)) + 1;
874 printf(_("Everything: %d kilobytes\n"), offset
>> 10);
876 /* Write the superblock now that we can fill in all of the fields. */
877 write_superblock(root_entry
, rom_image
+opt_pad
, offset
);
879 printf(_("Super block: %d bytes\n"),
880 sizeof(struct cramfs_super
));
882 /* Put the checksum in. */
883 crc
= crc32(crc
, (rom_image
+opt_pad
), (offset
-opt_pad
));
884 ((struct cramfs_super
*) (rom_image
+opt_pad
))->fsid
.crc
= crc
;
886 printf(_("CRC: %x\n"), crc
);
888 /* Check to make sure we allocated enough space. */
889 if (fslen_ub
< offset
) {
891 _("not enough space allocated for ROM image "
892 "(%Ld allocated, %d used)\n"),
897 written
= write(fd
, rom_image
, offset
);
902 if (offset
!= written
) {
903 fprintf(stderr
, _("ROM image write failed (%d %d)\n"),
908 /* (These warnings used to come at the start, but they scroll off the
909 screen too quickly.) */
910 if (warn_namelen
) /* (can't happen when reading from ext2fs) */
911 fprintf(stderr
, /* bytes, not chars: think UTF8. */
912 _("warning: filenames truncated to 255 bytes.\n"));
915 _("warning: files were skipped due to errors.\n"));
918 _("warning: file sizes truncated to %luMB "
919 "(minus 1 byte).\n"),
920 1L << (CRAMFS_SIZE_WIDTH
- 20));
921 if (warn_uid
) /* (not possible with current Linux versions) */
923 _("warning: uids truncated to %u bits. "
924 "(This may be a security concern.)\n"),
928 _("warning: gids truncated to %u bits. "
929 "(This may be a security concern.)\n"),
933 _("WARNING: device numbers truncated to %u bits. "
934 "This almost certainly means\n"
935 "that some device files will be wrong.\n"),
936 CRAMFS_OFFSET_WIDTH
);
938 (warn_namelen
|warn_skip
|warn_size
|warn_uid
|warn_gid
|warn_dev
))