2 * mkcramfs - make a cramfs file system
4 * Copyright (C) 1999-2002 Transmeta Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
22 * Old version would die on largish filesystems. Change to mmap the
23 * files one by one instaed of all simultaneously. - aeb, 2002-11-01
26 #include <sys/types.h>
31 #include <sys/fcntl.h>
45 /* Exit codes used by mkfs-type programs */
46 #define MKFS_OK 0 /* No errors */
47 #define MKFS_ERROR 8 /* Operational error */
48 #define MKFS_USAGE 16 /* Usage or syntax error */
50 /* The kernel only supports PAD_SIZE of 0 and 512. */
53 static const char *progname
= "mkcramfs";
54 static int verbose
= 0;
56 static unsigned int blksize
; /* settable via -b option */
57 static long total_blocks
= 0, total_nodes
= 1; /* pre-count the root node */
58 static int image_length
= 0;
61 * If opt_holes is set, then mkcramfs can create explicit holes in the
62 * data, which saves 26 bytes per hole (which is a lot smaller a
63 * saving than for most filesystems).
65 * Note that kernels up to at least 2.3.39 don't support cramfs holes,
66 * which is why this is turned off by default.
68 static int opt_edition
= 0;
69 static int opt_errors
= 0;
70 static int opt_holes
= 0;
71 static int opt_pad
= 0;
72 static char *opt_image
= NULL
;
73 static char *opt_name
= NULL
;
75 static int warn_dev
= 0;
76 static int warn_gid
= 0;
77 static int warn_namelen
= 0;
78 static int warn_skip
= 0;
79 static int warn_size
= 0;
80 static int warn_uid
= 0;
83 # define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
86 /* In-core version of inode / directory entry. */
90 unsigned int mode
, size
, uid
, gid
;
91 unsigned char md5sum
[16];
98 int fd
; /* temporarily open files while mmapped */
99 struct entry
*same
; /* points to other identical file */
100 unsigned int offset
; /* pointer to compressed data in archive */
101 unsigned int dir_offset
; /* offset of directory entry in archive */
104 struct entry
*child
; /* NULL for non-directory and empty dir */
109 * Width of various bitfields in struct cramfs_inode.
110 * Used only to generate warnings.
112 #define CRAMFS_SIZE_WIDTH 24
113 #define CRAMFS_UID_WIDTH 16
114 #define CRAMFS_GID_WIDTH 8
115 #define CRAMFS_OFFSET_WIDTH 26
117 /* Input status of 0 to print help and exit without an error. */
120 FILE *stream
= status
? stderr
: stdout
;
123 _("usage: %s [-h] [-v] [-b blksize] [-e edition] [-i file] "
124 "[-n name] dirname outfile\n"
125 " -h print this help\n"
127 " -E make all warnings errors "
128 "(non-zero exit status)\n"
129 " -b blksize use this blocksize, must equal page size\n"
130 " -e edition set edition number (part of fsid)\n"
131 " -i file insert a file image into the filesystem "
132 "(requires >= 2.4.0)\n"
133 " -n name set name of cramfs filesystem\n"
134 " -p pad by %d bytes for boot code\n"
135 " -s sort directory entries (old option, ignored)\n"
136 " -z make explicit holes (requires >= 2.3.39)\n"
137 " dirname root of the filesystem to be compressed\n"
138 " outfile output file\n"),
146 xmalloc (size_t size
) {
147 void *t
= malloc(size
);
150 exit(8); /* out of memory */
156 do_mmap(char *path
, unsigned int size
, unsigned int mode
){
164 start
= xmalloc(size
);
165 if (readlink(path
, start
, size
) < 0) {
173 fd
= open(path
, O_RDONLY
);
180 start
= mmap(NULL
, size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
181 if (-1 == (int) (long) start
) {
191 do_munmap(char *start
, unsigned int size
, unsigned int mode
){
198 /* compute md5sums, so that we do not have to compare every pair of files */
200 mdfile(struct entry
*e
) {
204 start
= do_mmap(e
->path
, e
->size
, e
->mode
);
209 MD5Update(&ctx
, start
, e
->size
);
210 MD5Final(e
->md5sum
, &ctx
);
212 do_munmap(start
, e
->size
, e
->mode
);
214 e
->flags
|= HAVE_MD5
;
218 /* md5 digests are equal; files are almost certainly the same,
219 but just to be sure, do the comparison */
221 identical_file(struct entry
*e1
, struct entry
*e2
){
222 char *start1
, *start2
;
225 start1
= do_mmap(e1
->path
, e1
->size
, e1
->mode
);
228 start2
= do_mmap(e2
->path
, e2
->size
, e2
->mode
);
231 equal
= !memcmp(start1
, start2
, e1
->size
);
232 do_munmap(start1
, e1
->size
, e1
->mode
);
233 do_munmap(start2
, e2
->size
, e2
->mode
);
238 * The longest file name component to allow for in the input directory tree.
239 * Ext2fs (and many others) allow up to 255 bytes. A couple of filesystems
240 * allow longer (e.g. smbfs 1024), but there isn't much use in supporting
241 * >255-byte names in the input directory tree given that such names get
242 * truncated to 255 bytes when written to cramfs.
244 #define MAX_INPUT_NAMELEN 255
246 static int find_identical_file(struct entry
*orig
, struct entry
*new)
252 if (orig
->size
== new->size
&& orig
->path
) {
258 if ((orig
->flags
& HAVE_MD5
) && (new->flags
& HAVE_MD5
) &&
259 !memcmp(orig
->md5sum
, new->md5sum
, 16) &&
260 identical_file(orig
, new)) {
265 return find_identical_file(orig
->child
, new) ||
266 find_identical_file(orig
->next
, new);
269 static void eliminate_doubles(struct entry
*root
, struct entry
*orig
) {
271 if (orig
->size
&& orig
->path
)
272 find_identical_file(root
,orig
);
273 eliminate_doubles(root
,orig
->child
);
274 eliminate_doubles(root
,orig
->next
);
279 * We define our own sorting function instead of using alphasort which
280 * uses strcoll and changes ordering based on locale information.
282 static int cramsort (const void *a
, const void *b
)
284 return strcmp ((*(const struct dirent
**) a
)->d_name
,
285 (*(const struct dirent
**) b
)->d_name
);
288 static unsigned int parse_directory(struct entry
*root_entry
, const char *name
, struct entry
**prev
, loff_t
*fslen_ub
)
290 struct dirent
**dirlist
;
291 int totalsize
= 0, dircount
, dirindex
;
292 char *path
, *endpath
;
293 size_t len
= strlen(name
);
295 /* Set up the path. */
296 /* TODO: Reuse the parent's buffer to save memcpy'ing and duplication. */
297 path
= xmalloc(len
+ 1 + MAX_INPUT_NAMELEN
+ 1);
298 memcpy(path
, name
, len
);
299 endpath
= path
+ len
;
303 /* read in the directory and sort */
304 dircount
= scandir(name
, &dirlist
, 0, cramsort
);
311 /* process directory */
312 for (dirindex
= 0; dirindex
< dircount
; dirindex
++) {
313 struct dirent
*dirent
;
319 dirent
= dirlist
[dirindex
];
321 /* Ignore "." and ".." - we won't be adding them
323 if (dirent
->d_name
[0] == '.') {
324 if (dirent
->d_name
[1] == '\0')
326 if (dirent
->d_name
[1] == '.') {
327 if (dirent
->d_name
[2] == '\0')
331 namelen
= strlen(dirent
->d_name
);
332 if (namelen
> MAX_INPUT_NAMELEN
) {
334 _("Very long (%u bytes) filename `%s' found.\n"
335 " Please increase MAX_INPUT_NAMELEN in "
336 "mkcramfs.c and recompile. Exiting.\n"),
337 namelen
, dirent
->d_name
);
340 memcpy(endpath
, dirent
->d_name
, namelen
+ 1);
342 if (lstat(path
, &st
) < 0) {
347 entry
= calloc(1, sizeof(struct entry
));
352 entry
->name
= strdup(dirent
->d_name
);
358 /* Can't happen when reading from ext2fs. */
360 /* TODO: we ought to avoid chopping in half
361 multi-byte UTF8 characters. */
362 entry
->name
[namelen
= 255] = '\0';
365 entry
->mode
= st
.st_mode
;
366 entry
->size
= st
.st_size
;
367 entry
->uid
= st
.st_uid
;
368 if (entry
->uid
>= 1 << CRAMFS_UID_WIDTH
)
370 entry
->gid
= st
.st_gid
;
371 if (entry
->gid
>= 1 << CRAMFS_GID_WIDTH
)
372 /* TODO: We ought to replace with a default
373 gid instead of truncating; otherwise there
374 are security problems. Maybe mode should
375 be &= ~070. Same goes for uid once Linux
376 supports >16-bit uids. */
378 size
= sizeof(struct cramfs_inode
) + ((namelen
+ 3) & ~3);
380 if (S_ISDIR(st
.st_mode
)) {
381 entry
->size
= parse_directory(root_entry
, path
, &entry
->child
, fslen_ub
);
382 } else if (S_ISREG(st
.st_mode
)) {
383 entry
->path
= strdup(path
);
385 if (entry
->size
>= (1 << CRAMFS_SIZE_WIDTH
)) {
387 entry
->size
= (1 << CRAMFS_SIZE_WIDTH
) - 1;
390 } else if (S_ISLNK(st
.st_mode
)) {
391 entry
->path
= strdup(path
);
392 } else if (S_ISFIFO(st
.st_mode
) || S_ISSOCK(st
.st_mode
)) {
393 /* maybe we should skip sockets */
396 entry
->size
= st
.st_rdev
;
397 if (entry
->size
& -(1<<CRAMFS_SIZE_WIDTH
))
401 if (S_ISREG(st
.st_mode
) || S_ISLNK(st
.st_mode
)) {
402 int blocks
= ((entry
->size
- 1) / blksize
+ 1);
404 /* block pointers & data expansion allowance + data */
406 *fslen_ub
+= (4+26)*blocks
+ entry
->size
+ 3;
409 /* Link it into the list */
415 free(dirlist
); /* allocated by scandir() with malloc() */
419 /* Returns sizeof(struct cramfs_super), which includes the root inode. */
420 static unsigned int write_superblock(struct entry
*root
, char *base
, int size
)
422 struct cramfs_super
*super
= (struct cramfs_super
*) base
;
423 unsigned int offset
= sizeof(struct cramfs_super
) + image_length
;
429 super
->magic
= CRAMFS_MAGIC
;
430 super
->flags
= CRAMFS_FLAG_FSID_VERSION_2
| CRAMFS_FLAG_SORTED_DIRS
;
432 super
->flags
|= CRAMFS_FLAG_HOLES
;
433 if (image_length
> 0)
434 super
->flags
|= CRAMFS_FLAG_SHIFTED_ROOT_OFFSET
;
436 memcpy(super
->signature
, CRAMFS_SIGNATURE
, sizeof(super
->signature
));
438 super
->fsid
.crc
= crc32(0L, Z_NULL
, 0);
439 super
->fsid
.edition
= opt_edition
;
440 super
->fsid
.blocks
= total_blocks
;
441 super
->fsid
.files
= total_nodes
;
443 memset(super
->name
, 0x00, sizeof(super
->name
));
445 strncpy(super
->name
, opt_name
, sizeof(super
->name
));
447 strncpy(super
->name
, "Compressed", sizeof(super
->name
));
449 super
->root
.mode
= root
->mode
;
450 super
->root
.uid
= root
->uid
;
451 super
->root
.gid
= root
->gid
;
452 super
->root
.size
= root
->size
;
453 super
->root
.offset
= offset
>> 2;
458 static void set_data_offset(struct entry
*entry
, char *base
, unsigned long offset
)
460 struct cramfs_inode
*inode
= (struct cramfs_inode
*) (base
+ entry
->dir_offset
);
461 if (offset
>= (1 << (2 + CRAMFS_OFFSET_WIDTH
))) {
462 fprintf(stderr
, _("filesystem too big. Exiting.\n"));
465 inode
->offset
= (offset
>> 2);
470 * We do a width-first printout of the directory
471 * entries, using a stack to remember the directories
474 static unsigned int write_directory_structure(struct entry
*entry
, char *base
, unsigned int offset
)
476 int stack_entries
= 0;
478 struct entry
**entry_stack
;
480 entry_stack
= xmalloc(stack_size
* sizeof(struct entry
*));
483 int dir_start
= stack_entries
;
485 struct cramfs_inode
*inode
=
486 (struct cramfs_inode
*) (base
+ offset
);
487 size_t len
= strlen(entry
->name
);
489 entry
->dir_offset
= offset
;
491 inode
->mode
= entry
->mode
;
492 inode
->uid
= entry
->uid
;
493 inode
->gid
= entry
->gid
;
494 inode
->size
= entry
->size
;
496 /* Non-empty directories, regfiles and symlinks will
497 write over inode->offset later. */
499 offset
+= sizeof(struct cramfs_inode
);
500 total_nodes
++; /* another node */
501 memcpy(base
+ offset
, entry
->name
, len
);
502 /* Pad up the name to a 4-byte boundary */
504 *(base
+ offset
+ len
) = '\0';
507 inode
->namelen
= len
>> 2;
511 printf(" %s\n", entry
->name
);
513 if (stack_entries
>= stack_size
) {
515 entry_stack
= realloc(entry_stack
, stack_size
* sizeof(struct entry
*));
518 exit(8); /* out of memory */
521 entry_stack
[stack_entries
] = entry
;
528 * Reverse the order the stack entries pushed during
529 * this directory, for a small optimization of disk
530 * access in the created fs. This change makes things
534 struct entry
**lo
= entry_stack
+ dir_start
;
535 struct entry
**hi
= entry_stack
+ stack_entries
;
545 /* Pop a subdirectory entry from the stack, and recurse. */
549 entry
= entry_stack
[stack_entries
];
551 set_data_offset(entry
, base
, offset
);
553 printf("'%s':\n", entry
->name
);
554 entry
= entry
->child
;
560 static int is_zero(char const *begin
, unsigned len
)
563 /* Returns non-zero iff the first LEN bytes from BEGIN are
565 return (len
-- == 0 ||
573 memcmp(begin
, begin
+ 4, len
) == 0))))))));
575 /* Never create holes. */
580 * One 4-byte pointer per block and then the actual blocked
581 * output. The first block does not need an offset pointer,
582 * as it will start immediately after the pointer block;
583 * so the i'th pointer points to the end of the i'th block
584 * (i.e. the start of the (i+1)'th block or past EOF).
586 * Note that size > 0, as a zero-sized file wouldn't ever
587 * have gotten here in the first place.
590 do_compress(char *base
, unsigned int offset
, char const *name
,
591 char *path
, unsigned int size
, unsigned int mode
)
593 unsigned long original_size
, original_offset
, new_size
, blocks
, curr
;
597 /* get uncompressed data */
598 start
= do_mmap(path
, size
, mode
);
603 original_size
= size
;
604 original_offset
= offset
;
605 blocks
= (size
- 1) / blksize
+ 1;
606 curr
= offset
+ 4 * blocks
;
608 total_blocks
+= blocks
;
611 unsigned long len
= 2 * blksize
;
612 unsigned int input
= size
;
616 if (!is_zero (p
, input
)) {
617 compress(base
+ curr
, &len
, p
, input
);
622 if (len
> blksize
*2) {
623 /* (I don't think this can happen with zlib.) */
624 printf(_("AIEEE: block \"compressed\" to > "
625 "2*blocklength (%ld)\n"),
630 *(u32
*) (base
+ offset
) = curr
;
634 do_munmap(start
, original_size
, mode
);
636 curr
= (curr
+ 3) & ~3;
637 new_size
= curr
- original_offset
;
638 /* TODO: Arguably, original_size in these 2 lines should be
639 st_blocks * 512. But if you say that, then perhaps
640 administrative data should also be included in both. */
641 change
= new_size
- original_size
;
643 printf(_("%6.2f%% (%+d bytes)\t%s\n"),
644 (change
* 100) / (double) original_size
, change
, name
);
651 * Traverse the entry tree, writing data for every item that has
652 * non-null entry->path (i.e. every symlink and non-empty
656 write_data(struct entry
*entry
, char *base
, unsigned int offset
) {
659 for (e
= entry
; e
; e
= e
->next
) {
662 set_data_offset(e
, base
, e
->same
->offset
);
663 e
->offset
= e
->same
->offset
;
665 set_data_offset(e
, base
, offset
);
667 offset
= do_compress(base
, offset
, e
->name
,
668 e
->path
, e
->size
,e
->mode
);
671 offset
= write_data(e
->child
, base
, offset
);
676 static unsigned int write_file(char *file
, char *base
, unsigned int offset
)
681 fd
= open(file
, O_RDONLY
);
686 buf
= mmap(NULL
, image_length
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
687 memcpy(base
+ offset
, buf
, image_length
);
688 munmap(buf
, image_length
);
690 /* Pad up the image_length to a 4-byte boundary */
691 while (image_length
& 3) {
692 *(base
+ offset
+ image_length
) = '\0';
695 return (offset
+ image_length
);
699 * Maximum size fs you can create is roughly 256MB. (The last file's
700 * data must begin within 256MB boundary but can extend beyond that.)
702 * Note that if you want it to fit in a ROM then you're limited to what the
703 * hardware and kernel can support (64MB?).
707 return (((1 << CRAMFS_OFFSET_WIDTH
) - 1) << 2) /* offset */
708 + (1 << CRAMFS_SIZE_WIDTH
) - 1 /* filesize */
709 + (1 << CRAMFS_SIZE_WIDTH
) * 4 / blksize
; /* block pointers */
715 * mkcramfs directory-name outfile
717 * where "directory-name" is simply the root of the directory
718 * tree that we want to generate a compressed filesystem out
721 int main(int argc
, char **argv
)
723 struct stat st
; /* used twice... */
724 struct entry
*root_entry
;
726 ssize_t offset
, written
;
728 /* initial guess (upper-bound) of required filesystem size */
729 loff_t fslen_ub
= sizeof(struct cramfs_super
);
730 unsigned int fslen_max
;
731 char const *dirname
, *outfile
;
732 u32 crc
= crc32(0L, Z_NULL
, 0);
735 blksize
= sysconf(_SC_PAGESIZE
);
741 if ((p
= strrchr(progname
, '/')) != NULL
)
745 /* command line options */
746 while ((c
= getopt(argc
, argv
, "hb:Ee:i:n:psVvz")) != EOF
) {
751 blksize
= atoi(optarg
);
759 opt_edition
= atoi(optarg
);
763 if (lstat(opt_image
, &st
) < 0) {
767 image_length
= st
.st_size
; /* may be padded later */
768 fslen_ub
+= (image_length
+ 3); /* 3 is for padding */
775 fslen_ub
+= PAD_SIZE
;
778 /* old option, ignored */
781 printf(_("%s (%s)\n"),
782 progname
, PACKAGE_STRING
);
793 if ((argc
- optind
) != 2)
795 dirname
= argv
[optind
];
796 outfile
= argv
[optind
+ 1];
798 if (stat(dirname
, &st
) < 0) {
802 fd
= open(outfile
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
804 root_entry
= calloc(1, sizeof(struct entry
));
809 root_entry
->mode
= st
.st_mode
;
810 root_entry
->uid
= st
.st_uid
;
811 root_entry
->gid
= st
.st_gid
;
813 root_entry
->size
= parse_directory(root_entry
, dirname
, &root_entry
->child
, &fslen_ub
);
815 /* always allocate a multiple of blksize bytes because that's
816 what we're going to write later on */
817 fslen_ub
= ((fslen_ub
- 1) | (blksize
- 1)) + 1;
818 fslen_max
= maxfslen();
820 if (fslen_ub
> fslen_max
) {
822 _("warning: guestimate of required size (upper bound) "
823 "is %LdMB, but maximum image size is %uMB. "
824 "We might die prematurely.\n"),
827 fslen_ub
= fslen_max
;
830 /* find duplicate files */
831 eliminate_doubles(root_entry
,root_entry
);
833 /* TODO: Why do we use a private/anonymous mapping here
834 followed by a write below, instead of just a shared mapping
835 and a couple of ftruncate calls? Is it just to save us
836 having to deal with removing the file afterwards? If we
837 really need this huge anonymous mapping, we ought to mmap
838 in smaller chunks, so that the user doesn't need nn MB of
839 RAM free. If the reason is to be able to write to
840 un-mmappable block devices, then we could try shared mmap
841 and revert to anonymous mmap if the shared mmap fails. */
842 rom_image
= mmap(NULL
,
844 PROT_READ
| PROT_WRITE
,
845 MAP_PRIVATE
| MAP_ANONYMOUS
,
848 if (-1 == (int) (long) rom_image
) {
849 perror("ROM image map");
853 /* Skip the first opt_pad bytes for boot loader code */
855 memset(rom_image
, 0x00, opt_pad
);
857 /* Skip the superblock and come back to write it later. */
858 offset
+= sizeof(struct cramfs_super
);
860 /* Insert a file image. */
863 printf(_("Including: %s\n"), opt_image
);
864 offset
= write_file(opt_image
, rom_image
, offset
);
867 offset
= write_directory_structure(root_entry
->child
, rom_image
, offset
);
869 printf(_("Directory data: %d bytes\n"), offset
);
871 offset
= write_data(root_entry
, rom_image
, offset
);
873 /* We always write a multiple of blksize bytes, so that
875 offset
= ((offset
- 1) | (blksize
- 1)) + 1;
877 printf(_("Everything: %d kilobytes\n"), offset
>> 10);
879 /* Write the superblock now that we can fill in all of the fields. */
880 write_superblock(root_entry
, rom_image
+opt_pad
, offset
);
882 printf(_("Super block: %d bytes\n"),
883 sizeof(struct cramfs_super
));
885 /* Put the checksum in. */
886 crc
= crc32(crc
, (rom_image
+opt_pad
), (offset
-opt_pad
));
887 ((struct cramfs_super
*) (rom_image
+opt_pad
))->fsid
.crc
= crc
;
889 printf(_("CRC: %x\n"), crc
);
891 /* Check to make sure we allocated enough space. */
892 if (fslen_ub
< offset
) {
894 _("not enough space allocated for ROM image "
895 "(%Ld allocated, %d used)\n"),
900 written
= write(fd
, rom_image
, offset
);
905 if (offset
!= written
) {
906 fprintf(stderr
, _("ROM image write failed (%d %d)\n"),
911 /* (These warnings used to come at the start, but they scroll off the
912 screen too quickly.) */
913 if (warn_namelen
) /* (can't happen when reading from ext2fs) */
914 fprintf(stderr
, /* bytes, not chars: think UTF8. */
915 _("warning: filenames truncated to 255 bytes.\n"));
918 _("warning: files were skipped due to errors.\n"));
921 _("warning: file sizes truncated to %luMB "
922 "(minus 1 byte).\n"),
923 1L << (CRAMFS_SIZE_WIDTH
- 20));
924 if (warn_uid
) /* (not possible with current Linux versions) */
926 _("warning: uids truncated to %u bits. "
927 "(This may be a security concern.)\n"),
931 _("warning: gids truncated to %u bits. "
932 "(This may be a security concern.)\n"),
936 _("WARNING: device numbers truncated to %u bits. "
937 "This almost certainly means\n"
938 "that some device files will be wrong.\n"),
939 CRAMFS_OFFSET_WIDTH
);
941 (warn_namelen
|warn_skip
|warn_size
|warn_uid
|warn_gid
|warn_dev
))