2 * mkcramfs - make a cramfs file system
4 * Copyright (C) 1999-2001 Transmeta Corporation
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License as published by
8 * the Free Software Foundation; either version 2 of the License, or
9 * (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, write to the Free Software
18 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
21 #include <sys/types.h>
26 #include <sys/fcntl.h>
38 #define PAD_SIZE 512 /* only 0 and 512 supported by kernel */
40 static const char *progname
= "mkcramfs";
41 static int verbose
= 0;
43 /* Input status of 0 to print help and exit without an error. */
44 static void usage(int status
)
46 FILE *stream
= status
? stderr
: stdout
;
49 _("usage: %s [-h] [-v] [-e edition] [-i file] [-n name] "
51 " -h print this help\n"
53 " -E make all warnings errors "
54 "(non-zero exit status)\n"
55 " -e edition set edition number (part of fsid)\n"
56 " -i file insert a file image into the filesystem "
57 "(requires >= 2.4.0)\n"
58 " -n name set name of cramfs filesystem\n"
59 " -p pad by %d bytes for boot code\n"
60 " -s sort directory entries (old option, ignored)\n"
61 " -z make explicit holes (requires >= 2.3.39)\n"
62 " dirname root of the filesystem to be compressed\n"
63 " outfile output file\n"),
69 #define PAGE_CACHE_SIZE (4096)
70 /* The kernel assumes PAGE_CACHE_SIZE as block size. */
71 static unsigned int blksize
= PAGE_CACHE_SIZE
;
72 static long total_blocks
= 0, total_nodes
= 1; /* pre-count the root node */
73 static int image_length
= 0;
76 * If opt_holes is set, then mkcramfs can create explicit holes in the
77 * data, which saves 26 bytes per hole (which is a lot smaller a
78 * saving than most most filesystems).
80 * Note that kernels up to at least 2.3.39 don't support cramfs holes,
81 * which is why this is turned off by default.
83 static int opt_edition
= 0;
84 static int opt_errors
= 0;
85 static int opt_holes
= 0;
86 static int opt_pad
= 0;
87 static char *opt_image
= NULL
;
88 static char *opt_name
= NULL
;
90 static int warn_dev
, warn_gid
, warn_namelen
, warn_skip
, warn_size
, warn_uid
;
93 # define MIN(_a,_b) ((_a) < (_b) ? (_a) : (_b))
96 /* In-core version of inode / directory entry. */
100 unsigned int mode
, size
, uid
, gid
;
104 /* points to other identical file */
106 unsigned int offset
; /* pointer to compressed data in archive */
107 unsigned int dir_offset
; /* Where in the archive is the directory entry? */
110 struct entry
*child
; /* null for non-directories and empty directories */
115 * Width of various bitfields in struct cramfs_inode.
116 * Used only to generate warnings.
118 #define CRAMFS_SIZE_WIDTH 24
119 #define CRAMFS_UID_WIDTH 16
120 #define CRAMFS_GID_WIDTH 8
121 #define CRAMFS_OFFSET_WIDTH 26
124 * The longest file name component to allow for in the input directory tree.
125 * Ext2fs (and many others) allow up to 255 bytes. A couple of filesystems
126 * allow longer (e.g. smbfs 1024), but there isn't much use in supporting
127 * >255-byte names in the input directory tree given that such names get
128 * truncated to 255 bytes when written to cramfs.
130 #define MAX_INPUT_NAMELEN 255
132 static int find_identical_file(struct entry
*orig
,struct entry
*newfile
)
134 if(orig
==newfile
) return 1;
136 if(orig
->size
==newfile
->size
&& orig
->uncompressed
&& !memcmp(orig
->uncompressed
,newfile
->uncompressed
,orig
->size
)) {
140 return find_identical_file(orig
->child
,newfile
) ||
141 find_identical_file(orig
->next
,newfile
);
144 static void eliminate_doubles(struct entry
*root
,struct entry
*orig
) {
146 if(orig
->size
&& orig
->uncompressed
)
147 find_identical_file(root
,orig
);
148 eliminate_doubles(root
,orig
->child
);
149 eliminate_doubles(root
,orig
->next
);
154 * We define our own sorting function instead of using alphasort which
155 * uses strcoll and changes ordering based on locale information.
157 static int cramsort (const void *a
, const void *b
)
159 return strcmp ((*(const struct dirent
**) a
)->d_name
,
160 (*(const struct dirent
**) b
)->d_name
);
163 static unsigned int parse_directory(struct entry
*root_entry
, const char *name
, struct entry
**prev
, loff_t
*fslen_ub
)
165 struct dirent
**dirlist
;
166 int totalsize
= 0, dircount
, dirindex
;
167 char *path
, *endpath
;
168 size_t len
= strlen(name
);
170 /* Set up the path. */
171 /* TODO: Reuse the parent's buffer to save memcpy'ing and duplication. */
172 path
= malloc(len
+ 1 + MAX_INPUT_NAMELEN
+ 1);
177 memcpy(path
, name
, len
);
178 endpath
= path
+ len
;
182 /* read in the directory and sort */
183 dircount
= scandir(name
, &dirlist
, 0, cramsort
);
190 /* process directory */
191 for (dirindex
= 0; dirindex
< dircount
; dirindex
++) {
192 struct dirent
*dirent
;
198 dirent
= dirlist
[dirindex
];
200 /* Ignore "." and ".." - we won't be adding them
202 if (dirent
->d_name
[0] == '.') {
203 if (dirent
->d_name
[1] == '\0')
205 if (dirent
->d_name
[1] == '.') {
206 if (dirent
->d_name
[2] == '\0')
210 namelen
= strlen(dirent
->d_name
);
211 if (namelen
> MAX_INPUT_NAMELEN
) {
213 _("Very long (%u bytes) filename `%s' found.\n"
214 " Please increase MAX_INPUT_NAMELEN in "
215 "mkcramfs.c and recompile. Exiting.\n"),
216 namelen
, dirent
->d_name
);
219 memcpy(endpath
, dirent
->d_name
, namelen
+ 1);
221 if (lstat(path
, &st
) < 0) {
226 entry
= calloc(1, sizeof(struct entry
));
231 entry
->name
= strdup(dirent
->d_name
);
237 /* Can't happen when reading from ext2fs. */
239 /* TODO: we ought to avoid chopping in half
240 multi-byte UTF8 characters. */
241 entry
->name
[namelen
= 255] = '\0';
244 entry
->mode
= st
.st_mode
;
245 entry
->size
= st
.st_size
;
246 entry
->uid
= st
.st_uid
;
247 if (entry
->uid
>= 1 << CRAMFS_UID_WIDTH
)
249 entry
->gid
= st
.st_gid
;
250 if (entry
->gid
>= 1 << CRAMFS_GID_WIDTH
)
251 /* TODO: We ought to replace with a default
252 gid instead of truncating; otherwise there
253 are security problems. Maybe mode should
254 be &= ~070. Same goes for uid once Linux
255 supports >16-bit uids. */
257 size
= sizeof(struct cramfs_inode
) + ((namelen
+ 3) & ~3);
259 if (S_ISDIR(st
.st_mode
)) {
260 entry
->size
= parse_directory(root_entry
, path
, &entry
->child
, fslen_ub
);
261 } else if (S_ISREG(st
.st_mode
)) {
262 /* TODO: We ought to open files in do_compress, one
263 at a time, instead of amassing all these memory
264 maps during parse_directory (which don't get used
265 until do_compress anyway). As it is, we tend to
266 get EMFILE errors (especially if mkcramfs is run
269 While we're at it, do analagously for symlinks
270 (which would just save a little memory). */
271 int fd
= open(path
, O_RDONLY
);
278 if ((entry
->size
>= 1 << CRAMFS_SIZE_WIDTH
)) {
280 entry
->size
= (1 << CRAMFS_SIZE_WIDTH
) - 1;
283 entry
->uncompressed
= mmap(NULL
, entry
->size
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
284 if (-1 == (int) (long) entry
->uncompressed
) {
290 } else if (S_ISLNK(st
.st_mode
)) {
291 entry
->uncompressed
= malloc(entry
->size
);
292 if (!entry
->uncompressed
) {
296 if (readlink(path
, entry
->uncompressed
, entry
->size
) < 0) {
301 } else if (S_ISFIFO(st
.st_mode
) || S_ISSOCK(st
.st_mode
)) {
302 /* maybe we should skip sockets */
305 entry
->size
= st
.st_rdev
;
306 if (entry
->size
& -(1<<CRAMFS_SIZE_WIDTH
))
310 if (S_ISREG(st
.st_mode
) || S_ISLNK(st
.st_mode
)) {
311 int blocks
= ((entry
->size
- 1) / blksize
+ 1);
313 /* block pointers & data expansion allowance + data */
315 *fslen_ub
+= (4+26)*blocks
+ entry
->size
+ 3;
318 /* Link it into the list */
324 free(dirlist
); /* allocated by scandir() with malloc() */
328 /* Returns sizeof(struct cramfs_super), which includes the root inode. */
329 static unsigned int write_superblock(struct entry
*root
, char *base
, int size
)
331 struct cramfs_super
*super
= (struct cramfs_super
*) base
;
332 unsigned int offset
= sizeof(struct cramfs_super
) + image_length
;
338 super
->magic
= CRAMFS_MAGIC
;
339 super
->flags
= CRAMFS_FLAG_FSID_VERSION_2
| CRAMFS_FLAG_SORTED_DIRS
;
341 super
->flags
|= CRAMFS_FLAG_HOLES
;
342 if (image_length
> 0)
343 super
->flags
|= CRAMFS_FLAG_SHIFTED_ROOT_OFFSET
;
345 memcpy(super
->signature
, CRAMFS_SIGNATURE
, sizeof(super
->signature
));
347 super
->fsid
.crc
= crc32(0L, Z_NULL
, 0);
348 super
->fsid
.edition
= opt_edition
;
349 super
->fsid
.blocks
= total_blocks
;
350 super
->fsid
.files
= total_nodes
;
352 memset(super
->name
, 0x00, sizeof(super
->name
));
354 strncpy(super
->name
, opt_name
, sizeof(super
->name
));
356 strncpy(super
->name
, "Compressed", sizeof(super
->name
));
358 super
->root
.mode
= root
->mode
;
359 super
->root
.uid
= root
->uid
;
360 super
->root
.gid
= root
->gid
;
361 super
->root
.size
= root
->size
;
362 super
->root
.offset
= offset
>> 2;
367 static void set_data_offset(struct entry
*entry
, char *base
, unsigned long offset
)
369 struct cramfs_inode
*inode
= (struct cramfs_inode
*) (base
+ entry
->dir_offset
);
370 if (offset
>= (1 << (2 + CRAMFS_OFFSET_WIDTH
))) {
371 fprintf(stderr
, _("filesystem too big. Exiting.\n"));
374 inode
->offset
= (offset
>> 2);
379 * We do a width-first printout of the directory
380 * entries, using a stack to remember the directories
383 #define MAXENTRIES (100)
384 static unsigned int write_directory_structure(struct entry
*entry
, char *base
, unsigned int offset
)
386 int stack_entries
= 0;
387 struct entry
*entry_stack
[MAXENTRIES
];
390 int dir_start
= stack_entries
;
392 struct cramfs_inode
*inode
=
393 (struct cramfs_inode
*) (base
+ offset
);
394 size_t len
= strlen(entry
->name
);
396 entry
->dir_offset
= offset
;
398 inode
->mode
= entry
->mode
;
399 inode
->uid
= entry
->uid
;
400 inode
->gid
= entry
->gid
;
401 inode
->size
= entry
->size
;
403 /* Non-empty directories, regfiles and symlinks will
404 write over inode->offset later. */
406 offset
+= sizeof(struct cramfs_inode
);
407 total_nodes
++; /* another node */
408 memcpy(base
+ offset
, entry
->name
, len
);
409 /* Pad up the name to a 4-byte boundary */
411 *(base
+ offset
+ len
) = '\0';
414 inode
->namelen
= len
>> 2;
418 printf(" %s\n", entry
->name
);
420 if (stack_entries
>= MAXENTRIES
) {
422 _("Exceeded MAXENTRIES. Raise"
423 " this value in mkcramfs.c "
424 "and recompile. Exiting.\n")
428 entry_stack
[stack_entries
] = entry
;
435 * Reverse the order the stack entries pushed during
436 * this directory, for a small optimization of disk
437 * access in the created fs. This change makes things
441 struct entry
**lo
= entry_stack
+ dir_start
;
442 struct entry
**hi
= entry_stack
+ stack_entries
;
452 /* Pop a subdirectory entry from the stack, and recurse. */
456 entry
= entry_stack
[stack_entries
];
458 set_data_offset(entry
, base
, offset
);
460 printf("'%s':\n", entry
->name
);
461 entry
= entry
->child
;
466 static int is_zero(char const *begin
, unsigned len
)
469 /* Returns non-zero iff the first LEN bytes from BEGIN are
471 return (len
-- == 0 ||
479 memcmp(begin
, begin
+ 4, len
) == 0))))))));
481 /* Never create holes. */
486 * One 4-byte pointer per block and then the actual blocked
487 * output. The first block does not need an offset pointer,
488 * as it will start immediately after the pointer block;
489 * so the i'th pointer points to the end of the i'th block
490 * (i.e. the start of the (i+1)'th block or past EOF).
492 * Note that size > 0, as a zero-sized file wouldn't ever
493 * have gotten here in the first place.
495 static unsigned int do_compress(char *base
, unsigned int offset
, char const *name
, char *uncompressed
, unsigned int size
)
497 unsigned long original_size
= size
;
498 unsigned long original_offset
= offset
;
499 unsigned long new_size
;
500 unsigned long blocks
= (size
- 1) / blksize
+ 1;
501 unsigned long curr
= offset
+ 4 * blocks
;
504 total_blocks
+= blocks
;
507 unsigned long len
= 2 * blksize
;
508 unsigned int input
= size
;
512 if (!is_zero (uncompressed
, input
)) {
513 compress(base
+ curr
, &len
, uncompressed
, input
);
516 uncompressed
+= input
;
518 if (len
> blksize
*2) {
519 /* (I don't think this can happen with zlib.) */
520 printf(_("AIEEE: block \"compressed\" to > "
521 "2*blocklength (%ld)\n"),
526 *(u32
*) (base
+ offset
) = curr
;
530 curr
= (curr
+ 3) & ~3;
531 new_size
= curr
- original_offset
;
532 /* TODO: Arguably, original_size in these 2 lines should be
533 st_blocks * 512. But if you say that then perhaps
534 administrative data should also be included in both. */
535 change
= new_size
- original_size
;
537 printf(_("%6.2f%% (%+d bytes)\t%s\n"),
538 (change
* 100) / (double) original_size
, change
, name
);
545 * Traverse the entry tree, writing data for every item that has
546 * non-null entry->compressed (i.e. every symlink and non-empty
549 static unsigned int write_data(struct entry
*entry
, char *base
, unsigned int offset
)
552 if (entry
->uncompressed
) {
554 set_data_offset(entry
, base
, entry
->same
->offset
);
555 entry
->offset
=entry
->same
->offset
;
557 set_data_offset(entry
, base
, offset
);
558 entry
->offset
=offset
;
559 offset
= do_compress(base
, offset
, entry
->name
, entry
->uncompressed
, entry
->size
);
562 else if (entry
->child
)
563 offset
= write_data(entry
->child
, base
, offset
);
569 static unsigned int write_file(char *file
, char *base
, unsigned int offset
)
574 fd
= open(file
, O_RDONLY
);
579 buf
= mmap(NULL
, image_length
, PROT_READ
, MAP_PRIVATE
, fd
, 0);
580 memcpy(base
+ offset
, buf
, image_length
);
581 munmap(buf
, image_length
);
583 /* Pad up the image_length to a 4-byte boundary */
584 while (image_length
& 3) {
585 *(base
+ offset
+ image_length
) = '\0';
588 return (offset
+ image_length
);
592 * Maximum size fs you can create is roughly 256MB. (The last file's
593 * data must begin within 256MB boundary but can extend beyond that.)
595 * Note that if you want it to fit in a ROM then you're limited to what the
596 * hardware and kernel can support (64MB?).
598 #define MAXFSLEN ((((1 << CRAMFS_OFFSET_WIDTH) - 1) << 2) /* offset */ \
599 + (1 << CRAMFS_SIZE_WIDTH) - 1 /* filesize */ \
600 + (1 << CRAMFS_SIZE_WIDTH) * 4 / PAGE_CACHE_SIZE /* block pointers */ )
606 * mkcramfs directory-name outfile
608 * where "directory-name" is simply the root of the directory
609 * tree that we want to generate a compressed filesystem out
612 int main(int argc
, char **argv
)
614 struct stat st
; /* used twice... */
615 struct entry
*root_entry
;
617 ssize_t offset
, written
;
619 /* initial guess (upper-bound) of required filesystem size */
620 loff_t fslen_ub
= sizeof(struct cramfs_super
);
621 char const *dirname
, *outfile
;
622 u32 crc
= crc32(0L, Z_NULL
, 0);
623 int c
; /* for getopt */
630 if ((p
= strrchr(progname
, '/')) != NULL
)
634 /* command line options */
635 while ((c
= getopt(argc
, argv
, "hEe:i:n:psVvz")) != EOF
) {
643 opt_edition
= atoi(optarg
);
647 if (lstat(opt_image
, &st
) < 0) {
651 image_length
= st
.st_size
; /* may be padded later */
652 fslen_ub
+= (image_length
+ 3); /* 3 is for padding */
659 fslen_ub
+= PAD_SIZE
;
662 /* old option, ignored */
665 printf(_("%s from %s\n"),
666 progname
, util_linux_version
);
677 if ((argc
- optind
) != 2)
679 dirname
= argv
[optind
];
680 outfile
= argv
[optind
+ 1];
682 if (stat(dirname
, &st
) < 0) {
686 fd
= open(outfile
, O_WRONLY
| O_CREAT
| O_TRUNC
, 0666);
688 root_entry
= calloc(1, sizeof(struct entry
));
693 root_entry
->mode
= st
.st_mode
;
694 root_entry
->uid
= st
.st_uid
;
695 root_entry
->gid
= st
.st_gid
;
697 root_entry
->size
= parse_directory(root_entry
, dirname
, &root_entry
->child
, &fslen_ub
);
699 /* always allocate a multiple of blksize bytes because that's
700 what we're going to write later on */
701 fslen_ub
= ((fslen_ub
- 1) | (blksize
- 1)) + 1;
703 if (fslen_ub
> MAXFSLEN
) {
705 _("warning: guestimate of required size (upper bound) "
706 "is %LdMB, but maximum image size is %uMB. "
707 "We might die prematurely.\n"),
713 /* find duplicate files. TODO: uses the most inefficient algorithm
715 eliminate_doubles(root_entry
,root_entry
);
717 /* TODO: Why do we use a private/anonymous mapping here
718 followed by a write below, instead of just a shared mapping
719 and a couple of ftruncate calls? Is it just to save us
720 having to deal with removing the file afterwards? If we
721 really need this huge anonymous mapping, we ought to mmap
722 in smaller chunks, so that the user doesn't need nn MB of
723 RAM free. If the reason is to be able to write to
724 un-mmappable block devices, then we could try shared mmap
725 and revert to anonymous mmap if the shared mmap fails. */
726 rom_image
= mmap(NULL
,
728 PROT_READ
| PROT_WRITE
,
729 MAP_PRIVATE
| MAP_ANONYMOUS
,
732 if (-1 == (int) (long) rom_image
) {
733 perror("ROM image map");
737 /* Skip the first opt_pad bytes for boot loader code */
739 memset(rom_image
, 0x00, opt_pad
);
741 /* Skip the superblock and come back to write it later. */
742 offset
+= sizeof(struct cramfs_super
);
744 /* Insert a file image. */
747 printf(_("Including: %s\n"), opt_image
);
748 offset
= write_file(opt_image
, rom_image
, offset
);
751 offset
= write_directory_structure(root_entry
->child
, rom_image
, offset
);
753 printf(_("Directory data: %d bytes\n"), offset
);
755 offset
= write_data(root_entry
, rom_image
, offset
);
757 /* We always write a multiple of blksize bytes, so that
759 offset
= ((offset
- 1) | (blksize
- 1)) + 1;
761 printf(_("Everything: %d kilobytes\n"), offset
>> 10);
763 /* Write the superblock now that we can fill in all of the fields. */
764 write_superblock(root_entry
, rom_image
+opt_pad
, offset
);
766 printf(_("Super block: %d bytes\n"),
767 sizeof(struct cramfs_super
));
769 /* Put the checksum in. */
770 crc
= crc32(crc
, (rom_image
+opt_pad
), (offset
-opt_pad
));
771 ((struct cramfs_super
*) (rom_image
+opt_pad
))->fsid
.crc
= crc
;
773 printf(_("CRC: %x\n"), crc
);
775 /* Check to make sure we allocated enough space. */
776 if (fslen_ub
< offset
) {
778 _("not enough space allocated for ROM image "
779 "(%Ld allocated, %d used)\n"),
784 written
= write(fd
, rom_image
, offset
);
789 if (offset
!= written
) {
790 fprintf(stderr
, _("ROM image write failed (%d %d)\n"),
795 /* (These warnings used to come at the start, but they scroll off the
796 screen too quickly.) */
797 if (warn_namelen
) /* (can't happen when reading from ext2fs) */
798 fprintf(stderr
, /* bytes, not chars: think UTF8. */
799 _("warning: filenames truncated to 255 bytes.\n"));
802 _("warning: files were skipped due to errors.\n"));
805 _("warning: file sizes truncated to %luMB "
806 "(minus 1 byte).\n"),
807 1L << (CRAMFS_SIZE_WIDTH
- 20));
808 if (warn_uid
) /* (not possible with current Linux versions) */
810 _("warning: uids truncated to %u bits. "
811 "(This may be a security concern.)\n"),
815 _("warning: gids truncated to %u bits. "
816 "(This may be a security concern.)\n"),
820 _("WARNING: device numbers truncated to %u bits. "
821 "This almost certainly means\n"
822 "that some device files will be wrong.\n"),
823 CRAMFS_OFFSET_WIDTH
);
825 (warn_namelen
|warn_skip
|warn_size
|warn_uid
|warn_gid
|warn_dev
))