2 * Copyright (c) 2005, 2006 Rene Scharfe
4 #include "git-compat-util.h"
9 #include "object-store.h"
10 #include "streaming.h"
11 #include "run-command.h"
13 #define RECORDSIZE (512)
14 #define BLOCKSIZE (RECORDSIZE * 20)
16 static char block
[BLOCKSIZE
];
17 static unsigned long offset
;
19 static int tar_umask
= 002;
21 static int write_tar_filter_archive(const struct archiver
*ar
,
22 struct archiver_args
*args
);
25 * This is the max value that a ustar size header can specify, as it is fixed
26 * at 11 octal digits. POSIX specifies that we switch to extended headers at
29 * Likewise for the mtime (which happens to use a buffer of the same size).
31 #if ULONG_MAX == 0xFFFFFFFF
32 #define USTAR_MAX_SIZE ULONG_MAX
34 #define USTAR_MAX_SIZE 077777777777UL
36 #if TIME_MAX == 0xFFFFFFFF
37 #define USTAR_MAX_MTIME TIME_MAX
39 #define USTAR_MAX_MTIME 077777777777ULL
42 static void tar_write_block(const void *buf
)
44 write_or_die(1, buf
, BLOCKSIZE
);
47 static void (*write_block
)(const void *) = tar_write_block
;
49 /* writes out the whole block, but only if it is full */
50 static void write_if_needed(void)
52 if (offset
== BLOCKSIZE
) {
59 * queues up writes, so that all our write(2) calls write exactly one
60 * full block; pads writes to RECORDSIZE
62 static void do_write_blocked(const void *data
, unsigned long size
)
64 const char *buf
= data
;
67 unsigned long chunk
= BLOCKSIZE
- offset
;
70 memcpy(block
+ offset
, buf
, chunk
);
76 while (size
>= BLOCKSIZE
) {
82 memcpy(block
+ offset
, buf
, size
);
87 static void finish_record(void)
90 tail
= offset
% RECORDSIZE
;
92 memset(block
+ offset
, 0, RECORDSIZE
- tail
);
93 offset
+= RECORDSIZE
- tail
;
98 static void write_blocked(const void *data
, unsigned long size
)
100 do_write_blocked(data
, size
);
105 * The end of tar archives is marked by 2*512 nul bytes and after that
106 * follows the rest of the block (if any).
108 static void write_trailer(void)
110 int tail
= BLOCKSIZE
- offset
;
111 memset(block
+ offset
, 0, tail
);
113 if (tail
< 2 * RECORDSIZE
) {
114 memset(block
, 0, offset
);
120 * queues up writes, so that all our write(2) calls write exactly one
121 * full block; pads writes to RECORDSIZE
123 static int stream_blocked(struct repository
*r
, const struct object_id
*oid
)
125 struct git_istream
*st
;
126 enum object_type type
;
131 st
= open_istream(r
, oid
, &type
, &sz
, NULL
);
133 return error(_("cannot stream blob %s"), oid_to_hex(oid
));
135 readlen
= read_istream(st
, buf
, sizeof(buf
));
138 do_write_blocked(buf
, readlen
);
147 * pax extended header records have the format "%u %s=%s\n". %u contains
148 * the size of the whole string (including the %u), the first %s is the
149 * keyword, the second one is the value. This function constructs such a
150 * string and appends it to a struct strbuf.
152 static void strbuf_append_ext_header(struct strbuf
*sb
, const char *keyword
,
153 const char *value
, size_t valuelen
)
155 size_t orig_len
= sb
->len
;
159 len
= 1 + 1 + strlen(keyword
) + 1 + valuelen
+ 1;
160 for (tmp
= 1; len
/ 10 >= tmp
; tmp
*= 10)
163 strbuf_grow(sb
, len
);
164 strbuf_addf(sb
, "%"PRIuMAX
" %s=", (uintmax_t)len
, keyword
);
165 strbuf_add(sb
, value
, valuelen
);
166 strbuf_addch(sb
, '\n');
168 if (len
!= sb
->len
- orig_len
)
169 BUG("pax extended header length miscalculated as %"PRIuMAX
170 ", should be %"PRIuMAX
,
171 (uintmax_t)len
, (uintmax_t)(sb
->len
- orig_len
));
175 * Like strbuf_append_ext_header, but for numeric values.
177 static void strbuf_append_ext_header_uint(struct strbuf
*sb
,
181 char buf
[40]; /* big enough for 2^128 in decimal, plus NUL */
184 len
= xsnprintf(buf
, sizeof(buf
), "%"PRIuMAX
, value
);
185 strbuf_append_ext_header(sb
, keyword
, buf
, len
);
188 static unsigned int ustar_header_chksum(const struct ustar_header
*header
)
190 const unsigned char *p
= (const unsigned char *)header
;
191 unsigned int chksum
= 0;
192 while (p
< (const unsigned char *)header
->chksum
)
194 chksum
+= sizeof(header
->chksum
) * ' ';
195 p
+= sizeof(header
->chksum
);
196 while (p
< (const unsigned char *)header
+ sizeof(struct ustar_header
))
201 static size_t get_path_prefix(const char *path
, size_t pathlen
, size_t maxlen
)
204 if (i
> 1 && path
[i
- 1] == '/')
210 } while (i
> 0 && path
[i
] != '/');
214 static void prepare_header(struct archiver_args
*args
,
215 struct ustar_header
*header
,
216 unsigned int mode
, unsigned long size
)
218 xsnprintf(header
->mode
, sizeof(header
->mode
), "%07o", mode
& 07777);
219 xsnprintf(header
->size
, sizeof(header
->size
), "%011"PRIoMAX
, S_ISREG(mode
) ? (uintmax_t)size
: (uintmax_t)0);
220 xsnprintf(header
->mtime
, sizeof(header
->mtime
), "%011lo", (unsigned long) args
->time
);
222 xsnprintf(header
->uid
, sizeof(header
->uid
), "%07o", 0);
223 xsnprintf(header
->gid
, sizeof(header
->gid
), "%07o", 0);
224 strlcpy(header
->uname
, "root", sizeof(header
->uname
));
225 strlcpy(header
->gname
, "root", sizeof(header
->gname
));
226 xsnprintf(header
->devmajor
, sizeof(header
->devmajor
), "%07o", 0);
227 xsnprintf(header
->devminor
, sizeof(header
->devminor
), "%07o", 0);
229 memcpy(header
->magic
, "ustar", 6);
230 memcpy(header
->version
, "00", 2);
232 xsnprintf(header
->chksum
, sizeof(header
->chksum
), "%07o", ustar_header_chksum(header
));
235 static void write_extended_header(struct archiver_args
*args
,
236 const struct object_id
*oid
,
237 const void *buffer
, unsigned long size
)
239 struct ustar_header header
;
241 memset(&header
, 0, sizeof(header
));
242 *header
.typeflag
= TYPEFLAG_EXT_HEADER
;
244 xsnprintf(header
.name
, sizeof(header
.name
), "%s.paxheader", oid_to_hex(oid
));
245 prepare_header(args
, &header
, mode
, size
);
246 write_blocked(&header
, sizeof(header
));
247 write_blocked(buffer
, size
);
250 static int write_tar_entry(struct archiver_args
*args
,
251 const struct object_id
*oid
,
252 const char *path
, size_t pathlen
,
254 void *buffer
, unsigned long size
)
256 struct ustar_header header
;
257 struct strbuf ext_header
= STRBUF_INIT
;
258 unsigned long size_in_header
;
261 memset(&header
, 0, sizeof(header
));
263 if (S_ISDIR(mode
) || S_ISGITLINK(mode
)) {
264 *header
.typeflag
= TYPEFLAG_DIR
;
265 mode
= (mode
| 0777) & ~tar_umask
;
266 } else if (S_ISLNK(mode
)) {
267 *header
.typeflag
= TYPEFLAG_LNK
;
269 } else if (S_ISREG(mode
)) {
270 *header
.typeflag
= TYPEFLAG_REG
;
271 mode
= (mode
| ((mode
& 0100) ? 0777 : 0666)) & ~tar_umask
;
273 return error(_("unsupported file mode: 0%o (SHA1: %s)"),
274 mode
, oid_to_hex(oid
));
276 if (pathlen
> sizeof(header
.name
)) {
277 size_t plen
= get_path_prefix(path
, pathlen
,
278 sizeof(header
.prefix
));
279 size_t rest
= pathlen
- plen
- 1;
280 if (plen
> 0 && rest
<= sizeof(header
.name
)) {
281 memcpy(header
.prefix
, path
, plen
);
282 memcpy(header
.name
, path
+ plen
+ 1, rest
);
284 xsnprintf(header
.name
, sizeof(header
.name
), "%s.data",
286 strbuf_append_ext_header(&ext_header
, "path",
290 memcpy(header
.name
, path
, pathlen
);
293 if (size
> sizeof(header
.linkname
)) {
294 xsnprintf(header
.linkname
, sizeof(header
.linkname
),
295 "see %s.paxheader", oid_to_hex(oid
));
296 strbuf_append_ext_header(&ext_header
, "linkpath",
299 memcpy(header
.linkname
, buffer
, size
);
302 size_in_header
= size
;
303 if (S_ISREG(mode
) && size
> USTAR_MAX_SIZE
) {
305 strbuf_append_ext_header_uint(&ext_header
, "size", size
);
308 prepare_header(args
, &header
, mode
, size_in_header
);
310 if (ext_header
.len
> 0) {
311 write_extended_header(args
, oid
, ext_header
.buf
,
314 strbuf_release(&ext_header
);
315 write_blocked(&header
, sizeof(header
));
316 if (S_ISREG(mode
) && size
> 0) {
318 write_blocked(buffer
, size
);
320 err
= stream_blocked(args
->repo
, oid
);
325 static void write_global_extended_header(struct archiver_args
*args
)
327 const struct object_id
*oid
= args
->commit_oid
;
328 struct strbuf ext_header
= STRBUF_INIT
;
329 struct ustar_header header
;
333 strbuf_append_ext_header(&ext_header
, "comment",
335 the_hash_algo
->hexsz
);
336 if (args
->time
> USTAR_MAX_MTIME
) {
337 strbuf_append_ext_header_uint(&ext_header
, "mtime",
339 args
->time
= USTAR_MAX_MTIME
;
345 memset(&header
, 0, sizeof(header
));
346 *header
.typeflag
= TYPEFLAG_GLOBAL_HEADER
;
348 xsnprintf(header
.name
, sizeof(header
.name
), "pax_global_header");
349 prepare_header(args
, &header
, mode
, ext_header
.len
);
350 write_blocked(&header
, sizeof(header
));
351 write_blocked(ext_header
.buf
, ext_header
.len
);
352 strbuf_release(&ext_header
);
355 static struct archiver
**tar_filters
;
356 static int nr_tar_filters
;
357 static int alloc_tar_filters
;
359 static struct archiver
*find_tar_filter(const char *name
, size_t len
)
362 for (i
= 0; i
< nr_tar_filters
; i
++) {
363 struct archiver
*ar
= tar_filters
[i
];
364 if (!strncmp(ar
->name
, name
, len
) && !ar
->name
[len
])
370 static int tar_filter_config(const char *var
, const char *value
,
378 if (parse_config_key(var
, "tar", &name
, &namelen
, &type
) < 0 || !name
)
381 ar
= find_tar_filter(name
, namelen
);
384 ar
->name
= xmemdupz(name
, namelen
);
385 ar
->write_archive
= write_tar_filter_archive
;
386 ar
->flags
= ARCHIVER_WANT_COMPRESSION_LEVELS
|
387 ARCHIVER_HIGH_COMPRESSION_LEVELS
;
388 ALLOC_GROW(tar_filters
, nr_tar_filters
+ 1, alloc_tar_filters
);
389 tar_filters
[nr_tar_filters
++] = ar
;
392 if (!strcmp(type
, "command")) {
394 return config_error_nonbool(var
);
395 free(ar
->filter_command
);
396 ar
->filter_command
= xstrdup(value
);
399 if (!strcmp(type
, "remote")) {
400 if (git_config_bool(var
, value
))
401 ar
->flags
|= ARCHIVER_REMOTE
;
403 ar
->flags
&= ~ARCHIVER_REMOTE
;
410 static int git_tar_config(const char *var
, const char *value
, void *cb
)
412 if (!strcmp(var
, "tar.umask")) {
413 if (value
&& !strcmp(value
, "user")) {
414 tar_umask
= umask(0);
417 tar_umask
= git_config_int(var
, value
);
422 return tar_filter_config(var
, value
, cb
);
425 static int write_tar_archive(const struct archiver
*ar UNUSED
,
426 struct archiver_args
*args
)
430 write_global_extended_header(args
);
431 err
= write_archive_entries(args
, write_tar_entry
);
437 static git_zstream gzstream
;
438 static unsigned char outbuf
[16384];
440 static void tgz_deflate(int flush
)
442 while (gzstream
.avail_in
|| flush
== Z_FINISH
) {
443 int status
= git_deflate(&gzstream
, flush
);
444 if (!gzstream
.avail_out
|| status
== Z_STREAM_END
) {
445 write_or_die(1, outbuf
, gzstream
.next_out
- outbuf
);
446 gzstream
.next_out
= outbuf
;
447 gzstream
.avail_out
= sizeof(outbuf
);
448 if (status
== Z_STREAM_END
)
451 if (status
!= Z_OK
&& status
!= Z_BUF_ERROR
)
452 die(_("deflate error (%d)"), status
);
456 static void tgz_write_block(const void *data
)
458 gzstream
.next_in
= (void *)data
;
459 gzstream
.avail_in
= BLOCKSIZE
;
460 tgz_deflate(Z_NO_FLUSH
);
463 static const char internal_gzip_command
[] = "git archive gzip";
465 static int write_tar_filter_archive(const struct archiver
*ar
,
466 struct archiver_args
*args
)
468 #if ZLIB_VERNUM >= 0x1221
469 struct gz_header_s gzhead
= { .os
= 3 }; /* Unix, for reproducibility */
471 struct strbuf cmd
= STRBUF_INIT
;
472 struct child_process filter
= CHILD_PROCESS_INIT
;
475 if (!ar
->filter_command
)
476 BUG("tar-filter archiver called with no filter defined");
478 if (!strcmp(ar
->filter_command
, internal_gzip_command
)) {
479 write_block
= tgz_write_block
;
480 git_deflate_init_gzip(&gzstream
, args
->compression_level
);
481 #if ZLIB_VERNUM >= 0x1221
482 if (deflateSetHeader(&gzstream
.z
, &gzhead
) != Z_OK
)
483 BUG("deflateSetHeader() called too late");
485 gzstream
.next_out
= outbuf
;
486 gzstream
.avail_out
= sizeof(outbuf
);
488 r
= write_tar_archive(ar
, args
);
490 tgz_deflate(Z_FINISH
);
491 git_deflate_end(&gzstream
);
495 strbuf_addstr(&cmd
, ar
->filter_command
);
496 if (args
->compression_level
>= 0)
497 strbuf_addf(&cmd
, " -%d", args
->compression_level
);
499 strvec_push(&filter
.args
, cmd
.buf
);
500 filter
.use_shell
= 1;
502 filter
.silent_exec_failure
= 1;
504 if (start_command(&filter
) < 0)
505 die_errno(_("unable to start '%s' filter"), cmd
.buf
);
507 if (dup2(filter
.in
, 1) < 0)
508 die_errno(_("unable to redirect descriptor"));
511 r
= write_tar_archive(ar
, args
);
514 if (finish_command(&filter
) != 0)
515 die(_("'%s' filter reported error"), cmd
.buf
);
517 strbuf_release(&cmd
);
521 static struct archiver tar_archiver
= {
523 .write_archive
= write_tar_archive
,
524 .flags
= ARCHIVER_REMOTE
,
527 void init_tar_archiver(void)
530 register_archiver(&tar_archiver
);
532 tar_filter_config("tar.tgz.command", internal_gzip_command
, NULL
);
533 tar_filter_config("tar.tgz.remote", "true", NULL
);
534 tar_filter_config("tar.tar.gz.command", internal_gzip_command
, NULL
);
535 tar_filter_config("tar.tar.gz.remote", "true", NULL
);
536 git_config(git_tar_config
, NULL
);
537 for (i
= 0; i
< nr_tar_filters
; i
++) {
538 /* omit any filters that never had a command configured */
539 if (tar_filters
[i
]->filter_command
)
540 register_archiver(tar_filters
[i
]);