7 #include "chunk-format.h"
8 #include "pack-mtimes.h"
10 #include "pack-objects.h"
12 void reset_pack_idx_option(struct pack_idx_option
*opts
)
14 memset(opts
, 0, sizeof(*opts
));
16 opts
->off32_limit
= 0x7fffffff;
19 static int sha1_compare(const void *_a
, const void *_b
)
21 struct pack_idx_entry
*a
= *(struct pack_idx_entry
**)_a
;
22 struct pack_idx_entry
*b
= *(struct pack_idx_entry
**)_b
;
23 return oidcmp(&a
->oid
, &b
->oid
);
26 static int cmp_uint32(const void *a_
, const void *b_
)
28 uint32_t a
= *((uint32_t *)a_
);
29 uint32_t b
= *((uint32_t *)b_
);
31 return (a
< b
) ? -1 : (a
!= b
);
34 static int need_large_offset(off_t offset
, const struct pack_idx_option
*opts
)
38 if ((offset
>> 31) || (opts
->off32_limit
< offset
))
40 if (!opts
->anomaly_nr
)
43 return !!bsearch(&ofsval
, opts
->anomaly
, opts
->anomaly_nr
,
44 sizeof(ofsval
), cmp_uint32
);
48 * The *sha1 contains the pack content SHA1 hash.
49 * The objects array passed in will be sorted by SHA1 on exit.
51 const char *write_idx_file(const char *index_name
, struct pack_idx_entry
**objects
,
52 int nr_objects
, const struct pack_idx_option
*opts
,
53 const unsigned char *sha1
)
56 struct pack_idx_entry
**sorted_by_sha
, **list
, **last
;
57 off_t last_obj_offset
= 0;
59 uint32_t index_version
;
62 sorted_by_sha
= objects
;
64 last
= sorted_by_sha
+ nr_objects
;
65 for (i
= 0; i
< nr_objects
; ++i
) {
66 if (objects
[i
]->offset
> last_obj_offset
)
67 last_obj_offset
= objects
[i
]->offset
;
69 QSORT(sorted_by_sha
, nr_objects
, sha1_compare
);
72 sorted_by_sha
= list
= last
= NULL
;
74 if (opts
->flags
& WRITE_IDX_VERIFY
) {
76 f
= hashfd_check(index_name
);
79 struct strbuf tmp_file
= STRBUF_INIT
;
80 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_idx_XXXXXX");
81 index_name
= strbuf_detach(&tmp_file
, NULL
);
84 fd
= xopen(index_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
86 f
= hashfd(fd
, index_name
);
89 /* if last object's offset is >= 2^31 we should use index V2 */
90 index_version
= need_large_offset(last_obj_offset
, opts
) ? 2 : opts
->version
;
92 /* index versions 2 and above need a header */
93 if (index_version
>= 2) {
94 struct pack_idx_header hdr
;
95 hdr
.idx_signature
= htonl(PACK_IDX_SIGNATURE
);
96 hdr
.idx_version
= htonl(index_version
);
97 hashwrite(f
, &hdr
, sizeof(hdr
));
101 * Write the first-level table (the list is sorted,
102 * but we use a 256-entry lookup to be able to avoid
103 * having to do eight extra binary search iterations).
105 for (i
= 0; i
< 256; i
++) {
106 struct pack_idx_entry
**next
= list
;
107 while (next
< last
) {
108 struct pack_idx_entry
*obj
= *next
;
109 if (obj
->oid
.hash
[0] != i
)
113 hashwrite_be32(f
, next
- sorted_by_sha
);
118 * Write the actual SHA1 entries..
120 list
= sorted_by_sha
;
121 for (i
= 0; i
< nr_objects
; i
++) {
122 struct pack_idx_entry
*obj
= *list
++;
123 if (index_version
< 2)
124 hashwrite_be32(f
, obj
->offset
);
125 hashwrite(f
, obj
->oid
.hash
, the_hash_algo
->rawsz
);
126 if ((opts
->flags
& WRITE_IDX_STRICT
) &&
127 (i
&& oideq(&list
[-2]->oid
, &obj
->oid
)))
128 die("The same object %s appears twice in the pack",
129 oid_to_hex(&obj
->oid
));
132 if (index_version
>= 2) {
133 unsigned int nr_large_offset
= 0;
135 /* write the crc32 table */
136 list
= sorted_by_sha
;
137 for (i
= 0; i
< nr_objects
; i
++) {
138 struct pack_idx_entry
*obj
= *list
++;
139 hashwrite_be32(f
, obj
->crc32
);
142 /* write the 32-bit offset table */
143 list
= sorted_by_sha
;
144 for (i
= 0; i
< nr_objects
; i
++) {
145 struct pack_idx_entry
*obj
= *list
++;
148 offset
= (need_large_offset(obj
->offset
, opts
)
149 ? (0x80000000 | nr_large_offset
++)
151 hashwrite_be32(f
, offset
);
154 /* write the large offset table */
155 list
= sorted_by_sha
;
156 while (nr_large_offset
) {
157 struct pack_idx_entry
*obj
= *list
++;
158 uint64_t offset
= obj
->offset
;
160 if (!need_large_offset(offset
, opts
))
162 hashwrite_be64(f
, offset
);
167 hashwrite(f
, sha1
, the_hash_algo
->rawsz
);
168 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
169 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
170 ((opts
->flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
174 static int pack_order_cmp(const void *va
, const void *vb
, void *ctx
)
176 struct pack_idx_entry
**objects
= ctx
;
178 off_t oa
= objects
[*(uint32_t*)va
]->offset
;
179 off_t ob
= objects
[*(uint32_t*)vb
]->offset
;
188 static void write_rev_header(struct hashfile
*f
)
190 hashwrite_be32(f
, RIDX_SIGNATURE
);
191 hashwrite_be32(f
, RIDX_VERSION
);
192 hashwrite_be32(f
, oid_version(the_hash_algo
));
195 static void write_rev_index_positions(struct hashfile
*f
,
196 uint32_t *pack_order
,
200 for (i
= 0; i
< nr_objects
; i
++)
201 hashwrite_be32(f
, pack_order
[i
]);
204 static void write_rev_trailer(struct hashfile
*f
, const unsigned char *hash
)
206 hashwrite(f
, hash
, the_hash_algo
->rawsz
);
209 const char *write_rev_file(const char *rev_name
,
210 struct pack_idx_entry
**objects
,
212 const unsigned char *hash
,
215 uint32_t *pack_order
;
219 if (!(flags
& WRITE_REV
) && !(flags
& WRITE_REV_VERIFY
))
222 ALLOC_ARRAY(pack_order
, nr_objects
);
223 for (i
= 0; i
< nr_objects
; i
++)
225 QSORT_S(pack_order
, nr_objects
, pack_order_cmp
, objects
);
227 ret
= write_rev_file_order(rev_name
, pack_order
, nr_objects
, hash
,
235 const char *write_rev_file_order(const char *rev_name
,
236 uint32_t *pack_order
,
238 const unsigned char *hash
,
244 if ((flags
& WRITE_REV
) && (flags
& WRITE_REV_VERIFY
))
245 die(_("cannot both write and verify reverse index"));
247 if (flags
& WRITE_REV
) {
249 struct strbuf tmp_file
= STRBUF_INIT
;
250 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_rev_XXXXXX");
251 rev_name
= strbuf_detach(&tmp_file
, NULL
);
254 fd
= xopen(rev_name
, O_CREAT
|O_EXCL
|O_WRONLY
, 0600);
256 f
= hashfd(fd
, rev_name
);
257 } else if (flags
& WRITE_REV_VERIFY
) {
259 if (stat(rev_name
, &statbuf
)) {
260 if (errno
== ENOENT
) {
261 /* .rev files are optional */
264 die_errno(_("could not stat: %s"), rev_name
);
266 f
= hashfd_check(rev_name
);
272 write_rev_index_positions(f
, pack_order
, nr_objects
);
273 write_rev_trailer(f
, hash
);
275 if (rev_name
&& adjust_shared_perm(rev_name
) < 0)
276 die(_("failed to make %s readable"), rev_name
);
278 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
279 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
|
280 ((flags
& WRITE_IDX_VERIFY
) ? 0 : CSUM_FSYNC
));
285 static void write_mtimes_header(struct hashfile
*f
)
287 hashwrite_be32(f
, MTIMES_SIGNATURE
);
288 hashwrite_be32(f
, MTIMES_VERSION
);
289 hashwrite_be32(f
, oid_version(the_hash_algo
));
293 * Writes the object mtimes of "objects" for use in a .mtimes file.
294 * Note that objects must be in lexicographic (index) order, which is
295 * the expected ordering of these values in the .mtimes file.
297 static void write_mtimes_objects(struct hashfile
*f
,
298 struct packing_data
*to_pack
,
299 struct pack_idx_entry
**objects
,
303 for (i
= 0; i
< nr_objects
; i
++) {
304 struct object_entry
*e
= (struct object_entry
*)objects
[i
];
305 hashwrite_be32(f
, oe_cruft_mtime(to_pack
, e
));
309 static void write_mtimes_trailer(struct hashfile
*f
, const unsigned char *hash
)
311 hashwrite(f
, hash
, the_hash_algo
->rawsz
);
314 static const char *write_mtimes_file(struct packing_data
*to_pack
,
315 struct pack_idx_entry
**objects
,
317 const unsigned char *hash
)
319 struct strbuf tmp_file
= STRBUF_INIT
;
320 const char *mtimes_name
;
325 BUG("cannot call write_mtimes_file with NULL packing_data");
327 fd
= odb_mkstemp(&tmp_file
, "pack/tmp_mtimes_XXXXXX");
328 mtimes_name
= strbuf_detach(&tmp_file
, NULL
);
329 f
= hashfd(fd
, mtimes_name
);
331 write_mtimes_header(f
);
332 write_mtimes_objects(f
, to_pack
, objects
, nr_objects
);
333 write_mtimes_trailer(f
, hash
);
335 if (adjust_shared_perm(mtimes_name
) < 0)
336 die(_("failed to make %s readable"), mtimes_name
);
338 finalize_hashfile(f
, NULL
, FSYNC_COMPONENT_PACK_METADATA
,
339 CSUM_HASH_IN_STREAM
| CSUM_CLOSE
| CSUM_FSYNC
);
344 off_t
write_pack_header(struct hashfile
*f
, uint32_t nr_entries
)
346 struct pack_header hdr
;
348 hdr
.hdr_signature
= htonl(PACK_SIGNATURE
);
349 hdr
.hdr_version
= htonl(PACK_VERSION
);
350 hdr
.hdr_entries
= htonl(nr_entries
);
351 hashwrite(f
, &hdr
, sizeof(hdr
));
356 * Update pack header with object_count and compute new SHA1 for pack data
357 * associated to pack_fd, and write that SHA1 at the end. That new SHA1
358 * is also returned in new_pack_sha1.
360 * If partial_pack_sha1 is non null, then the SHA1 of the existing pack
361 * (without the header update) is computed and validated against the
362 * one provided in partial_pack_sha1. The validation is performed at
363 * partial_pack_offset bytes in the pack file. The SHA1 of the remaining
364 * data (i.e. from partial_pack_offset to the end) is then computed and
365 * returned in partial_pack_sha1.
367 * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and
368 * partial_pack_sha1 can refer to the same buffer if the caller is not
369 * interested in the resulting SHA1 of pack data above partial_pack_offset.
371 void fixup_pack_header_footer(int pack_fd
,
372 unsigned char *new_pack_hash
,
373 const char *pack_name
,
374 uint32_t object_count
,
375 unsigned char *partial_pack_hash
,
376 off_t partial_pack_offset
)
378 int aligned_sz
, buf_sz
= 8 * 1024;
379 git_hash_ctx old_hash_ctx
, new_hash_ctx
;
380 struct pack_header hdr
;
384 the_hash_algo
->init_fn(&old_hash_ctx
);
385 the_hash_algo
->init_fn(&new_hash_ctx
);
387 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
388 die_errno("Failed seeking to start of '%s'", pack_name
);
389 read_result
= read_in_full(pack_fd
, &hdr
, sizeof(hdr
));
391 die_errno("Unable to reread header of '%s'", pack_name
);
392 else if (read_result
!= sizeof(hdr
))
393 die_errno("Unexpected short read for header of '%s'",
395 if (lseek(pack_fd
, 0, SEEK_SET
) != 0)
396 die_errno("Failed seeking to start of '%s'", pack_name
);
397 the_hash_algo
->update_fn(&old_hash_ctx
, &hdr
, sizeof(hdr
));
398 hdr
.hdr_entries
= htonl(object_count
);
399 the_hash_algo
->update_fn(&new_hash_ctx
, &hdr
, sizeof(hdr
));
400 write_or_die(pack_fd
, &hdr
, sizeof(hdr
));
401 partial_pack_offset
-= sizeof(hdr
);
403 buf
= xmalloc(buf_sz
);
404 aligned_sz
= buf_sz
- sizeof(hdr
);
407 m
= (partial_pack_hash
&& partial_pack_offset
< aligned_sz
) ?
408 partial_pack_offset
: aligned_sz
;
409 n
= xread(pack_fd
, buf
, m
);
413 die_errno("Failed to checksum '%s'", pack_name
);
414 the_hash_algo
->update_fn(&new_hash_ctx
, buf
, n
);
420 if (!partial_pack_hash
)
423 the_hash_algo
->update_fn(&old_hash_ctx
, buf
, n
);
424 partial_pack_offset
-= n
;
425 if (partial_pack_offset
== 0) {
426 unsigned char hash
[GIT_MAX_RAWSZ
];
427 the_hash_algo
->final_fn(hash
, &old_hash_ctx
);
428 if (!hasheq(hash
, partial_pack_hash
))
429 die("Unexpected checksum for %s "
430 "(disk corruption?)", pack_name
);
432 * Now let's compute the SHA1 of the remainder of the
433 * pack, which also means making partial_pack_offset
434 * big enough not to matter anymore.
436 the_hash_algo
->init_fn(&old_hash_ctx
);
437 partial_pack_offset
= ~partial_pack_offset
;
438 partial_pack_offset
-= MSB(partial_pack_offset
, 1);
443 if (partial_pack_hash
)
444 the_hash_algo
->final_fn(partial_pack_hash
, &old_hash_ctx
);
445 the_hash_algo
->final_fn(new_pack_hash
, &new_hash_ctx
);
446 write_or_die(pack_fd
, new_pack_hash
, the_hash_algo
->rawsz
);
447 fsync_component_or_die(FSYNC_COMPONENT_PACK
, pack_fd
, pack_name
);
450 char *index_pack_lockfile(int ip_out
, int *is_well_formed
)
452 char packname
[GIT_MAX_HEXSZ
+ 6];
453 const int len
= the_hash_algo
->hexsz
+ 6;
456 * The first thing we expect from index-pack's output
457 * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where
458 * %40s is the newly created pack SHA1 name. In the "keep"
459 * case, we need it to remove the corresponding .keep file
460 * later on. If we don't get that then tough luck with it.
462 if (read_in_full(ip_out
, packname
, len
) == len
&& packname
[len
-1] == '\n') {
468 if (skip_prefix(packname
, "keep\t", &name
))
469 return xstrfmt("%s/pack/pack-%s.keep",
470 get_object_directory(), name
);
479 * The per-object header is a pretty dense thing, which is
480 * - first byte: low four bits are "size", then three bits of "type",
481 * and the high bit is "size continues".
482 * - each byte afterwards: low seven bits are size continuation,
483 * with the high bit being "size continues"
485 int encode_in_pack_object_header(unsigned char *hdr
, int hdr_len
,
486 enum object_type type
, uintmax_t size
)
491 if (type
< OBJ_COMMIT
|| type
> OBJ_REF_DELTA
)
492 die("bad type %d", type
);
494 c
= (type
<< 4) | (size
& 15);
498 die("object size is too enormous to format");
508 struct hashfile
*create_tmp_packfile(char **pack_tmp_name
)
510 struct strbuf tmpname
= STRBUF_INIT
;
513 fd
= odb_mkstemp(&tmpname
, "pack/tmp_pack_XXXXXX");
514 *pack_tmp_name
= strbuf_detach(&tmpname
, NULL
);
515 return hashfd(fd
, *pack_tmp_name
);
518 static void rename_tmp_packfile(struct strbuf
*name_prefix
, const char *source
,
521 size_t name_prefix_len
= name_prefix
->len
;
523 strbuf_addstr(name_prefix
, ext
);
524 if (rename(source
, name_prefix
->buf
))
525 die_errno("unable to rename temporary file to '%s'",
527 strbuf_setlen(name_prefix
, name_prefix_len
);
530 void rename_tmp_packfile_idx(struct strbuf
*name_buffer
,
533 rename_tmp_packfile(name_buffer
, *idx_tmp_name
, "idx");
536 void stage_tmp_packfiles(struct strbuf
*name_buffer
,
537 const char *pack_tmp_name
,
538 struct pack_idx_entry
**written_list
,
540 struct packing_data
*to_pack
,
541 struct pack_idx_option
*pack_idx_opts
,
542 unsigned char hash
[],
545 const char *rev_tmp_name
= NULL
;
546 const char *mtimes_tmp_name
= NULL
;
548 if (adjust_shared_perm(pack_tmp_name
))
549 die_errno("unable to make temporary pack file readable");
551 *idx_tmp_name
= (char *)write_idx_file(NULL
, written_list
, nr_written
,
552 pack_idx_opts
, hash
);
553 if (adjust_shared_perm(*idx_tmp_name
))
554 die_errno("unable to make temporary index file readable");
556 rev_tmp_name
= write_rev_file(NULL
, written_list
, nr_written
, hash
,
557 pack_idx_opts
->flags
);
559 if (pack_idx_opts
->flags
& WRITE_MTIMES
) {
560 mtimes_tmp_name
= write_mtimes_file(to_pack
, written_list
,
565 rename_tmp_packfile(name_buffer
, pack_tmp_name
, "pack");
567 rename_tmp_packfile(name_buffer
, rev_tmp_name
, "rev");
569 rename_tmp_packfile(name_buffer
, mtimes_tmp_name
, "mtimes");
572 void write_promisor_file(const char *promisor_name
, struct ref
**sought
, int nr_sought
)
575 FILE *output
= xfopen(promisor_name
, "w");
577 for (i
= 0; i
< nr_sought
; i
++)
578 fprintf(output
, "%s %s\n", oid_to_hex(&sought
[i
]->old_oid
),
581 err
= ferror(output
);
582 err
|= fclose(output
);
584 die(_("could not write '%s' promisor file"), promisor_name
);