]>
Commit | Line | Data |
---|---|---|
1 | #include "cache.h" | |
2 | #include "pack.h" | |
3 | #include "csum-file.h" | |
4 | ||
5 | void reset_pack_idx_option(struct pack_idx_option *opts) | |
6 | { | |
7 | memset(opts, 0, sizeof(*opts)); | |
8 | opts->version = 2; | |
9 | opts->off32_limit = 0x7fffffff; | |
10 | } | |
11 | ||
12 | static int sha1_compare(const void *_a, const void *_b) | |
13 | { | |
14 | struct pack_idx_entry *a = *(struct pack_idx_entry **)_a; | |
15 | struct pack_idx_entry *b = *(struct pack_idx_entry **)_b; | |
16 | return oidcmp(&a->oid, &b->oid); | |
17 | } | |
18 | ||
19 | static int cmp_uint32(const void *a_, const void *b_) | |
20 | { | |
21 | uint32_t a = *((uint32_t *)a_); | |
22 | uint32_t b = *((uint32_t *)b_); | |
23 | ||
24 | return (a < b) ? -1 : (a != b); | |
25 | } | |
26 | ||
27 | static int need_large_offset(off_t offset, const struct pack_idx_option *opts) | |
28 | { | |
29 | uint32_t ofsval; | |
30 | ||
31 | if ((offset >> 31) || (opts->off32_limit < offset)) | |
32 | return 1; | |
33 | if (!opts->anomaly_nr) | |
34 | return 0; | |
35 | ofsval = offset; | |
36 | return !!bsearch(&ofsval, opts->anomaly, opts->anomaly_nr, | |
37 | sizeof(ofsval), cmp_uint32); | |
38 | } | |
39 | ||
40 | /* | |
41 | * The *sha1 contains the pack content SHA1 hash. | |
42 | * The objects array passed in will be sorted by SHA1 on exit. | |
43 | */ | |
44 | const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, | |
45 | int nr_objects, const struct pack_idx_option *opts, | |
46 | const unsigned char *sha1) | |
47 | { | |
48 | struct hashfile *f; | |
49 | struct pack_idx_entry **sorted_by_sha, **list, **last; | |
50 | off_t last_obj_offset = 0; | |
51 | uint32_t array[256]; | |
52 | int i, fd; | |
53 | uint32_t index_version; | |
54 | ||
55 | if (nr_objects) { | |
56 | sorted_by_sha = objects; | |
57 | list = sorted_by_sha; | |
58 | last = sorted_by_sha + nr_objects; | |
59 | for (i = 0; i < nr_objects; ++i) { | |
60 | if (objects[i]->offset > last_obj_offset) | |
61 | last_obj_offset = objects[i]->offset; | |
62 | } | |
63 | QSORT(sorted_by_sha, nr_objects, sha1_compare); | |
64 | } | |
65 | else | |
66 | sorted_by_sha = list = last = NULL; | |
67 | ||
68 | if (opts->flags & WRITE_IDX_VERIFY) { | |
69 | assert(index_name); | |
70 | f = hashfd_check(index_name); | |
71 | } else { | |
72 | if (!index_name) { | |
73 | struct strbuf tmp_file = STRBUF_INIT; | |
74 | fd = odb_mkstemp(&tmp_file, "pack/tmp_idx_XXXXXX"); | |
75 | index_name = strbuf_detach(&tmp_file, NULL); | |
76 | } else { | |
77 | unlink(index_name); | |
78 | fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600); | |
79 | if (fd < 0) | |
80 | die_errno("unable to create '%s'", index_name); | |
81 | } | |
82 | f = hashfd(fd, index_name); | |
83 | } | |
84 | ||
85 | /* if last object's offset is >= 2^31 we should use index V2 */ | |
86 | index_version = need_large_offset(last_obj_offset, opts) ? 2 : opts->version; | |
87 | ||
88 | /* index versions 2 and above need a header */ | |
89 | if (index_version >= 2) { | |
90 | struct pack_idx_header hdr; | |
91 | hdr.idx_signature = htonl(PACK_IDX_SIGNATURE); | |
92 | hdr.idx_version = htonl(index_version); | |
93 | hashwrite(f, &hdr, sizeof(hdr)); | |
94 | } | |
95 | ||
96 | /* | |
97 | * Write the first-level table (the list is sorted, | |
98 | * but we use a 256-entry lookup to be able to avoid | |
99 | * having to do eight extra binary search iterations). | |
100 | */ | |
101 | for (i = 0; i < 256; i++) { | |
102 | struct pack_idx_entry **next = list; | |
103 | while (next < last) { | |
104 | struct pack_idx_entry *obj = *next; | |
105 | if (obj->oid.hash[0] != i) | |
106 | break; | |
107 | next++; | |
108 | } | |
109 | array[i] = htonl(next - sorted_by_sha); | |
110 | list = next; | |
111 | } | |
112 | hashwrite(f, array, 256 * 4); | |
113 | ||
114 | /* | |
115 | * Write the actual SHA1 entries.. | |
116 | */ | |
117 | list = sorted_by_sha; | |
118 | for (i = 0; i < nr_objects; i++) { | |
119 | struct pack_idx_entry *obj = *list++; | |
120 | if (index_version < 2) | |
121 | hashwrite_be32(f, obj->offset); | |
122 | hashwrite(f, obj->oid.hash, the_hash_algo->rawsz); | |
123 | if ((opts->flags & WRITE_IDX_STRICT) && | |
124 | (i && oideq(&list[-2]->oid, &obj->oid))) | |
125 | die("The same object %s appears twice in the pack", | |
126 | oid_to_hex(&obj->oid)); | |
127 | } | |
128 | ||
129 | if (index_version >= 2) { | |
130 | unsigned int nr_large_offset = 0; | |
131 | ||
132 | /* write the crc32 table */ | |
133 | list = sorted_by_sha; | |
134 | for (i = 0; i < nr_objects; i++) { | |
135 | struct pack_idx_entry *obj = *list++; | |
136 | hashwrite_be32(f, obj->crc32); | |
137 | } | |
138 | ||
139 | /* write the 32-bit offset table */ | |
140 | list = sorted_by_sha; | |
141 | for (i = 0; i < nr_objects; i++) { | |
142 | struct pack_idx_entry *obj = *list++; | |
143 | uint32_t offset; | |
144 | ||
145 | offset = (need_large_offset(obj->offset, opts) | |
146 | ? (0x80000000 | nr_large_offset++) | |
147 | : obj->offset); | |
148 | hashwrite_be32(f, offset); | |
149 | } | |
150 | ||
151 | /* write the large offset table */ | |
152 | list = sorted_by_sha; | |
153 | while (nr_large_offset) { | |
154 | struct pack_idx_entry *obj = *list++; | |
155 | uint64_t offset = obj->offset; | |
156 | uint32_t split[2]; | |
157 | ||
158 | if (!need_large_offset(offset, opts)) | |
159 | continue; | |
160 | split[0] = htonl(offset >> 32); | |
161 | split[1] = htonl(offset & 0xffffffff); | |
162 | hashwrite(f, split, 8); | |
163 | nr_large_offset--; | |
164 | } | |
165 | } | |
166 | ||
167 | hashwrite(f, sha1, the_hash_algo->rawsz); | |
168 | finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE | | |
169 | ((opts->flags & WRITE_IDX_VERIFY) | |
170 | ? 0 : CSUM_FSYNC)); | |
171 | return index_name; | |
172 | } | |
173 | ||
174 | off_t write_pack_header(struct hashfile *f, uint32_t nr_entries) | |
175 | { | |
176 | struct pack_header hdr; | |
177 | ||
178 | hdr.hdr_signature = htonl(PACK_SIGNATURE); | |
179 | hdr.hdr_version = htonl(PACK_VERSION); | |
180 | hdr.hdr_entries = htonl(nr_entries); | |
181 | hashwrite(f, &hdr, sizeof(hdr)); | |
182 | return sizeof(hdr); | |
183 | } | |
184 | ||
185 | /* | |
186 | * Update pack header with object_count and compute new SHA1 for pack data | |
187 | * associated to pack_fd, and write that SHA1 at the end. That new SHA1 | |
188 | * is also returned in new_pack_sha1. | |
189 | * | |
190 | * If partial_pack_sha1 is non null, then the SHA1 of the existing pack | |
191 | * (without the header update) is computed and validated against the | |
192 | * one provided in partial_pack_sha1. The validation is performed at | |
193 | * partial_pack_offset bytes in the pack file. The SHA1 of the remaining | |
194 | * data (i.e. from partial_pack_offset to the end) is then computed and | |
195 | * returned in partial_pack_sha1. | |
196 | * | |
197 | * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and | |
198 | * partial_pack_sha1 can refer to the same buffer if the caller is not | |
199 | * interested in the resulting SHA1 of pack data above partial_pack_offset. | |
200 | */ | |
201 | void fixup_pack_header_footer(int pack_fd, | |
202 | unsigned char *new_pack_hash, | |
203 | const char *pack_name, | |
204 | uint32_t object_count, | |
205 | unsigned char *partial_pack_hash, | |
206 | off_t partial_pack_offset) | |
207 | { | |
208 | int aligned_sz, buf_sz = 8 * 1024; | |
209 | git_hash_ctx old_hash_ctx, new_hash_ctx; | |
210 | struct pack_header hdr; | |
211 | char *buf; | |
212 | ssize_t read_result; | |
213 | ||
214 | the_hash_algo->init_fn(&old_hash_ctx); | |
215 | the_hash_algo->init_fn(&new_hash_ctx); | |
216 | ||
217 | if (lseek(pack_fd, 0, SEEK_SET) != 0) | |
218 | die_errno("Failed seeking to start of '%s'", pack_name); | |
219 | read_result = read_in_full(pack_fd, &hdr, sizeof(hdr)); | |
220 | if (read_result < 0) | |
221 | die_errno("Unable to reread header of '%s'", pack_name); | |
222 | else if (read_result != sizeof(hdr)) | |
223 | die_errno("Unexpected short read for header of '%s'", | |
224 | pack_name); | |
225 | if (lseek(pack_fd, 0, SEEK_SET) != 0) | |
226 | die_errno("Failed seeking to start of '%s'", pack_name); | |
227 | the_hash_algo->update_fn(&old_hash_ctx, &hdr, sizeof(hdr)); | |
228 | hdr.hdr_entries = htonl(object_count); | |
229 | the_hash_algo->update_fn(&new_hash_ctx, &hdr, sizeof(hdr)); | |
230 | write_or_die(pack_fd, &hdr, sizeof(hdr)); | |
231 | partial_pack_offset -= sizeof(hdr); | |
232 | ||
233 | buf = xmalloc(buf_sz); | |
234 | aligned_sz = buf_sz - sizeof(hdr); | |
235 | for (;;) { | |
236 | ssize_t m, n; | |
237 | m = (partial_pack_hash && partial_pack_offset < aligned_sz) ? | |
238 | partial_pack_offset : aligned_sz; | |
239 | n = xread(pack_fd, buf, m); | |
240 | if (!n) | |
241 | break; | |
242 | if (n < 0) | |
243 | die_errno("Failed to checksum '%s'", pack_name); | |
244 | the_hash_algo->update_fn(&new_hash_ctx, buf, n); | |
245 | ||
246 | aligned_sz -= n; | |
247 | if (!aligned_sz) | |
248 | aligned_sz = buf_sz; | |
249 | ||
250 | if (!partial_pack_hash) | |
251 | continue; | |
252 | ||
253 | the_hash_algo->update_fn(&old_hash_ctx, buf, n); | |
254 | partial_pack_offset -= n; | |
255 | if (partial_pack_offset == 0) { | |
256 | unsigned char hash[GIT_MAX_RAWSZ]; | |
257 | the_hash_algo->final_fn(hash, &old_hash_ctx); | |
258 | if (!hasheq(hash, partial_pack_hash)) | |
259 | die("Unexpected checksum for %s " | |
260 | "(disk corruption?)", pack_name); | |
261 | /* | |
262 | * Now let's compute the SHA1 of the remainder of the | |
263 | * pack, which also means making partial_pack_offset | |
264 | * big enough not to matter anymore. | |
265 | */ | |
266 | the_hash_algo->init_fn(&old_hash_ctx); | |
267 | partial_pack_offset = ~partial_pack_offset; | |
268 | partial_pack_offset -= MSB(partial_pack_offset, 1); | |
269 | } | |
270 | } | |
271 | free(buf); | |
272 | ||
273 | if (partial_pack_hash) | |
274 | the_hash_algo->final_fn(partial_pack_hash, &old_hash_ctx); | |
275 | the_hash_algo->final_fn(new_pack_hash, &new_hash_ctx); | |
276 | write_or_die(pack_fd, new_pack_hash, the_hash_algo->rawsz); | |
277 | fsync_or_die(pack_fd, pack_name); | |
278 | } | |
279 | ||
280 | char *index_pack_lockfile(int ip_out) | |
281 | { | |
282 | char packname[GIT_MAX_HEXSZ + 6]; | |
283 | const int len = the_hash_algo->hexsz + 6; | |
284 | ||
285 | /* | |
286 | * The first thing we expect from index-pack's output | |
287 | * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where | |
288 | * %40s is the newly created pack SHA1 name. In the "keep" | |
289 | * case, we need it to remove the corresponding .keep file | |
290 | * later on. If we don't get that then tough luck with it. | |
291 | */ | |
292 | if (read_in_full(ip_out, packname, len) == len && packname[len-1] == '\n') { | |
293 | const char *name; | |
294 | packname[len-1] = 0; | |
295 | if (skip_prefix(packname, "keep\t", &name)) | |
296 | return xstrfmt("%s/pack/pack-%s.keep", | |
297 | get_object_directory(), name); | |
298 | } | |
299 | return NULL; | |
300 | } | |
301 | ||
302 | /* | |
303 | * The per-object header is a pretty dense thing, which is | |
304 | * - first byte: low four bits are "size", then three bits of "type", | |
305 | * and the high bit is "size continues". | |
306 | * - each byte afterwards: low seven bits are size continuation, | |
307 | * with the high bit being "size continues" | |
308 | */ | |
309 | int encode_in_pack_object_header(unsigned char *hdr, int hdr_len, | |
310 | enum object_type type, uintmax_t size) | |
311 | { | |
312 | int n = 1; | |
313 | unsigned char c; | |
314 | ||
315 | if (type < OBJ_COMMIT || type > OBJ_REF_DELTA) | |
316 | die("bad type %d", type); | |
317 | ||
318 | c = (type << 4) | (size & 15); | |
319 | size >>= 4; | |
320 | while (size) { | |
321 | if (n == hdr_len) | |
322 | die("object size is too enormous to format"); | |
323 | *hdr++ = c | 0x80; | |
324 | c = size & 0x7f; | |
325 | size >>= 7; | |
326 | n++; | |
327 | } | |
328 | *hdr = c; | |
329 | return n; | |
330 | } | |
331 | ||
332 | struct hashfile *create_tmp_packfile(char **pack_tmp_name) | |
333 | { | |
334 | struct strbuf tmpname = STRBUF_INIT; | |
335 | int fd; | |
336 | ||
337 | fd = odb_mkstemp(&tmpname, "pack/tmp_pack_XXXXXX"); | |
338 | *pack_tmp_name = strbuf_detach(&tmpname, NULL); | |
339 | return hashfd(fd, *pack_tmp_name); | |
340 | } | |
341 | ||
342 | void finish_tmp_packfile(struct strbuf *name_buffer, | |
343 | const char *pack_tmp_name, | |
344 | struct pack_idx_entry **written_list, | |
345 | uint32_t nr_written, | |
346 | struct pack_idx_option *pack_idx_opts, | |
347 | unsigned char hash[]) | |
348 | { | |
349 | const char *idx_tmp_name; | |
350 | int basename_len = name_buffer->len; | |
351 | ||
352 | if (adjust_shared_perm(pack_tmp_name)) | |
353 | die_errno("unable to make temporary pack file readable"); | |
354 | ||
355 | idx_tmp_name = write_idx_file(NULL, written_list, nr_written, | |
356 | pack_idx_opts, hash); | |
357 | if (adjust_shared_perm(idx_tmp_name)) | |
358 | die_errno("unable to make temporary index file readable"); | |
359 | ||
360 | strbuf_addf(name_buffer, "%s.pack", hash_to_hex(hash)); | |
361 | ||
362 | if (rename(pack_tmp_name, name_buffer->buf)) | |
363 | die_errno("unable to rename temporary pack file"); | |
364 | ||
365 | strbuf_setlen(name_buffer, basename_len); | |
366 | ||
367 | strbuf_addf(name_buffer, "%s.idx", hash_to_hex(hash)); | |
368 | if (rename(idx_tmp_name, name_buffer->buf)) | |
369 | die_errno("unable to rename temporary index file"); | |
370 | ||
371 | strbuf_setlen(name_buffer, basename_len); | |
372 | ||
373 | free((void *)idx_tmp_name); | |
374 | } |