]>
Commit | Line | Data |
---|---|---|
1 | #include "cache.h" | |
2 | #include "pack.h" | |
3 | #include "csum-file.h" | |
4 | ||
5 | void reset_pack_idx_option(struct pack_idx_option *opts) | |
6 | { | |
7 | memset(opts, 0, sizeof(*opts)); | |
8 | opts->version = 2; | |
9 | opts->off32_limit = 0x7fffffff; | |
10 | } | |
11 | ||
12 | static int sha1_compare(const void *_a, const void *_b) | |
13 | { | |
14 | struct pack_idx_entry *a = *(struct pack_idx_entry **)_a; | |
15 | struct pack_idx_entry *b = *(struct pack_idx_entry **)_b; | |
16 | return oidcmp(&a->oid, &b->oid); | |
17 | } | |
18 | ||
19 | static int cmp_uint32(const void *a_, const void *b_) | |
20 | { | |
21 | uint32_t a = *((uint32_t *)a_); | |
22 | uint32_t b = *((uint32_t *)b_); | |
23 | ||
24 | return (a < b) ? -1 : (a != b); | |
25 | } | |
26 | ||
27 | static int need_large_offset(off_t offset, const struct pack_idx_option *opts) | |
28 | { | |
29 | uint32_t ofsval; | |
30 | ||
31 | if ((offset >> 31) || (opts->off32_limit < offset)) | |
32 | return 1; | |
33 | if (!opts->anomaly_nr) | |
34 | return 0; | |
35 | ofsval = offset; | |
36 | return !!bsearch(&ofsval, opts->anomaly, opts->anomaly_nr, | |
37 | sizeof(ofsval), cmp_uint32); | |
38 | } | |
39 | ||
40 | /* | |
41 | * On entry *sha1 contains the pack content SHA1 hash, on exit it is | |
42 | * the SHA1 hash of sorted object names. The objects array passed in | |
43 | * will be sorted by SHA1 on exit. | |
44 | */ | |
45 | const char *write_idx_file(const char *index_name, struct pack_idx_entry **objects, | |
46 | int nr_objects, const struct pack_idx_option *opts, | |
47 | const unsigned char *sha1) | |
48 | { | |
49 | struct hashfile *f; | |
50 | struct pack_idx_entry **sorted_by_sha, **list, **last; | |
51 | off_t last_obj_offset = 0; | |
52 | uint32_t array[256]; | |
53 | int i, fd; | |
54 | uint32_t index_version; | |
55 | ||
56 | if (nr_objects) { | |
57 | sorted_by_sha = objects; | |
58 | list = sorted_by_sha; | |
59 | last = sorted_by_sha + nr_objects; | |
60 | for (i = 0; i < nr_objects; ++i) { | |
61 | if (objects[i]->offset > last_obj_offset) | |
62 | last_obj_offset = objects[i]->offset; | |
63 | } | |
64 | QSORT(sorted_by_sha, nr_objects, sha1_compare); | |
65 | } | |
66 | else | |
67 | sorted_by_sha = list = last = NULL; | |
68 | ||
69 | if (opts->flags & WRITE_IDX_VERIFY) { | |
70 | assert(index_name); | |
71 | f = hashfd_check(index_name); | |
72 | } else { | |
73 | if (!index_name) { | |
74 | struct strbuf tmp_file = STRBUF_INIT; | |
75 | fd = odb_mkstemp(&tmp_file, "pack/tmp_idx_XXXXXX"); | |
76 | index_name = strbuf_detach(&tmp_file, NULL); | |
77 | } else { | |
78 | unlink(index_name); | |
79 | fd = open(index_name, O_CREAT|O_EXCL|O_WRONLY, 0600); | |
80 | if (fd < 0) | |
81 | die_errno("unable to create '%s'", index_name); | |
82 | } | |
83 | f = hashfd(fd, index_name); | |
84 | } | |
85 | ||
86 | /* if last object's offset is >= 2^31 we should use index V2 */ | |
87 | index_version = need_large_offset(last_obj_offset, opts) ? 2 : opts->version; | |
88 | ||
89 | /* index versions 2 and above need a header */ | |
90 | if (index_version >= 2) { | |
91 | struct pack_idx_header hdr; | |
92 | hdr.idx_signature = htonl(PACK_IDX_SIGNATURE); | |
93 | hdr.idx_version = htonl(index_version); | |
94 | hashwrite(f, &hdr, sizeof(hdr)); | |
95 | } | |
96 | ||
97 | /* | |
98 | * Write the first-level table (the list is sorted, | |
99 | * but we use a 256-entry lookup to be able to avoid | |
100 | * having to do eight extra binary search iterations). | |
101 | */ | |
102 | for (i = 0; i < 256; i++) { | |
103 | struct pack_idx_entry **next = list; | |
104 | while (next < last) { | |
105 | struct pack_idx_entry *obj = *next; | |
106 | if (obj->oid.hash[0] != i) | |
107 | break; | |
108 | next++; | |
109 | } | |
110 | array[i] = htonl(next - sorted_by_sha); | |
111 | list = next; | |
112 | } | |
113 | hashwrite(f, array, 256 * 4); | |
114 | ||
115 | /* | |
116 | * Write the actual SHA1 entries.. | |
117 | */ | |
118 | list = sorted_by_sha; | |
119 | for (i = 0; i < nr_objects; i++) { | |
120 | struct pack_idx_entry *obj = *list++; | |
121 | if (index_version < 2) { | |
122 | uint32_t offset = htonl(obj->offset); | |
123 | hashwrite(f, &offset, 4); | |
124 | } | |
125 | hashwrite(f, obj->oid.hash, the_hash_algo->rawsz); | |
126 | if ((opts->flags & WRITE_IDX_STRICT) && | |
127 | (i && oideq(&list[-2]->oid, &obj->oid))) | |
128 | die("The same object %s appears twice in the pack", | |
129 | oid_to_hex(&obj->oid)); | |
130 | } | |
131 | ||
132 | if (index_version >= 2) { | |
133 | unsigned int nr_large_offset = 0; | |
134 | ||
135 | /* write the crc32 table */ | |
136 | list = sorted_by_sha; | |
137 | for (i = 0; i < nr_objects; i++) { | |
138 | struct pack_idx_entry *obj = *list++; | |
139 | uint32_t crc32_val = htonl(obj->crc32); | |
140 | hashwrite(f, &crc32_val, 4); | |
141 | } | |
142 | ||
143 | /* write the 32-bit offset table */ | |
144 | list = sorted_by_sha; | |
145 | for (i = 0; i < nr_objects; i++) { | |
146 | struct pack_idx_entry *obj = *list++; | |
147 | uint32_t offset; | |
148 | ||
149 | offset = (need_large_offset(obj->offset, opts) | |
150 | ? (0x80000000 | nr_large_offset++) | |
151 | : obj->offset); | |
152 | offset = htonl(offset); | |
153 | hashwrite(f, &offset, 4); | |
154 | } | |
155 | ||
156 | /* write the large offset table */ | |
157 | list = sorted_by_sha; | |
158 | while (nr_large_offset) { | |
159 | struct pack_idx_entry *obj = *list++; | |
160 | uint64_t offset = obj->offset; | |
161 | uint32_t split[2]; | |
162 | ||
163 | if (!need_large_offset(offset, opts)) | |
164 | continue; | |
165 | split[0] = htonl(offset >> 32); | |
166 | split[1] = htonl(offset & 0xffffffff); | |
167 | hashwrite(f, split, 8); | |
168 | nr_large_offset--; | |
169 | } | |
170 | } | |
171 | ||
172 | hashwrite(f, sha1, the_hash_algo->rawsz); | |
173 | finalize_hashfile(f, NULL, CSUM_HASH_IN_STREAM | CSUM_CLOSE | | |
174 | ((opts->flags & WRITE_IDX_VERIFY) | |
175 | ? 0 : CSUM_FSYNC)); | |
176 | return index_name; | |
177 | } | |
178 | ||
179 | off_t write_pack_header(struct hashfile *f, uint32_t nr_entries) | |
180 | { | |
181 | struct pack_header hdr; | |
182 | ||
183 | hdr.hdr_signature = htonl(PACK_SIGNATURE); | |
184 | hdr.hdr_version = htonl(PACK_VERSION); | |
185 | hdr.hdr_entries = htonl(nr_entries); | |
186 | hashwrite(f, &hdr, sizeof(hdr)); | |
187 | return sizeof(hdr); | |
188 | } | |
189 | ||
190 | /* | |
191 | * Update pack header with object_count and compute new SHA1 for pack data | |
192 | * associated to pack_fd, and write that SHA1 at the end. That new SHA1 | |
193 | * is also returned in new_pack_sha1. | |
194 | * | |
195 | * If partial_pack_sha1 is non null, then the SHA1 of the existing pack | |
196 | * (without the header update) is computed and validated against the | |
197 | * one provided in partial_pack_sha1. The validation is performed at | |
198 | * partial_pack_offset bytes in the pack file. The SHA1 of the remaining | |
199 | * data (i.e. from partial_pack_offset to the end) is then computed and | |
200 | * returned in partial_pack_sha1. | |
201 | * | |
202 | * Note that new_pack_sha1 is updated last, so both new_pack_sha1 and | |
203 | * partial_pack_sha1 can refer to the same buffer if the caller is not | |
204 | * interested in the resulting SHA1 of pack data above partial_pack_offset. | |
205 | */ | |
206 | void fixup_pack_header_footer(int pack_fd, | |
207 | unsigned char *new_pack_hash, | |
208 | const char *pack_name, | |
209 | uint32_t object_count, | |
210 | unsigned char *partial_pack_hash, | |
211 | off_t partial_pack_offset) | |
212 | { | |
213 | int aligned_sz, buf_sz = 8 * 1024; | |
214 | git_hash_ctx old_hash_ctx, new_hash_ctx; | |
215 | struct pack_header hdr; | |
216 | char *buf; | |
217 | ssize_t read_result; | |
218 | ||
219 | the_hash_algo->init_fn(&old_hash_ctx); | |
220 | the_hash_algo->init_fn(&new_hash_ctx); | |
221 | ||
222 | if (lseek(pack_fd, 0, SEEK_SET) != 0) | |
223 | die_errno("Failed seeking to start of '%s'", pack_name); | |
224 | read_result = read_in_full(pack_fd, &hdr, sizeof(hdr)); | |
225 | if (read_result < 0) | |
226 | die_errno("Unable to reread header of '%s'", pack_name); | |
227 | else if (read_result != sizeof(hdr)) | |
228 | die_errno("Unexpected short read for header of '%s'", | |
229 | pack_name); | |
230 | if (lseek(pack_fd, 0, SEEK_SET) != 0) | |
231 | die_errno("Failed seeking to start of '%s'", pack_name); | |
232 | the_hash_algo->update_fn(&old_hash_ctx, &hdr, sizeof(hdr)); | |
233 | hdr.hdr_entries = htonl(object_count); | |
234 | the_hash_algo->update_fn(&new_hash_ctx, &hdr, sizeof(hdr)); | |
235 | write_or_die(pack_fd, &hdr, sizeof(hdr)); | |
236 | partial_pack_offset -= sizeof(hdr); | |
237 | ||
238 | buf = xmalloc(buf_sz); | |
239 | aligned_sz = buf_sz - sizeof(hdr); | |
240 | for (;;) { | |
241 | ssize_t m, n; | |
242 | m = (partial_pack_hash && partial_pack_offset < aligned_sz) ? | |
243 | partial_pack_offset : aligned_sz; | |
244 | n = xread(pack_fd, buf, m); | |
245 | if (!n) | |
246 | break; | |
247 | if (n < 0) | |
248 | die_errno("Failed to checksum '%s'", pack_name); | |
249 | the_hash_algo->update_fn(&new_hash_ctx, buf, n); | |
250 | ||
251 | aligned_sz -= n; | |
252 | if (!aligned_sz) | |
253 | aligned_sz = buf_sz; | |
254 | ||
255 | if (!partial_pack_hash) | |
256 | continue; | |
257 | ||
258 | the_hash_algo->update_fn(&old_hash_ctx, buf, n); | |
259 | partial_pack_offset -= n; | |
260 | if (partial_pack_offset == 0) { | |
261 | unsigned char hash[GIT_MAX_RAWSZ]; | |
262 | the_hash_algo->final_fn(hash, &old_hash_ctx); | |
263 | if (!hasheq(hash, partial_pack_hash)) | |
264 | die("Unexpected checksum for %s " | |
265 | "(disk corruption?)", pack_name); | |
266 | /* | |
267 | * Now let's compute the SHA1 of the remainder of the | |
268 | * pack, which also means making partial_pack_offset | |
269 | * big enough not to matter anymore. | |
270 | */ | |
271 | the_hash_algo->init_fn(&old_hash_ctx); | |
272 | partial_pack_offset = ~partial_pack_offset; | |
273 | partial_pack_offset -= MSB(partial_pack_offset, 1); | |
274 | } | |
275 | } | |
276 | free(buf); | |
277 | ||
278 | if (partial_pack_hash) | |
279 | the_hash_algo->final_fn(partial_pack_hash, &old_hash_ctx); | |
280 | the_hash_algo->final_fn(new_pack_hash, &new_hash_ctx); | |
281 | write_or_die(pack_fd, new_pack_hash, the_hash_algo->rawsz); | |
282 | fsync_or_die(pack_fd, pack_name); | |
283 | } | |
284 | ||
285 | char *index_pack_lockfile(int ip_out) | |
286 | { | |
287 | char packname[GIT_MAX_HEXSZ + 6]; | |
288 | const int len = the_hash_algo->hexsz + 6; | |
289 | ||
290 | /* | |
291 | * The first thing we expect from index-pack's output | |
292 | * is "pack\t%40s\n" or "keep\t%40s\n" (46 bytes) where | |
293 | * %40s is the newly created pack SHA1 name. In the "keep" | |
294 | * case, we need it to remove the corresponding .keep file | |
295 | * later on. If we don't get that then tough luck with it. | |
296 | */ | |
297 | if (read_in_full(ip_out, packname, len) == len && packname[len-1] == '\n') { | |
298 | const char *name; | |
299 | packname[len-1] = 0; | |
300 | if (skip_prefix(packname, "keep\t", &name)) | |
301 | return xstrfmt("%s/pack/pack-%s.keep", | |
302 | get_object_directory(), name); | |
303 | } | |
304 | return NULL; | |
305 | } | |
306 | ||
307 | /* | |
308 | * The per-object header is a pretty dense thing, which is | |
309 | * - first byte: low four bits are "size", then three bits of "type", | |
310 | * and the high bit is "size continues". | |
311 | * - each byte afterwards: low seven bits are size continuation, | |
312 | * with the high bit being "size continues" | |
313 | */ | |
314 | int encode_in_pack_object_header(unsigned char *hdr, int hdr_len, | |
315 | enum object_type type, uintmax_t size) | |
316 | { | |
317 | int n = 1; | |
318 | unsigned char c; | |
319 | ||
320 | if (type < OBJ_COMMIT || type > OBJ_REF_DELTA) | |
321 | die("bad type %d", type); | |
322 | ||
323 | c = (type << 4) | (size & 15); | |
324 | size >>= 4; | |
325 | while (size) { | |
326 | if (n == hdr_len) | |
327 | die("object size is too enormous to format"); | |
328 | *hdr++ = c | 0x80; | |
329 | c = size & 0x7f; | |
330 | size >>= 7; | |
331 | n++; | |
332 | } | |
333 | *hdr = c; | |
334 | return n; | |
335 | } | |
336 | ||
337 | struct hashfile *create_tmp_packfile(char **pack_tmp_name) | |
338 | { | |
339 | struct strbuf tmpname = STRBUF_INIT; | |
340 | int fd; | |
341 | ||
342 | fd = odb_mkstemp(&tmpname, "pack/tmp_pack_XXXXXX"); | |
343 | *pack_tmp_name = strbuf_detach(&tmpname, NULL); | |
344 | return hashfd(fd, *pack_tmp_name); | |
345 | } | |
346 | ||
347 | void finish_tmp_packfile(struct strbuf *name_buffer, | |
348 | const char *pack_tmp_name, | |
349 | struct pack_idx_entry **written_list, | |
350 | uint32_t nr_written, | |
351 | struct pack_idx_option *pack_idx_opts, | |
352 | unsigned char sha1[]) | |
353 | { | |
354 | const char *idx_tmp_name; | |
355 | int basename_len = name_buffer->len; | |
356 | ||
357 | if (adjust_shared_perm(pack_tmp_name)) | |
358 | die_errno("unable to make temporary pack file readable"); | |
359 | ||
360 | idx_tmp_name = write_idx_file(NULL, written_list, nr_written, | |
361 | pack_idx_opts, sha1); | |
362 | if (adjust_shared_perm(idx_tmp_name)) | |
363 | die_errno("unable to make temporary index file readable"); | |
364 | ||
365 | strbuf_addf(name_buffer, "%s.pack", sha1_to_hex(sha1)); | |
366 | ||
367 | if (rename(pack_tmp_name, name_buffer->buf)) | |
368 | die_errno("unable to rename temporary pack file"); | |
369 | ||
370 | strbuf_setlen(name_buffer, basename_len); | |
371 | ||
372 | strbuf_addf(name_buffer, "%s.idx", sha1_to_hex(sha1)); | |
373 | if (rename(idx_tmp_name, name_buffer->buf)) | |
374 | die_errno("unable to rename temporary index file"); | |
375 | ||
376 | strbuf_setlen(name_buffer, basename_len); | |
377 | ||
378 | free((void *)idx_tmp_name); | |
379 | } |