]> git.ipfire.org Git - thirdparty/git.git/blame - builtin-unpack-objects.c
Merge branch 'master' of git://repo.or.cz/git/fastimport
[thirdparty/git.git] / builtin-unpack-objects.c
CommitLineData
64413630 1#include "builtin.h"
bad50dc8 2#include "cache.h"
74536958 3#include "object.h"
8ee378a0 4#include "delta.h"
a733cb60 5#include "pack.h"
8e440259
PE
6#include "blob.h"
7#include "commit.h"
8#include "tag.h"
9#include "tree.h"
bad50dc8 10
3b67d291
JH
11static int dry_run, quiet, recover, has_errors;
12static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [-r] < pack-file";
bad50dc8 13
67e5a5ec
LT
14/* We always read in 4kB chunks. */
15static unsigned char buffer[4096];
209c554a 16static unsigned long offset, len, consumed_bytes;
67e5a5ec 17static SHA_CTX ctx;
bad50dc8 18
67e5a5ec
LT
19/*
20 * Make sure at least "min" bytes are available in the buffer, and
21 * return the pointer to the buffer.
22 */
79a65697 23static void *fill(int min)
67e5a5ec
LT
24{
25 if (min <= len)
26 return buffer + offset;
67e5a5ec
LT
27 if (min > sizeof(buffer))
28 die("cannot fill %d bytes", min);
29 if (offset) {
30 SHA1_Update(&ctx, buffer, offset);
79a65697 31 memmove(buffer, buffer + offset, len);
67e5a5ec
LT
32 offset = 0;
33 }
34 do {
1c15afb9 35 int ret = xread(0, buffer + len, sizeof(buffer) - len);
67e5a5ec
LT
36 if (ret <= 0) {
37 if (!ret)
38 die("early EOF");
67e5a5ec
LT
39 die("read error on input: %s", strerror(errno));
40 }
41 len += ret;
42 } while (len < min);
43 return buffer;
44}
74536958 45
67e5a5ec
LT
46static void use(int bytes)
47{
48 if (bytes > len)
49 die("used more bytes than were available");
50 len -= bytes;
51 offset += bytes;
209c554a 52 consumed_bytes += bytes;
67e5a5ec 53}
bad50dc8 54
67e5a5ec 55static void *get_data(unsigned long size)
bad50dc8 56{
67e5a5ec
LT
57 z_stream stream;
58 void *buf = xmalloc(size);
59
67e5a5ec
LT
60 memset(&stream, 0, sizeof(stream));
61
62 stream.next_out = buf;
63 stream.avail_out = size;
64 stream.next_in = fill(1);
65 stream.avail_in = len;
66 inflateInit(&stream);
67
68 for (;;) {
69 int ret = inflate(&stream, 0);
70 use(len - stream.avail_in);
71 if (stream.total_out == size && ret == Z_STREAM_END)
72 break;
f986f2c8
JH
73 if (ret != Z_OK) {
74 error("inflate returned %d\n", ret);
75 free(buf);
76 buf = NULL;
3b67d291 77 if (!recover)
f986f2c8
JH
78 exit(1);
79 has_errors = 1;
80 break;
81 }
67e5a5ec
LT
82 stream.next_in = fill(1);
83 stream.avail_in = len;
84 }
ee639140 85 inflateEnd(&stream);
67e5a5ec 86 return buf;
bad50dc8
LT
87}
88
67e5a5ec
LT
89struct delta_info {
90 unsigned char base_sha1[20];
209c554a 91 unsigned long base_offset;
67e5a5ec
LT
92 unsigned long size;
93 void *delta;
209c554a 94 unsigned nr;
67e5a5ec
LT
95 struct delta_info *next;
96};
97
98static struct delta_info *delta_list;
99
209c554a
NP
100static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
101 unsigned long base_offset,
102 void *delta, unsigned long size)
bad50dc8 103{
67e5a5ec 104 struct delta_info *info = xmalloc(sizeof(*info));
bad50dc8 105
e702496e 106 hashcpy(info->base_sha1, base_sha1);
209c554a 107 info->base_offset = base_offset;
67e5a5ec
LT
108 info->size = size;
109 info->delta = delta;
209c554a 110 info->nr = nr;
67e5a5ec
LT
111 info->next = delta_list;
112 delta_list = info;
bad50dc8
LT
113}
114
209c554a
NP
115struct obj_info {
116 unsigned long offset;
117 unsigned char sha1[20];
118};
119
120static struct obj_info *obj_list;
67e5a5ec 121
209c554a
NP
122static void added_object(unsigned nr, const char *type, void *data,
123 unsigned long size);
124
125static void write_object(unsigned nr, void *buf, unsigned long size,
126 const char *type)
cca7081a 127{
209c554a 128 if (write_sha1_file(buf, size, type, obj_list[nr].sha1) < 0)
cca7081a 129 die("failed to write object");
209c554a 130 added_object(nr, type, buf, size);
cca7081a
LT
131}
132
209c554a 133static void resolve_delta(unsigned nr, const char *type,
f986f2c8
JH
134 void *base, unsigned long base_size,
135 void *delta, unsigned long delta_size)
bad50dc8 136{
67e5a5ec
LT
137 void *result;
138 unsigned long result_size;
bad50dc8 139
67e5a5ec
LT
140 result = patch_delta(base, base_size,
141 delta, delta_size,
142 &result_size);
143 if (!result)
144 die("failed to apply delta");
145 free(delta);
209c554a 146 write_object(nr, result, result_size, type);
67e5a5ec 147 free(result);
bad50dc8
LT
148}
149
209c554a
NP
150static void added_object(unsigned nr, const char *type, void *data,
151 unsigned long size)
74536958 152{
67e5a5ec
LT
153 struct delta_info **p = &delta_list;
154 struct delta_info *info;
155
156 while ((info = *p) != NULL) {
209c554a
NP
157 if (!hashcmp(info->base_sha1, obj_list[nr].sha1) ||
158 info->base_offset == obj_list[nr].offset) {
67e5a5ec
LT
159 *p = info->next;
160 p = &delta_list;
209c554a
NP
161 resolve_delta(info->nr, type, data, size,
162 info->delta, info->size);
67e5a5ec
LT
163 free(info);
164 continue;
165 }
166 p = &info->next;
167 }
168}
169
209c554a
NP
170static void unpack_non_delta_entry(enum object_type kind, unsigned long size,
171 unsigned nr)
67e5a5ec
LT
172{
173 void *buf = get_data(size);
cca7081a 174 const char *type;
8ee378a0 175
8ee378a0 176 switch (kind) {
8e440259
PE
177 case OBJ_COMMIT: type = commit_type; break;
178 case OBJ_TREE: type = tree_type; break;
179 case OBJ_BLOB: type = blob_type; break;
180 case OBJ_TAG: type = tag_type; break;
67e5a5ec 181 default: die("bad type %d", kind);
8ee378a0 182 }
f986f2c8 183 if (!dry_run && buf)
209c554a 184 write_object(nr, buf, size, type);
67e5a5ec 185 free(buf);
8ee378a0
JH
186}
187
209c554a
NP
188static void unpack_delta_entry(enum object_type kind, unsigned long delta_size,
189 unsigned nr)
8ee378a0 190{
67e5a5ec
LT
191 void *delta_data, *base;
192 unsigned long base_size;
8ee378a0 193 char type[20];
67e5a5ec 194 unsigned char base_sha1[20];
8ee378a0 195
209c554a
NP
196 if (kind == OBJ_REF_DELTA) {
197 hashcpy(base_sha1, fill(20));
198 use(20);
199 delta_data = get_data(delta_size);
200 if (dry_run || !delta_data) {
201 free(delta_data);
202 return;
203 }
204 if (!has_sha1_file(base_sha1)) {
205 hashcpy(obj_list[nr].sha1, null_sha1);
206 add_delta_to_list(nr, base_sha1, 0, delta_data, delta_size);
207 return;
208 }
209 } else {
210 unsigned base_found = 0;
211 unsigned char *pack, c;
212 unsigned long base_offset;
213 unsigned lo, mid, hi;
c4fb06c0 214
209c554a
NP
215 pack = fill(1);
216 c = *pack;
217 use(1);
218 base_offset = c & 127;
219 while (c & 128) {
220 base_offset += 1;
221 if (!base_offset || base_offset & ~(~0UL >> 7))
222 die("offset value overflow for delta base object");
223 pack = fill(1);
224 c = *pack;
225 use(1);
226 base_offset = (base_offset << 7) + (c & 127);
227 }
228 base_offset = obj_list[nr].offset - base_offset;
8ee378a0 229
209c554a
NP
230 delta_data = get_data(delta_size);
231 if (dry_run || !delta_data) {
232 free(delta_data);
233 return;
234 }
235 lo = 0;
236 hi = nr;
237 while (lo < hi) {
238 mid = (lo + hi)/2;
239 if (base_offset < obj_list[mid].offset) {
240 hi = mid;
241 } else if (base_offset > obj_list[mid].offset) {
242 lo = mid + 1;
243 } else {
244 hashcpy(base_sha1, obj_list[mid].sha1);
245 base_found = !is_null_sha1(base_sha1);
246 break;
247 }
248 }
249 if (!base_found) {
250 /* The delta base object is itself a delta that
251 has not been resolved yet. */
252 hashcpy(obj_list[nr].sha1, null_sha1);
253 add_delta_to_list(nr, null_sha1, base_offset, delta_data, delta_size);
254 return;
255 }
8ee378a0 256 }
209c554a 257
c4fb06c0 258 base = read_sha1_file(base_sha1, type, &base_size);
f986f2c8
JH
259 if (!base) {
260 error("failed to read delta-pack base object %s",
261 sha1_to_hex(base_sha1));
3b67d291 262 if (!recover)
f986f2c8
JH
263 exit(1);
264 has_errors = 1;
265 return;
266 }
209c554a 267 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
ee639140 268 free(base);
8ee378a0
JH
269}
270
d36f7b80 271static void unpack_one(unsigned nr, unsigned total)
8ee378a0 272{
01247d87 273 unsigned shift;
a733cb60 274 unsigned char *pack, c;
67e5a5ec
LT
275 unsigned long size;
276 enum object_type type;
74536958 277
209c554a
NP
278 obj_list[nr].offset = consumed_bytes;
279
67e5a5ec
LT
280 pack = fill(1);
281 c = *pack;
282 use(1);
a733cb60
LT
283 type = (c >> 4) & 7;
284 size = (c & 15);
01247d87 285 shift = 4;
a733cb60 286 while (c & 0x80) {
67e5a5ec 287 pack = fill(1);
209c554a 288 c = *pack;
67e5a5ec 289 use(1);
01247d87
LT
290 size += (c & 0x7f) << shift;
291 shift += 7;
a733cb60 292 }
d36f7b80
LT
293 if (!quiet) {
294 static unsigned long last_sec;
295 static unsigned last_percent;
296 struct timeval now;
209c554a 297 unsigned percentage = ((nr+1) * 100) / total;
d36f7b80
LT
298
299 gettimeofday(&now, NULL);
300 if (percentage != last_percent || now.tv_sec != last_sec) {
301 last_sec = now.tv_sec;
302 last_percent = percentage;
209c554a
NP
303 fprintf(stderr, "%4u%% (%u/%u) done\r",
304 percentage, (nr+1), total);
d36f7b80
LT
305 }
306 }
a733cb60
LT
307 switch (type) {
308 case OBJ_COMMIT:
309 case OBJ_TREE:
310 case OBJ_BLOB:
311 case OBJ_TAG:
209c554a 312 unpack_non_delta_entry(type, size, nr);
a733cb60 313 return;
eb32d236 314 case OBJ_REF_DELTA:
209c554a
NP
315 case OBJ_OFS_DELTA:
316 unpack_delta_entry(type, size, nr);
a733cb60 317 return;
67e5a5ec 318 default:
f986f2c8
JH
319 error("bad object type %d", type);
320 has_errors = 1;
3b67d291 321 if (recover)
f986f2c8
JH
322 return;
323 exit(1);
74536958
LT
324 }
325}
326
74536958
LT
327static void unpack_all(void)
328{
67e5a5ec
LT
329 int i;
330 struct pack_header *hdr = fill(sizeof(struct pack_header));
67e5a5ec
LT
331 unsigned nr_objects = ntohl(hdr->hdr_entries);
332
333 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
334 die("bad pack file");
d60fc1c8
NP
335 if (!pack_version_ok(hdr->hdr_version))
336 die("unknown pack file version %d", ntohl(hdr->hdr_version));
67e5a5ec
LT
337 fprintf(stderr, "Unpacking %d objects\n", nr_objects);
338
209c554a 339 obj_list = xmalloc(nr_objects * sizeof(*obj_list));
67e5a5ec
LT
340 use(sizeof(struct pack_header));
341 for (i = 0; i < nr_objects; i++)
209c554a 342 unpack_one(i, nr_objects);
67e5a5ec
LT
343 if (delta_list)
344 die("unresolved deltas left after unpacking");
74536958
LT
345}
346
64413630 347int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
bad50dc8
LT
348{
349 int i;
67e5a5ec 350 unsigned char sha1[20];
bad50dc8 351
8e273641 352 git_config(git_default_config);
53228a5f 353
476e8011
JH
354 quiet = !isatty(2);
355
bad50dc8
LT
356 for (i = 1 ; i < argc; i++) {
357 const char *arg = argv[i];
358
359 if (*arg == '-') {
74536958
LT
360 if (!strcmp(arg, "-n")) {
361 dry_run = 1;
362 continue;
363 }
d36f7b80
LT
364 if (!strcmp(arg, "-q")) {
365 quiet = 1;
366 continue;
367 }
f986f2c8 368 if (!strcmp(arg, "-r")) {
3b67d291 369 recover = 1;
f986f2c8
JH
370 continue;
371 }
bed006fb
NP
372 if (!strncmp(arg, "--pack_header=", 14)) {
373 struct pack_header *hdr;
374 char *c;
375
376 hdr = (struct pack_header *)buffer;
377 hdr->hdr_signature = htonl(PACK_SIGNATURE);
378 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
379 if (*c != ',')
380 die("bad %s", arg);
381 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
382 if (*c)
383 die("bad %s", arg);
384 len = sizeof(*hdr);
385 continue;
386 }
bad50dc8
LT
387 usage(unpack_usage);
388 }
67e5a5ec
LT
389
390 /* We don't take any non-flag arguments now.. Maybe some day */
bad50dc8 391 usage(unpack_usage);
67e5a5ec
LT
392 }
393 SHA1_Init(&ctx);
74536958 394 unpack_all();
67e5a5ec
LT
395 SHA1_Update(&ctx, buffer, offset);
396 SHA1_Final(sha1, &ctx);
a89fccd2 397 if (hashcmp(fill(20), sha1))
67e5a5ec
LT
398 die("final sha1 did not match");
399 use(20);
400
401 /* Write the last part of the buffer to stdout */
402 while (len) {
1c15afb9
JH
403 int ret = xwrite(1, buffer + offset, len);
404 if (ret <= 0)
67e5a5ec 405 break;
67e5a5ec
LT
406 len -= ret;
407 offset += ret;
408 }
409
410 /* All done */
d36f7b80
LT
411 if (!quiet)
412 fprintf(stderr, "\n");
f986f2c8 413 return has_errors;
bad50dc8 414}