2 * Copyright (c) 2011, Google Inc.
6 #include "repository.h"
7 #include "object-store.h"
8 #include "replace-object.h"
18 typedef int (*open_istream_fn
)(struct git_istream
*,
21 const struct object_id
*,
23 typedef int (*close_istream_fn
)(struct git_istream
*);
24 typedef ssize_t (*read_istream_fn
)(struct git_istream
*, char *, size_t);
27 close_istream_fn close
;
31 #define open_method_decl(name) \
32 int open_istream_ ##name \
33 (struct git_istream *st, struct repository *r, \
34 struct object_info *oi, const struct object_id *oid, \
35 enum object_type *type)
37 #define close_method_decl(name) \
38 int close_istream_ ##name \
39 (struct git_istream *st)
41 #define read_method_decl(name) \
42 ssize_t read_istream_ ##name \
43 (struct git_istream *st, char *buf, size_t sz)
45 #define FILTER_BUFFER (1024*16)
47 struct filtered_istream
{
48 struct git_istream
*upstream
;
49 struct stream_filter
*filter
;
50 char ibuf
[FILTER_BUFFER
];
51 char obuf
[FILTER_BUFFER
];
58 const struct stream_vtbl
*vtbl
;
59 unsigned long size
; /* inflated size of full object */
61 enum { z_unused
, z_used
, z_done
, z_error
} z_state
;
65 char *buf
; /* from read_object() */
66 unsigned long read_ptr
;
71 unsigned long mapsize
;
78 struct packed_git
*pack
;
82 struct filtered_istream filtered
;
86 /*****************************************************************
90 *****************************************************************/
92 static void close_deflated_stream(struct git_istream
*st
)
94 if (st
->z_state
== z_used
)
95 git_inflate_end(&st
->z
);
99 /*****************************************************************
103 *****************************************************************/
105 static close_method_decl(filtered
)
107 free_stream_filter(st
->u
.filtered
.filter
);
108 return close_istream(st
->u
.filtered
.upstream
);
111 static read_method_decl(filtered
)
113 struct filtered_istream
*fs
= &(st
->u
.filtered
);
117 /* do we already have filtered output? */
118 if (fs
->o_ptr
< fs
->o_end
) {
119 size_t to_move
= fs
->o_end
- fs
->o_ptr
;
122 memcpy(buf
+ filled
, fs
->obuf
+ fs
->o_ptr
, to_move
);
123 fs
->o_ptr
+= to_move
;
128 fs
->o_end
= fs
->o_ptr
= 0;
130 /* do we have anything to feed the filter with? */
131 if (fs
->i_ptr
< fs
->i_end
) {
132 size_t to_feed
= fs
->i_end
- fs
->i_ptr
;
133 size_t to_receive
= FILTER_BUFFER
;
134 if (stream_filter(fs
->filter
,
135 fs
->ibuf
+ fs
->i_ptr
, &to_feed
,
136 fs
->obuf
, &to_receive
))
138 fs
->i_ptr
= fs
->i_end
- to_feed
;
139 fs
->o_end
= FILTER_BUFFER
- to_receive
;
143 /* tell the filter to drain upon no more input */
144 if (fs
->input_finished
) {
145 size_t to_receive
= FILTER_BUFFER
;
146 if (stream_filter(fs
->filter
,
148 fs
->obuf
, &to_receive
))
150 fs
->o_end
= FILTER_BUFFER
- to_receive
;
155 fs
->i_end
= fs
->i_ptr
= 0;
157 /* refill the input from the upstream */
158 if (!fs
->input_finished
) {
159 fs
->i_end
= read_istream(fs
->upstream
, fs
->ibuf
, FILTER_BUFFER
);
165 fs
->input_finished
= 1;
170 static struct stream_vtbl filtered_vtbl
= {
171 close_istream_filtered
,
172 read_istream_filtered
,
175 static struct git_istream
*attach_stream_filter(struct git_istream
*st
,
176 struct stream_filter
*filter
)
178 struct git_istream
*ifs
= xmalloc(sizeof(*ifs
));
179 struct filtered_istream
*fs
= &(ifs
->u
.filtered
);
181 ifs
->vtbl
= &filtered_vtbl
;
184 fs
->i_end
= fs
->i_ptr
= 0;
185 fs
->o_end
= fs
->o_ptr
= 0;
186 fs
->input_finished
= 0;
187 ifs
->size
= -1; /* unknown */
191 /*****************************************************************
193 * Loose object stream
195 *****************************************************************/
197 static read_method_decl(loose
)
199 size_t total_read
= 0;
201 switch (st
->z_state
) {
210 if (st
->u
.loose
.hdr_used
< st
->u
.loose
.hdr_avail
) {
211 size_t to_copy
= st
->u
.loose
.hdr_avail
- st
->u
.loose
.hdr_used
;
214 memcpy(buf
, st
->u
.loose
.hdr
+ st
->u
.loose
.hdr_used
, to_copy
);
215 st
->u
.loose
.hdr_used
+= to_copy
;
216 total_read
+= to_copy
;
219 while (total_read
< sz
) {
222 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
223 st
->z
.avail_out
= sz
- total_read
;
224 status
= git_inflate(&st
->z
, Z_FINISH
);
226 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
228 if (status
== Z_STREAM_END
) {
229 git_inflate_end(&st
->z
);
230 st
->z_state
= z_done
;
233 if (status
!= Z_OK
&& (status
!= Z_BUF_ERROR
|| total_read
< sz
)) {
234 git_inflate_end(&st
->z
);
235 st
->z_state
= z_error
;
242 static close_method_decl(loose
)
244 close_deflated_stream(st
);
245 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
249 static struct stream_vtbl loose_vtbl
= {
254 static open_method_decl(loose
)
256 st
->u
.loose
.mapped
= map_loose_object(r
, oid
, &st
->u
.loose
.mapsize
);
257 if (!st
->u
.loose
.mapped
)
259 if ((unpack_loose_header(&st
->z
,
263 sizeof(st
->u
.loose
.hdr
)) < 0) ||
264 (parse_loose_header(st
->u
.loose
.hdr
, &st
->size
) < 0)) {
265 git_inflate_end(&st
->z
);
266 munmap(st
->u
.loose
.mapped
, st
->u
.loose
.mapsize
);
270 st
->u
.loose
.hdr_used
= strlen(st
->u
.loose
.hdr
) + 1;
271 st
->u
.loose
.hdr_avail
= st
->z
.total_out
;
272 st
->z_state
= z_used
;
274 st
->vtbl
= &loose_vtbl
;
279 /*****************************************************************
281 * Non-delta packed object stream
283 *****************************************************************/
285 static read_method_decl(pack_non_delta
)
287 size_t total_read
= 0;
289 switch (st
->z_state
) {
291 memset(&st
->z
, 0, sizeof(st
->z
));
292 git_inflate_init(&st
->z
);
293 st
->z_state
= z_used
;
303 while (total_read
< sz
) {
305 struct pack_window
*window
= NULL
;
306 unsigned char *mapped
;
308 mapped
= use_pack(st
->u
.in_pack
.pack
, &window
,
309 st
->u
.in_pack
.pos
, &st
->z
.avail_in
);
311 st
->z
.next_out
= (unsigned char *)buf
+ total_read
;
312 st
->z
.avail_out
= sz
- total_read
;
313 st
->z
.next_in
= mapped
;
314 status
= git_inflate(&st
->z
, Z_FINISH
);
316 st
->u
.in_pack
.pos
+= st
->z
.next_in
- mapped
;
317 total_read
= st
->z
.next_out
- (unsigned char *)buf
;
320 if (status
== Z_STREAM_END
) {
321 git_inflate_end(&st
->z
);
322 st
->z_state
= z_done
;
327 * Unlike the loose object case, we do not have to worry here
328 * about running out of input bytes and spinning infinitely. If
329 * we get Z_BUF_ERROR due to too few input bytes, then we'll
330 * replenish them in the next use_pack() call when we loop. If
331 * we truly hit the end of the pack (i.e., because it's corrupt
332 * or truncated), then use_pack() catches that and will die().
334 if (status
!= Z_OK
&& status
!= Z_BUF_ERROR
) {
335 git_inflate_end(&st
->z
);
336 st
->z_state
= z_error
;
343 static close_method_decl(pack_non_delta
)
345 close_deflated_stream(st
);
349 static struct stream_vtbl pack_non_delta_vtbl
= {
350 close_istream_pack_non_delta
,
351 read_istream_pack_non_delta
,
354 static open_method_decl(pack_non_delta
)
356 struct pack_window
*window
;
357 enum object_type in_pack_type
;
359 st
->u
.in_pack
.pack
= oi
->u
.packed
.pack
;
360 st
->u
.in_pack
.pos
= oi
->u
.packed
.offset
;
363 in_pack_type
= unpack_object_header(st
->u
.in_pack
.pack
,
368 switch (in_pack_type
) {
370 return -1; /* we do not do deltas for now */
377 st
->z_state
= z_unused
;
378 st
->vtbl
= &pack_non_delta_vtbl
;
383 /*****************************************************************
387 *****************************************************************/
389 static close_method_decl(incore
)
391 free(st
->u
.incore
.buf
);
395 static read_method_decl(incore
)
397 size_t read_size
= sz
;
398 size_t remainder
= st
->size
- st
->u
.incore
.read_ptr
;
400 if (remainder
<= read_size
)
401 read_size
= remainder
;
403 memcpy(buf
, st
->u
.incore
.buf
+ st
->u
.incore
.read_ptr
, read_size
);
404 st
->u
.incore
.read_ptr
+= read_size
;
409 static struct stream_vtbl incore_vtbl
= {
410 close_istream_incore
,
414 static open_method_decl(incore
)
416 st
->u
.incore
.buf
= read_object_file_extended(r
, oid
, type
, &st
->size
, 0);
417 st
->u
.incore
.read_ptr
= 0;
418 st
->vtbl
= &incore_vtbl
;
420 return st
->u
.incore
.buf
? 0 : -1;
423 /*****************************************************************************
424 * static helpers variables and functions for users of streaming interface
425 *****************************************************************************/
427 static open_istream_fn open_istream_tbl
[] = {
430 open_istream_pack_non_delta
,
433 static enum input_source
istream_source(struct repository
*r
,
434 const struct object_id
*oid
,
435 enum object_type
*type
,
436 struct object_info
*oi
)
443 status
= oid_object_info_extended(r
, oid
, oi
, 0);
447 switch (oi
->whence
) {
451 if (!oi
->u
.packed
.is_delta
&& big_file_threshold
< size
)
452 return pack_non_delta
;
460 /****************************************************************
461 * Users of streaming interface
462 ****************************************************************/
464 int close_istream(struct git_istream
*st
)
466 int r
= st
->vtbl
->close(st
);
471 ssize_t
read_istream(struct git_istream
*st
, void *buf
, size_t sz
)
473 return st
->vtbl
->read(st
, buf
, sz
);
476 struct git_istream
*open_istream(struct repository
*r
,
477 const struct object_id
*oid
,
478 enum object_type
*type
,
480 struct stream_filter
*filter
)
482 struct git_istream
*st
;
483 struct object_info oi
= OBJECT_INFO_INIT
;
484 const struct object_id
*real
= lookup_replace_object(r
, oid
);
485 enum input_source src
= istream_source(r
, real
, type
, &oi
);
490 st
= xmalloc(sizeof(*st
));
491 if (open_istream_tbl
[src
](st
, r
, &oi
, real
, type
)) {
492 if (open_istream_incore(st
, r
, &oi
, real
, type
)) {
498 /* Add "&& !is_null_stream_filter(filter)" for performance */
499 struct git_istream
*nst
= attach_stream_filter(st
, filter
);
511 int stream_blob_to_fd(int fd
, const struct object_id
*oid
, struct stream_filter
*filter
,
514 struct git_istream
*st
;
515 enum object_type type
;
520 st
= open_istream(the_repository
, oid
, &type
, &sz
, filter
);
523 free_stream_filter(filter
);
526 if (type
!= OBJ_BLOB
)
530 ssize_t wrote
, holeto
;
531 ssize_t readlen
= read_istream(st
, buf
, sizeof(buf
));
537 if (can_seek
&& sizeof(buf
) == readlen
) {
538 for (holeto
= 0; holeto
< readlen
; holeto
++)
541 if (readlen
== holeto
) {
547 if (kept
&& lseek(fd
, kept
, SEEK_CUR
) == (off_t
) -1)
551 wrote
= write_in_full(fd
, buf
, readlen
);
556 if (kept
&& (lseek(fd
, kept
- 1, SEEK_CUR
) == (off_t
) -1 ||
557 xwrite(fd
, "", 1) != 1))