Merge branch 'ps/object-read-stream'

The "git_istream" abstraction has been revamped to make it easier
to interface with pluggable object database design.

* ps/object-read-stream:
  streaming: drop redundant type and size pointers
  streaming: move into object database subsystem
  streaming: refactor interface to be object-database-centric
  streaming: move logic to read packed objects streams into backend
  streaming: move logic to read loose objects streams into backend
  streaming: make the `odb_read_stream` definition public
  streaming: get rid of `the_repository`
  streaming: rely on object sources to create object stream
  packfile: introduce function to read object info from a store
  streaming: move zlib stream into backends
  streaming: create structure for filtered object streams
  streaming: create structure for packed object streams
  streaming: create structure for loose object streams
  streaming: create structure for in-core object streams
  streaming: allocate stream inside the backend-specific logic
  streaming: explicitly pass packfile info when streaming a packed object
  streaming: propagate final object type via the stream
  streaming: drop the `open()` callback function
  streaming: rename `git_istream` into `odb_read_stream`

diff --cc Makefile
Simple merge

diff --cc builtin/fsck.c
Simple merge

diff --cc builtin/index-pack.c
Simple merge

diff --cc builtin/pack-objects.c
Simple merge

diff --cc meson.build
Simple merge

diff --cc object-file.c
Simple merge

diff --cc odb.c
Simple merge

diff --cc packfile.c
index 3d8b994a617e81e08b06d5b3b3aaa2999c1f73a0,7a16aaa90d0a2f817d2de8a5b2aac5b594f7ecfc..c88bd92619913bf78f5dde5c13e0ea81a28e9fd5
--- 1/packfile.c
--- 2/packfile.c
+++ b/packfile.c
@@@ -2105,9 -2049,11 +2090,11 @@@ static int fill_pack_entry(const struc
        return 1;
  }
  
- int find_pack_entry(struct repository *r, const struct object_id *oid, struct pack_entry *e)
+ static int find_pack_entry(struct repository *r,
+                          const struct object_id *oid,
+                          struct pack_entry *e)
  {
 -      struct list_head *pos;
 +      struct packfile_list_entry *l;
  
        packfile_store_prepare(r->objects->packfiles);
  
@@@ -2394,9 -2401,136 +2432,136 @@@ void packfile_store_free(struct packfil
  
  void packfile_store_close(struct packfile_store *store)
  {
 -      for (struct packed_git *p = store->packs; p; p = p->next) {
 -              if (p->do_not_close)
 +      for (struct packfile_list_entry *e = store->packs.head; e; e = e->next) {
 +              if (e->pack->do_not_close)
                        BUG("want to close pack marked 'do-not-close'");
 -              close_pack(p);
 +              close_pack(e->pack);
        }
  }
+ 
+ struct odb_packed_read_stream {
+       struct odb_read_stream base;
+       struct packed_git *pack;
+       git_zstream z;
+       enum {
+               ODB_PACKED_READ_STREAM_UNINITIALIZED,
+               ODB_PACKED_READ_STREAM_INUSE,
+               ODB_PACKED_READ_STREAM_DONE,
+               ODB_PACKED_READ_STREAM_ERROR,
+       } z_state;
+       off_t pos;
+ };
+ 
+ static ssize_t read_istream_pack_non_delta(struct odb_read_stream *_st, char *buf,
+                                          size_t sz)
+ {
+       struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
+       size_t total_read = 0;
+ 
+       switch (st->z_state) {
+       case ODB_PACKED_READ_STREAM_UNINITIALIZED:
+               memset(&st->z, 0, sizeof(st->z));
+               git_inflate_init(&st->z);
+               st->z_state = ODB_PACKED_READ_STREAM_INUSE;
+               break;
+       case ODB_PACKED_READ_STREAM_DONE:
+               return 0;
+       case ODB_PACKED_READ_STREAM_ERROR:
+               return -1;
+       case ODB_PACKED_READ_STREAM_INUSE:
+               break;
+       }
+ 
+       while (total_read < sz) {
+               int status;
+               struct pack_window *window = NULL;
+               unsigned char *mapped;
+ 
+               mapped = use_pack(st->pack, &window,
+                                 st->pos, &st->z.avail_in);
+ 
+               st->z.next_out = (unsigned char *)buf + total_read;
+               st->z.avail_out = sz - total_read;
+               st->z.next_in = mapped;
+               status = git_inflate(&st->z, Z_FINISH);
+ 
+               st->pos += st->z.next_in - mapped;
+               total_read = st->z.next_out - (unsigned char *)buf;
+               unuse_pack(&window);
+ 
+               if (status == Z_STREAM_END) {
+                       git_inflate_end(&st->z);
+                       st->z_state = ODB_PACKED_READ_STREAM_DONE;
+                       break;
+               }
+ 
+               /*
+                * Unlike the loose object case, we do not have to worry here
+                * about running out of input bytes and spinning infinitely. If
+                * we get Z_BUF_ERROR due to too few input bytes, then we'll
+                * replenish them in the next use_pack() call when we loop. If
+                * we truly hit the end of the pack (i.e., because it's corrupt
+                * or truncated), then use_pack() catches that and will die().
+                */
+               if (status != Z_OK && status != Z_BUF_ERROR) {
+                       git_inflate_end(&st->z);
+                       st->z_state = ODB_PACKED_READ_STREAM_ERROR;
+                       return -1;
+               }
+       }
+       return total_read;
+ }
+ 
+ static int close_istream_pack_non_delta(struct odb_read_stream *_st)
+ {
+       struct odb_packed_read_stream *st = (struct odb_packed_read_stream *)_st;
+       if (st->z_state == ODB_PACKED_READ_STREAM_INUSE)
+               git_inflate_end(&st->z);
+       return 0;
+ }
+ 
+ int packfile_store_read_object_stream(struct odb_read_stream **out,
+                                     struct packfile_store *store,
+                                     const struct object_id *oid)
+ {
+       struct odb_packed_read_stream *stream;
+       struct pack_window *window = NULL;
+       struct object_info oi = OBJECT_INFO_INIT;
+       enum object_type in_pack_type;
+       unsigned long size;
+ 
+       oi.sizep = &size;
+ 
+       if (packfile_store_read_object_info(store, oid, &oi, 0) ||
+           oi.u.packed.is_delta ||
+           repo_settings_get_big_file_threshold(store->odb->repo) >= size)
+               return -1;
+ 
+       in_pack_type = unpack_object_header(oi.u.packed.pack,
+                                           &window,
+                                           &oi.u.packed.offset,
+                                           &size);
+       unuse_pack(&window);
+       switch (in_pack_type) {
+       default:
+               return -1; /* we do not do deltas for now */
+       case OBJ_COMMIT:
+       case OBJ_TREE:
+       case OBJ_BLOB:
+       case OBJ_TAG:
+               break;
+       }
+ 
+       CALLOC_ARRAY(stream, 1);
+       stream->base.close = close_istream_pack_non_delta;
+       stream->base.read = read_istream_pack_non_delta;
+       stream->base.type = in_pack_type;
+       stream->base.size = size;
+       stream->z_state = ODB_PACKED_READ_STREAM_UNINITIALIZED;
+       stream->pack = oi.u.packed.pack;
+       stream->pos = oi.u.packed.offset;
+ 
+       *out = &stream->base;
+ 
+       return 0;
+ }

diff --cc packfile.h
index 20e1cf17b26083c3dd278f188c0ae4ddcd22b7ce,3fcc5ae6e08c4b670f6d4bae9b62f882e84d3818..59d162a3f415e50d05af84a77beed7fedbbb6fc2
--- 1/packfile.h
--- 2/packfile.h
+++ b/packfile.h
@@@ -9,8 -8,12 +9,9 @@@
  
  /* in odb.h */
  struct object_info;
+ struct odb_read_stream;
  
  struct packed_git {
 -      struct hashmap_entry packmap_ent;
 -      struct packed_git *next;
 -      struct list_head mru;
        struct pack_window *windows;
        off_t pack_size;
        const void *index_data;
@@@ -174,9 -143,23 +175,24 @@@ void packfile_store_add_pack(struct pac
   * repository.
   */
  #define repo_for_each_pack(repo, p) \
 -      for (p = packfile_store_get_packs(repo->objects->packfiles); p; p = p->next)
 +      for (struct packfile_list_entry *e = packfile_store_get_packs(repo->objects->packfiles); \
 +           ((p) = (e ? e->pack : NULL)); e = e->next)
  
+ int packfile_store_read_object_stream(struct odb_read_stream **out,
+                                     struct packfile_store *store,
+                                     const struct object_id *oid);
+ 
+ /*
+  * Try to read the object identified by its ID from the object store and
+  * populate the object info with its data. Returns 1 in case the object was
+  * not found, 0 if it was and read successfully, and a negative error code in
+  * case the object was corrupted.
+  */
+ int packfile_store_read_object_info(struct packfile_store *store,
+                                   const struct object_id *oid,
+                                   struct object_info *oi,
+                                   unsigned flags);
+ 
  /*
   * Get all packs managed by the given store, including packfiles that are
   * referenced by multi-pack indices.