builtin/cat-file: use bitmaps to efficiently filter by object type

While it is now possible to filter objects by type, this mechanism is
for now mostly a convenience. Most importantly, we still have to iterate
through the whole packfile to find all objects of a specific type. This
can be prohibitively expensive depending on the size of the packfiles.

It isn't really possible to do better than this when only considering a
packfile itself, as the order of objects is not fixed. But when we have
a packfile with a corresponding bitmap, either because the packfile
itself has one or because the multi-pack index has a bitmap for it, then
we can use these bitmaps to improve the runtime.

While bitmaps are typically used to compute reachability of objects,
they also contain one bitmap per object type that encodes which object
has what type. So instead of reading through the whole packfile(s), we
can use the bitmaps and iterate through the type-specific bitmap.
Typically, only a subset of packfiles will have a bitmap. But this isn't
really much of a problem: we can use bitmaps when available, and then
use the non-bitmap walk for every packfile that isn't covered by one.

Overall, this leads to quite a significant speedup depending on how many
objects of a certain type exist. The following benchmarks have been
executed in the Chromium repository, which has a 50GB packfile with
almost 25 million objects. As expected, there isn't really much of a
change in performance without an object filter:

    Benchmark 1: cat-file with no-filter (revision = HEAD~)
      Time (mean ± σ):     89.675 s ±  4.527 s    [User: 40.807 s, System: 10.782 s]
      Range (min … max):   83.052 s … 96.084 s    10 runs

    Benchmark 2: cat-file with no-filter (revision = HEAD)
      Time (mean ± σ):     88.991 s ±  2.488 s    [User: 42.278 s, System: 10.305 s]
      Range (min … max):   82.843 s … 91.271 s    10 runs

    Summary
      cat-file with no-filter (revision = HEAD) ran
        1.01 ± 0.06 times faster than cat-file with no-filter (revision = HEAD~)

We still have to scan through all objects as we yield all of them, so
using the bitmap in this case doesn't really buy us anything. What is
noticeable in this benchmark is that we're I/O-bound, not CPU-bound, as
can be seen from the user/system runtimes, which combined are way lower
than the overall benchmarked runtime.

But when we do use a filter we can see a significant improvement:

    Benchmark 1: cat-file with filter=object:type=commit (revision = HEAD~)
      Time (mean ± σ):     86.444 s ±  4.081 s    [User: 36.830 s, System: 11.312 s]
      Range (min … max):   80.305 s … 93.104 s    10 runs

    Benchmark 2: cat-file with filter=object:type=commit (revision = HEAD)
      Time (mean ± σ):      2.089 s ±  0.015 s    [User: 1.872 s, System: 0.207 s]
      Range (min … max):    2.073 s …  2.119 s    10 runs

    Summary
      cat-file with filter=object:type=commit (revision = HEAD) ran
       41.38 ± 1.98 times faster than cat-file with filter=object:type=commit (revision = HEAD~)

This is because we don't have to scan through all packfiles anymore, but
can instead directly look up relevant objects.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/builtin/cat-file.c b/builtin/cat-file.c
index b0c758eca022a88b1583cebf970696200e4dd9d8..ead7554a57aa6ae90b0a7bdbaf67668213d33910 100644 (file)
--- a/builtin/cat-file.c
+++ b/builtin/cat-file.c
@@ -21,6 +21,7 @@
 #include "streaming.h"
 #include "oid-array.h"
 #include "packfile.h"
+#include "pack-bitmap.h"
 #include "object-file.h"
 #include "object-name.h"
 #include "object-store-ll.h"
@@ -825,7 +826,20 @@ static int batch_one_object_packed(const struct object_id *oid,
                                 payload->payload);
 }
 
-static void batch_each_object(for_each_object_fn callback,
+static int batch_one_object_bitmapped(const struct object_id *oid,
+                                     enum object_type type UNUSED,
+                                     int flags UNUSED,
+                                     uint32_t hash UNUSED,
+                                     struct packed_git *pack,
+                                     off_t offset,
+                                     void *_payload)
+{
+       struct for_each_object_payload *payload = _payload;
+       return payload->callback(oid, pack, offset, payload->payload);
+}
+
+static void batch_each_object(struct batch_options *opt,
+                             for_each_object_fn callback,
                              unsigned flags,
                              void *_payload)
 {
@@ -833,9 +847,27 @@ static void batch_each_object(for_each_object_fn callback,
                .callback = callback,
                .payload = _payload,
        };
+       struct bitmap_index *bitmap = prepare_bitmap_git(the_repository);
+
        for_each_loose_object(batch_one_object_loose, &payload, 0);
-       for_each_packed_object(the_repository, batch_one_object_packed,
-                              &payload, flags);
+
+       if (bitmap && !for_each_bitmapped_object(bitmap, &opt->objects_filter,
+                                                batch_one_object_bitmapped, &payload)) {
+               struct packed_git *pack;
+
+               for (pack = get_all_packs(the_repository); pack; pack = pack->next) {
+                       if (bitmap_index_contains_pack(bitmap, pack) ||
+                           open_pack_index(pack))
+                               continue;
+                       for_each_object_in_pack(pack, batch_one_object_packed,
+                                               &payload, flags);
+               }
+       } else {
+               for_each_packed_object(the_repository, batch_one_object_packed,
+                                      &payload, flags);
+       }
+
+       free_bitmap_index(bitmap);
 }
 
 static int batch_objects(struct batch_options *opt)
@@ -892,14 +924,14 @@ static int batch_objects(struct batch_options *opt)
 
                        cb.seen = &seen;
 
-                       batch_each_object(batch_unordered_object,
+                       batch_each_object(opt, batch_unordered_object,
                                          FOR_EACH_OBJECT_PACK_ORDER, &cb);
 
                        oidset_clear(&seen);
                } else {
                        struct oid_array sa = OID_ARRAY_INIT;
 
-                       batch_each_object(collect_object, 0, &sa);
+                       batch_each_object(opt, collect_object, 0, &sa);
                        oid_array_for_each_unique(&sa, batch_object_cb, &cb);
 
                        oid_array_clear(&sa);