odb: introduce object filters to `odb_for_each_object()`

The function `for_each_bitmapped_object()` can be used to iterate
through all objects covered by a bitmap. The benefit of this function is
that it allows the caller to efficiently handle some object filters. For
example, this can be used to filter out objects of a specific type with
some simple bitmap operations. But callers are currently required to
manually wire up the use of bitmaps though, and to do so they have to
reach into internals of a given object database source.

Introduce a new `struct odb_for_each_object_options::filter` field so
that the interface becomes generic. When set, then a backend may
optionally use the filter to skip some objects that it would have
otherwise yielded.

Note that the respective backends are free to ignore this field if they
cannot meaningfully optimize for a given filter, and consequently
callers need to verify whether they actually want the returned objects.
While annoying, we cannot easily lift this restriction anyway as the
object filter infrastructure supports some filters that cannot be
answered by the object database alone.

An alternative might be to limit the filters to only those that _can_ be
answered by backends. But ultimately, the filters that can be answered
efficiently by the "packed" backend are completely disjunct from those
that can be answered by the "loose" backend, and consequently the set of
filters supported by all backends would be empty. Furthermore, it would
require us to make assumptions about capabilities of future backends,
which may be able to efficiently handle more filters than current ones.
So in the end, this alternative would only limit us artificially.

Implement the logic for the "packed" source. Note that we use the new
function `prepare_bitmap_git_for_source()` to open the bitmap: as the
backend operates on a single object source, we must only use bitmaps
that belong to that specific source. Otherwise we might yield objects
that are not part of the source at all, and with multiple sources we
would enumerate the same bitmap once per source.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/odb.h b/odb.h
index a1e222f605d5f89e4a91299c2aea150c1b64afe5..67d0b349426f0c2219984d1a872dfd4d7c4732b0 100644 (file)
--- a/odb.h
+++ b/odb.h
@@ -8,6 +8,7 @@
 #include "thread-utils.h"
 
 struct cached_object_entry;
+struct list_objects_filter_options;
 struct odb_source_inmemory;
 struct packed_git;
 struct repository;
@@ -490,6 +491,17 @@ struct odb_for_each_object_options {
         */
        const struct object_id *prefix;
        size_t prefix_hex_len;
+
+       /*
+        * Optional object filter that allows backends to skip yielding
+        * objects that are excluded by the filter as an optimization. The
+        * filter is a best-effort hint: backends may use it to skip
+        * excluded objects (e.g. by consulting a reachability bitmap), but
+        * are also free to ignore it entirely and yield every object. As a
+        * consequence, callers must re-apply the filter on yielded objects
+        * if they require strict filtering semantics.
+        */
+       const struct list_objects_filter_options *filter;
 };
 
 /*

diff --git a/odb/source-packed.c b/odb/source-packed.c
index 9cfa02b7a2e53ef514cc50e90070b5be7582eff5..477739505307810cdd857526db8d2c114d43cdf9 100644 (file)
--- a/odb/source-packed.c
+++ b/odb/source-packed.c
@@ -3,11 +3,13 @@
 #include "chdir-notify.h"
 #include "dir.h"
 #include "git-zlib.h"
+#include "list-objects-filter-options.h"
 #include "mergesort.h"
 #include "midx.h"
 #include "odb/source-packed.h"
 #include "odb/streaming.h"
 #include "packfile.h"
+#include "pack-bitmap.h"
 
 static int find_pack_entry(struct odb_source_packed *store,
                           const struct object_id *oid,
@@ -315,6 +317,37 @@ out:
        return ret;
 }
 
+struct bitmapped_for_each_object_data {
+       struct odb_source_packed *packed;
+       const struct object_info *request;
+       const struct odb_for_each_object_options *opts;
+       odb_for_each_object_cb cb;
+       void *cb_data;
+};
+
+static int bitmapped_for_each_object(const struct object_id *oid,
+                                    enum object_type type UNUSED,
+                                    int flags UNUSED,
+                                    uint32_t hash UNUSED,
+                                    struct packed_git *pack,
+                                    off_t offset,
+                                    void *cb_data)
+{
+       struct bitmapped_for_each_object_data *data = cb_data;
+
+       if (should_exclude_pack(pack, data->opts->flags))
+               return 0;
+
+       if (data->request) {
+               struct object_info oi = *data->request;
+               if (packed_object_info(data->packed, pack, offset, &oi) < 0)
+                       return -1;
+               return data->cb(oid, &oi, data->cb_data);
+       }
+
+       return data->cb(oid, NULL, data->cb_data);
+}
+
 static int odb_source_packed_for_each_object(struct odb_source *source,
                                             const struct object_info *request,
                                             odb_for_each_object_cb cb,
@@ -328,12 +361,33 @@ static int odb_source_packed_for_each_object(struct odb_source *source,
                .cb = cb,
                .cb_data = cb_data,
        };
+       struct bitmap_index *bitmap = NULL;
        struct packfile_list_entry *e;
        int pack_errors = 0, ret;
 
        if (opts->prefix)
                return odb_source_packed_for_each_prefixed_object(packed, opts, &data);
 
+       if (opts->filter &&
+           opts->filter->choice != LOFC_DISABLED &&
+           can_filter_bitmap(opts->filter))
+               bitmap = prepare_bitmap_git_for_source(packed);
+       if (bitmap) {
+               struct bitmapped_for_each_object_data bitmap_data = {
+                       .packed = packed,
+                       .request = request,
+                       .opts = opts,
+                       .cb = cb,
+                       .cb_data = cb_data,
+               };
+
+               ret = for_each_bitmapped_object(bitmap, opts->filter,
+                                               bitmapped_for_each_object,
+                                               &bitmap_data);
+               if (ret)
+                       goto out;
+       }
+
        packed->skip_mru_updates = true;
 
        for (e = packfile_store_get_packs(packed); e; e = e->next) {
@@ -342,6 +396,13 @@ static int odb_source_packed_for_each_object(struct odb_source *source,
                if (should_exclude_pack(p, opts->flags))
                        continue;
 
+               /*
+                * Objects covered by the bitmap have already been yielded
+                * above; skip them here to avoid duplicates.
+                */
+               if (bitmap && bitmap_index_contains_pack(bitmap, p))
+                       continue;
+
                if (open_pack_index(p)) {
                        pack_errors = 1;
                        continue;
@@ -357,6 +418,7 @@ static int odb_source_packed_for_each_object(struct odb_source *source,
 
 out:
        packed->skip_mru_updates = false;
+       free_bitmap_index(bitmap);
 
        if (!ret && pack_errors)
                ret = -1;

diff --git a/pack-bitmap.c b/pack-bitmap.c
index 09ba15d26be88a8da76ddff1c37bf4c016647ba6..f55a0859eaf5288c411d7d9323c160b56206ace5 100644 (file)
--- a/pack-bitmap.c
+++ b/pack-bitmap.c
@@ -2039,12 +2039,11 @@ static int filter_bitmap(struct bitmap_index *bitmap_git,
        return -1;
 }
 
-static int can_filter_bitmap(const struct list_objects_filter_options *filter)
+bool can_filter_bitmap(const struct list_objects_filter_options *filter)
 {
        return !filter_bitmap(NULL, NULL, NULL, filter);
 }
 
-
 static void filter_packed_objects_from_bitmap(struct bitmap_index *bitmap_git,
                                              struct bitmap *result)
 {

diff --git a/pack-bitmap.h b/pack-bitmap.h
index 9f20fb6e563091825811bbd825bb78b14e310c61..1385027c1ff5fa0f99bd6d08d23d6fbf4338d354 100644 (file)
--- a/pack-bitmap.h
+++ b/pack-bitmap.h
@@ -92,6 +92,9 @@ int test_bitmap_pseudo_merge_objects(struct repository *r, uint32_t n);
 
 struct list_objects_filter_options;
 
+/* Check whether the filter can be computed via the bitmap. */
+bool can_filter_bitmap(const struct list_objects_filter_options *filter);
+
 /*
  * Filter bitmapped objects and iterate through all resulting objects,
  * executing `show_reach` for each of them. Returns `-1` in case the filter is