backfill: accept revision arguments

The existing implementation of 'git backfill' only includes downloading
missing blobs reachable from HEAD. Advanced uses may desire more general
commit limiting options, such as '--all' for all references, specifying a
commit range via negative references, or specifying a recency of use such as
with '--since=<date>'.

All of these options are available if we use setup_revisions() to parse the
unknown arguments with the revision machinery. This opens up a large number
of possibilities, only a small set of which are tested here.

For documentation, we avoid duplicating the option documentation and instead
link to the documentation of 'git rev-list'.

Note that these arguments currently allow specifying a pathspec, which
modifies the commit history checks but does not limit the paths used in the
backfill logic. This will be updated in a future change.

Signed-off-by: Derrick Stolee <stolee@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-backfill.adoc b/Documentation/git-backfill.adoc
index b8394dcf22b6e1e195b9bab202161cf3d1543a03..246ab417c24a100e2d874917717c02197c24a29e 100644 (file)
--- a/Documentation/git-backfill.adoc
+++ b/Documentation/git-backfill.adoc
@@ -63,9 +63,12 @@ OPTIONS
        current sparse-checkout. If the sparse-checkout feature is enabled,
        then `--sparse` is assumed and can be disabled with `--no-sparse`.
 
+You may also specify the commit limiting options from linkgit:git-rev-list[1].
+
 SEE ALSO
 --------
-linkgit:git-clone[1].
+linkgit:git-clone[1],
+linkgit:git-rev-list[1]
 
 GIT
 ---

diff --git a/builtin/backfill.c b/builtin/backfill.c
index e9a33e81be4cdb03f6eb33833800843e15327e56..b98b0b591fe3d10777a3659691f25220951a43cf 100644 (file)
--- a/builtin/backfill.c
+++ b/builtin/backfill.c
@@ -35,6 +35,7 @@ struct backfill_context {
        struct oid_array current_batch;
        size_t min_batch_size;
        int sparse;
+       struct rev_info revs;
 };
 
 static void backfill_context_clear(struct backfill_context *ctx)
@@ -79,7 +80,6 @@ static int fill_missing_blobs(const char *path UNUSED,
 
 static int do_backfill(struct backfill_context *ctx)
 {
-       struct rev_info revs;
        struct path_walk_info info = PATH_WALK_INFO_INIT;
        int ret;
 
@@ -91,13 +91,14 @@ static int do_backfill(struct backfill_context *ctx)
                }
        }
 
-       repo_init_revisions(ctx->repo, &revs, "");
-       handle_revision_arg("HEAD", &revs, 0, 0);
+       /* Walk from HEAD if otherwise unspecified. */
+       if (!ctx->revs.pending.nr)
+               add_head_to_pending(&ctx->revs);
 
        info.blobs = 1;
        info.tags = info.commits = info.trees = 0;
 
-       info.revs = &revs;
+       info.revs = &ctx->revs;
        info.path_fn = fill_missing_blobs;
        info.path_fn_data = ctx;
 
@@ -108,7 +109,6 @@ static int do_backfill(struct backfill_context *ctx)
                download_batch(ctx);
 
        path_walk_info_clear(&info);
-       release_revisions(&revs);
        return ret;
 }
 
@@ -120,6 +120,7 @@ int cmd_backfill(int argc, const char **argv, const char *prefix, struct reposit
                .current_batch = OID_ARRAY_INIT,
                .min_batch_size = 50000,
                .sparse = 0,
+               .revs = REV_INFO_INIT,
        };
        struct option options[] = {
                OPT_UNSIGNED(0, "min-batch-size", &ctx.min_batch_size,
@@ -134,7 +135,12 @@ int cmd_backfill(int argc, const char **argv, const char *prefix, struct reposit
                                         builtin_backfill_usage, options);
 
        argc = parse_options(argc, argv, prefix, options, builtin_backfill_usage,
-                            0);
+                            PARSE_OPT_KEEP_UNKNOWN_OPT |
+                            PARSE_OPT_KEEP_ARGV0 |
+                            PARSE_OPT_KEEP_DASHDASH);
+
+       repo_init_revisions(repo, &ctx.revs, prefix);
+       argc = setup_revisions(argc, argv, &ctx.revs, NULL);
 
        repo_config(repo, git_default_config, NULL);
 
@@ -143,5 +149,6 @@ int cmd_backfill(int argc, const char **argv, const char *prefix, struct reposit
 
        result = do_backfill(&ctx);
        backfill_context_clear(&ctx);
+       release_revisions(&ctx.revs);
        return result;
 }

diff --git a/t/t5620-backfill.sh b/t/t5620-backfill.sh
index 1331949be47ea864a79c7d2f21399ef402125249..db66d8b614dd19472fe027f1177fdf85597cddf7 100755 (executable)
--- a/t/t5620-backfill.sh
+++ b/t/t5620-backfill.sh
@@ -224,6 +224,162 @@ test_expect_success 'backfill --sparse without cone mode (negative)' '
        test_line_count = 12 missing
 '
 
+test_expect_success 'backfill with revision range' '
+       test_when_finished rm -rf backfill-revs &&
+       git clone --no-checkout --filter=blob:none              \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv.bare" backfill-revs &&
+
+       # No blobs yet
+       git -C backfill-revs rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 48 missing &&
+
+       git -C backfill-revs backfill HEAD~2..HEAD &&
+
+       # 30 objects downloaded.
+       git -C backfill-revs rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 18 missing
+'
+
+test_expect_success 'backfill with revisions over stdin' '
+       test_when_finished rm -rf backfill-revs &&
+       git clone --no-checkout --filter=blob:none              \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv.bare" backfill-revs &&
+
+       # No blobs yet
+       git -C backfill-revs rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 48 missing &&
+
+       cat >in <<-EOF &&
+       HEAD
+       ^HEAD~2
+       EOF
+
+       git -C backfill-revs backfill --stdin <in &&
+
+       # 30 objects downloaded.
+       git -C backfill-revs rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 18 missing
+'
+
+test_expect_success 'backfill with prefix pathspec' '
+       test_when_finished rm -rf backfill-path &&
+       git clone --bare --filter=blob:none                     \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv.bare" backfill-path &&
+
+       # No blobs yet
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 48 missing &&
+
+       # TODO: The pathspec should limit the downloaded blobs to
+       # only those matching the prefix "d/f", but currently all
+       # blobs are downloaded.
+       git -C backfill-path backfill HEAD -- d/f &&
+
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 0 missing
+'
+
+test_expect_success 'backfill with multiple pathspecs' '
+       test_when_finished rm -rf backfill-path &&
+       git clone --bare --filter=blob:none                     \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv.bare" backfill-path &&
+
+       # No blobs yet
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 48 missing &&
+
+       # TODO: The pathspecs should limit the downloaded blobs to
+       # only those matching "d/f" or "a", but currently all blobs
+       # are downloaded.
+       git -C backfill-path backfill HEAD -- d/f a &&
+
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 0 missing
+'
+
+test_expect_success 'backfill with wildcard pathspec' '
+       test_when_finished rm -rf backfill-path &&
+       git clone --bare --filter=blob:none                     \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv.bare" backfill-path &&
+
+       # No blobs yet
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 48 missing &&
+
+       # TODO: The wildcard pathspec should limit downloaded blobs,
+       # but currently all blobs are downloaded.
+       git -C backfill-path backfill HEAD -- "d/file.*.txt" &&
+
+       git -C backfill-path rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 0 missing
+'
+
+test_expect_success 'backfill with --all' '
+       test_when_finished rm -rf backfill-all &&
+       git clone --no-checkout --filter=blob:none              \
+               "file://$(pwd)/srv-revs.bare" backfill-all &&
+
+       # All blobs from all refs are missing
+       git -C backfill-all rev-list --quiet --objects --all --missing=print >missing &&
+       test_line_count = 54 missing &&
+
+       # Backfill from HEAD gets main blobs only
+       git -C backfill-all backfill HEAD &&
+
+       # Other branch blobs still missing
+       git -C backfill-all rev-list --quiet --objects --all --missing=print >missing &&
+       test_line_count = 2 missing &&
+
+       # Backfill with --all gets everything
+       git -C backfill-all backfill --all &&
+
+       git -C backfill-all rev-list --quiet --objects --all --missing=print >missing &&
+       test_line_count = 0 missing
+'
+
+test_expect_success 'backfill with --first-parent' '
+       test_when_finished rm -rf backfill-fp &&
+       git clone --no-checkout --filter=blob:none              \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv-revs.bare" backfill-fp &&
+
+       git -C backfill-fp rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 52 missing &&
+
+       # --first-parent skips the side branch commits, so
+       # s/file.{1,2}.txt v1 blobs (only in side commit 1) are missed.
+       git -C backfill-fp backfill --first-parent HEAD &&
+
+       git -C backfill-fp rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 2 missing
+'
+
+test_expect_success 'backfill with --since' '
+       test_when_finished rm -rf backfill-since &&
+       git clone --no-checkout --filter=blob:none              \
+               --single-branch --branch=main                   \
+               "file://$(pwd)/srv-revs.bare" backfill-since &&
+
+       git -C backfill-since rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 52 missing &&
+
+       # Use a cutoff between commits 4 and 5 (between v1 and v2
+       # iterations). Commits 5-8 still carry v1 of files 2-4 in
+       # their trees, but v1 of file.1.txt is only in commits 1-4.
+       SINCE=$(git -C backfill-since log --first-parent --reverse \
+               --format=%ct HEAD~1 | sed -n 5p) &&
+       git -C backfill-since backfill --since="@$((SINCE - 1))" HEAD &&
+
+       # 6 missing: v1 of file.1.txt in all 6 directories
+       git -C backfill-since rev-list --quiet --objects --missing=print HEAD >missing &&
+       test_line_count = 6 missing
+'
+
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd