promisor-remote: teach lazy-fetch in any repo

This is one step towards supporting partial clone submodules.

Even after this patch, we will still lack partial clone submodules
support, primarily because a lot of Git code that accesses submodule
objects does so by adding their object stores as alternates, meaning
that any lazy fetches that would occur in the submodule would be done
based on the config of the superproject, not of the submodule. This also
prevents testing of the functionality in this patch by user-facing
commands. So for now, test this mechanism using a test helper.

Besides that, there is some code that uses the wrapper functions
like has_promisor_remote(). Those will need to be checked to see if they
could support the non-wrapper functions instead (and thus support any
repository, not just the_repository).

Signed-off-by: Jonathan Tan <jonathantanmy@google.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
Reviewed-by: Elijah Newren <newren@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Makefile b/Makefile
index c3565fc0f8f7df81232deca663bf55e14737ae97..f6653bcd5e38b62e81bbda6365c89b53eb4bbcff 100644 (file)
--- a/Makefile
+++ b/Makefile
@@ -725,6 +725,7 @@ TEST_BUILTINS_OBJS += test-oidmap.o
 TEST_BUILTINS_OBJS += test-online-cpus.o
 TEST_BUILTINS_OBJS += test-parse-options.o
 TEST_BUILTINS_OBJS += test-parse-pathspec-file.o
+TEST_BUILTINS_OBJS += test-partial-clone.o
 TEST_BUILTINS_OBJS += test-path-utils.o
 TEST_BUILTINS_OBJS += test-pcre2-config.o
 TEST_BUILTINS_OBJS += test-pkt-line.o

diff --git a/object-file.c b/object-file.c
index f233b440b22c061cbdac1db019b1e8b58c093324..ebf273e9e774cfed12643098071553a2b2ca0329 100644 (file)
--- a/object-file.c
+++ b/object-file.c
@@ -1570,15 +1570,12 @@ static int do_oid_object_info_extended(struct repository *r,
                }
 
                /* Check if it is a missing object */
-               if (fetch_if_missing && has_promisor_remote() &&
-                   !already_retried && r == the_repository &&
+               if (fetch_if_missing && repo_has_promisor_remote(r) &&
+                   !already_retried &&
                    !(flags & OBJECT_INFO_SKIP_FETCH_OBJECT)) {
                        /*
                         * TODO Investigate checking promisor_remote_get_direct()
                         * TODO return value and stopping on error here.
-                        * TODO Pass a repository struct through
-                        * promisor_remote_get_direct(), such that arbitrary
-                        * repositories work.
                         */
                        promisor_remote_get_direct(r, real, 1);
                        already_retried = 1;

diff --git a/promisor-remote.c b/promisor-remote.c
index 1e00e16b0fbef136302f4fa363e347af467a647f..c088dcbff348a50b18206cdd304f6ce4fc86e17d 100644 (file)
--- a/promisor-remote.c
+++ b/promisor-remote.c
@@ -10,7 +10,8 @@ struct promisor_remote_config {
        struct promisor_remote **promisors_tail;
 };
 
-static int fetch_objects(const char *remote_name,
+static int fetch_objects(struct repository *repo,
+                        const char *remote_name,
                         const struct object_id *oids,
                         int oid_nr)
 {
@@ -20,6 +21,8 @@ static int fetch_objects(const char *remote_name,
 
        child.git_cmd = 1;
        child.in = -1;
+       if (repo != the_repository)
+               prepare_other_repo_env(&child.env_array, repo->gitdir);
        strvec_pushl(&child.args, "-c", "fetch.negotiationAlgorithm=noop",
                     "fetch", remote_name, "--no-tags",
                     "--no-write-fetch-head", "--recurse-submodules=no",
@@ -240,10 +243,8 @@ int promisor_remote_get_direct(struct repository *repo,
 
        promisor_remote_init(repo);
 
-       if (repo != the_repository)
-               BUG("only the_repository is supported for now");
        for (r = repo->promisor_remote_config->promisors; r; r = r->next) {
-               if (fetch_objects(r->name, remaining_oids, remaining_nr) < 0) {
+               if (fetch_objects(repo, r->name, remaining_oids, remaining_nr) < 0) {
                        if (remaining_nr == 1)
                                continue;
                        remaining_nr = remove_fetched_oids(repo, &remaining_oids,

diff --git a/t/helper/test-partial-clone.c b/t/helper/test-partial-clone.c
new file mode 100644 (file)
index 0000000..3f102cf
--- /dev/null
+++ b/t/helper/test-partial-clone.c
@@ -0,0 +1,43 @@
+#include "cache.h"
+#include "test-tool.h"
+#include "repository.h"
+#include "object-store.h"
+
+/*
+ * Prints the size of the object corresponding to the given hash in a specific
+ * gitdir. This is similar to "git -C gitdir cat-file -s", except that this
+ * exercises the code that accesses the object of an arbitrary repository that
+ * is not the_repository. ("git -C gitdir" makes it so that the_repository is
+ * the one in gitdir.)
+ */
+static void object_info(const char *gitdir, const char *oid_hex)
+{
+       struct repository r;
+       struct object_id oid;
+       unsigned long size;
+       struct object_info oi = {.sizep = &size};
+       const char *p;
+
+       if (repo_init(&r, gitdir, NULL))
+               die("could not init repo");
+       if (parse_oid_hex(oid_hex, &oid, &p))
+               die("could not parse oid");
+       if (oid_object_info_extended(&r, &oid, &oi, 0))
+               die("could not obtain object info");
+       printf("%d\n", (int) size);
+}
+
+int cmd__partial_clone(int argc, const char **argv)
+{
+       setup_git_directory();
+
+       if (argc < 4)
+               die("too few arguments");
+
+       if (!strcmp(argv[1], "object-info"))
+               object_info(argv[2], argv[3]);
+       else
+               die("invalid argument '%s'", argv[1]);
+
+       return 0;
+}

diff --git a/t/helper/test-tool.c b/t/helper/test-tool.c
index c5bd0c6d4c7abc0649e526d94924b05b0127ea53..b21e8f15190228f8664ad21405c6c7bd37f9a6e9 100644 (file)
--- a/t/helper/test-tool.c
+++ b/t/helper/test-tool.c
@@ -46,6 +46,7 @@ static struct test_cmd cmds[] = {
        { "online-cpus", cmd__online_cpus },
        { "parse-options", cmd__parse_options },
        { "parse-pathspec-file", cmd__parse_pathspec_file },
+       { "partial-clone", cmd__partial_clone },
        { "path-utils", cmd__path_utils },
        { "pcre2-config", cmd__pcre2_config },
        { "pkt-line", cmd__pkt_line },

diff --git a/t/helper/test-tool.h b/t/helper/test-tool.h
index e8069a3b2222b9982bb346f033f08c796f0b205d..f845ced4b3a58881980e7f7c2e3b4d72b941a069 100644 (file)
--- a/t/helper/test-tool.h
+++ b/t/helper/test-tool.h
@@ -35,6 +35,7 @@ int cmd__oidmap(int argc, const char **argv);
 int cmd__online_cpus(int argc, const char **argv);
 int cmd__parse_options(int argc, const char **argv);
 int cmd__parse_pathspec_file(int argc, const char** argv);
+int cmd__partial_clone(int argc, const char **argv);
 int cmd__path_utils(int argc, const char **argv);
 int cmd__pcre2_config(int argc, const char **argv);
 int cmd__pkt_line(int argc, const char **argv);

diff --git a/t/t0410-partial-clone.sh b/t/t0410-partial-clone.sh
index 584a039b851dbbf727d35d469e7d05895f9ec4e8..a211a66c67d8e6c9c73f3feb1d40b2d75b0bf420 100755 (executable)
--- a/t/t0410-partial-clone.sh
+++ b/t/t0410-partial-clone.sh
@@ -604,6 +604,29 @@ test_expect_success 'do not fetch when checking existence of tree we construct o
        git -C repo cherry-pick side1
 '
 
+test_expect_success 'lazy-fetch when accessing object not in the_repository' '
+       rm -rf full partial.git &&
+       test_create_repo full &&
+       test_commit -C full create-a-file file.txt &&
+
+       test_config -C full uploadpack.allowfilter 1 &&
+       test_config -C full uploadpack.allowanysha1inwant 1 &&
+       git clone --filter=blob:none --bare "file://$(pwd)/full" partial.git &&
+       FILE_HASH=$(git -C full rev-parse HEAD:file.txt) &&
+
+       # Sanity check that the file is missing
+       git -C partial.git rev-list --objects --missing=print HEAD >out &&
+       grep "[?]$FILE_HASH" out &&
+
+       git -C full cat-file -s "$FILE_HASH" >expect &&
+       test-tool partial-clone object-info partial.git "$FILE_HASH" >actual &&
+       test_cmp expect actual &&
+
+       # Sanity check that the file is now present
+       git -C partial.git rev-list --objects --missing=print HEAD >out &&
+       ! grep "[?]$FILE_HASH" out
+'
+
 . "$TEST_DIRECTORY"/lib-httpd.sh
 start_httpd