pack-refs: teach --exclude option to exclude refs from being packed

At GitLab, we have a system that creates ephemeral internal refs that
don't live long before getting deleted. Having an option to exclude
certain refs from a packed-refs file allows these internal references to
be deleted much more efficiently.

Add an --exclude option to the pack-refs builtin, and use the ref
exclusions API to exclude certain refs from being packed into the final
packed-refs file

Signed-off-by: John Cai <johncai86@gmail.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

diff --git a/Documentation/git-pack-refs.txt b/Documentation/git-pack-refs.txt
index 22f006650062c83b175b54bbb1ca2fda655db4bf..546aa122dff5a600d163b85793b89e3ef5e6a8da 100644 (file)
--- a/Documentation/git-pack-refs.txt
+++ b/Documentation/git-pack-refs.txt
@@ -8,7 +8,7 @@ git-pack-refs - Pack heads and tags for efficient repository access
 SYNOPSIS
 --------
 [verse]
-'git pack-refs' [--all] [--no-prune]
+'git pack-refs' [--all] [--no-prune] [--exclude <pattern>]
 
 DESCRIPTION
 -----------
@@ -60,6 +60,16 @@ with many branches of historical interests.
 The command usually removes loose refs under `$GIT_DIR/refs`
 hierarchy after packing them.  This option tells it not to.
 
+--exclude <pattern>::
+
+Do not pack refs matching the given `glob(7)` pattern. Repetitions of this option
+accumulate exclusion patterns. Use `--no-exclude` to clear and reset the list of
+patterns. If a ref is already packed, including it with `--exclude` will not
+unpack it.
+
+When used with `--all`, pack only loose refs which do not match any of
+the provided `--exclude` patterns.
+
 
 BUGS
 ----

diff --git a/builtin/pack-refs.c b/builtin/pack-refs.c
index 9833815fb30a3aae55c9512dc53094d1862e4be6..1d1a64fe3863bc32041838ed34f1b9707dfcd583 100644 (file)
--- a/builtin/pack-refs.c
+++ b/builtin/pack-refs.c
@@ -4,22 +4,34 @@
 #include "parse-options.h"
 #include "refs.h"
 #include "repository.h"
+#include "revision.h"
 
 static char const * const pack_refs_usage[] = {
-       N_("git pack-refs [--all] [--no-prune]"),
+       N_("git pack-refs [--all] [--no-prune] [--exclude <pattern>]"),
        NULL
 };
 
 int cmd_pack_refs(int argc, const char **argv, const char *prefix)
 {
        unsigned int flags = PACK_REFS_PRUNE;
+       static struct ref_exclusions excludes = REF_EXCLUSIONS_INIT;
+       struct pack_refs_opts pack_refs_opts = {.exclusions = &excludes, .flags = flags};
+       static struct string_list option_excluded_refs = STRING_LIST_INIT_NODUP;
+       struct string_list_item *item;
+
        struct option opts[] = {
-               OPT_BIT(0, "all",   &flags, N_("pack everything"), PACK_REFS_ALL),
-               OPT_BIT(0, "prune", &flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE),
+               OPT_BIT(0, "all",   &pack_refs_opts.flags, N_("pack everything"), PACK_REFS_ALL),
+               OPT_BIT(0, "prune", &pack_refs_opts.flags, N_("prune loose refs (default)"), PACK_REFS_PRUNE),
+               OPT_STRING_LIST(0, "exclude", &option_excluded_refs, N_("pattern"),
+                       N_("references to exclude")),
                OPT_END(),
        };
        git_config(git_default_config, NULL);
        if (parse_options(argc, argv, prefix, opts, pack_refs_usage, 0))
                usage_with_options(pack_refs_usage, opts);
-       return refs_pack_refs(get_main_ref_store(the_repository), flags);
+
+       for_each_string_list_item(item, &option_excluded_refs)
+               add_ref_exclusion(pack_refs_opts.exclusions, item->string);
+
+       return refs_pack_refs(get_main_ref_store(the_repository), &pack_refs_opts);
 }

diff --git a/refs.c b/refs.c
index d2a98e1c21f4e9e0be61070aeec48446b0335efc..881a0da65cf3388638218c08fc1410a989d5c017 100644 (file)
--- a/refs.c
+++ b/refs.c
@@ -2132,9 +2132,9 @@ void base_ref_store_init(struct ref_store *refs, struct repository *repo,
 }
 
 /* backend functions */
-int refs_pack_refs(struct ref_store *refs, unsigned int flags)
+int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts)
 {
-       return refs->be->pack_refs(refs, flags);
+       return refs->be->pack_refs(refs, opts);
 }
 
 int peel_iterated_oid(const struct object_id *base, struct object_id *peeled)

diff --git a/refs.h b/refs.h
index 123cfa4424453fbda6503dbbb66d28f092815154..46020bd335c03bfb2f829008c636a4dfa5a45600 100644 (file)
--- a/refs.h
+++ b/refs.h
@@ -63,6 +63,11 @@ struct worktree;
 #define RESOLVE_REF_NO_RECURSE 0x02
 #define RESOLVE_REF_ALLOW_BAD_NAME 0x04
 
+struct pack_refs_opts {
+       unsigned int flags;
+       struct ref_exclusions *exclusions;
+};
+
 const char *refs_resolve_ref_unsafe(struct ref_store *refs,
                                    const char *refname,
                                    int resolve_flags,
@@ -405,7 +410,7 @@ void warn_dangling_symrefs(FILE *fp, const char *msg_fmt,
  * Write a packed-refs file for the current repository.
  * flags: Combination of the above PACK_REFS_* flags.
  */
-int refs_pack_refs(struct ref_store *refs, unsigned int flags);
+int refs_pack_refs(struct ref_store *refs, struct pack_refs_opts *opts);
 
 /*
  * Setup reflog before using. Fill in err and return -1 on failure.

diff --git a/refs/debug.c b/refs/debug.c
index adc34c836fce48f34e811af27c04e79aa50e617b..63f434ea26b168cd2468d385bd2469e5c5e5455d 100644 (file)
--- a/refs/debug.c
+++ b/refs/debug.c
@@ -122,10 +122,10 @@ static int debug_initial_transaction_commit(struct ref_store *refs,
        return res;
 }
 
-static int debug_pack_refs(struct ref_store *ref_store, unsigned int flags)
+static int debug_pack_refs(struct ref_store *ref_store, struct pack_refs_opts *opts)
 {
        struct debug_ref_store *drefs = (struct debug_ref_store *)ref_store;
-       int res = drefs->refs->be->pack_refs(drefs->refs, flags);
+       int res = drefs->refs->be->pack_refs(drefs->refs, opts);
        trace_printf_key(&trace_refs, "pack_refs: %d\n", res);
        return res;
 }

diff --git a/refs/files-backend.c b/refs/files-backend.c
index d0581ee41ac6181961567627f292ad6a0dbcdc14..8620e01362508fb356dda19bb0df2d89bbf8a219 100644 (file)
--- a/refs/files-backend.c
+++ b/refs/files-backend.c
@@ -19,6 +19,7 @@
 #include "../worktree.h"
 #include "../wrapper.h"
 #include "../write-or-die.h"
+#include "../revision.h"
 
 /*
  * This backend uses the following flags in `ref_update::flags` for
@@ -1173,15 +1174,18 @@ static void prune_refs(struct files_ref_store *refs, struct ref_to_prune **refs_
  */
 static int should_pack_ref(const char *refname,
                           const struct object_id *oid, unsigned int ref_flags,
-                          unsigned int pack_flags)
+                          struct pack_refs_opts *opts)
 {
        /* Do not pack per-worktree refs: */
        if (parse_worktree_ref(refname, NULL, NULL, NULL) !=
            REF_WORKTREE_SHARED)
                return 0;
 
+       if (ref_excluded(opts->exclusions, refname))
+               return 0;
+
        /* Do not pack non-tags unless PACK_REFS_ALL is set: */
-       if (!(pack_flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/"))
+       if (!(opts->flags & PACK_REFS_ALL) && !starts_with(refname, "refs/tags/"))
                return 0;
 
        /* Do not pack symbolic refs: */
@@ -1195,7 +1199,8 @@ static int should_pack_ref(const char *refname,
        return 1;
 }
 
-static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
+static int files_pack_refs(struct ref_store *ref_store,
+                          struct pack_refs_opts *opts)
 {
        struct files_ref_store *refs =
                files_downcast(ref_store, REF_STORE_WRITE | REF_STORE_ODB,
@@ -1220,8 +1225,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
                 * in the packed ref cache. If the reference should be
                 * pruned, also add it to refs_to_prune.
                 */
-               if (!should_pack_ref(iter->refname, iter->oid, iter->flags,
-                                    flags))
+               if (!should_pack_ref(iter->refname, iter->oid, iter->flags, opts))
                        continue;
 
                /*
@@ -1235,7 +1239,7 @@ static int files_pack_refs(struct ref_store *ref_store, unsigned int flags)
                            iter->refname, err.buf);
 
                /* Schedule the loose reference for pruning if requested. */
-               if ((flags & PACK_REFS_PRUNE)) {
+               if ((opts->flags & PACK_REFS_PRUNE)) {
                        struct ref_to_prune *n;
                        FLEX_ALLOC_STR(n, name, iter->refname);
                        oidcpy(&n->oid, iter->oid);

diff --git a/refs/packed-backend.c b/refs/packed-backend.c
index 34c0c4e20f1dc40dbde73f6110f0d21ff990ca1a..87ba5f520e4cf331882e6afc5c025eaa58ef5d24 100644 (file)
--- a/refs/packed-backend.c
+++ b/refs/packed-backend.c
@@ -1576,7 +1576,7 @@ static int packed_delete_refs(struct ref_store *ref_store, const char *msg,
 }
 
 static int packed_pack_refs(struct ref_store *ref_store UNUSED,
-                           unsigned int flags UNUSED)
+                           struct pack_refs_opts *pack_opts UNUSED)
 {
        /*
         * Packed refs are already packed. It might be that loose refs

diff --git a/refs/refs-internal.h b/refs/refs-internal.h
index a85d113123c99b55e1e7047d334c7608843eb2d6..f72b7be89413a95fcc72a8289ce87603b10aa967 100644 (file)
--- a/refs/refs-internal.h
+++ b/refs/refs-internal.h
@@ -547,7 +547,8 @@ typedef int ref_transaction_commit_fn(struct ref_store *refs,
                                      struct ref_transaction *transaction,
                                      struct strbuf *err);
 
-typedef int pack_refs_fn(struct ref_store *ref_store, unsigned int flags);
+typedef int pack_refs_fn(struct ref_store *ref_store,
+                        struct pack_refs_opts *opts);
 typedef int create_symref_fn(struct ref_store *ref_store,
                             const char *ref_target,
                             const char *refs_heads_master,

diff --git a/revision.h b/revision.h
index e8f6de968490340cd2dcd9927d1961e6c9519855..ac7b08bbfd810bdfd5122920b8bd273d4bf8d4db 100644 (file)
--- a/revision.h
+++ b/revision.h
@@ -86,7 +86,7 @@ struct rev_cmdline_info {
 struct ref_exclusions {
        /*
         * Excluded refs is a list of wildmatch patterns. If any of the
-        * patterns matches, the reference will be excluded.
+        * patterns match, the reference will be excluded.
         */
        struct string_list excluded_refs;
 

diff --git a/t/helper/test-ref-store.c b/t/helper/test-ref-store.c
index 6d8f844e9c7dc4889d6670e86a26fcb59f87bbb1..de4197708d914633fbd05c07a76a7502e95f555a 100644 (file)
--- a/t/helper/test-ref-store.c
+++ b/t/helper/test-ref-store.c
@@ -116,8 +116,9 @@ static struct flag_definition pack_flags[] = { FLAG_DEF(PACK_REFS_PRUNE),
 static int cmd_pack_refs(struct ref_store *refs, const char **argv)
 {
        unsigned int flags = arg_flags(*argv++, "flags", pack_flags);
+       struct pack_refs_opts pack_opts = { .flags = flags };
 
-       return refs_pack_refs(refs, flags);
+       return refs_pack_refs(refs, &pack_opts);
 }
 
 static int cmd_create_symref(struct ref_store *refs, const char **argv)

diff --git a/t/t3210-pack-refs.sh b/t/t3210-pack-refs.sh
index 07a0ff93defebe395fe49344834b2fc627c7f81d..925b90cd3ba66b8b46ffb56f9fb6d6fd8a8a7f6d 100755 (executable)
--- a/t/t3210-pack-refs.sh
+++ b/t/t3210-pack-refs.sh
@@ -108,6 +108,22 @@ test_expect_success \
      git branch -d n/o/p &&
      git branch n'
 
+test_expect_success 'test excluded refs are not packed' '
+       git branch dont_pack1 &&
+       git branch dont_pack2 &&
+       git branch pack_this &&
+       git pack-refs --all --exclude "refs/heads/dont_pack*" &&
+       test -f .git/refs/heads/dont_pack1 &&
+       test -f .git/refs/heads/dont_pack2 &&
+       ! test -f .git/refs/heads/pack_this'
+
+test_expect_success 'test --no-exclude refs clears excluded refs' '
+       git branch dont_pack3 &&
+       git branch dont_pack4 &&
+       git pack-refs --all --exclude "refs/heads/dont_pack*" --no-exclude &&
+       ! test -f .git/refs/heads/dont_pack3 &&
+       ! test -f .git/refs/heads/dont_pack4'
+
 test_expect_success \
        'see if up-to-date packed refs are preserved' \
        'git branch q &&