2 * git gc builtin command
4 * Cleanup unreachable files and optimize the repository.
6 * Copyright (c) 2007 James Bowes
8 * Based on git-gc.sh, which is
10 * Copyright (c) 2006 Shawn O. Pearce
14 #include "repository.h"
18 #include "parse-options.h"
19 #include "run-command.h"
23 #include "commit-graph.h"
25 #include "object-store.h"
27 #include "pack-objects.h"
30 #include "promisor-remote.h"
33 #include "object-store.h"
35 #define FAILED_RUN "failed to run %s"
37 static const char * const builtin_gc_usage
[] = {
38 N_("git gc [<options>]"),
42 static int pack_refs
= 1;
43 static int prune_reflogs
= 1;
44 static int aggressive_depth
= 50;
45 static int aggressive_window
= 250;
46 static int gc_auto_threshold
= 6700;
47 static int gc_auto_pack_limit
= 50;
48 static int detach_auto
= 1;
49 static timestamp_t gc_log_expire_time
;
50 static const char *gc_log_expire
= "1.day.ago";
51 static const char *prune_expire
= "2.weeks.ago";
52 static const char *prune_worktrees_expire
= "3.months.ago";
53 static unsigned long big_pack_threshold
;
54 static unsigned long max_delta_cache_size
= DEFAULT_DELTA_CACHE_SIZE
;
56 static struct strvec pack_refs_cmd
= STRVEC_INIT
;
57 static struct strvec reflog
= STRVEC_INIT
;
58 static struct strvec repack
= STRVEC_INIT
;
59 static struct strvec prune
= STRVEC_INIT
;
60 static struct strvec prune_worktrees
= STRVEC_INIT
;
61 static struct strvec rerere
= STRVEC_INIT
;
63 static struct tempfile
*pidfile
;
64 static struct lock_file log_lock
;
66 static struct string_list pack_garbage
= STRING_LIST_INIT_DUP
;
68 static void clean_pack_garbage(void)
71 for (i
= 0; i
< pack_garbage
.nr
; i
++)
72 unlink_or_warn(pack_garbage
.items
[i
].string
);
73 string_list_clear(&pack_garbage
, 0);
76 static void report_pack_garbage(unsigned seen_bits
, const char *path
)
78 if (seen_bits
== PACKDIR_FILE_IDX
)
79 string_list_append(&pack_garbage
, path
);
82 static void process_log_file(void)
85 if (fstat(get_lock_file_fd(&log_lock
), &st
)) {
87 * Perhaps there was an i/o error or another
88 * unlikely situation. Try to make a note of
89 * this in gc.log along with any existing
92 int saved_errno
= errno
;
93 fprintf(stderr
, _("Failed to fstat %s: %s"),
94 get_tempfile_path(log_lock
.tempfile
),
95 strerror(saved_errno
));
97 commit_lock_file(&log_lock
);
99 } else if (st
.st_size
) {
100 /* There was some error recorded in the lock file */
101 commit_lock_file(&log_lock
);
103 /* No error, clean up any old gc.log */
104 unlink(git_path("gc.log"));
105 rollback_lock_file(&log_lock
);
109 static void process_log_file_at_exit(void)
115 static void process_log_file_on_signal(int signo
)
122 static int gc_config_is_timestamp_never(const char *var
)
127 if (!git_config_get_value(var
, &value
) && value
) {
128 if (parse_expiry_date(value
, &expire
))
129 die(_("failed to parse '%s' value '%s'"), var
, value
);
135 static void gc_config(void)
139 if (!git_config_get_value("gc.packrefs", &value
)) {
140 if (value
&& !strcmp(value
, "notbare"))
143 pack_refs
= git_config_bool("gc.packrefs", value
);
146 if (gc_config_is_timestamp_never("gc.reflogexpire") &&
147 gc_config_is_timestamp_never("gc.reflogexpireunreachable"))
150 git_config_get_int("gc.aggressivewindow", &aggressive_window
);
151 git_config_get_int("gc.aggressivedepth", &aggressive_depth
);
152 git_config_get_int("gc.auto", &gc_auto_threshold
);
153 git_config_get_int("gc.autopacklimit", &gc_auto_pack_limit
);
154 git_config_get_bool("gc.autodetach", &detach_auto
);
155 git_config_get_expiry("gc.pruneexpire", &prune_expire
);
156 git_config_get_expiry("gc.worktreepruneexpire", &prune_worktrees_expire
);
157 git_config_get_expiry("gc.logexpiry", &gc_log_expire
);
159 git_config_get_ulong("gc.bigpackthreshold", &big_pack_threshold
);
160 git_config_get_ulong("pack.deltacachesize", &max_delta_cache_size
);
162 git_config(git_default_config
, NULL
);
165 static int too_many_loose_objects(void)
168 * Quickly check if a "gc" is needed, by estimating how
169 * many loose objects there are. Because SHA-1 is evenly
170 * distributed, we can check only one and get a reasonable
178 const unsigned hexsz_loose
= the_hash_algo
->hexsz
- 2;
180 dir
= opendir(git_path("objects/17"));
184 auto_threshold
= DIV_ROUND_UP(gc_auto_threshold
, 256);
185 while ((ent
= readdir(dir
)) != NULL
) {
186 if (strspn(ent
->d_name
, "0123456789abcdef") != hexsz_loose
||
187 ent
->d_name
[hexsz_loose
] != '\0')
189 if (++num_loose
> auto_threshold
) {
198 static struct packed_git
*find_base_packs(struct string_list
*packs
,
201 struct packed_git
*p
, *base
= NULL
;
203 for (p
= get_all_packs(the_repository
); p
; p
= p
->next
) {
207 if (p
->pack_size
>= limit
)
208 string_list_append(packs
, p
->pack_name
);
209 } else if (!base
|| base
->pack_size
< p
->pack_size
) {
215 string_list_append(packs
, base
->pack_name
);
220 static int too_many_packs(void)
222 struct packed_git
*p
;
225 if (gc_auto_pack_limit
<= 0)
228 for (cnt
= 0, p
= get_all_packs(the_repository
); p
; p
= p
->next
) {
234 * Perhaps check the size of the pack and count only
235 * very small ones here?
239 return gc_auto_pack_limit
< cnt
;
242 static uint64_t total_ram(void)
244 #if defined(HAVE_SYSINFO)
249 #elif defined(HAVE_BSD_SYSCTL) && (defined(HW_MEMSIZE) || defined(HW_PHYSMEM))
250 int64_t physical_memory
;
255 # if defined(HW_MEMSIZE)
260 length
= sizeof(int64_t);
261 if (!sysctl(mib
, 2, &physical_memory
, &length
, NULL
, 0))
262 return physical_memory
;
263 #elif defined(GIT_WINDOWS_NATIVE)
264 MEMORYSTATUSEX memInfo
;
266 memInfo
.dwLength
= sizeof(MEMORYSTATUSEX
);
267 if (GlobalMemoryStatusEx(&memInfo
))
268 return memInfo
.ullTotalPhys
;
273 static uint64_t estimate_repack_memory(struct packed_git
*pack
)
275 unsigned long nr_objects
= approximate_object_count();
276 size_t os_cache
, heap
;
278 if (!pack
|| !nr_objects
)
282 * First we have to scan through at least one pack.
283 * Assume enough room in OS file cache to keep the entire pack
284 * or we may accidentally evict data of other processes from
287 os_cache
= pack
->pack_size
+ pack
->index_size
;
288 /* then pack-objects needs lots more for book keeping */
289 heap
= sizeof(struct object_entry
) * nr_objects
;
291 * internal rev-list --all --objects takes up some memory too,
292 * let's say half of it is for blobs
294 heap
+= sizeof(struct blob
) * nr_objects
/ 2;
296 * and the other half is for trees (commits and tags are
297 * usually insignificant)
299 heap
+= sizeof(struct tree
) * nr_objects
/ 2;
300 /* and then obj_hash[], underestimated in fact */
301 heap
+= sizeof(struct object
*) * nr_objects
;
302 /* revindex is used also */
303 heap
+= sizeof(struct revindex_entry
) * nr_objects
;
305 * read_sha1_file() (either at delta calculation phase, or
306 * writing phase) also fills up the delta base cache
308 heap
+= delta_base_cache_limit
;
309 /* and of course pack-objects has its own delta cache */
310 heap
+= max_delta_cache_size
;
312 return os_cache
+ heap
;
315 static int keep_one_pack(struct string_list_item
*item
, void *data
)
317 strvec_pushf(&repack
, "--keep-pack=%s", basename(item
->string
));
321 static void add_repack_all_option(struct string_list
*keep_pack
)
323 if (prune_expire
&& !strcmp(prune_expire
, "now"))
324 strvec_push(&repack
, "-a");
326 strvec_push(&repack
, "-A");
328 strvec_pushf(&repack
, "--unpack-unreachable=%s", prune_expire
);
332 for_each_string_list(keep_pack
, keep_one_pack
, NULL
);
335 static void add_repack_incremental_option(void)
337 strvec_push(&repack
, "--no-write-bitmap-index");
340 static int need_to_gc(void)
343 * Setting gc.auto to 0 or negative can disable the
346 if (gc_auto_threshold
<= 0)
350 * If there are too many loose objects, but not too many
351 * packs, we run "repack -d -l". If there are too many packs,
352 * we run "repack -A -d -l". Otherwise we tell the caller
355 if (too_many_packs()) {
356 struct string_list keep_pack
= STRING_LIST_INIT_NODUP
;
358 if (big_pack_threshold
) {
359 find_base_packs(&keep_pack
, big_pack_threshold
);
360 if (keep_pack
.nr
>= gc_auto_pack_limit
) {
361 big_pack_threshold
= 0;
362 string_list_clear(&keep_pack
, 0);
363 find_base_packs(&keep_pack
, 0);
366 struct packed_git
*p
= find_base_packs(&keep_pack
, 0);
367 uint64_t mem_have
, mem_want
;
369 mem_have
= total_ram();
370 mem_want
= estimate_repack_memory(p
);
373 * Only allow 1/2 of memory for pack-objects, leave
374 * the rest for the OS and other processes in the
377 if (!mem_have
|| mem_want
< mem_have
/ 2)
378 string_list_clear(&keep_pack
, 0);
381 add_repack_all_option(&keep_pack
);
382 string_list_clear(&keep_pack
, 0);
383 } else if (too_many_loose_objects())
384 add_repack_incremental_option();
388 if (run_hook_le(NULL
, "pre-auto-gc", NULL
))
393 /* return NULL on success, else hostname running the gc */
394 static const char *lock_repo_for_gc(int force
, pid_t
* ret_pid
)
396 struct lock_file lock
= LOCK_INIT
;
397 char my_host
[HOST_NAME_MAX
+ 1];
398 struct strbuf sb
= STRBUF_INIT
;
405 if (is_tempfile_active(pidfile
))
409 if (xgethostname(my_host
, sizeof(my_host
)))
410 xsnprintf(my_host
, sizeof(my_host
), "unknown");
412 pidfile_path
= git_pathdup("gc.pid");
413 fd
= hold_lock_file_for_update(&lock
, pidfile_path
,
416 static char locking_host
[HOST_NAME_MAX
+ 1];
417 static char *scan_fmt
;
421 scan_fmt
= xstrfmt("%s %%%ds", "%"SCNuMAX
, HOST_NAME_MAX
);
422 fp
= fopen(pidfile_path
, "r");
423 memset(locking_host
, 0, sizeof(locking_host
));
426 !fstat(fileno(fp
), &st
) &&
428 * 12 hour limit is very generous as gc should
429 * never take that long. On the other hand we
430 * don't really need a strict limit here,
431 * running gc --auto one day late is not a big
432 * problem. --force can be used in manual gc
433 * after the user verifies that no gc is
436 time(NULL
) - st
.st_mtime
<= 12 * 3600 &&
437 fscanf(fp
, scan_fmt
, &pid
, locking_host
) == 2 &&
438 /* be gentle to concurrent "gc" on remote hosts */
439 (strcmp(locking_host
, my_host
) || !kill(pid
, 0) || errno
== EPERM
);
444 rollback_lock_file(&lock
);
451 strbuf_addf(&sb
, "%"PRIuMAX
" %s",
452 (uintmax_t) getpid(), my_host
);
453 write_in_full(fd
, sb
.buf
, sb
.len
);
455 commit_lock_file(&lock
);
456 pidfile
= register_tempfile(pidfile_path
);
462 * Returns 0 if there was no previous error and gc can proceed, 1 if
463 * gc should not proceed due to an error in the last run. Prints a
464 * message and returns -1 if an error occurred while reading gc.log
466 static int report_last_gc_error(void)
468 struct strbuf sb
= STRBUF_INIT
;
472 char *gc_log_path
= git_pathdup("gc.log");
474 if (stat(gc_log_path
, &st
)) {
478 ret
= error_errno(_("cannot stat '%s'"), gc_log_path
);
482 if (st
.st_mtime
< gc_log_expire_time
)
485 len
= strbuf_read_file(&sb
, gc_log_path
, 0);
487 ret
= error_errno(_("cannot read '%s'"), gc_log_path
);
490 * A previous gc failed. Report the error, and don't
491 * bother with an automatic gc run since it is likely
492 * to fail in the same way.
494 warning(_("The last gc run reported the following. "
495 "Please correct the root cause\n"
497 "Automatic cleanup will not be performed "
498 "until the file is removed.\n\n"
500 gc_log_path
, sb
.buf
);
509 static void gc_before_repack(void)
512 * We may be called twice, as both the pre- and
513 * post-daemonized phases will call us, but running these
514 * commands more than once is pointless and wasteful.
520 if (pack_refs
&& run_command_v_opt(pack_refs_cmd
.v
, RUN_GIT_CMD
))
521 die(FAILED_RUN
, pack_refs_cmd
.v
[0]);
523 if (prune_reflogs
&& run_command_v_opt(reflog
.v
, RUN_GIT_CMD
))
524 die(FAILED_RUN
, reflog
.v
[0]);
527 int cmd_gc(int argc
, const char **argv
, const char *prefix
)
536 int keep_base_pack
= -1;
539 struct option builtin_gc_options
[] = {
540 OPT__QUIET(&quiet
, N_("suppress progress reporting")),
541 { OPTION_STRING
, 0, "prune", &prune_expire
, N_("date"),
542 N_("prune unreferenced objects"),
543 PARSE_OPT_OPTARG
, NULL
, (intptr_t)prune_expire
},
544 OPT_BOOL(0, "aggressive", &aggressive
, N_("be more thorough (increased runtime)")),
545 OPT_BOOL_F(0, "auto", &auto_gc
, N_("enable auto-gc mode"),
546 PARSE_OPT_NOCOMPLETE
),
547 OPT_BOOL_F(0, "force", &force
,
548 N_("force running gc even if there may be another gc running"),
549 PARSE_OPT_NOCOMPLETE
),
550 OPT_BOOL(0, "keep-largest-pack", &keep_base_pack
,
551 N_("repack all other packs except the largest pack")),
555 if (argc
== 2 && !strcmp(argv
[1], "-h"))
556 usage_with_options(builtin_gc_usage
, builtin_gc_options
);
558 strvec_pushl(&pack_refs_cmd
, "pack-refs", "--all", "--prune", NULL
);
559 strvec_pushl(&reflog
, "reflog", "expire", "--all", NULL
);
560 strvec_pushl(&repack
, "repack", "-d", "-l", NULL
);
561 strvec_pushl(&prune
, "prune", "--expire", NULL
);
562 strvec_pushl(&prune_worktrees
, "worktree", "prune", "--expire", NULL
);
563 strvec_pushl(&rerere
, "rerere", "gc", NULL
);
565 /* default expiry time, overwritten in gc_config */
567 if (parse_expiry_date(gc_log_expire
, &gc_log_expire_time
))
568 die(_("failed to parse gc.logexpiry value %s"), gc_log_expire
);
571 pack_refs
= !is_bare_repository();
573 argc
= parse_options(argc
, argv
, prefix
, builtin_gc_options
,
574 builtin_gc_usage
, 0);
576 usage_with_options(builtin_gc_usage
, builtin_gc_options
);
578 if (prune_expire
&& parse_expiry_date(prune_expire
, &dummy
))
579 die(_("failed to parse prune expiry value %s"), prune_expire
);
582 strvec_push(&repack
, "-f");
583 if (aggressive_depth
> 0)
584 strvec_pushf(&repack
, "--depth=%d", aggressive_depth
);
585 if (aggressive_window
> 0)
586 strvec_pushf(&repack
, "--window=%d", aggressive_window
);
589 strvec_push(&repack
, "-q");
593 * Auto-gc should be least intrusive as possible.
599 fprintf(stderr
, _("Auto packing the repository in background for optimum performance.\n"));
601 fprintf(stderr
, _("Auto packing the repository for optimum performance.\n"));
602 fprintf(stderr
, _("See \"git help gc\" for manual housekeeping.\n"));
605 int ret
= report_last_gc_error();
607 /* an I/O error occurred, already reported */
610 /* Last gc --auto failed. Skip this one. */
613 if (lock_repo_for_gc(force
, &pid
))
615 gc_before_repack(); /* dies on failure */
616 delete_tempfile(&pidfile
);
619 * failure to daemonize is ok, we'll continue
622 daemonized
= !daemonize();
625 struct string_list keep_pack
= STRING_LIST_INIT_NODUP
;
627 if (keep_base_pack
!= -1) {
629 find_base_packs(&keep_pack
, 0);
630 } else if (big_pack_threshold
) {
631 find_base_packs(&keep_pack
, big_pack_threshold
);
634 add_repack_all_option(&keep_pack
);
635 string_list_clear(&keep_pack
, 0);
638 name
= lock_repo_for_gc(force
, &pid
);
641 return 0; /* be quiet on --auto */
642 die(_("gc is already running on machine '%s' pid %"PRIuMAX
" (use --force if not)"),
643 name
, (uintmax_t)pid
);
647 hold_lock_file_for_update(&log_lock
,
650 dup2(get_lock_file_fd(&log_lock
), 2);
651 sigchain_push_common(process_log_file_on_signal
);
652 atexit(process_log_file_at_exit
);
657 if (!repository_format_precious_objects
) {
658 close_object_store(the_repository
->objects
);
659 if (run_command_v_opt(repack
.v
, RUN_GIT_CMD
))
660 die(FAILED_RUN
, repack
.v
[0]);
663 strvec_push(&prune
, prune_expire
);
665 strvec_push(&prune
, "--no-progress");
666 if (has_promisor_remote())
668 "--exclude-promisor-objects");
669 if (run_command_v_opt(prune
.v
, RUN_GIT_CMD
))
670 die(FAILED_RUN
, prune
.v
[0]);
674 if (prune_worktrees_expire
) {
675 strvec_push(&prune_worktrees
, prune_worktrees_expire
);
676 if (run_command_v_opt(prune_worktrees
.v
, RUN_GIT_CMD
))
677 die(FAILED_RUN
, prune_worktrees
.v
[0]);
680 if (run_command_v_opt(rerere
.v
, RUN_GIT_CMD
))
681 die(FAILED_RUN
, rerere
.v
[0]);
683 report_garbage
= report_pack_garbage
;
684 reprepare_packed_git(the_repository
);
685 if (pack_garbage
.nr
> 0) {
686 close_object_store(the_repository
->objects
);
687 clean_pack_garbage();
690 prepare_repo_settings(the_repository
);
691 if (the_repository
->settings
.gc_write_commit_graph
== 1)
692 write_commit_graph_reachable(the_repository
->objects
->odb
,
693 !quiet
&& !daemonized
? COMMIT_GRAPH_WRITE_PROGRESS
: 0,
696 if (auto_gc
&& too_many_loose_objects())
697 warning(_("There are too many unreachable loose objects; "
698 "run 'git prune' to remove them."));
701 unlink(git_path("gc.log"));
706 static const char * const builtin_maintenance_run_usage
[] = {
707 N_("git maintenance run [--auto] [--[no-]quiet] [--task=<task>]"),
711 struct maintenance_run_opts
{
716 /* Remember to update object flag allocation in object.h */
719 struct cg_auto_data
{
720 int num_not_in_graph
;
724 static int dfs_on_ref(const char *refname
,
725 const struct object_id
*oid
, int flags
,
728 struct cg_auto_data
*data
= (struct cg_auto_data
*)cb_data
;
730 struct object_id peeled
;
731 struct commit_list
*stack
= NULL
;
732 struct commit
*commit
;
734 if (!peel_ref(refname
, &peeled
))
736 if (oid_object_info(the_repository
, oid
, NULL
) != OBJ_COMMIT
)
739 commit
= lookup_commit(the_repository
, oid
);
742 if (parse_commit(commit
) ||
743 commit_graph_position(commit
) != COMMIT_NOT_FROM_GRAPH
)
746 data
->num_not_in_graph
++;
748 if (data
->num_not_in_graph
>= data
->limit
)
751 commit_list_append(commit
, &stack
);
753 while (!result
&& stack
) {
754 struct commit_list
*parent
;
756 commit
= pop_commit(&stack
);
758 for (parent
= commit
->parents
; parent
; parent
= parent
->next
) {
759 if (parse_commit(parent
->item
) ||
760 commit_graph_position(parent
->item
) != COMMIT_NOT_FROM_GRAPH
||
761 parent
->item
->object
.flags
& SEEN
)
764 parent
->item
->object
.flags
|= SEEN
;
765 data
->num_not_in_graph
++;
767 if (data
->num_not_in_graph
>= data
->limit
) {
772 commit_list_append(parent
->item
, &stack
);
776 free_commit_list(stack
);
780 static int should_write_commit_graph(void)
783 struct cg_auto_data data
;
785 data
.num_not_in_graph
= 0;
787 git_config_get_int("maintenance.commit-graph.auto",
795 result
= for_each_ref(dfs_on_ref
, &data
);
797 clear_commit_marks_all(SEEN
);
802 static int run_write_commit_graph(struct maintenance_run_opts
*opts
)
804 struct child_process child
= CHILD_PROCESS_INIT
;
807 strvec_pushl(&child
.args
, "commit-graph", "write",
808 "--split", "--reachable", NULL
);
811 strvec_push(&child
.args
, "--no-progress");
813 return !!run_command(&child
);
816 static int maintenance_task_commit_graph(struct maintenance_run_opts
*opts
)
818 prepare_repo_settings(the_repository
);
819 if (!the_repository
->settings
.core_commit_graph
)
822 close_object_store(the_repository
->objects
);
823 if (run_write_commit_graph(opts
)) {
824 error(_("failed to write commit-graph"));
831 static int fetch_remote(const char *remote
, struct maintenance_run_opts
*opts
)
833 struct child_process child
= CHILD_PROCESS_INIT
;
836 strvec_pushl(&child
.args
, "fetch", remote
, "--prune", "--no-tags",
837 "--no-write-fetch-head", "--recurse-submodules=no",
841 strvec_push(&child
.args
, "--quiet");
843 strvec_pushf(&child
.args
, "+refs/heads/*:refs/prefetch/%s/*", remote
);
845 return !!run_command(&child
);
848 static int append_remote(struct remote
*remote
, void *cbdata
)
850 struct string_list
*remotes
= (struct string_list
*)cbdata
;
852 string_list_append(remotes
, remote
->name
);
856 static int maintenance_task_prefetch(struct maintenance_run_opts
*opts
)
859 struct string_list_item
*item
;
860 struct string_list remotes
= STRING_LIST_INIT_DUP
;
862 if (for_each_remote(append_remote
, &remotes
)) {
863 error(_("failed to fill remotes"));
868 for_each_string_list_item(item
, &remotes
)
869 result
|= fetch_remote(item
->string
, opts
);
872 string_list_clear(&remotes
, 0);
876 static int maintenance_task_gc(struct maintenance_run_opts
*opts
)
878 struct child_process child
= CHILD_PROCESS_INIT
;
881 strvec_push(&child
.args
, "gc");
884 strvec_push(&child
.args
, "--auto");
886 strvec_push(&child
.args
, "--quiet");
888 strvec_push(&child
.args
, "--no-quiet");
890 close_object_store(the_repository
->objects
);
891 return run_command(&child
);
894 static int prune_packed(struct maintenance_run_opts
*opts
)
896 struct child_process child
= CHILD_PROCESS_INIT
;
899 strvec_push(&child
.args
, "prune-packed");
902 strvec_push(&child
.args
, "--quiet");
904 return !!run_command(&child
);
907 struct write_loose_object_data
{
913 static int loose_object_auto_limit
= 100;
915 static int loose_object_count(const struct object_id
*oid
,
919 int *count
= (int*)data
;
920 if (++(*count
) >= loose_object_auto_limit
)
925 static int loose_object_auto_condition(void)
929 git_config_get_int("maintenance.loose-objects.auto",
930 &loose_object_auto_limit
);
932 if (!loose_object_auto_limit
)
934 if (loose_object_auto_limit
< 0)
937 return for_each_loose_file_in_objdir(the_repository
->objects
->odb
->path
,
942 static int bail_on_loose(const struct object_id
*oid
,
949 static int write_loose_object_to_stdin(const struct object_id
*oid
,
953 struct write_loose_object_data
*d
= (struct write_loose_object_data
*)data
;
955 fprintf(d
->in
, "%s\n", oid_to_hex(oid
));
957 return ++(d
->count
) > d
->batch_size
;
960 static int pack_loose(struct maintenance_run_opts
*opts
)
962 struct repository
*r
= the_repository
;
964 struct write_loose_object_data data
;
965 struct child_process pack_proc
= CHILD_PROCESS_INIT
;
968 * Do not start pack-objects process
969 * if there are no loose objects.
971 if (!for_each_loose_file_in_objdir(r
->objects
->odb
->path
,
976 pack_proc
.git_cmd
= 1;
978 strvec_push(&pack_proc
.args
, "pack-objects");
980 strvec_push(&pack_proc
.args
, "--quiet");
981 strvec_pushf(&pack_proc
.args
, "%s/pack/loose", r
->objects
->odb
->path
);
985 if (start_command(&pack_proc
)) {
986 error(_("failed to start 'git pack-objects' process"));
990 data
.in
= xfdopen(pack_proc
.in
, "w");
992 data
.batch_size
= 50000;
994 for_each_loose_file_in_objdir(r
->objects
->odb
->path
,
995 write_loose_object_to_stdin
,
1002 if (finish_command(&pack_proc
)) {
1003 error(_("failed to finish 'git pack-objects' process"));
1010 static int maintenance_task_loose_objects(struct maintenance_run_opts
*opts
)
1012 return prune_packed(opts
) || pack_loose(opts
);
1015 static int incremental_repack_auto_condition(void)
1017 struct packed_git
*p
;
1019 int incremental_repack_auto_limit
= 10;
1022 if (git_config_get_bool("core.multiPackIndex", &enabled
) ||
1026 git_config_get_int("maintenance.incremental-repack.auto",
1027 &incremental_repack_auto_limit
);
1029 if (!incremental_repack_auto_limit
)
1031 if (incremental_repack_auto_limit
< 0)
1034 for (p
= get_packed_git(the_repository
);
1035 count
< incremental_repack_auto_limit
&& p
;
1037 if (!p
->multi_pack_index
)
1041 return count
>= incremental_repack_auto_limit
;
1044 static int multi_pack_index_write(struct maintenance_run_opts
*opts
)
1046 struct child_process child
= CHILD_PROCESS_INIT
;
1049 strvec_pushl(&child
.args
, "multi-pack-index", "write", NULL
);
1052 strvec_push(&child
.args
, "--no-progress");
1054 if (run_command(&child
))
1055 return error(_("failed to write multi-pack-index"));
1060 static int multi_pack_index_expire(struct maintenance_run_opts
*opts
)
1062 struct child_process child
= CHILD_PROCESS_INIT
;
1065 strvec_pushl(&child
.args
, "multi-pack-index", "expire", NULL
);
1068 strvec_push(&child
.args
, "--no-progress");
1070 close_object_store(the_repository
->objects
);
1072 if (run_command(&child
))
1073 return error(_("'git multi-pack-index expire' failed"));
1078 #define TWO_GIGABYTES (INT32_MAX)
1080 static off_t
get_auto_pack_size(void)
1083 * The "auto" value is special: we optimize for
1084 * one large pack-file (i.e. from a clone) and
1085 * expect the rest to be small and they can be
1088 * The strategy we select here is to select a
1089 * size that is one more than the second largest
1090 * pack-file. This ensures that we will repack
1091 * at least two packs if there are three or more
1095 off_t second_largest_size
= 0;
1097 struct packed_git
*p
;
1098 struct repository
*r
= the_repository
;
1100 reprepare_packed_git(r
);
1101 for (p
= get_all_packs(r
); p
; p
= p
->next
) {
1102 if (p
->pack_size
> max_size
) {
1103 second_largest_size
= max_size
;
1104 max_size
= p
->pack_size
;
1105 } else if (p
->pack_size
> second_largest_size
)
1106 second_largest_size
= p
->pack_size
;
1109 result_size
= second_largest_size
+ 1;
1111 /* But limit ourselves to a batch size of 2g */
1112 if (result_size
> TWO_GIGABYTES
)
1113 result_size
= TWO_GIGABYTES
;
1118 static int multi_pack_index_repack(struct maintenance_run_opts
*opts
)
1120 struct child_process child
= CHILD_PROCESS_INIT
;
1123 strvec_pushl(&child
.args
, "multi-pack-index", "repack", NULL
);
1126 strvec_push(&child
.args
, "--no-progress");
1128 strvec_pushf(&child
.args
, "--batch-size=%"PRIuMAX
,
1129 (uintmax_t)get_auto_pack_size());
1131 close_object_store(the_repository
->objects
);
1133 if (run_command(&child
))
1134 return error(_("'git multi-pack-index repack' failed"));
1139 static int maintenance_task_incremental_repack(struct maintenance_run_opts
*opts
)
1141 prepare_repo_settings(the_repository
);
1142 if (!the_repository
->settings
.core_multi_pack_index
) {
1143 warning(_("skipping incremental-repack task because core.multiPackIndex is disabled"));
1147 if (multi_pack_index_write(opts
))
1149 if (multi_pack_index_expire(opts
))
1151 if (multi_pack_index_repack(opts
))
1156 typedef int maintenance_task_fn(struct maintenance_run_opts
*opts
);
1159 * An auto condition function returns 1 if the task should run
1160 * and 0 if the task should NOT run. See needs_to_gc() for an
1163 typedef int maintenance_auto_fn(void);
1165 struct maintenance_task
{
1167 maintenance_task_fn
*fn
;
1168 maintenance_auto_fn
*auto_condition
;
1171 /* -1 if not selected. */
1175 enum maintenance_task_label
{
1178 TASK_INCREMENTAL_REPACK
,
1182 /* Leave as final value */
1186 static struct maintenance_task tasks
[] = {
1189 maintenance_task_prefetch
,
1191 [TASK_LOOSE_OBJECTS
] = {
1193 maintenance_task_loose_objects
,
1194 loose_object_auto_condition
,
1196 [TASK_INCREMENTAL_REPACK
] = {
1197 "incremental-repack",
1198 maintenance_task_incremental_repack
,
1199 incremental_repack_auto_condition
,
1203 maintenance_task_gc
,
1207 [TASK_COMMIT_GRAPH
] = {
1209 maintenance_task_commit_graph
,
1210 should_write_commit_graph
,
1214 static int compare_tasks_by_selection(const void *a_
, const void *b_
)
1216 const struct maintenance_task
*a
, *b
;
1218 a
= (const struct maintenance_task
*)&a_
;
1219 b
= (const struct maintenance_task
*)&b_
;
1221 return b
->selected_order
- a
->selected_order
;
1224 static int maintenance_run_tasks(struct maintenance_run_opts
*opts
)
1226 int i
, found_selected
= 0;
1228 struct lock_file lk
;
1229 struct repository
*r
= the_repository
;
1230 char *lock_path
= xstrfmt("%s/maintenance", r
->objects
->odb
->path
);
1232 if (hold_lock_file_for_update(&lk
, lock_path
, LOCK_NO_DEREF
) < 0) {
1234 * Another maintenance command is running.
1236 * If --auto was provided, then it is likely due to a
1237 * recursive process stack. Do not report an error in
1240 if (!opts
->auto_flag
&& !opts
->quiet
)
1241 warning(_("lock file '%s' exists, skipping maintenance"),
1248 for (i
= 0; !found_selected
&& i
< TASK__COUNT
; i
++)
1249 found_selected
= tasks
[i
].selected_order
>= 0;
1252 QSORT(tasks
, TASK__COUNT
, compare_tasks_by_selection
);
1254 for (i
= 0; i
< TASK__COUNT
; i
++) {
1255 if (found_selected
&& tasks
[i
].selected_order
< 0)
1258 if (!found_selected
&& !tasks
[i
].enabled
)
1261 if (opts
->auto_flag
&&
1262 (!tasks
[i
].auto_condition
||
1263 !tasks
[i
].auto_condition()))
1266 trace2_region_enter("maintenance", tasks
[i
].name
, r
);
1267 if (tasks
[i
].fn(opts
)) {
1268 error(_("task '%s' failed"), tasks
[i
].name
);
1271 trace2_region_leave("maintenance", tasks
[i
].name
, r
);
1274 rollback_lock_file(&lk
);
1278 static void initialize_task_config(void)
1281 struct strbuf config_name
= STRBUF_INIT
;
1284 for (i
= 0; i
< TASK__COUNT
; i
++) {
1287 strbuf_setlen(&config_name
, 0);
1288 strbuf_addf(&config_name
, "maintenance.%s.enabled",
1291 if (!git_config_get_bool(config_name
.buf
, &config_value
))
1292 tasks
[i
].enabled
= config_value
;
1295 strbuf_release(&config_name
);
1298 static int task_option_parse(const struct option
*opt
,
1299 const char *arg
, int unset
)
1301 int i
, num_selected
= 0;
1302 struct maintenance_task
*task
= NULL
;
1304 BUG_ON_OPT_NEG(unset
);
1306 for (i
= 0; i
< TASK__COUNT
; i
++) {
1307 if (tasks
[i
].selected_order
>= 0)
1309 if (!strcasecmp(tasks
[i
].name
, arg
)) {
1315 error(_("'%s' is not a valid task"), arg
);
1319 if (task
->selected_order
>= 0) {
1320 error(_("task '%s' cannot be selected multiple times"), arg
);
1324 task
->selected_order
= num_selected
+ 1;
1329 static int maintenance_run(int argc
, const char **argv
, const char *prefix
)
1332 struct maintenance_run_opts opts
;
1333 struct option builtin_maintenance_run_options
[] = {
1334 OPT_BOOL(0, "auto", &opts
.auto_flag
,
1335 N_("run tasks based on the state of the repository")),
1336 OPT_BOOL(0, "quiet", &opts
.quiet
,
1337 N_("do not report progress or other information over stderr")),
1338 OPT_CALLBACK_F(0, "task", NULL
, N_("task"),
1339 N_("run a specific task"),
1340 PARSE_OPT_NONEG
, task_option_parse
),
1343 memset(&opts
, 0, sizeof(opts
));
1345 opts
.quiet
= !isatty(2);
1346 initialize_task_config();
1348 for (i
= 0; i
< TASK__COUNT
; i
++)
1349 tasks
[i
].selected_order
= -1;
1351 argc
= parse_options(argc
, argv
, prefix
,
1352 builtin_maintenance_run_options
,
1353 builtin_maintenance_run_usage
,
1354 PARSE_OPT_STOP_AT_NON_OPTION
);
1357 usage_with_options(builtin_maintenance_run_usage
,
1358 builtin_maintenance_run_options
);
1359 return maintenance_run_tasks(&opts
);
1362 static const char builtin_maintenance_usage
[] = N_("git maintenance run [<options>]");
1364 int cmd_maintenance(int argc
, const char **argv
, const char *prefix
)
1367 (argc
== 2 && !strcmp(argv
[1], "-h")))
1368 usage(builtin_maintenance_usage
);
1370 if (!strcmp(argv
[1], "run"))
1371 return maintenance_run(argc
- 1, argv
+ 1, prefix
);
1373 die(_("invalid subcommand: %s"), argv
[1]);