duplicates. (If a given OID is given more than once, it is marked as
preferred if at least one instance of it begins with the special `+`
marker).
+
+ --incremental::
+ Write an incremental MIDX file containing only objects
+ and packs not present in an existing MIDX layer.
+ Migrates non-incremental MIDXs to incremental ones when
+ necessary. Incompatible with `--bitmap`.
--
verify::
have no objects referenced by the MIDX (with the exception of
`.keep` packs and cruft packs). Rewrite the MIDX file afterward
to remove all references to these pack-files.
++
+NOTE: this mode is incompatible with incremental MIDX files.
repack::
Create a new pack-file containing objects in small pack-files
+
If `repack.packKeptObjects` is `false`, then any pack-files with an
associated `.keep` file will not be selected for the batch to repack.
-
++
+NOTE: this mode is incompatible with incremental MIDX files.
EXAMPLES
--------
MIDX_WRITE_BITMAP | MIDX_WRITE_REV_INDEX),
OPT_BIT(0, "progress", &opts.flags,
N_("force progress reporting"), MIDX_PROGRESS),
+ OPT_BIT(0, "incremental", &opts.flags,
+ N_("write a new incremental MIDX"), MIDX_WRITE_INCREMENTAL),
OPT_BOOL(0, "stdin-packs", &opts.stdin_packs,
N_("write multi-pack index containing only given indexes")),
OPT_FILENAME(0, "refs-snapshot", &opts.refs_snapshot,
if (run_update_server_info)
update_server_info(0);
- if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0))
- write_midx_file(get_object_directory(), NULL, NULL, 0);
+ if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX, 0)) {
+ unsigned flags = 0;
+ if (git_env_bool(GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL, 0))
+ flags |= MIDX_WRITE_INCREMENTAL;
+ write_midx_file(get_object_directory(), NULL, NULL, flags);
+ }
cleanup:
string_list_clear(&names, 1);
export GIT_TEST_COMMIT_GRAPH=1
export GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=1
export GIT_TEST_MULTI_PACK_INDEX=1
+ export GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=1
export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=master
export GIT_TEST_NO_WRITE_REV_INDEX=1
export GIT_TEST_CHECKOUT_WORKERS=2
#include "refs.h"
#include "revision.h"
#include "list-objects.h"
+#include "path.h"
+#include "pack-revindex.h"
#define PACK_EXPIRED UINT_MAX
#define BITMAP_POS_UNKNOWN (~((uint32_t)0))
extern int midx_checksum_valid(struct multi_pack_index *m);
extern void clear_midx_files_ext(const char *object_dir, const char *ext,
- unsigned char *keep_hash);
+ const char *keep_hash);
+extern void clear_incremental_midx_files_ext(const char *object_dir,
+ const char *ext,
+ const char **keep_hashes,
+ uint32_t hashes_nr);
extern int cmp_idx_or_pack_name(const char *idx_or_pack_name,
const char *idx_name);
size_t nr;
size_t alloc;
struct multi_pack_index *m;
+ struct multi_pack_index *base_midx;
struct progress *progress;
unsigned pack_paths_checked;
int preferred_pack_idx;
+ int incremental;
+ uint32_t num_multi_pack_indexes_before;
+
struct string_list *to_include;
};
*/
if (ctx->m && midx_contains_pack(ctx->m, file_name))
return 0;
+ else if (ctx->base_midx && midx_contains_pack(ctx->base_midx,
+ file_name))
+ return 0;
else if (ctx->to_include &&
!string_list_has_string(ctx->to_include, file_name))
return 0;
for (cur_fanout = 0; cur_fanout < 256; cur_fanout++) {
fanout.nr = 0;
- if (ctx->m)
+ if (ctx->m && !ctx->incremental)
midx_fanout_add_midx_fanout(&fanout, ctx->m, cur_fanout,
ctx->preferred_pack_idx);
if (cur_object && oideq(&fanout.entries[cur_object - 1].oid,
&fanout.entries[cur_object].oid))
continue;
+ if (ctx->incremental && ctx->base_midx &&
+ midx_has_oid(ctx->base_midx,
+ &fanout.entries[cur_object].oid))
+ continue;
ALLOC_GROW(ctx->entries, st_add(ctx->entries_nr, 1),
alloc_objects);
void *data)
{
struct write_midx_context *ctx = data;
- uint32_t i;
+ uint32_t i, nr_base;
+
+ if (ctx->incremental && ctx->base_midx)
+ nr_base = ctx->base_midx->num_objects +
+ ctx->base_midx->num_objects_in_base;
+ else
+ nr_base = 0;
for (i = 0; i < ctx->entries_nr; i++)
- hashwrite_be32(f, ctx->pack_order[i]);
+ hashwrite_be32(f, ctx->pack_order[i] + nr_base);
return 0;
}
static uint32_t *midx_pack_order(struct write_midx_context *ctx)
{
struct midx_pack_order_data *data;
- uint32_t *pack_order;
+ uint32_t *pack_order, base_objects = 0;
uint32_t i;
trace2_region_enter("midx", "midx_pack_order", the_repository);
+ if (ctx->incremental && ctx->base_midx)
+ base_objects = ctx->base_midx->num_objects +
+ ctx->base_midx->num_objects_in_base;
+
+ ALLOC_ARRAY(pack_order, ctx->entries_nr);
ALLOC_ARRAY(data, ctx->entries_nr);
+
for (i = 0; i < ctx->entries_nr; i++) {
struct pack_midx_entry *e = &ctx->entries[i];
data[i].nr = i;
QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
- ALLOC_ARRAY(pack_order, ctx->entries_nr);
for (i = 0; i < ctx->entries_nr; i++) {
struct pack_midx_entry *e = &ctx->entries[data[i].nr];
struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]];
if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
- pack->bitmap_pos = i;
+ pack->bitmap_pos = i + base_objects;
pack->bitmap_nr++;
pack_order[i] = data[i].nr;
}
prepare_packing_data(the_repository, pdata);
for (i = 0; i < ctx->entries_nr; i++) {
- struct pack_midx_entry *from = &ctx->entries[ctx->pack_order[i]];
+ uint32_t pos = ctx->pack_order[i];
+ struct pack_midx_entry *from = &ctx->entries[pos];
struct object_entry *to = packlist_alloc(pdata, &from->oid);
oe_set_in_pack(pdata, to,
static int fill_packs_from_midx(struct write_midx_context *ctx,
const char *preferred_pack_name, uint32_t flags)
{
- uint32_t i;
+ struct multi_pack_index *m;
- for (i = 0; i < ctx->m->num_packs; i++) {
- ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
+ for (m = ctx->m; m; m = m->base_midx) {
+ uint32_t i;
+
+ for (i = 0; i < m->num_packs; i++) {
+ ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
- if (flags & MIDX_WRITE_REV_INDEX || preferred_pack_name) {
/*
* If generating a reverse index, need to have
* packed_git's loaded to compare their
* mtimes and object count.
*
- *
* If a preferred pack is specified, need to
* have packed_git's loaded to ensure the chosen
* preferred pack has a non-zero object count.
*/
- if (prepare_midx_pack(the_repository, ctx->m, i))
- return error(_("could not load pack"));
+ if (flags & MIDX_WRITE_REV_INDEX ||
+ preferred_pack_name) {
+ if (prepare_midx_pack(the_repository, m,
+ m->num_packs_in_base + i)) {
+ error(_("could not load pack"));
+ return 1;
+ }
+
+ if (open_pack_index(m->packs[i]))
+ die(_("could not open index for %s"),
+ m->packs[i]->pack_name);
+ }
+
+ fill_pack_info(&ctx->info[ctx->nr++], m->packs[i],
+ m->pack_names[i],
+ m->num_packs_in_base + i);
+ }
+ }
+ return 0;
+}
- if (open_pack_index(ctx->m->packs[i]))
- die(_("could not open index for %s"),
- ctx->m->packs[i]->pack_name);
+static struct {
+ const char *non_split;
+ const char *split;
+} midx_exts[] = {
+ {NULL, MIDX_EXT_MIDX},
+ {MIDX_EXT_BITMAP, MIDX_EXT_BITMAP},
+ {MIDX_EXT_REV, MIDX_EXT_REV},
+};
+
+static int link_midx_to_chain(struct multi_pack_index *m)
+{
+ struct strbuf from = STRBUF_INIT;
+ struct strbuf to = STRBUF_INIT;
+ int ret = 0;
+ size_t i;
+
+ if (!m || m->has_chain) {
+ /*
+ * Either no MIDX previously existed, or it was already
+ * part of a MIDX chain. In both cases, we have nothing
+ * to link, so return early.
+ */
+ goto done;
+ }
+
+ for (i = 0; i < ARRAY_SIZE(midx_exts); i++) {
+ const unsigned char *hash = get_midx_checksum(m);
+
+ get_midx_filename_ext(&from, m->object_dir, hash,
+ midx_exts[i].non_split);
+ get_split_midx_filename_ext(&to, m->object_dir, hash,
+ midx_exts[i].split);
+
+ if (link(from.buf, to.buf) < 0 && errno != ENOENT) {
+ ret = error_errno(_("unable to link '%s' to '%s'"),
+ from.buf, to.buf);
+ goto done;
}
- fill_pack_info(&ctx->info[ctx->nr++], ctx->m->packs[i],
- ctx->m->pack_names[i], i);
+ strbuf_reset(&from);
+ strbuf_reset(&to);
}
- return 0;
+done:
+ strbuf_release(&from);
+ strbuf_release(&to);
+ return ret;
+}
+
+static void clear_midx_files(const char *object_dir,
+ const char **hashes,
+ uint32_t hashes_nr,
+ unsigned incremental)
+{
+ /*
+ * if incremental:
+ * - remove all non-incremental MIDX files
+ * - remove any incremental MIDX files not in the current one
+ *
+ * if non-incremental:
+ * - remove all incremental MIDX files
+ * - remove any non-incremental MIDX files not matching the current
+ * hash
+ */
+ struct strbuf buf = STRBUF_INIT;
+ const char *exts[] = { MIDX_EXT_BITMAP, MIDX_EXT_REV, MIDX_EXT_MIDX };
+ uint32_t i, j;
+
+ for (i = 0; i < ARRAY_SIZE(exts); i++) {
+ clear_incremental_midx_files_ext(object_dir, exts[i],
+ hashes, hashes_nr);
+ for (j = 0; j < hashes_nr; j++)
+ clear_midx_files_ext(object_dir, exts[i], hashes[j]);
+ }
+
+ if (incremental)
+ get_midx_filename(&buf, object_dir);
+ else
+ get_midx_chain_filename(&buf, object_dir);
+
+ if (unlink(buf.buf) && errno != ENOENT)
+ die_errno(_("failed to clear multi-pack-index at %s"), buf.buf);
+
+ strbuf_release(&buf);
}
static int write_midx_internal(const char *object_dir,
uint32_t i, start_pack;
struct hashfile *f = NULL;
struct lock_file lk;
+ struct tempfile *incr;
struct write_midx_context ctx = { 0 };
int bitmapped_packs_concat_len = 0;
int pack_name_concat_len = 0;
int dropped_packs = 0;
int result = 0;
+ const char **keep_hashes = NULL;
struct chunkfile *cf;
trace2_region_enter("midx", "write_midx_internal", the_repository);
- get_midx_filename(&midx_name, object_dir);
+ ctx.incremental = !!(flags & MIDX_WRITE_INCREMENTAL);
+ if (ctx.incremental && (flags & MIDX_WRITE_BITMAP))
+ die(_("cannot write incremental MIDX with bitmap"));
+
+ if (ctx.incremental)
+ strbuf_addf(&midx_name,
+ "%s/pack/multi-pack-index.d/tmp_midx_XXXXXX",
+ object_dir);
+ else
+ get_midx_filename(&midx_name, object_dir);
if (safe_create_leading_directories(midx_name.buf))
die_errno(_("unable to create leading directories of %s"),
midx_name.buf);
- if (!packs_to_include) {
- /*
- * Only reference an existing MIDX when not filtering which
- * packs to include, since all packs and objects are copied
- * blindly from an existing MIDX if one is present.
- */
- ctx.m = lookup_multi_pack_index(the_repository, object_dir);
- }
+ if (!packs_to_include || ctx.incremental) {
+ struct multi_pack_index *m = lookup_multi_pack_index(the_repository,
+ object_dir);
+ if (m && !midx_checksum_valid(m)) {
+ warning(_("ignoring existing multi-pack-index; checksum mismatch"));
+ m = NULL;
+ }
- if (ctx.m && !midx_checksum_valid(ctx.m)) {
- warning(_("ignoring existing multi-pack-index; checksum mismatch"));
- ctx.m = NULL;
+ if (m) {
+ /*
+ * Only reference an existing MIDX when not filtering
+ * which packs to include, since all packs and objects
+ * are copied blindly from an existing MIDX if one is
+ * present.
+ */
+ if (ctx.incremental)
+ ctx.base_midx = m;
+ else if (!packs_to_include)
+ ctx.m = m;
+ }
}
ctx.nr = 0;
- ctx.alloc = ctx.m ? ctx.m->num_packs : 16;
+ ctx.alloc = ctx.m ? ctx.m->num_packs + ctx.m->num_packs_in_base : 16;
ctx.info = NULL;
ALLOC_ARRAY(ctx.info, ctx.alloc);
- if (ctx.m && fill_packs_from_midx(&ctx, preferred_pack_name,
- flags) < 0) {
- result = 1;
+ if (ctx.incremental) {
+ struct multi_pack_index *m = ctx.base_midx;
+ while (m) {
+ ctx.num_multi_pack_indexes_before++;
+ m = m->base_midx;
+ }
+ } else if (ctx.m && fill_packs_from_midx(&ctx, preferred_pack_name,
+ flags) < 0) {
goto cleanup;
}
for_each_file_in_pack_dir(object_dir, add_pack_to_midx, &ctx);
stop_progress(&ctx.progress);
- if ((ctx.m && ctx.nr == ctx.m->num_packs) &&
+ if ((ctx.m && ctx.nr == ctx.m->num_packs + ctx.m->num_packs_in_base) &&
+ !ctx.incremental &&
!(packs_to_include || packs_to_drop)) {
struct bitmap_index *bitmap_git;
int bitmap_exists;
* corresponding bitmap (or one wasn't requested).
*/
if (!want_bitmap)
- clear_midx_files_ext(object_dir, ".bitmap",
- NULL);
+ clear_midx_files_ext(object_dir, "bitmap", NULL);
goto cleanup;
}
}
+ if (ctx.incremental && !ctx.nr)
+ goto cleanup; /* nothing to do */
+
if (preferred_pack_name) {
ctx.preferred_pack_idx = -1;
pack_name_concat_len += MIDX_CHUNK_ALIGNMENT -
(pack_name_concat_len % MIDX_CHUNK_ALIGNMENT);
- hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
- f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
+ if (ctx.incremental) {
+ struct strbuf lock_name = STRBUF_INIT;
+
+ get_midx_chain_filename(&lock_name, object_dir);
+ hold_lock_file_for_update(&lk, lock_name.buf, LOCK_DIE_ON_ERROR);
+ strbuf_release(&lock_name);
+
+ incr = mks_tempfile_m(midx_name.buf, 0444);
+ if (!incr) {
+ error(_("unable to create temporary MIDX layer"));
+ return -1;
+ }
+
+ if (adjust_shared_perm(get_tempfile_path(incr))) {
+ error(_("unable to adjust shared permissions for '%s'"),
+ get_tempfile_path(incr));
+ return -1;
+ }
+
+ f = hashfd(get_tempfile_fd(incr), get_tempfile_path(incr));
+ } else {
+ hold_lock_file_for_update(&lk, midx_name.buf, LOCK_DIE_ON_ERROR);
+ f = hashfd(get_lock_file_fd(&lk), get_lock_file_path(&lk));
+ }
if (ctx.nr - dropped_packs == 0) {
error(_("no pack files to index."));
* have been freed in the previous if block.
*/
- if (ctx.m)
+ CALLOC_ARRAY(keep_hashes, ctx.num_multi_pack_indexes_before + 1);
+
+ if (ctx.incremental) {
+ FILE *chainf = fdopen_lock_file(&lk, "w");
+ struct strbuf final_midx_name = STRBUF_INIT;
+ struct multi_pack_index *m = ctx.base_midx;
+
+ if (!chainf) {
+ error_errno(_("unable to open multi-pack-index chain file"));
+ return -1;
+ }
+
+ if (link_midx_to_chain(ctx.base_midx) < 0)
+ return -1;
+
+ get_split_midx_filename_ext(&final_midx_name, object_dir,
+ midx_hash, MIDX_EXT_MIDX);
+
+ if (rename_tempfile(&incr, final_midx_name.buf) < 0) {
+ error_errno(_("unable to rename new multi-pack-index layer"));
+ return -1;
+ }
+
+ keep_hashes[ctx.num_multi_pack_indexes_before] =
+ xstrdup(hash_to_hex(midx_hash));
+
+ for (i = 0; i < ctx.num_multi_pack_indexes_before; i++) {
+ uint32_t j = ctx.num_multi_pack_indexes_before - i - 1;
+
+ keep_hashes[j] = xstrdup(hash_to_hex(get_midx_checksum(m)));
+ m = m->base_midx;
+ }
+
+ for (i = 0; i < ctx.num_multi_pack_indexes_before + 1; i++)
+ fprintf(get_lock_file_fp(&lk), "%s\n", keep_hashes[i]);
+ } else {
+ keep_hashes[ctx.num_multi_pack_indexes_before] =
+ xstrdup(hash_to_hex(midx_hash));
+ }
+
+ if (ctx.m || ctx.base_midx)
close_object_store(the_repository->objects);
if (commit_lock_file(&lk) < 0)
die_errno(_("could not write multi-pack-index"));
- clear_midx_files_ext(object_dir, ".bitmap", midx_hash);
- clear_midx_files_ext(object_dir, ".rev", midx_hash);
+ clear_midx_files(object_dir, keep_hashes,
+ ctx.num_multi_pack_indexes_before + 1,
+ ctx.incremental);
cleanup:
for (i = 0; i < ctx.nr; i++) {
free(ctx.entries);
free(ctx.pack_perm);
free(ctx.pack_order);
+ if (keep_hashes) {
+ for (i = 0; i < ctx.num_multi_pack_indexes_before + 1; i++)
+ free((char *)keep_hashes[i]);
+ free(keep_hashes);
+ }
strbuf_release(&midx_name);
trace2_region_leave("midx", "write_midx_internal", the_repository);
if (!m)
return 0;
+ if (m->base_midx)
+ die(_("cannot expire packs from an incremental multi-pack-index"));
+
CALLOC_ARRAY(count, m->num_packs);
if (flags & MIDX_PROGRESS)
if (!m)
return 0;
+ if (m->base_midx)
+ die(_("cannot repack an incremental multi-pack-index"));
CALLOC_ARRAY(include_pack, m->num_packs);
int midx_checksum_valid(struct multi_pack_index *m);
void clear_midx_files_ext(const char *object_dir, const char *ext,
- unsigned char *keep_hash);
+ const char *keep_hash);
+void clear_incremental_midx_files_ext(const char *object_dir, const char *ext,
+ char **keep_hashes,
+ uint32_t hashes_nr);
int cmp_idx_or_pack_name(const char *idx_or_pack_name,
const char *idx_name);
return 0;
}
+int midx_has_oid(struct multi_pack_index *m, const struct object_id *oid)
+{
+ return bsearch_midx(oid, m, NULL);
+}
+
struct object_id *nth_midxed_object_oid(struct object_id *oid,
struct multi_pack_index *m,
uint32_t n)
}
struct clear_midx_data {
- char *keep;
+ char **keep;
+ uint32_t keep_nr;
const char *ext;
};
const char *file_name, void *_data)
{
struct clear_midx_data *data = _data;
+ uint32_t i;
if (!(starts_with(file_name, "multi-pack-index-") &&
ends_with(file_name, data->ext)))
return;
- if (data->keep && !strcmp(data->keep, file_name))
- return;
-
+ for (i = 0; i < data->keep_nr; i++) {
+ if (!strcmp(data->keep[i], file_name))
+ return;
+ }
if (unlink(full_path))
die_errno(_("failed to remove %s"), full_path);
}
void clear_midx_files_ext(const char *object_dir, const char *ext,
- unsigned char *keep_hash)
+ const char *keep_hash)
{
struct clear_midx_data data;
memset(&data, 0, sizeof(struct clear_midx_data));
- if (keep_hash)
- data.keep = xstrfmt("multi-pack-index-%s%s",
- hash_to_hex(keep_hash), ext);
+ if (keep_hash) {
+ ALLOC_ARRAY(data.keep, 1);
+
+ data.keep[0] = xstrfmt("multi-pack-index-%s.%s", keep_hash, ext);
+ data.keep_nr = 1;
+ }
data.ext = ext;
for_each_file_in_pack_dir(object_dir,
clear_midx_file_ext,
&data);
+ if (keep_hash)
+ free(data.keep[0]);
+ free(data.keep);
+}
+
+void clear_incremental_midx_files_ext(const char *object_dir, const char *ext,
+ char **keep_hashes,
+ uint32_t hashes_nr)
+{
+ struct clear_midx_data data;
+ uint32_t i;
+
+ memset(&data, 0, sizeof(struct clear_midx_data));
+
+ ALLOC_ARRAY(data.keep, hashes_nr);
+ for (i = 0; i < hashes_nr; i++)
+ data.keep[i] = xstrfmt("multi-pack-index-%s.%s", keep_hashes[i],
+ ext);
+ data.keep_nr = hashes_nr;
+ data.ext = ext;
+
+ for_each_file_in_pack_subdir(object_dir, "multi-pack-index.d",
+ clear_midx_file_ext, &data);
+
+ for (i = 0; i < hashes_nr; i++)
+ free(data.keep[i]);
free(data.keep);
}
if (remove_path(midx.buf))
die(_("failed to clear multi-pack-index at %s"), midx.buf);
- clear_midx_files_ext(r->objects->odb->path, ".bitmap", NULL);
- clear_midx_files_ext(r->objects->odb->path, ".rev", NULL);
+ clear_midx_files_ext(r->objects->odb->path, MIDX_EXT_BITMAP, NULL);
+ clear_midx_files_ext(r->objects->odb->path, MIDX_EXT_REV, NULL);
strbuf_release(&midx);
}
#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
#define GIT_TEST_MULTI_PACK_INDEX "GIT_TEST_MULTI_PACK_INDEX"
+#define GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL \
+ "GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL"
struct multi_pack_index {
struct multi_pack_index *next;
#define MIDX_WRITE_BITMAP (1 << 2)
#define MIDX_WRITE_BITMAP_HASH_CACHE (1 << 3)
#define MIDX_WRITE_BITMAP_LOOKUP_TABLE (1 << 4)
+#define MIDX_WRITE_INCREMENTAL (1 << 5)
#define MIDX_EXT_REV "rev"
#define MIDX_EXT_BITMAP "bitmap"
uint32_t *result);
int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m,
uint32_t *result);
+int midx_has_oid(struct multi_pack_index *m, const struct object_id *oid);
off_t nth_midxed_offset(struct multi_pack_index *m, uint32_t pos);
uint32_t nth_midxed_pack_int_id(struct multi_pack_index *m, uint32_t pos);
struct object_id *nth_midxed_object_oid(struct object_id *oid,
report_helper(list, seen_bits, first, list->nr);
}
-void for_each_file_in_pack_dir(const char *objdir,
- each_file_in_pack_dir_fn fn,
- void *data)
+void for_each_file_in_pack_subdir(const char *objdir,
+ const char *subdir,
+ each_file_in_pack_dir_fn fn,
+ void *data)
{
struct strbuf path = STRBUF_INIT;
size_t dirnamelen;
strbuf_addstr(&path, objdir);
strbuf_addstr(&path, "/pack");
+ if (subdir)
+ strbuf_addf(&path, "/%s", subdir);
dir = opendir(path.buf);
if (!dir) {
if (errno != ENOENT)
strbuf_release(&path);
}
+void for_each_file_in_pack_dir(const char *objdir,
+ each_file_in_pack_dir_fn fn,
+ void *data)
+{
+ for_each_file_in_pack_subdir(objdir, NULL, fn, data);
+}
+
struct prepare_pack_data {
struct repository *r;
struct string_list *garbage;
typedef void each_file_in_pack_dir_fn(const char *full_path, size_t full_path_len,
const char *file_name, void *data);
+void for_each_file_in_pack_subdir(const char *objdir,
+ const char *subdir,
+ each_file_in_pack_dir_fn fn,
+ void *data);
void for_each_file_in_pack_dir(const char *objdir,
each_file_in_pack_dir_fn fn,
void *data);
index to be written after every 'git repack' command, and overrides the
'core.multiPackIndex' setting to true.
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=<boolean>, when true, sets
+the '--incremental' option on all invocations of 'git multi-pack-index
+write'.
+
GIT_TEST_SIDEBAND_ALL=<boolean>, when true, overrides the
'uploadpack.allowSidebandAll' setting to true, and when false, forces
fetch-pack to not request sideband-all (even if the server advertises
# Helpers for scripts testing bitmap functionality; see t5310 for
# example usage.
+. "$TEST_DIRECTORY"/lib-midx.sh
+
objdir=.git/objects
midx=$objdir/pack/multi-pack-index
test_cmp expect actual
}
-midx_checksum () {
- test-tool read-midx --checksum "$1"
-}
-
# midx_pack_source <obj>
midx_pack_source () {
test-tool read-midx --show-objects .git/objects | grep "^$1 " | cut -f2
test_cmp expect actual &&
git multi-pack-index --object-dir=$1 verify
}
+
+midx_checksum () {
+ test-tool read-midx --checksum "$1"
+}
+
+midx_git_two_modes () {
+ git -c core.multiPackIndex=false $1 >expect &&
+ git -c core.multiPackIndex=true $1 >actual &&
+ if [ "$2" = "sorted" ]
+ then
+ sort <expect >expect.sorted &&
+ mv expect.sorted expect &&
+ sort <actual >actual.sorted &&
+ mv actual.sorted actual
+ fi &&
+ test_cmp expect actual
+}
+
+compare_results_with_midx () {
+ MSG=$1
+ test_expect_success "check normal git operations: $MSG" '
+ midx_git_two_modes "rev-list --objects --all" &&
+ midx_git_two_modes "log --raw" &&
+ midx_git_two_modes "count-objects --verbose" &&
+ midx_git_two_modes "cat-file --batch-all-objects --batch-check" &&
+ midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted
+ '
+}
test_description='multi-pack-indexes'
. ./test-lib.sh
. "$TEST_DIRECTORY"/lib-chunk.sh
+. "$TEST_DIRECTORY"/lib-midx.sh
GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
objdir=.git/objects
HASH_LEN=$(test_oid rawsz)
midx_read_expect 1 18 4 $objdir
'
-midx_git_two_modes () {
- git -c core.multiPackIndex=false $1 >expect &&
- git -c core.multiPackIndex=true $1 >actual &&
- if [ "$2" = "sorted" ]
- then
- sort <expect >expect.sorted &&
- mv expect.sorted expect &&
- sort <actual >actual.sorted &&
- mv actual.sorted actual
- fi &&
- test_cmp expect actual
-}
-
-compare_results_with_midx () {
- MSG=$1
- test_expect_success "check normal git operations: $MSG" '
- midx_git_two_modes "rev-list --objects --all" &&
- midx_git_two_modes "log --raw" &&
- midx_git_two_modes "count-objects --verbose" &&
- midx_git_two_modes "cat-file --batch-all-objects --batch-check" &&
- midx_git_two_modes "cat-file --batch-all-objects --batch-check --unordered" sorted
- '
-}
-
test_expect_success 'write midx with one v2 pack' '
git pack-objects --index-version=2,0x40 $objdir/pack/test <obj-list &&
git multi-pack-index --object-dir=$objdir write &&
# We'll be writing our own MIDX, so avoid getting confused by the
# automatic ones.
GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
# This test exercise multi-pack bitmap functionality where the object order is
# stored and read from a special chunk within the MIDX, so use the default
# We'll be writing our own MIDX, so avoid getting confused by the automatic
# ones.
GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
# Unlike t5326, this test exercise multi-pack bitmap functionality where the
# object order is stored in a separate .rev file.
. ./test-lib.sh
. "$TEST_DIRECTORY"/lib-bitmap.sh
+GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
objdir=.git/objects
packdir=$objdir/pack
--- /dev/null
+#!/bin/sh
+
+test_description='incremental multi-pack-index'
+. ./test-lib.sh
+. "$TEST_DIRECTORY"/lib-midx.sh
+
+GIT_TEST_MULTI_PACK_INDEX=0
+export GIT_TEST_MULTI_PACK_INDEX
+
+objdir=.git/objects
+packdir=$objdir/pack
+midxdir=$packdir/multi-pack-index.d
+midx_chain=$midxdir/multi-pack-index-chain
+
+test_expect_success 'convert non-incremental MIDX to incremental' '
+ test_commit base &&
+ git repack -ad &&
+ git multi-pack-index write &&
+
+ test_path_is_file $packdir/multi-pack-index &&
+ old_hash="$(midx_checksum $objdir)" &&
+
+ test_commit other &&
+ git repack -d &&
+ git multi-pack-index write --incremental &&
+
+ test_path_is_missing $packdir/multi-pack-index &&
+ test_path_is_file $midx_chain &&
+ test_line_count = 2 $midx_chain &&
+ grep $old_hash $midx_chain
+'
+
+compare_results_with_midx 'incremental MIDX'
+
+test_expect_success 'convert incremental to non-incremental' '
+ test_commit squash &&
+ git repack -d &&
+ git multi-pack-index write &&
+
+ test_path_is_file $packdir/multi-pack-index &&
+ test_dir_is_empty $midxdir
+'
+
+compare_results_with_midx 'non-incremental MIDX conversion'
+
+test_done
. "${TEST_DIRECTORY}/lib-midx.sh"
. "${TEST_DIRECTORY}/lib-terminal.sh"
+GIT_TEST_MULTI_PACK_INDEX=0
+GIT_TEST_MULTI_PACK_INDEX_WRITE_INCREMENTAL=0
+
commit_and_pack () {
test_commit "$@" 1>&2 &&
incrpackid=$(git pack-objects --all --unpacked --incremental .git/objects/pack/pack </dev/null) &&
(
cd member &&
test_commit "object" &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adl --write-bitmap-index 2>err &&
+ git repack -Adl --write-bitmap-index 2>err &&
cat >expect <<-EOF &&
warning: disabling bitmap writing, as some objects are not being packed
EOF
test_expect_success '--write-midx unchanged' '
(
cd midx &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack &&
+ git repack &&
test_path_is_missing $midx &&
test_path_is_missing $midx-*.bitmap &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack --write-midx &&
+ git repack --write-midx &&
test_path_is_file $midx &&
test_path_is_missing $midx-*.bitmap &&
cd midx &&
test_commit loose &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack --write-midx &&
+ git repack --write-midx &&
test_path_is_file $midx &&
test_path_is_missing $midx-*.bitmap &&
test_expect_success '--write-midx with -b' '
(
cd midx &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -mb &&
+ git repack -mb &&
test_path_is_file $midx &&
test_path_is_file $midx-*.bitmap &&
cd midx &&
test_commit repack &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Ad --write-midx &&
+ git repack -Ad --write-midx &&
test_path_is_file $midx &&
test_path_is_missing $midx-*.bitmap &&
cd midx &&
test_commit repack-2 &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb --write-midx &&
+ git repack -Adb --write-midx &&
checksum=$(midx_checksum $objdir) &&
test_path_is_file $midx &&
test_path_is_file $midx-$checksum.bitmap &&
test_commit repack-3 &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb --write-midx &&
+ git repack -Adb --write-midx &&
test_path_is_file $midx &&
test_path_is_missing $midx-$checksum.bitmap &&
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
test_commit repack-4 &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Adb &&
+ git repack -Adb &&
find $objdir/pack -type f -name "multi-pack-index*" >files &&
test_must_be_empty files
git log --format="create refs/tags/%s/%s %H" HEAD >refs &&
git update-ref --stdin <refs &&
- GIT_TEST_MULTI_PACK_INDEX=0 \
git repack --write-midx --write-bitmap-index &&
test_path_is_file $midx &&
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
(
cd repo &&
test_commit base &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -Ab &&
+ git repack -Ab &&
pack_bitmap=$(ls $objdir/pack/pack-*.bitmap) &&
test_path_is_file "$pack_bitmap" &&
test_commit tip &&
- GIT_TEST_MULTI_PACK_INDEX=0 git repack -bm &&
+ git repack -bm &&
test_path_is_file $midx &&
test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
keep="$objdir/pack/pack-$one.keep" &&
touch "$keep" &&
- GIT_TEST_MULTI_PACK_INDEX=0 \
git repack --write-midx --write-bitmap-index --geometric=2 -d \
--pack-kept-objects &&
projects / thirdparty / git.git / commitdiff
