#include "refs.h"
#include "revision.h"
#include "list-objects.h"
+#include "pack-revindex.h"
#define MIDX_SIGNATURE 0x4d494458 /* "MIDX" */
#define MIDX_VERSION 1
#define MIDX_CHUNK_ALIGNMENT 4
#define MIDX_CHUNKID_PACKNAMES 0x504e414d /* "PNAM" */
+#define MIDX_CHUNKID_BITMAPPEDPACKS 0x42544d50 /* "BTMP" */
#define MIDX_CHUNKID_OIDFANOUT 0x4f494446 /* "OIDF" */
#define MIDX_CHUNKID_OIDLOOKUP 0x4f49444c /* "OIDL" */
#define MIDX_CHUNKID_OBJECTOFFSETS 0x4f4f4646 /* "OOFF" */
#define MIDX_CHUNK_FANOUT_SIZE (sizeof(uint32_t) * 256)
#define MIDX_CHUNK_OFFSET_WIDTH (2 * sizeof(uint32_t))
#define MIDX_CHUNK_LARGE_OFFSET_WIDTH (sizeof(uint64_t))
+#define MIDX_CHUNK_BITMAPPED_PACKS_WIDTH (2 * sizeof(uint32_t))
#define MIDX_LARGE_OFFSET_NEEDED 0x80000000
#define PACK_EXPIRED UINT_MAX
static int midx_read_oid_fanout(const unsigned char *chunk_start,
size_t chunk_size, void *data)
{
+ int i;
struct multi_pack_index *m = data;
m->chunk_oid_fanout = (uint32_t *)chunk_start;
error(_("multi-pack-index OID fanout is of the wrong size"));
return 1;
}
+ for (i = 0; i < 255; i++) {
+ uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
+ uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i+1]);
+
+ if (oid_fanout1 > oid_fanout2) {
+ error(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
+ i, oid_fanout1, oid_fanout2, i + 1);
+ return 1;
+ }
+ }
m->num_objects = ntohl(m->chunk_oid_fanout[255]);
return 0;
}
m->num_packs = get_be32(m->data + MIDX_BYTE_NUM_PACKS);
+ m->preferred_pack_idx = -1;
+
cf = init_chunkfile(NULL);
if (read_table_of_contents(cf, m->data, midx_size,
pair_chunk(cf, MIDX_CHUNKID_LARGEOFFSETS, &m->chunk_large_offsets,
&m->chunk_large_offsets_len);
+ pair_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS,
+ (const unsigned char **)&m->chunk_bitmapped_packs,
+ &m->chunk_bitmapped_packs_len);
if (git_env_bool("GIT_TEST_MIDX_READ_RIDX", 1))
pair_chunk(cf, MIDX_CHUNKID_REVINDEX, &m->chunk_revindex,
return 0;
}
+int nth_bitmapped_pack(struct repository *r, struct multi_pack_index *m,
+ struct bitmapped_pack *bp, uint32_t pack_int_id)
+{
+ if (!m->chunk_bitmapped_packs)
+ return error(_("MIDX does not contain the BTMP chunk"));
+
+ if (prepare_midx_pack(r, m, pack_int_id))
+ return error(_("could not load bitmapped pack %"PRIu32), pack_int_id);
+
+ bp->p = m->packs[pack_int_id];
+ bp->bitmap_pos = get_be32((char *)m->chunk_bitmapped_packs +
+ MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id);
+ bp->bitmap_nr = get_be32((char *)m->chunk_bitmapped_packs +
+ MIDX_CHUNK_BITMAPPED_PACKS_WIDTH * pack_int_id +
+ sizeof(uint32_t));
+ bp->pack_int_id = pack_int_id;
+
+ return 0;
+}
+
int bsearch_midx(const struct object_id *oid, struct multi_pack_index *m, uint32_t *result)
{
return bsearch_hash(oid->hash, m->chunk_oid_fanout, m->chunk_oid_lookup,
return strcmp(idx_or_pack_name, idx_name);
}
-int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
+int midx_locate_pack(struct multi_pack_index *m, const char *idx_or_pack_name,
+ uint32_t *pos)
{
uint32_t first = 0, last = m->num_packs;
current = m->pack_names[mid];
cmp = cmp_idx_or_pack_name(idx_or_pack_name, current);
- if (!cmp)
+ if (!cmp) {
+ if (pos)
+ *pos = mid;
return 1;
+ }
if (cmp > 0) {
first = mid + 1;
continue;
return 0;
}
+int midx_contains_pack(struct multi_pack_index *m, const char *idx_or_pack_name)
+{
+ return midx_locate_pack(m, idx_or_pack_name, NULL);
+}
+
+int midx_preferred_pack(struct multi_pack_index *m, uint32_t *pack_int_id)
+{
+ if (m->preferred_pack_idx == -1) {
+ if (load_midx_revindex(m) < 0) {
+ m->preferred_pack_idx = -2;
+ return -1;
+ }
+
+ m->preferred_pack_idx =
+ nth_midxed_pack_int_id(m, pack_pos_to_midx(m, 0));
+ } else if (m->preferred_pack_idx == -2)
+ return -1; /* no revindex */
+
+ *pack_int_id = m->preferred_pack_idx;
+ return 0;
+}
+
int prepare_multi_pack_index_one(struct repository *r, const char *object_dir, int local)
{
struct multi_pack_index *m;
return MIDX_HEADER_SIZE;
}
+#define BITMAP_POS_UNKNOWN (~((uint32_t)0))
+
struct pack_info {
uint32_t orig_pack_int_id;
char *pack_name;
struct packed_git *p;
+
+ uint32_t bitmap_pos;
+ uint32_t bitmap_nr;
+
unsigned expired : 1;
};
+static void fill_pack_info(struct pack_info *info,
+ struct packed_git *p, const char *pack_name,
+ uint32_t orig_pack_int_id)
+{
+ memset(info, 0, sizeof(struct pack_info));
+
+ info->orig_pack_int_id = orig_pack_int_id;
+ info->pack_name = xstrdup(pack_name);
+ info->p = p;
+ info->bitmap_pos = BITMAP_POS_UNKNOWN;
+}
+
static int pack_info_compare(const void *_a, const void *_b)
{
struct pack_info *a = (struct pack_info *)_a;
const char *file_name, void *data)
{
struct write_midx_context *ctx = data;
+ struct packed_git *p;
if (ends_with(file_name, ".idx")) {
display_progress(ctx->progress, ++ctx->pack_paths_checked);
ALLOC_GROW(ctx->info, ctx->nr + 1, ctx->alloc);
- ctx->info[ctx->nr].p = add_packed_git(full_path,
- full_path_len,
- 0);
-
- if (!ctx->info[ctx->nr].p) {
+ p = add_packed_git(full_path, full_path_len, 0);
+ if (!p) {
warning(_("failed to add packfile '%s'"),
full_path);
return;
}
- if (open_pack_index(ctx->info[ctx->nr].p)) {
+ if (open_pack_index(p)) {
warning(_("failed to open pack-index '%s'"),
full_path);
- close_pack(ctx->info[ctx->nr].p);
- FREE_AND_NULL(ctx->info[ctx->nr].p);
+ close_pack(p);
+ free(p);
return;
}
- ctx->info[ctx->nr].pack_name = xstrdup(file_name);
- ctx->info[ctx->nr].orig_pack_int_id = ctx->nr;
- ctx->info[ctx->nr].expired = 0;
+ fill_pack_info(&ctx->info[ctx->nr], p, file_name, ctx->nr);
ctx->nr++;
}
}
return 0;
}
+static int write_midx_bitmapped_packs(struct hashfile *f, void *data)
+{
+ struct write_midx_context *ctx = data;
+ size_t i;
+
+ for (i = 0; i < ctx->nr; i++) {
+ struct pack_info *pack = &ctx->info[i];
+ if (pack->expired)
+ continue;
+
+ if (pack->bitmap_pos == BITMAP_POS_UNKNOWN && pack->bitmap_nr)
+ BUG("pack '%s' has no bitmap position, but has %d bitmapped object(s)",
+ pack->pack_name, pack->bitmap_nr);
+
+ hashwrite_be32(f, pack->bitmap_pos);
+ hashwrite_be32(f, pack->bitmap_nr);
+ }
+ return 0;
+}
+
static int write_midx_oid_fanout(struct hashfile *f,
void *data)
{
QSORT(data, ctx->entries_nr, midx_pack_order_cmp);
ALLOC_ARRAY(pack_order, ctx->entries_nr);
- for (i = 0; i < ctx->entries_nr; i++)
+ for (i = 0; i < ctx->entries_nr; i++) {
+ struct pack_midx_entry *e = &ctx->entries[data[i].nr];
+ struct pack_info *pack = &ctx->info[ctx->pack_perm[e->pack_int_id]];
+ if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
+ pack->bitmap_pos = i;
+ pack->bitmap_nr++;
pack_order[i] = data[i].nr;
+ }
+ for (i = 0; i < ctx->nr; i++) {
+ struct pack_info *pack = &ctx->info[ctx->pack_perm[i]];
+ if (pack->bitmap_pos == BITMAP_POS_UNKNOWN)
+ pack->bitmap_pos = 0;
+ }
free(data);
trace2_region_leave("midx", "midx_pack_order", the_repository);
struct hashfile *f = NULL;
struct lock_file lk;
struct write_midx_context ctx = { 0 };
+ int bitmapped_packs_concat_len = 0;
int pack_name_concat_len = 0;
int dropped_packs = 0;
int result = 0;
for (i = 0; i < ctx.m->num_packs; i++) {
ALLOC_GROW(ctx.info, ctx.nr + 1, ctx.alloc);
- ctx.info[ctx.nr].orig_pack_int_id = i;
- ctx.info[ctx.nr].pack_name = xstrdup(ctx.m->pack_names[i]);
- ctx.info[ctx.nr].p = ctx.m->packs[i];
- ctx.info[ctx.nr].expired = 0;
-
if (flags & MIDX_WRITE_REV_INDEX) {
/*
* If generating a reverse index, need to have
if (open_pack_index(ctx.m->packs[i]))
die(_("could not open index for %s"),
ctx.m->packs[i]->pack_name);
- ctx.info[ctx.nr].p = ctx.m->packs[i];
}
- ctx.nr++;
+ fill_pack_info(&ctx.info[ctx.nr++], ctx.m->packs[i],
+ ctx.m->pack_names[i], i);
}
}
}
for (i = 0; i < ctx.nr; i++) {
- if (!ctx.info[i].expired)
- pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
+ if (ctx.info[i].expired)
+ continue;
+ pack_name_concat_len += strlen(ctx.info[i].pack_name) + 1;
+ bitmapped_packs_concat_len += 2 * sizeof(uint32_t);
}
/* Check that the preferred pack wasn't expired (if given). */
add_chunk(cf, MIDX_CHUNKID_REVINDEX,
st_mult(ctx.entries_nr, sizeof(uint32_t)),
write_midx_revindex);
+ add_chunk(cf, MIDX_CHUNKID_BITMAPPEDPACKS,
+ bitmapped_packs_concat_len,
+ write_midx_bitmapped_packs);
}
write_midx_header(f, get_num_chunks(cf), ctx.nr - dropped_packs);
flags) < 0) {
error(_("could not write multi-pack bitmap"));
result = 1;
+ clear_packing_data(&pdata);
+ free(commits);
goto cleanup;
}
+
+ clear_packing_data(&pdata);
+ free(commits);
}
/*
* NOTE: Do not use ctx.entries beyond this point, since it might
}
stop_progress(&progress);
- for (i = 0; i < 255; i++) {
- uint32_t oid_fanout1 = ntohl(m->chunk_oid_fanout[i]);
- uint32_t oid_fanout2 = ntohl(m->chunk_oid_fanout[i + 1]);
-
- if (oid_fanout1 > oid_fanout2)
- midx_report(_("oid fanout out of order: fanout[%d] = %"PRIx32" > %"PRIx32" = fanout[%d]"),
- i, oid_fanout1, oid_fanout2, i + 1);
- }
-
if (m->num_objects == 0) {
midx_report(_("the midx contains no oid"));
/*
projects / thirdparty / git.git / blobdiff