#include "shallow.h"
#include "json-writer.h"
#include "trace2.h"
+#include "chunk-format.h"
void git_test_write_commit_graph_or_die(void)
{
#define GRAPH_CHUNKID_BLOOMINDEXES 0x42494458 /* "BIDX" */
#define GRAPH_CHUNKID_BLOOMDATA 0x42444154 /* "BDAT" */
#define GRAPH_CHUNKID_BASE 0x42415345 /* "BASE" */
-#define MAX_NUM_CHUNKS 9
#define GRAPH_DATA_WIDTH (the_hash_algo->rawsz + 16)
#define GRAPH_HEADER_SIZE 8
#define GRAPH_FANOUT_SIZE (4 * 256)
-#define GRAPH_CHUNKLOOKUP_WIDTH 12
-#define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * GRAPH_CHUNKLOOKUP_WIDTH \
+#define GRAPH_MIN_SIZE (GRAPH_HEADER_SIZE + 4 * CHUNK_TOC_ENTRY_SIZE \
+ GRAPH_FANOUT_SIZE + the_hash_algo->rawsz)
#define CORRECTED_COMMIT_DATE_OFFSET_OVERFLOW (1ULL << 31)
static struct commit_graph_data_slab commit_graph_data_slab =
COMMIT_SLAB_INIT(1, commit_graph_data_slab);
+static int get_configured_generation_version(struct repository *r)
+{
+ int version = 2;
+ repo_config_get_int(r, "commitgraph.generationversion", &version);
+ return version;
+}
+
uint32_t commit_graph_position(const struct commit *c)
{
struct commit_graph_data *data =
if (read_replace_refs) {
prepare_replace_object(r);
- if (hashmap_get_size(&r->objects->replace_map->map)) {
- warning(_("repository contains replace objects; "
- "skipping commit-graph"));
+ if (hashmap_get_size(&r->objects->replace_map->map))
return 0;
- }
}
prepare_commit_graft(r);
if (r->parsed_objects &&
- (r->parsed_objects->grafts_nr || r->parsed_objects->substituted_parent)) {
- warning(_("repository contains (deprecated) grafts; "
- "skipping commit-graph"));
+ (r->parsed_objects->grafts_nr || r->parsed_objects->substituted_parent))
return 0;
- }
- if (is_repository_shallow(r)) {
- warning(_("repository is shallow; skipping commit-graph"));
+ if (is_repository_shallow(r))
return 0;
- }
return 1;
}
return 0;
}
+static int graph_read_oid_lookup(const unsigned char *chunk_start,
+ size_t chunk_size, void *data)
+{
+ struct commit_graph *g = data;
+ g->chunk_oid_lookup = chunk_start;
+ g->num_commits = chunk_size / g->hash_len;
+ return 0;
+}
+
+static int graph_read_bloom_data(const unsigned char *chunk_start,
+ size_t chunk_size, void *data)
+{
+ struct commit_graph *g = data;
+ uint32_t hash_version;
+ g->chunk_bloom_data = chunk_start;
+ hash_version = get_be32(chunk_start);
+
+ if (hash_version != 1)
+ return 0;
+
+ g->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
+ g->bloom_filter_settings->hash_version = hash_version;
+ g->bloom_filter_settings->num_hashes = get_be32(chunk_start + 4);
+ g->bloom_filter_settings->bits_per_entry = get_be32(chunk_start + 8);
+ g->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES;
+
+ return 0;
+}
+
struct commit_graph *parse_commit_graph(struct repository *r,
void *graph_map, size_t graph_size)
{
- const unsigned char *data, *chunk_lookup;
- uint32_t i;
+ const unsigned char *data;
struct commit_graph *graph;
- uint64_t next_chunk_offset;
uint32_t graph_signature;
unsigned char graph_version, hash_version;
+ struct chunkfile *cf = NULL;
if (!graph_map)
return NULL;
graph->data_len = graph_size;
if (graph_size < GRAPH_HEADER_SIZE +
- (graph->num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH +
+ (graph->num_chunks + 1) * CHUNK_TOC_ENTRY_SIZE +
GRAPH_FANOUT_SIZE + the_hash_algo->rawsz) {
error(_("commit-graph file is too small to hold %u chunks"),
graph->num_chunks);
return NULL;
}
- chunk_lookup = data + 8;
- next_chunk_offset = get_be64(chunk_lookup + 4);
- for (i = 0; i < graph->num_chunks; i++) {
- uint32_t chunk_id;
- uint64_t chunk_offset = next_chunk_offset;
- int chunk_repeated = 0;
-
- chunk_id = get_be32(chunk_lookup + 0);
-
- chunk_lookup += GRAPH_CHUNKLOOKUP_WIDTH;
- next_chunk_offset = get_be64(chunk_lookup + 4);
-
- if (chunk_offset > graph_size - the_hash_algo->rawsz) {
- error(_("commit-graph improper chunk offset %08x%08x"), (uint32_t)(chunk_offset >> 32),
- (uint32_t)chunk_offset);
- goto free_and_return;
- }
-
- switch (chunk_id) {
- case GRAPH_CHUNKID_OIDFANOUT:
- if (graph->chunk_oid_fanout)
- chunk_repeated = 1;
- else
- graph->chunk_oid_fanout = (uint32_t*)(data + chunk_offset);
- break;
-
- case GRAPH_CHUNKID_OIDLOOKUP:
- if (graph->chunk_oid_lookup)
- chunk_repeated = 1;
- else {
- graph->chunk_oid_lookup = data + chunk_offset;
- graph->num_commits = (next_chunk_offset - chunk_offset)
- / graph->hash_len;
- }
- break;
-
- case GRAPH_CHUNKID_DATA:
- if (graph->chunk_commit_data)
- chunk_repeated = 1;
- else
- graph->chunk_commit_data = data + chunk_offset;
- break;
-
- case GRAPH_CHUNKID_GENERATION_DATA:
- if (graph->chunk_generation_data)
- chunk_repeated = 1;
- else
- graph->chunk_generation_data = data + chunk_offset;
- break;
-
- case GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW:
- if (graph->chunk_generation_data_overflow)
- chunk_repeated = 1;
- else
- graph->chunk_generation_data_overflow = data + chunk_offset;
- break;
-
- case GRAPH_CHUNKID_EXTRAEDGES:
- if (graph->chunk_extra_edges)
- chunk_repeated = 1;
- else
- graph->chunk_extra_edges = data + chunk_offset;
- break;
-
- case GRAPH_CHUNKID_BASE:
- if (graph->chunk_base_graphs)
- chunk_repeated = 1;
- else
- graph->chunk_base_graphs = data + chunk_offset;
- break;
-
- case GRAPH_CHUNKID_BLOOMINDEXES:
- if (graph->chunk_bloom_indexes)
- chunk_repeated = 1;
- else if (r->settings.commit_graph_read_changed_paths)
- graph->chunk_bloom_indexes = data + chunk_offset;
- break;
+ cf = init_chunkfile(NULL);
- case GRAPH_CHUNKID_BLOOMDATA:
- if (graph->chunk_bloom_data)
- chunk_repeated = 1;
- else if (r->settings.commit_graph_read_changed_paths) {
- uint32_t hash_version;
- graph->chunk_bloom_data = data + chunk_offset;
- hash_version = get_be32(data + chunk_offset);
+ if (read_table_of_contents(cf, graph->data, graph_size,
+ GRAPH_HEADER_SIZE, graph->num_chunks))
+ goto free_and_return;
- if (hash_version != 1)
- break;
+ pair_chunk(cf, GRAPH_CHUNKID_OIDFANOUT,
+ (const unsigned char **)&graph->chunk_oid_fanout);
+ read_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, graph_read_oid_lookup, graph);
+ pair_chunk(cf, GRAPH_CHUNKID_DATA, &graph->chunk_commit_data);
+ pair_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES, &graph->chunk_extra_edges);
+ pair_chunk(cf, GRAPH_CHUNKID_BASE, &graph->chunk_base_graphs);
- graph->bloom_filter_settings = xmalloc(sizeof(struct bloom_filter_settings));
- graph->bloom_filter_settings->hash_version = hash_version;
- graph->bloom_filter_settings->num_hashes = get_be32(data + chunk_offset + 4);
- graph->bloom_filter_settings->bits_per_entry = get_be32(data + chunk_offset + 8);
- graph->bloom_filter_settings->max_changed_paths = DEFAULT_BLOOM_MAX_CHANGES;
- }
- break;
- }
+ if (get_configured_generation_version(r) >= 2) {
+ pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
+ &graph->chunk_generation_data);
+ pair_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
+ &graph->chunk_generation_data_overflow);
+ }
- if (chunk_repeated) {
- error(_("commit-graph chunk id %08x appears multiple times"), chunk_id);
- goto free_and_return;
- }
+ if (r->settings.commit_graph_read_changed_paths) {
+ pair_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
+ &graph->chunk_bloom_indexes);
+ read_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
+ graph_read_bloom_data, graph);
}
if (graph->chunk_bloom_indexes && graph->chunk_bloom_data) {
if (verify_commit_graph_lite(graph))
goto free_and_return;
+ free_chunkfile(cf);
return graph;
free_and_return:
+ free_chunkfile(cf);
free(graph->bloom_filter_settings);
free(graph);
return NULL;
return NULL;
count = st.st_size / (the_hash_algo->hexsz + 1);
- oids = xcalloc(count, sizeof(struct object_id));
+ CALLOC_ARRAY(oids, count);
prepare_alt_odb(r);
};
static int write_graph_chunk_fanout(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
int i, count = 0;
struct commit **list = ctx->commits.list;
}
static int write_graph_chunk_oids(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
struct commit **list = ctx->commits.list;
int count;
for (count = 0; count < ctx->commits.nr; count++, list++) {
}
static int write_graph_chunk_data(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
uint32_t num_extra_edges = 0;
}
static int write_graph_chunk_generation_data(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
int i, num_generation_data_overflows = 0;
for (i = 0; i < ctx->commits.nr; i++) {
}
static int write_graph_chunk_generation_data_overflow(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
int i;
for (i = 0; i < ctx->commits.nr; i++) {
struct commit *c = ctx->commits.list[i];
}
static int write_graph_chunk_extra_edges(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
struct commit_list *parent;
}
static int write_graph_chunk_bloom_indexes(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
uint32_t cur_pos = 0;
}
static int write_graph_chunk_bloom_data(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
struct commit **list = ctx->commits.list;
struct commit **last = ctx->commits.list + ctx->commits.nr;
}
static int write_graph_chunk_base(struct hashfile *f,
- struct write_commit_graph_context *ctx)
+ void *data)
{
+ struct write_commit_graph_context *ctx = data;
int num = write_graph_chunk_base_1(f, ctx->new_base_graph);
if (num != ctx->num_commit_graphs_after - 1) {
return 0;
}
-typedef int (*chunk_write_fn)(struct hashfile *f,
- struct write_commit_graph_context *ctx);
-
-struct chunk_info {
- uint32_t id;
- uint64_t size;
- chunk_write_fn write_fn;
-};
-
static int write_commit_graph_file(struct write_commit_graph_context *ctx)
{
uint32_t i;
int fd;
struct hashfile *f;
struct lock_file lk = LOCK_INIT;
- struct chunk_info chunks[MAX_NUM_CHUNKS + 1];
const unsigned hashsz = the_hash_algo->rawsz;
struct strbuf progress_title = STRBUF_INIT;
- int num_chunks = 3;
- uint64_t chunk_offset;
struct object_id file_hash;
+ struct chunkfile *cf;
if (ctx->split) {
struct strbuf tmp_file = STRBUF_INIT;
f = hashfd(fd, get_lock_file_path(&lk));
}
- chunks[0].id = GRAPH_CHUNKID_OIDFANOUT;
- chunks[0].size = GRAPH_FANOUT_SIZE;
- chunks[0].write_fn = write_graph_chunk_fanout;
- chunks[1].id = GRAPH_CHUNKID_OIDLOOKUP;
- chunks[1].size = hashsz * ctx->commits.nr;
- chunks[1].write_fn = write_graph_chunk_oids;
- chunks[2].id = GRAPH_CHUNKID_DATA;
- chunks[2].size = (hashsz + 16) * ctx->commits.nr;
- chunks[2].write_fn = write_graph_chunk_data;
-
- if (git_env_bool(GIT_TEST_COMMIT_GRAPH_NO_GDAT, 0))
- ctx->write_generation_data = 0;
- if (ctx->write_generation_data) {
- chunks[num_chunks].id = GRAPH_CHUNKID_GENERATION_DATA;
- chunks[num_chunks].size = sizeof(uint32_t) * ctx->commits.nr;
- chunks[num_chunks].write_fn = write_graph_chunk_generation_data;
- num_chunks++;
- }
- if (ctx->num_generation_data_overflows) {
- chunks[num_chunks].id = GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW;
- chunks[num_chunks].size = sizeof(timestamp_t) * ctx->num_generation_data_overflows;
- chunks[num_chunks].write_fn = write_graph_chunk_generation_data_overflow;
- num_chunks++;
- }
- if (ctx->num_extra_edges) {
- chunks[num_chunks].id = GRAPH_CHUNKID_EXTRAEDGES;
- chunks[num_chunks].size = 4 * ctx->num_extra_edges;
- chunks[num_chunks].write_fn = write_graph_chunk_extra_edges;
- num_chunks++;
- }
+ cf = init_chunkfile(f);
+
+ add_chunk(cf, GRAPH_CHUNKID_OIDFANOUT, GRAPH_FANOUT_SIZE,
+ write_graph_chunk_fanout);
+ add_chunk(cf, GRAPH_CHUNKID_OIDLOOKUP, hashsz * ctx->commits.nr,
+ write_graph_chunk_oids);
+ add_chunk(cf, GRAPH_CHUNKID_DATA, (hashsz + 16) * ctx->commits.nr,
+ write_graph_chunk_data);
+
+ if (ctx->write_generation_data)
+ add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA,
+ sizeof(uint32_t) * ctx->commits.nr,
+ write_graph_chunk_generation_data);
+ if (ctx->num_generation_data_overflows)
+ add_chunk(cf, GRAPH_CHUNKID_GENERATION_DATA_OVERFLOW,
+ sizeof(timestamp_t) * ctx->num_generation_data_overflows,
+ write_graph_chunk_generation_data_overflow);
+ if (ctx->num_extra_edges)
+ add_chunk(cf, GRAPH_CHUNKID_EXTRAEDGES,
+ 4 * ctx->num_extra_edges,
+ write_graph_chunk_extra_edges);
if (ctx->changed_paths) {
- chunks[num_chunks].id = GRAPH_CHUNKID_BLOOMINDEXES;
- chunks[num_chunks].size = sizeof(uint32_t) * ctx->commits.nr;
- chunks[num_chunks].write_fn = write_graph_chunk_bloom_indexes;
- num_chunks++;
- chunks[num_chunks].id = GRAPH_CHUNKID_BLOOMDATA;
- chunks[num_chunks].size = sizeof(uint32_t) * 3
- + ctx->total_bloom_filter_data_size;
- chunks[num_chunks].write_fn = write_graph_chunk_bloom_data;
- num_chunks++;
- }
- if (ctx->num_commit_graphs_after > 1) {
- chunks[num_chunks].id = GRAPH_CHUNKID_BASE;
- chunks[num_chunks].size = hashsz * (ctx->num_commit_graphs_after - 1);
- chunks[num_chunks].write_fn = write_graph_chunk_base;
- num_chunks++;
- }
-
- chunks[num_chunks].id = 0;
- chunks[num_chunks].size = 0;
+ add_chunk(cf, GRAPH_CHUNKID_BLOOMINDEXES,
+ sizeof(uint32_t) * ctx->commits.nr,
+ write_graph_chunk_bloom_indexes);
+ add_chunk(cf, GRAPH_CHUNKID_BLOOMDATA,
+ sizeof(uint32_t) * 3
+ + ctx->total_bloom_filter_data_size,
+ write_graph_chunk_bloom_data);
+ }
+ if (ctx->num_commit_graphs_after > 1)
+ add_chunk(cf, GRAPH_CHUNKID_BASE,
+ hashsz * (ctx->num_commit_graphs_after - 1),
+ write_graph_chunk_base);
hashwrite_be32(f, GRAPH_SIGNATURE);
hashwrite_u8(f, GRAPH_VERSION);
hashwrite_u8(f, oid_version());
- hashwrite_u8(f, num_chunks);
+ hashwrite_u8(f, get_num_chunks(cf));
hashwrite_u8(f, ctx->num_commit_graphs_after - 1);
- chunk_offset = 8 + (num_chunks + 1) * GRAPH_CHUNKLOOKUP_WIDTH;
- for (i = 0; i <= num_chunks; i++) {
- uint32_t chunk_write[3];
-
- chunk_write[0] = htonl(chunks[i].id);
- chunk_write[1] = htonl(chunk_offset >> 32);
- chunk_write[2] = htonl(chunk_offset & 0xffffffff);
- hashwrite(f, chunk_write, 12);
-
- chunk_offset += chunks[i].size;
- }
-
if (ctx->report_progress) {
strbuf_addf(&progress_title,
Q_("Writing out commit graph in %d pass",
"Writing out commit graph in %d passes",
- num_chunks),
- num_chunks);
+ get_num_chunks(cf)),
+ get_num_chunks(cf));
ctx->progress = start_delayed_progress(
progress_title.buf,
- num_chunks * ctx->commits.nr);
+ get_num_chunks(cf) * ctx->commits.nr);
}
- for (i = 0; i < num_chunks; i++) {
- uint64_t start_offset = f->total + f->offset;
-
- if (chunks[i].write_fn(f, ctx))
- return -1;
-
- if (f->total + f->offset != start_offset + chunks[i].size)
- BUG("expected to write %"PRId64" bytes to chunk %"PRIx32", but wrote %"PRId64" instead",
- chunks[i].size, chunks[i].id,
- f->total + f->offset - start_offset);
- }
+ write_chunkfile(cf, ctx);
stop_progress(&ctx->progress);
strbuf_release(&progress_title);
close_commit_graph(ctx->r->objects);
finalize_hashfile(f, file_hash.hash, CSUM_HASH_IN_STREAM | CSUM_FSYNC);
+ free_chunkfile(cf);
if (ctx->split) {
FILE *chainf = fdopen_lock_file(&lk, "w");
enum commit_graph_write_flags flags,
const struct commit_graph_opts *opts)
{
+ struct repository *r = the_repository;
struct write_commit_graph_context *ctx;
uint32_t i;
int res = 0;
struct bloom_filter_settings bloom_settings = DEFAULT_BLOOM_FILTER_SETTINGS;
struct topo_level_slab topo_levels;
- prepare_repo_settings(the_repository);
- if (!the_repository->settings.core_commit_graph) {
+ prepare_repo_settings(r);
+ if (!r->settings.core_commit_graph) {
warning(_("attempting to write a commit-graph, but 'core.commitGraph' is disabled"));
return 0;
}
- if (!commit_graph_compatible(the_repository))
+ if (!commit_graph_compatible(r))
return 0;
- ctx = xcalloc(1, sizeof(struct write_commit_graph_context));
- ctx->r = the_repository;
+ CALLOC_ARRAY(ctx, 1);
+ ctx->r = r;
ctx->odb = odb;
ctx->append = flags & COMMIT_GRAPH_WRITE_APPEND ? 1 : 0;
ctx->report_progress = flags & COMMIT_GRAPH_WRITE_PROGRESS ? 1 : 0;
ctx->split = flags & COMMIT_GRAPH_WRITE_SPLIT ? 1 : 0;
ctx->opts = opts;
ctx->total_bloom_filter_data_size = 0;
- ctx->write_generation_data = 1;
+ ctx->write_generation_data = (get_configured_generation_version(r) == 2);
ctx->num_generation_data_overflows = 0;
bloom_settings.bits_per_entry = git_env_ulong("GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY",
projects / thirdparty / git.git / blobdiff