[thirdparty/git.git] / sparse-index.c

#include "cache.h"
#include "repository.h"
#include "sparse-index.h"
#include "tree.h"
#include "pathspec.h"
#include "trace2.h"
#include "cache-tree.h"
#include "config.h"
#include "dir.h"
#include "fsmonitor.h"

static struct cache_entry *construct_sparse_dir_entry(
				struct index_state *istate,
				const char *sparse_dir,
				struct cache_tree *tree)
{
	struct cache_entry *de;

	de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0);

	de->ce_flags |= CE_SKIP_WORKTREE;
	return de;
}

/*
 * Returns the number of entries "inserted" into the index.
 */
static int convert_to_sparse_rec(struct index_state *istate,
				 int num_converted,
				 int start, int end,
				 const char *ct_path, size_t ct_pathlen,
				 struct cache_tree *ct)
{
	int i, can_convert = 1;
	int start_converted = num_converted;
	enum pattern_match_result match;
	int dtype = DT_UNKNOWN;
	struct strbuf child_path = STRBUF_INIT;
	struct pattern_list *pl = istate->sparse_checkout_patterns;

	/*
	 * Is the current path outside of the sparse cone?
	 * Then check if the region can be replaced by a sparse
	 * directory entry (everything is sparse and merged).
	 */
	match = path_matches_pattern_list(ct_path, ct_pathlen,
					  NULL, &dtype, pl, istate);
	if (match != NOT_MATCHED)
		can_convert = 0;

	for (i = start; can_convert && i < end; i++) {
		struct cache_entry *ce = istate->cache[i];

		if (ce_stage(ce) ||
		    S_ISGITLINK(ce->ce_mode) ||
		    !(ce->ce_flags & CE_SKIP_WORKTREE))
			can_convert = 0;
	}

	if (can_convert) {
		struct cache_entry *se;
		se = construct_sparse_dir_entry(istate, ct_path, ct);

		istate->cache[num_converted++] = se;
		return 1;
	}

	for (i = start; i < end; ) {
		int count, span, pos = -1;
		const char *base, *slash;
		struct cache_entry *ce = istate->cache[i];

		/*
		 * Detect if this is a normal entry outside of any subtree
		 * entry.
		 */
		base = ce->name + ct_pathlen;
		slash = strchr(base, '/');

		if (slash)
			pos = cache_tree_subtree_pos(ct, base, slash - base);

		if (pos < 0) {
			istate->cache[num_converted++] = ce;
			i++;
			continue;
		}

		strbuf_setlen(&child_path, 0);
		strbuf_add(&child_path, ce->name, slash - ce->name + 1);

		span = ct->down[pos]->cache_tree->entry_count;
		count = convert_to_sparse_rec(istate,
					      num_converted, i, i + span,
					      child_path.buf, child_path.len,
					      ct->down[pos]->cache_tree);
		num_converted += count;
		i += span;
	}

	strbuf_release(&child_path);
	return num_converted - start_converted;
}

int set_sparse_index_config(struct repository *repo, int enable)
{
	int res;
	char *config_path = repo_git_path(repo, "config.worktree");
	res = git_config_set_in_file_gently(config_path,
					    "index.sparse",
					    enable ? "true" : NULL);
	free(config_path);

	prepare_repo_settings(repo);
	repo->settings.sparse_index = enable;
	return res;
}

static int index_has_unmerged_entries(struct index_state *istate)
{
	int i;
	for (i = 0; i < istate->cache_nr; i++) {
		if (ce_stage(istate->cache[i]))
			return 1;
	}

	return 0;
}

int convert_to_sparse(struct index_state *istate)
{
	int test_env;
	if (istate->split_index || istate->sparse_index ||
	    !core_apply_sparse_checkout || !core_sparse_checkout_cone)
		return 0;

	if (!istate->repo)
		istate->repo = the_repository;

	/*
	 * The GIT_TEST_SPARSE_INDEX environment variable triggers the
	 * index.sparse config variable to be on.
	 */
	test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
	if (test_env >= 0)
		set_sparse_index_config(istate->repo, test_env);

	/*
	 * Only convert to sparse if index.sparse is set.
	 */
	prepare_repo_settings(istate->repo);
	if (!istate->repo->settings.sparse_index)
		return 0;

	if (!istate->sparse_checkout_patterns) {
		istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list));
		if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0)
			return 0;
	}

	if (!istate->sparse_checkout_patterns->use_cone_patterns) {
		warning(_("attempting to use sparse-index without cone mode"));
		return -1;
	}

	/*
	 * NEEDSWORK: If we have unmerged entries, then stay full.
	 * Unmerged entries prevent the cache-tree extension from working.
	 */
	if (index_has_unmerged_entries(istate))
		return 0;

	/* Clear and recompute the cache-tree */
	cache_tree_free(&istate->cache_tree);
	if (cache_tree_update(istate, 0)) {
		warning(_("unable to update cache-tree, staying full"));
		return -1;
	}

	remove_fsmonitor(istate);

	trace2_region_enter("index", "convert_to_sparse", istate->repo);
	istate->cache_nr = convert_to_sparse_rec(istate,
						 0, 0, istate->cache_nr,
						 "", 0, istate->cache_tree);

	/* Clear and recompute the cache-tree */
	cache_tree_free(&istate->cache_tree);
	cache_tree_update(istate, 0);

	istate->fsmonitor_has_run_once = 0;
	FREE_AND_NULL(istate->fsmonitor_dirty);
	FREE_AND_NULL(istate->fsmonitor_last_update);

	istate->sparse_index = 1;
	trace2_region_leave("index", "convert_to_sparse", istate->repo);
	return 0;
}

static void set_index_entry(struct index_state *istate, int nr, struct cache_entry *ce)
{
	ALLOC_GROW(istate->cache, nr + 1, istate->cache_alloc);

	istate->cache[nr] = ce;
	add_name_hash(istate, ce);
}

static int add_path_to_index(const struct object_id *oid,
			     struct strbuf *base, const char *path,
			     unsigned int mode, void *context)
{
	struct index_state *istate = (struct index_state *)context;
	struct cache_entry *ce;
	size_t len = base->len;

	if (S_ISDIR(mode))
		return READ_TREE_RECURSIVE;

	strbuf_addstr(base, path);

	ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0);
	ce->ce_flags |= CE_SKIP_WORKTREE | CE_EXTENDED;
	set_index_entry(istate, istate->cache_nr++, ce);

	strbuf_setlen(base, len);
	return 0;
}

void ensure_full_index(struct index_state *istate)
{
	int i;
	struct index_state *full;
	struct strbuf base = STRBUF_INIT;

	if (!istate || !istate->sparse_index)
		return;

	if (!istate->repo)
		istate->repo = the_repository;

	trace2_region_enter("index", "ensure_full_index", istate->repo);

	/* initialize basics of new index */
	full = xcalloc(1, sizeof(struct index_state));
	memcpy(full, istate, sizeof(struct index_state));

	/* then change the necessary things */
	full->sparse_index = 0;
	full->cache_alloc = (3 * istate->cache_alloc) / 2;
	full->cache_nr = 0;
	ALLOC_ARRAY(full->cache, full->cache_alloc);

	for (i = 0; i < istate->cache_nr; i++) {
		struct cache_entry *ce = istate->cache[i];
		struct tree *tree;
		struct pathspec ps;

		if (!S_ISSPARSEDIR(ce->ce_mode)) {
			set_index_entry(full, full->cache_nr++, ce);
			continue;
		}
		if (!(ce->ce_flags & CE_SKIP_WORKTREE))
			warning(_("index entry is a directory, but not sparse (%08x)"),
				ce->ce_flags);

		/* recursively walk into cd->name */
		tree = lookup_tree(istate->repo, &ce->oid);

		memset(&ps, 0, sizeof(ps));
		ps.recursive = 1;
		ps.has_wildcard = 1;
		ps.max_depth = -1;

		strbuf_setlen(&base, 0);
		strbuf_add(&base, ce->name, strlen(ce->name));

		read_tree_at(istate->repo, tree, &base, &ps,
			     add_path_to_index, full);

		/* free directory entries. full entries are re-used */
		discard_cache_entry(ce);
	}

	/* Copy back into original index. */
	memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
	istate->sparse_index = 0;
	free(istate->cache);
	istate->cache = full->cache;
	istate->cache_nr = full->cache_nr;
	istate->cache_alloc = full->cache_alloc;
	istate->fsmonitor_has_run_once = 0;
	FREE_AND_NULL(istate->fsmonitor_dirty);
	FREE_AND_NULL(istate->fsmonitor_last_update);

	strbuf_release(&base);
	free(full);

	/* Clear and recompute the cache-tree */
	cache_tree_free(&istate->cache_tree);
	cache_tree_update(istate, 0);

	trace2_region_leave("index", "ensure_full_index", istate->repo);
}

/*
 * This static global helps avoid infinite recursion between
 * expand_to_path() and index_file_exists().
 */
static int in_expand_to_path = 0;

void expand_to_path(struct index_state *istate,
		    const char *path, size_t pathlen, int icase)
{
	struct strbuf path_mutable = STRBUF_INIT;
	size_t substr_len;

	/* prevent extra recursion */
	if (in_expand_to_path)
		return;

	if (!istate || !istate->sparse_index)
		return;

	if (!istate->repo)
		istate->repo = the_repository;

	in_expand_to_path = 1;

	/*
	 * We only need to actually expand a region if the
	 * following are both true:
	 *
	 * 1. 'path' is not already in the index.
	 * 2. Some parent directory of 'path' is a sparse directory.
	 */

	if (index_file_exists(istate, path, pathlen, icase))
		goto cleanup;

	strbuf_add(&path_mutable, path, pathlen);
	strbuf_addch(&path_mutable, '/');

	/* Check the name hash for all parent directories */
	substr_len = 0;
	while (substr_len < pathlen) {
		char temp;
		char *replace = strchr(path_mutable.buf + substr_len, '/');

		if (!replace)
			break;

		/* replace the character _after_ the slash */
		replace++;
		temp = *replace;
		*replace = '\0';
		if (index_file_exists(istate, path_mutable.buf,
				      path_mutable.len, icase)) {
			/*
			 * We found a parent directory in the name-hash
			 * hashtable, because only sparse directory entries
			 * have a trailing '/' character.  Since "path" wasn't
			 * in the index, perhaps it exists within this
			 * sparse-directory.  Expand accordingly.
			 */
			ensure_full_index(istate);
			break;
		}

		*replace = temp;
		substr_len = replace - path_mutable.buf;
	}

cleanup:
	strbuf_release(&path_mutable);
	in_expand_to_path = 0;
}
Commit	Line	Data
3964fc2a DS	1	#include "cache.h"
	2	#include "repository.h"
	3	#include "sparse-index.h"
4300f844 DS	4	#include "tree.h"
	5	#include "pathspec.h"
	6	#include "trace2.h"
6e773527 DS	7	#include "cache-tree.h"
	8	#include "config.h"
	9	#include "dir.h"
	10	#include "fsmonitor.h"
	11
	12	static struct cache_entry *construct_sparse_dir_entry(
	13	struct index_state *istate,
	14	const char *sparse_dir,
	15	struct cache_tree *tree)
	16	{
	17	struct cache_entry *de;
	18
	19	de = make_cache_entry(istate, S_IFDIR, &tree->oid, sparse_dir, 0, 0);
	20
	21	de->ce_flags \|= CE_SKIP_WORKTREE;
	22	return de;
	23	}
	24
	25	/*
	26	* Returns the number of entries "inserted" into the index.
	27	*/
	28	static int convert_to_sparse_rec(struct index_state *istate,
	29	int num_converted,
	30	int start, int end,
	31	const char *ct_path, size_t ct_pathlen,
	32	struct cache_tree *ct)
	33	{
	34	int i, can_convert = 1;
	35	int start_converted = num_converted;
	36	enum pattern_match_result match;
4279cb1c	37	int dtype = DT_UNKNOWN;
6e773527 DS	38	struct strbuf child_path = STRBUF_INIT;
	39	struct pattern_list *pl = istate->sparse_checkout_patterns;
	40
	41	/*
	42	* Is the current path outside of the sparse cone?
	43	* Then check if the region can be replaced by a sparse
	44	* directory entry (everything is sparse and merged).
	45	*/
	46	match = path_matches_pattern_list(ct_path, ct_pathlen,
	47	NULL, &dtype, pl, istate);
	48	if (match != NOT_MATCHED)
	49	can_convert = 0;
	50
	51	for (i = start; can_convert && i < end; i++) {
	52	struct cache_entry *ce = istate->cache[i];
	53
	54	if (ce_stage(ce) \|\|
f442313e	55	S_ISGITLINK(ce->ce_mode) \|\|
6e773527 DS	56	!(ce->ce_flags & CE_SKIP_WORKTREE))
	57	can_convert = 0;
	58	}
	59
	60	if (can_convert) {
	61	struct cache_entry *se;
	62	se = construct_sparse_dir_entry(istate, ct_path, ct);
	63
	64	istate->cache[num_converted++] = se;
	65	return 1;
	66	}
	67
	68	for (i = start; i < end; ) {
	69	int count, span, pos = -1;
	70	const char base, slash;
	71	struct cache_entry *ce = istate->cache[i];
	72
	73	/*
	74	* Detect if this is a normal entry outside of any subtree
	75	* entry.
	76	*/
	77	base = ce->name + ct_pathlen;
	78	slash = strchr(base, '/');
	79
	80	if (slash)
	81	pos = cache_tree_subtree_pos(ct, base, slash - base);
	82
	83	if (pos < 0) {
	84	istate->cache[num_converted++] = ce;
	85	i++;
	86	continue;
	87	}
	88
	89	strbuf_setlen(&child_path, 0);
	90	strbuf_add(&child_path, ce->name, slash - ce->name + 1);
	91
	92	span = ct->down[pos]->cache_tree->entry_count;
	93	count = convert_to_sparse_rec(istate,
	94	num_converted, i, i + span,
	95	child_path.buf, child_path.len,
	96	ct->down[pos]->cache_tree);
	97	num_converted += count;
	98	i += span;
	99	}
	100
	101	strbuf_release(&child_path);
	102	return num_converted - start_converted;
	103	}
	104
b79f9c07	105	int set_sparse_index_config(struct repository *repo, int enable)
58300f47	106	{
122ba1f7 DS	107	int res;
	108	char *config_path = repo_git_path(repo, "config.worktree");
	109	res = git_config_set_in_file_gently(config_path,
	110	"index.sparse",
	111	enable ? "true" : NULL);
	112	free(config_path);
58300f47	113
122ba1f7 DS	114	prepare_repo_settings(repo);
	115	repo->settings.sparse_index = enable;
	116	return res;
58300f47 DS	117	}
58300f47 DS	118
fc6609d1 DS	119	static int index_has_unmerged_entries(struct index_state *istate)
	120	{
	121	int i;
	122	for (i = 0; i < istate->cache_nr; i++) {
	123	if (ce_stage(istate->cache[i]))
	124	return 1;
	125	}
	126
	127	return 0;
	128	}
	129
6e773527 DS	130	int convert_to_sparse(struct index_state *istate)
6e773527 DS	131	{
122ba1f7	132	int test_env;
6e773527 DS	133	if (istate->split_index \|\| istate->sparse_index \|\|
	134	!core_apply_sparse_checkout \|\| !core_sparse_checkout_cone)
	135	return 0;
	136
58300f47 DS	137	if (!istate->repo)
	138	istate->repo = the_repository;
	139
	140	/*
	141	* The GIT_TEST_SPARSE_INDEX environment variable triggers the
	142	* index.sparse config variable to be on.
	143	*/
122ba1f7 DS	144	test_env = git_env_bool("GIT_TEST_SPARSE_INDEX", -1);
	145	if (test_env >= 0)
	146	set_sparse_index_config(istate->repo, test_env);
58300f47	147
6e773527	148	/*
58300f47	149	* Only convert to sparse if index.sparse is set.
6e773527	150	*/
58300f47 DS	151	prepare_repo_settings(istate->repo);
58300f47 DS	152	if (!istate->repo->settings.sparse_index)
6e773527 DS	153	return 0;
	154
	155	if (!istate->sparse_checkout_patterns) {
	156	istate->sparse_checkout_patterns = xcalloc(1, sizeof(struct pattern_list));
	157	if (get_sparse_checkout_patterns(istate->sparse_checkout_patterns) < 0)
	158	return 0;
	159	}
	160
	161	if (!istate->sparse_checkout_patterns->use_cone_patterns) {
	162	warning(_("attempting to use sparse-index without cone mode"));
	163	return -1;
	164	}
	165
fc6609d1 DS	166	/*
	167	* NEEDSWORK: If we have unmerged entries, then stay full.
	168	* Unmerged entries prevent the cache-tree extension from working.
	169	*/
	170	if (index_has_unmerged_entries(istate))
	171	return 0;
	172
f934f1b4 DS	173	/* Clear and recompute the cache-tree */
f934f1b4 DS	174	cache_tree_free(&istate->cache_tree);
6e773527 DS	175	if (cache_tree_update(istate, 0)) {
	176	warning(_("unable to update cache-tree, staying full"));
	177	return -1;
	178	}
	179
	180	remove_fsmonitor(istate);
	181
	182	trace2_region_enter("index", "convert_to_sparse", istate->repo);
	183	istate->cache_nr = convert_to_sparse_rec(istate,
	184	0, 0, istate->cache_nr,
	185	"", 0, istate->cache_tree);
2de37c53 DS	186
	187	/* Clear and recompute the cache-tree */
	188	cache_tree_free(&istate->cache_tree);
	189	cache_tree_update(istate, 0);
	190
f8fe49e5 DS	191	istate->fsmonitor_has_run_once = 0;
	192	FREE_AND_NULL(istate->fsmonitor_dirty);
	193	FREE_AND_NULL(istate->fsmonitor_last_update);
	194
6e773527 DS	195	istate->sparse_index = 1;
	196	trace2_region_leave("index", "convert_to_sparse", istate->repo);
	197	return 0;
	198	}
4300f844 DS	199
	200	static void set_index_entry(struct index_state istate, int nr, struct cache_entry ce)
	201	{
	202	ALLOC_GROW(istate->cache, nr + 1, istate->cache_alloc);
	203
	204	istate->cache[nr] = ce;
	205	add_name_hash(istate, ce);
	206	}
	207
	208	static int add_path_to_index(const struct object_id *oid,
	209	struct strbuf base, const char path,
	210	unsigned int mode, void *context)
	211	{
	212	struct index_state istate = (struct index_state )context;
	213	struct cache_entry *ce;
	214	size_t len = base->len;
	215
	216	if (S_ISDIR(mode))
	217	return READ_TREE_RECURSIVE;
	218
	219	strbuf_addstr(base, path);
	220
	221	ce = make_cache_entry(istate, mode, oid, base->buf, 0, 0);
47410778	222	ce->ce_flags \|= CE_SKIP_WORKTREE \| CE_EXTENDED;
4300f844 DS	223	set_index_entry(istate, istate->cache_nr++, ce);
	224
	225	strbuf_setlen(base, len);
	226	return 0;
	227	}
3964fc2a DS	228
	229	void ensure_full_index(struct index_state *istate)
	230	{
4300f844 DS	231	int i;
	232	struct index_state *full;
	233	struct strbuf base = STRBUF_INIT;
	234
	235	if (!istate \|\| !istate->sparse_index)
	236	return;
	237
	238	if (!istate->repo)
	239	istate->repo = the_repository;
	240
	241	trace2_region_enter("index", "ensure_full_index", istate->repo);
	242
	243	/* initialize basics of new index */
	244	full = xcalloc(1, sizeof(struct index_state));
	245	memcpy(full, istate, sizeof(struct index_state));
	246
	247	/* then change the necessary things */
	248	full->sparse_index = 0;
	249	full->cache_alloc = (3 * istate->cache_alloc) / 2;
	250	full->cache_nr = 0;
	251	ALLOC_ARRAY(full->cache, full->cache_alloc);
	252
	253	for (i = 0; i < istate->cache_nr; i++) {
	254	struct cache_entry *ce = istate->cache[i];
	255	struct tree *tree;
	256	struct pathspec ps;
	257
	258	if (!S_ISSPARSEDIR(ce->ce_mode)) {
	259	set_index_entry(full, full->cache_nr++, ce);
	260	continue;
	261	}
	262	if (!(ce->ce_flags & CE_SKIP_WORKTREE))
	263	warning(_("index entry is a directory, but not sparse (%08x)"),
	264	ce->ce_flags);
	265
	266	/* recursively walk into cd->name */
	267	tree = lookup_tree(istate->repo, &ce->oid);
	268
	269	memset(&ps, 0, sizeof(ps));
	270	ps.recursive = 1;
	271	ps.has_wildcard = 1;
	272	ps.max_depth = -1;
	273
	274	strbuf_setlen(&base, 0);
	275	strbuf_add(&base, ce->name, strlen(ce->name));
	276
	277	read_tree_at(istate->repo, tree, &base, &ps,
	278	add_path_to_index, full);
	279
	280	/* free directory entries. full entries are re-used */
	281	discard_cache_entry(ce);
	282	}
	283
	284	/* Copy back into original index. */
	285	memcpy(&istate->name_hash, &full->name_hash, sizeof(full->name_hash));
	286	istate->sparse_index = 0;
	287	free(istate->cache);
	288	istate->cache = full->cache;
	289	istate->cache_nr = full->cache_nr;
	290	istate->cache_alloc = full->cache_alloc;
f8fe49e5 DS	291	istate->fsmonitor_has_run_once = 0;
	292	FREE_AND_NULL(istate->fsmonitor_dirty);
	293	FREE_AND_NULL(istate->fsmonitor_last_update);
4300f844 DS	294
	295	strbuf_release(&base);
	296	free(full);
	297
2de37c53 DS	298	/* Clear and recompute the cache-tree */
	299	cache_tree_free(&istate->cache_tree);
	300	cache_tree_update(istate, 0);
	301
4300f844	302	trace2_region_leave("index", "ensure_full_index", istate->repo);
3964fc2a	303	}
71f82d03 DS	304
	305	/*
	306	* This static global helps avoid infinite recursion between
	307	* expand_to_path() and index_file_exists().
	308	*/
	309	static int in_expand_to_path = 0;
	310
	311	void expand_to_path(struct index_state *istate,
	312	const char *path, size_t pathlen, int icase)
	313	{
	314	struct strbuf path_mutable = STRBUF_INIT;
	315	size_t substr_len;
	316
	317	/* prevent extra recursion */
	318	if (in_expand_to_path)
	319	return;
	320
	321	if (!istate \|\| !istate->sparse_index)
	322	return;
	323
	324	if (!istate->repo)
	325	istate->repo = the_repository;
	326
	327	in_expand_to_path = 1;
	328
	329	/*
	330	* We only need to actually expand a region if the
	331	* following are both true:
	332	*
	333	* 1. 'path' is not already in the index.
	334	* 2. Some parent directory of 'path' is a sparse directory.
	335	*/
	336
	337	if (index_file_exists(istate, path, pathlen, icase))
	338	goto cleanup;
	339
	340	strbuf_add(&path_mutable, path, pathlen);
	341	strbuf_addch(&path_mutable, '/');
	342
	343	/* Check the name hash for all parent directories */
	344	substr_len = 0;
	345	while (substr_len < pathlen) {
	346	char temp;
	347	char *replace = strchr(path_mutable.buf + substr_len, '/');
	348
	349	if (!replace)
	350	break;
	351
	352	/* replace the character _after_ the slash */
	353	replace++;
	354	temp = *replace;
	355	*replace = '\0';
	356	if (index_file_exists(istate, path_mutable.buf,
	357	path_mutable.len, icase)) {
	358	/*
	359	* We found a parent directory in the name-hash
	360	* hashtable, because only sparse directory entries
	361	* have a trailing '/' character. Since "path" wasn't
	362	* in the index, perhaps it exists within this
	363	* sparse-directory. Expand accordingly.
	364	*/
	365	ensure_full_index(istate);
	366	break;
	367	}
368
369	*replace = temp;
370	substr_len = replace - path_mutable.buf;
371	}
372
373	cleanup:
374	strbuf_release(&path_mutable);
375	in_expand_to_path = 0;
376	}