[thirdparty/git.git] / t / t5310-pack-bitmaps.sh

#!/bin/sh

test_description='exercise basic bitmap functionality'
. ./test-lib.sh

objpath () {
	echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
}

# show objects present in pack ($1 should be associated *.idx)
list_packed_objects () {
	git show-index <"$1" | cut -d' ' -f2
}

# has_any pattern-file content-file
# tests whether content-file has any entry from pattern-file with entries being
# whole lines.
has_any () {
	grep -Ff "$1" "$2"
}

test_expect_success 'setup repo with moderate-sized history' '
	for i in $(test_seq 1 10); do
		test_commit $i
	done &&
	git checkout -b other HEAD~5 &&
	for i in $(test_seq 1 10); do
		test_commit side-$i
	done &&
	git checkout master &&
	bitmaptip=$(git rev-parse master) &&
	blob=$(echo tagged-blob | git hash-object -w --stdin) &&
	git tag tagged-blob $blob &&
	git config repack.writebitmaps true &&
	git config pack.writebitmaphashcache true
'

test_expect_success 'full repack creates bitmaps' '
	git repack -ad &&
	ls .git/objects/pack/ | grep bitmap >output &&
	test_line_count = 1 output
'

test_expect_success 'rev-list --test-bitmap verifies bitmaps' '
	git rev-list --test-bitmap HEAD
'

rev_list_tests() {
	state=$1

	test_expect_success "counting commits via bitmap ($state)" '
		git rev-list --count HEAD >expect &&
		git rev-list --use-bitmap-index --count HEAD >actual &&
		test_cmp expect actual
	'

	test_expect_success "counting partial commits via bitmap ($state)" '
		git rev-list --count HEAD~5..HEAD >expect &&
		git rev-list --use-bitmap-index --count HEAD~5..HEAD >actual &&
		test_cmp expect actual
	'

	test_expect_success "counting commits with limit ($state)" '
		git rev-list --count -n 1 HEAD >expect &&
		git rev-list --use-bitmap-index --count -n 1 HEAD >actual &&
		test_cmp expect actual
	'

	test_expect_success "counting non-linear history ($state)" '
		git rev-list --count other...master >expect &&
		git rev-list --use-bitmap-index --count other...master >actual &&
		test_cmp expect actual
	'

	test_expect_success "counting commits with limiting ($state)" '
		git rev-list --count HEAD -- 1.t >expect &&
		git rev-list --use-bitmap-index --count HEAD -- 1.t >actual &&
		test_cmp expect actual
	'

	test_expect_success "enumerate --objects ($state)" '
		git rev-list --objects --use-bitmap-index HEAD >tmp &&
		cut -d" " -f1 <tmp >tmp2 &&
		sort <tmp2 >actual &&
		git rev-list --objects HEAD >tmp &&
		cut -d" " -f1 <tmp >tmp2 &&
		sort <tmp2 >expect &&
		test_cmp expect actual
	'

	test_expect_success "bitmap --objects handles non-commit objects ($state)" '
		git rev-list --objects --use-bitmap-index HEAD tagged-blob >actual &&
		grep $blob actual
	'
}

rev_list_tests 'full bitmap'

test_expect_success 'clone from bitmapped repository' '
	git clone --no-local --bare . clone.git &&
	git rev-parse HEAD >expect &&
	git --git-dir=clone.git rev-parse HEAD >actual &&
	test_cmp expect actual
'

test_expect_success 'setup further non-bitmapped commits' '
	for i in $(test_seq 1 10); do
		test_commit further-$i
	done
'

rev_list_tests 'partial bitmap'

test_expect_success 'fetch (partial bitmap)' '
	git --git-dir=clone.git fetch origin master:master &&
	git rev-parse HEAD >expect &&
	git --git-dir=clone.git rev-parse HEAD >actual &&
	test_cmp expect actual
'

test_expect_success 'incremental repack cannot create bitmaps' '
	test_commit more-1 &&
	find .git/objects/pack -name "*.bitmap" >expect &&
	git repack -d &&
	find .git/objects/pack -name "*.bitmap" >actual &&
	test_cmp expect actual
'

test_expect_success 'incremental repack can disable bitmaps' '
	test_commit more-2 &&
	git repack -d --no-write-bitmap-index
'

test_expect_success 'pack-objects respects --local (non-local loose)' '
	git init --bare alt.git &&
	echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
	echo content1 >file1 &&
	# non-local loose object which is not present in bitmapped pack
	altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
	# non-local loose object which is also present in bitmapped pack
	git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
	git add file1 &&
	test_tick &&
	git commit -m commit_file1 &&
	echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
	git index-pack 1.pack &&
	list_packed_objects 1.idx >1.objects &&
	printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
	! has_any nonlocal-loose 1.objects
'

test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
	echo content2 >file2 &&
	blob2=$(git hash-object -w file2) &&
	git add file2 &&
	test_tick &&
	git commit -m commit_file2 &&
	printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
	pack2=$(git pack-objects pack2 <keepobjects) &&
	mv pack2-$pack2.* .git/objects/pack/ &&
	>.git/objects/pack/pack2-$pack2.keep &&
	rm $(objpath $blob2) &&
	echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
	git index-pack 2a.pack &&
	list_packed_objects 2a.idx >2a.objects &&
	! has_any keepobjects 2a.objects
'

test_expect_success 'pack-objects respects --local (non-local pack)' '
	mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
	echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
	git index-pack 2b.pack &&
	list_packed_objects 2b.idx >2b.objects &&
	! has_any keepobjects 2b.objects
'

test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
	ls .git/objects/pack/ | grep bitmap >output &&
	test_line_count = 1 output &&
	packbitmap=$(basename $(cat output) .bitmap) &&
	list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
	test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
	>.git/objects/pack/$packbitmap.keep &&
	echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
	git index-pack 3a.pack &&
	list_packed_objects 3a.idx >3a.objects &&
	! has_any packbitmap.objects 3a.objects
'

test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
	mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
	test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
	echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
	git index-pack 3b.pack &&
	list_packed_objects 3b.idx >3b.objects &&
	! has_any packbitmap.objects 3b.objects
'

test_expect_success 'full repack, reusing previous bitmaps' '
	git repack -ad &&
	ls .git/objects/pack/ | grep bitmap >output &&
	test_line_count = 1 output
'

test_expect_success 'fetch (full bitmap)' '
	git --git-dir=clone.git fetch origin master:master &&
	git rev-parse HEAD >expect &&
	git --git-dir=clone.git rev-parse HEAD >actual &&
	test_cmp expect actual
'

test_expect_success 'create objects for missing-HAVE tests' '
	blob=$(echo "missing have" | git hash-object -w --stdin) &&
	tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
	parent=$(echo parent | git commit-tree $tree) &&
	commit=$(echo commit | git commit-tree $tree -p $parent) &&
	cat >revs <<-EOF
	HEAD
	^HEAD^
	^$commit
	EOF
'

test_expect_success 'pack-objects respects --incremental' '
	cat >revs2 <<-EOF &&
	HEAD
	$commit
	EOF
	git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
	git index-pack 4.pack &&
	list_packed_objects 4.idx >4.objects &&
	test_line_count = 4 4.objects &&
	git rev-list --objects $commit >revlist &&
	cut -d" " -f1 revlist |sort >objects &&
	test_cmp 4.objects objects
'

test_expect_success 'pack with missing blob' '
	rm $(objpath $blob) &&
	git pack-objects --stdout --revs <revs >/dev/null
'

test_expect_success 'pack with missing tree' '
	rm $(objpath $tree) &&
	git pack-objects --stdout --revs <revs >/dev/null
'

test_expect_success 'pack with missing parent' '
	rm $(objpath $parent) &&
	git pack-objects --stdout --revs <revs >/dev/null
'

test_lazy_prereq JGIT '
	type jgit
'

test_expect_success JGIT 'we can read jgit bitmaps' '
	git clone . compat-jgit &&
	(
		cd compat-jgit &&
		rm -f .git/objects/pack/*.bitmap &&
		jgit gc &&
		git rev-list --test-bitmap HEAD
	)
'

test_expect_success JGIT 'jgit can read our bitmaps' '
	git clone . compat-us &&
	(
		cd compat-us &&
		git repack -adb &&
		# jgit gc will barf if it does not like our bitmaps
		jgit gc
	)
'

test_expect_success 'splitting packs does not generate bogus bitmaps' '
	test-genrandom foo $((1024 * 1024)) >rand &&
	git add rand &&
	git commit -m "commit with big file" &&
	git -c pack.packSizeLimit=500k repack -adb &&
	git init --bare no-bitmaps.git &&
	git -C no-bitmaps.git fetch .. HEAD
'

test_done
Commit	Line	Data
212f2ffb JK	1	#!/bin/sh
	2
	3	test_description='exercise basic bitmap functionality'
	4	. ./test-lib.sh
	5
2db1a43f VM	6	objpath () {
	7	echo ".git/objects/$(echo "$1" \| sed -e 's\|\(..\)\|\1/\|')"
	8	}
	9
702d1b95 KS	10	# show objects present in pack ($1 should be associated *.idx)
	11	list_packed_objects () {
	12	git show-index <"$1" \| cut -d' ' -f2
	13	}
	14
	15	# has_any pattern-file content-file
	16	# tests whether content-file has any entry from pattern-file with entries being
	17	# whole lines.
	18	has_any () {
	19	grep -Ff "$1" "$2"
	20	}
	21
212f2ffb JK	22	test_expect_success 'setup repo with moderate-sized history' '
	23	for i in $(test_seq 1 10); do
	24	test_commit $i
	25	done &&
	26	git checkout -b other HEAD~5 &&
	27	for i in $(test_seq 1 10); do
	28	test_commit side-$i
	29	done &&
	30	git checkout master &&
702d1b95	31	bitmaptip=$(git rev-parse master) &&
212f2ffb JK	32	blob=$(echo tagged-blob \| git hash-object -w --stdin) &&
212f2ffb JK	33	git tag tagged-blob $blob &&
71d76cb4	34	git config repack.writebitmaps true &&
ae4f07fb	35	git config pack.writebitmaphashcache true
212f2ffb JK	36	'
	37
	38	test_expect_success 'full repack creates bitmaps' '
	39	git repack -ad &&
	40	ls .git/objects/pack/ \| grep bitmap >output &&
	41	test_line_count = 1 output
	42	'
	43
	44	test_expect_success 'rev-list --test-bitmap verifies bitmaps' '
	45	git rev-list --test-bitmap HEAD
	46	'
	47
	48	rev_list_tests() {
	49	state=$1
	50
	51	test_expect_success "counting commits via bitmap ($state)" '
	52	git rev-list --count HEAD >expect &&
	53	git rev-list --use-bitmap-index --count HEAD >actual &&
	54	test_cmp expect actual
	55	'
	56
	57	test_expect_success "counting partial commits via bitmap ($state)" '
	58	git rev-list --count HEAD~5..HEAD >expect &&
	59	git rev-list --use-bitmap-index --count HEAD~5..HEAD >actual &&
	60	test_cmp expect actual
	61	'
	62
5c9f9bf3 JK	63	test_expect_success "counting commits with limit ($state)" '
	64	git rev-list --count -n 1 HEAD >expect &&
	65	git rev-list --use-bitmap-index --count -n 1 HEAD >actual &&
	66	test_cmp expect actual
	67	'
	68
212f2ffb JK	69	test_expect_success "counting non-linear history ($state)" '
	70	git rev-list --count other...master >expect &&
	71	git rev-list --use-bitmap-index --count other...master >actual &&
	72	test_cmp expect actual
	73	'
	74
c8a70d35 JK	75	test_expect_success "counting commits with limiting ($state)" '
	76	git rev-list --count HEAD -- 1.t >expect &&
	77	git rev-list --use-bitmap-index --count HEAD -- 1.t >actual &&
	78	test_cmp expect actual
	79	'
	80
212f2ffb JK	81	test_expect_success "enumerate --objects ($state)" '
	82	git rev-list --objects --use-bitmap-index HEAD >tmp &&
	83	cut -d" " -f1 <tmp >tmp2 &&
	84	sort <tmp2 >actual &&
	85	git rev-list --objects HEAD >tmp &&
	86	cut -d" " -f1 <tmp >tmp2 &&
	87	sort <tmp2 >expect &&
	88	test_cmp expect actual
	89	'
	90
	91	test_expect_success "bitmap --objects handles non-commit objects ($state)" '
	92	git rev-list --objects --use-bitmap-index HEAD tagged-blob >actual &&
	93	grep $blob actual
	94	'
	95	}
	96
	97	rev_list_tests 'full bitmap'
	98
	99	test_expect_success 'clone from bitmapped repository' '
	100	git clone --no-local --bare . clone.git &&
	101	git rev-parse HEAD >expect &&
	102	git --git-dir=clone.git rev-parse HEAD >actual &&
	103	test_cmp expect actual
	104	'
	105
	106	test_expect_success 'setup further non-bitmapped commits' '
	107	for i in $(test_seq 1 10); do
	108	test_commit further-$i
	109	done
	110	'
	111
	112	rev_list_tests 'partial bitmap'
	113
	114	test_expect_success 'fetch (partial bitmap)' '
	115	git --git-dir=clone.git fetch origin master:master &&
	116	git rev-parse HEAD >expect &&
	117	git --git-dir=clone.git rev-parse HEAD >actual &&
	118	test_cmp expect actual
	119	'
	120
	121	test_expect_success 'incremental repack cannot create bitmaps' '
	122	test_commit more-1 &&
373c67da JK	123	find .git/objects/pack -name "*.bitmap" >expect &&
	124	git repack -d &&
	125	find .git/objects/pack -name "*.bitmap" >actual &&
	126	test_cmp expect actual
212f2ffb JK	127	'
	128
	129	test_expect_success 'incremental repack can disable bitmaps' '
	130	test_commit more-2 &&
	131	git repack -d --no-write-bitmap-index
	132	'
	133
702d1b95 KS	134	test_expect_success 'pack-objects respects --local (non-local loose)' '
	135	git init --bare alt.git &&
	136	echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
	137	echo content1 >file1 &&
	138	# non-local loose object which is not present in bitmapped pack
	139	altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
	140	# non-local loose object which is also present in bitmapped pack
	141	git cat-file blob $blob \| GIT_DIR=alt.git git hash-object -w --stdin &&
	142	git add file1 &&
	143	test_tick &&
	144	git commit -m commit_file1 &&
	145	echo HEAD \| git pack-objects --local --stdout --revs >1.pack &&
	146	git index-pack 1.pack &&
	147	list_packed_objects 1.idx >1.objects &&
	148	printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
	149	! has_any nonlocal-loose 1.objects
	150	'
	151
	152	test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
	153	echo content2 >file2 &&
	154	blob2=$(git hash-object -w file2) &&
	155	git add file2 &&
	156	test_tick &&
	157	git commit -m commit_file2 &&
	158	printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
	159	pack2=$(git pack-objects pack2 <keepobjects) &&
	160	mv pack2-$pack2.* .git/objects/pack/ &&
	161	>.git/objects/pack/pack2-$pack2.keep &&
	162	rm $(objpath $blob2) &&
	163	echo HEAD \| git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
	164	git index-pack 2a.pack &&
	165	list_packed_objects 2a.idx >2a.objects &&
	166	! has_any keepobjects 2a.objects
	167	'
	168
	169	test_expect_success 'pack-objects respects --local (non-local pack)' '
	170	mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
	171	echo HEAD \| git pack-objects --local --stdout --revs >2b.pack &&
	172	git index-pack 2b.pack &&
	173	list_packed_objects 2b.idx >2b.objects &&
	174	! has_any keepobjects 2b.objects
	175	'
	176
	177	test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
	178	ls .git/objects/pack/ \| grep bitmap >output &&
	179	test_line_count = 1 output &&
	180	packbitmap=$(basename $(cat output) .bitmap) &&
	181	list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
	182	test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
	183	>.git/objects/pack/$packbitmap.keep &&
	184	echo HEAD \| git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
	185	git index-pack 3a.pack &&
	186	list_packed_objects 3a.idx >3a.objects &&
	187	! has_any packbitmap.objects 3a.objects
	188	'
	189
	190	test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
	191	mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
	192	test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
	193	echo HEAD \| git pack-objects --local --stdout --revs >3b.pack &&
	194	git index-pack 3b.pack &&
	195	list_packed_objects 3b.idx >3b.objects &&
	196	! has_any packbitmap.objects 3b.objects
	197	'
198
212f2ffb JK	199	test_expect_success 'full repack, reusing previous bitmaps' '
	200	git repack -ad &&
	201	ls .git/objects/pack/ \| grep bitmap >output &&
	202	test_line_count = 1 output
	203	'
	204
	205	test_expect_success 'fetch (full bitmap)' '
	206	git --git-dir=clone.git fetch origin master:master &&
	207	git rev-parse HEAD >expect &&
	208	git --git-dir=clone.git rev-parse HEAD >actual &&
	209	test_cmp expect actual
	210	'
	211
2db1a43f VM	212	test_expect_success 'create objects for missing-HAVE tests' '
	213	blob=$(echo "missing have" \| git hash-object -w --stdin) &&
	214	tree=$(printf "100644 blob $blob\tfile\n" \| git mktree) &&
	215	parent=$(echo parent \| git commit-tree $tree) &&
	216	commit=$(echo commit \| git commit-tree $tree -p $parent) &&
	217	cat >revs <<-EOF
	218	HEAD
	219	^HEAD^
	220	^$commit
	221	EOF
	222	'
	223
702d1b95 KS	224	test_expect_success 'pack-objects respects --incremental' '
	225	cat >revs2 <<-EOF &&
	226	HEAD
	227	$commit
	228	EOF
	229	git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
	230	git index-pack 4.pack &&
	231	list_packed_objects 4.idx >4.objects &&
	232	test_line_count = 4 4.objects &&
	233	git rev-list --objects $commit >revlist &&
	234	cut -d" " -f1 revlist \|sort >objects &&
	235	test_cmp 4.objects objects
	236	'
	237
2db1a43f VM	238	test_expect_success 'pack with missing blob' '
	239	rm $(objpath $blob) &&
	240	git pack-objects --stdout --revs <revs >/dev/null
	241	'
	242
	243	test_expect_success 'pack with missing tree' '
	244	rm $(objpath $tree) &&
	245	git pack-objects --stdout --revs <revs >/dev/null
	246	'
	247
	248	test_expect_success 'pack with missing parent' '
	249	rm $(objpath $parent) &&
	250	git pack-objects --stdout --revs <revs >/dev/null
	251	'
	252
212f2ffb JK	253	test_lazy_prereq JGIT '
	254	type jgit
	255	'
	256
	257	test_expect_success JGIT 'we can read jgit bitmaps' '
	258	git clone . compat-jgit &&
	259	(
	260	cd compat-jgit &&
	261	rm -f .git/objects/pack/*.bitmap &&
	262	jgit gc &&
	263	git rev-list --test-bitmap HEAD
	264	)
	265	'
	266
	267	test_expect_success JGIT 'jgit can read our bitmaps' '
	268	git clone . compat-us &&
	269	(
	270	cd compat-us &&
	271	git repack -adb &&
	272	# jgit gc will barf if it does not like our bitmaps
	273	jgit gc
	274	)
	275	'
	276
21134714 JK	277	test_expect_success 'splitting packs does not generate bogus bitmaps' '
	278	test-genrandom foo $((1024 * 1024)) >rand &&
	279	git add rand &&
	280	git commit -m "commit with big file" &&
	281	git -c pack.packSizeLimit=500k repack -adb &&
	282	git init --bare no-bitmaps.git &&
	283	git -C no-bitmaps.git fetch .. HEAD
	284	'
	285
212f2ffb	286	test_done