[thirdparty/git.git] / t / t5326-multi-pack-bitmaps.sh

#!/bin/sh

test_description='exercise basic multi-pack bitmap functionality'
. ./test-lib.sh
. "${TEST_DIRECTORY}/lib-bitmap.sh"

# We'll be writing our own midx and bitmaps, so avoid getting confused by the
# automatic ones.
GIT_TEST_MULTI_PACK_INDEX=0
GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0

# This test exercise multi-pack bitmap functionality where the object order is
# stored and read from a special chunk within the MIDX, so use the default
# behavior here.
sane_unset GIT_TEST_MIDX_WRITE_REV
sane_unset GIT_TEST_MIDX_READ_RIDX

bitmap_reuse_tests() {
	from=$1
	to=$2
	writeLookupTable=false

	for i in $3-${$#}
	do
		case $i in
		"pack.writeBitmapLookupTable") writeLookupTable=true;;
		esac
	done

	test_expect_success "setup pack reuse tests ($from -> $to)" '
		rm -fr repo &&
		git init repo &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
			test_commit_bulk 16 &&
			git tag old-tip &&

			git config core.multiPackIndex true &&
			if test "MIDX" = "$from"
			then
				git repack -Ad &&
				git multi-pack-index write --bitmap
			else
				git repack -Adb
			fi
		)
	'

	test_expect_success "build bitmap from existing ($from -> $to)" '
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
			test_commit_bulk --id=further 16 &&
			git tag new-tip &&

			if test "MIDX" = "$to"
			then
				git repack -d &&
				git multi-pack-index write --bitmap
			else
				git repack -Adb
			fi
		)
	'

	test_expect_success "verify resulting bitmaps ($from -> $to)" '
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
			git for-each-ref &&
			git rev-list --test-bitmap refs/tags/old-tip &&
			git rev-list --test-bitmap refs/tags/new-tip
		)
	'
}

test_midx_bitmap_cases () {
	writeLookupTable=false
	writeBitmapLookupTable=

	for i in "$@"
	do
		case $i in
		"pack.writeBitmapLookupTable")
			writeLookupTable=true
			writeBitmapLookupTable="$i"
			;;
		esac
	done

	test_expect_success 'setup test_repository' '
		rm -rf * .git &&
		git init &&
		git config pack.writeBitmapLookupTable '"$writeLookupTable"'
	'

	midx_bitmap_core

	bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
	bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
	bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"

	test_expect_success 'missing object closure fails gracefully' '
		rm -fr repo &&
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit loose &&
			test_commit packed &&

			# Do not pass "--revs"; we want a pack without the "loose"
			# commit.
			git pack-objects $objdir/pack/pack <<-EOF &&
			$(git rev-parse packed)
			EOF

			test_must_fail git multi-pack-index write --bitmap 2>err &&
			grep "doesn.t have full closure" err &&
			test_path_is_missing $midx
		)
	'

	midx_bitmap_partial_tests

	test_expect_success 'removing a MIDX clears stale bitmaps' '
		rm -fr repo &&
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
			test_commit base &&
			git repack &&
			git multi-pack-index write --bitmap &&

			# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
			stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
			rm $midx &&

			# Then write a new MIDX.
			test_commit new &&
			git repack &&
			git multi-pack-index write --bitmap &&

			test_path_is_file $midx &&
			test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
			test_path_is_missing $stale_bitmap
		)
	'

	test_expect_success 'pack.preferBitmapTips' '
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit_bulk --message="%s" 103 &&

			git log --format="%H" >commits.raw &&
			sort <commits.raw >commits &&

			git log --format="create refs/tags/%s %H" HEAD >refs &&
			git update-ref --stdin <refs &&

			git multi-pack-index write --bitmap &&
			test_path_is_file $midx &&
			test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

			test-tool bitmap list-commits | sort >bitmaps &&
			comm -13 bitmaps commits >before &&
			test_line_count = 1 before &&

			perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
				<before | git update-ref --stdin &&

			rm -fr $midx-$(midx_checksum $objdir).bitmap &&
			rm -fr $midx &&

			git -c pack.preferBitmapTips=refs/tags/include \
				multi-pack-index write --bitmap &&
			test-tool bitmap list-commits | sort >bitmaps &&
			comm -13 bitmaps commits >after &&

			! test_cmp before after
		)
	'

	test_expect_success 'writing a bitmap with --refs-snapshot' '
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit one &&
			test_commit two &&

			git rev-parse one >snapshot &&

			git repack -ad &&

			# First, write a MIDX which see both refs/tags/one and
			# refs/tags/two (causing both of those commits to receive
			# bitmaps).
			git multi-pack-index write --bitmap &&

			test_path_is_file $midx &&
			test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

			test-tool bitmap list-commits | sort >bitmaps &&
			grep "$(git rev-parse one)" bitmaps &&
			grep "$(git rev-parse two)" bitmaps &&

			rm -fr $midx-$(midx_checksum $objdir).bitmap &&
			rm -fr $midx &&

			# Then again, but with a refs snapshot which only sees
			# refs/tags/one.
			git multi-pack-index write --bitmap --refs-snapshot=snapshot &&

			test_path_is_file $midx &&
			test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

			test-tool bitmap list-commits | sort >bitmaps &&
			grep "$(git rev-parse one)" bitmaps &&
			! grep "$(git rev-parse two)" bitmaps
		)
	'

	test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit_bulk --message="%s" 103 &&

			git log --format="%H" >commits.raw &&
			sort <commits.raw >commits &&

			git log --format="create refs/tags/%s %H" HEAD >refs &&
			git update-ref --stdin <refs &&

			git multi-pack-index write --bitmap &&
			test_path_is_file $midx &&
			test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

			test-tool bitmap list-commits | sort >bitmaps &&
			comm -13 bitmaps commits >before &&
			test_line_count = 1 before &&

			(
				grep -vf before commits.raw &&
				# mark missing commits as preferred
				sed "s/^/+/" before
			) >snapshot &&

			rm -fr $midx-$(midx_checksum $objdir).bitmap &&
			rm -fr $midx &&

			git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
			test-tool bitmap list-commits | sort >bitmaps &&
			comm -13 bitmaps commits >after &&

			! test_cmp before after
		)
	'

	test_expect_success 'hash-cache values are propagated from pack bitmaps' '
		rm -fr repo &&
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit base &&
			test_commit base2 &&
			git repack -adb &&

			test-tool bitmap dump-hashes >pack.raw &&
			test_file_not_empty pack.raw &&
			sort pack.raw >pack.hashes &&

			test_commit new &&
			git repack &&
			git multi-pack-index write --bitmap &&

			test-tool bitmap dump-hashes >midx.raw &&
			sort midx.raw >midx.hashes &&

			# ensure that every namehash in the pack bitmap can be found in
			# the midx bitmap (i.e., that there are no oid-namehash pairs
			# unique to the pack bitmap).
			comm -23 pack.hashes midx.hashes >dropped.hashes &&
			test_must_be_empty dropped.hashes
		)
	'

	test_expect_success 'no .bitmap is written without any objects' '
		rm -fr repo &&
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
			cat >packs <<-EOF &&
			pack-$empty.idx
			EOF

			git multi-pack-index write --bitmap --stdin-packs \
				<packs 2>err &&

			grep "bitmap without any objects" err &&

			test_path_is_file $midx &&
			test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
		)
	'

	test_expect_success 'graceful fallback when missing reverse index' '
		rm -fr repo &&
		git init repo &&
		test_when_finished "rm -fr repo" &&
		(
			cd repo &&
			git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&

			test_commit base &&

			# write a pack and MIDX bitmap containing base
			git repack -adb &&
			git multi-pack-index write --bitmap &&

			GIT_TEST_MIDX_READ_RIDX=0 \
				git rev-list --use-bitmap-index HEAD 2>err &&
			! grep "ignoring extra bitmap file" err
		)
	'
}

test_midx_bitmap_cases

test_midx_bitmap_cases "pack.writeBitmapLookupTable"

test_expect_success 'multi-pack-index write writes lookup table if enabled' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
		cd repo &&
		test_commit base &&
		git config pack.writeBitmapLookupTable true &&
		git repack -ad &&
		GIT_TRACE2_EVENT="$(pwd)/trace" \
			git multi-pack-index write --bitmap &&
		grep "\"label\":\"writing_lookup_table\"" trace
	)
'

test_expect_success 'preferred pack change with existing MIDX bitmap' '
	git init preferred-pack-with-existing &&
	(
		cd preferred-pack-with-existing &&

		test_commit base &&
		test_commit other &&

		git rev-list --objects --no-object-names base >p1.objects &&
		git rev-list --objects --no-object-names other >p2.objects &&

		p1="$(git pack-objects "$objdir/pack/pack" \
			--delta-base-offset <p1.objects)" &&
		p2="$(git pack-objects "$objdir/pack/pack" \
			--delta-base-offset <p2.objects)" &&

		# Generate a MIDX containing the first two packs,
		# marking p1 as preferred, and ensure that it can be
		# successfully cloned.
		git multi-pack-index write --bitmap \
			--preferred-pack="pack-$p1.pack" &&
		test_path_is_file $midx &&
		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
		git clone --no-local . clone1 &&

		# Then generate a new pack which sorts ahead of any
		# existing pack (by tweaking the pack prefix).
		test_commit foo &&
		git pack-objects --all --unpacked $objdir/pack/pack0 &&

		# Generate a new MIDX which changes the preferred pack
		# to a pack contained in the existing MIDX.
		git multi-pack-index write --bitmap \
			--preferred-pack="pack-$p2.pack" &&
		test_path_is_file $midx &&
		test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&

		# When the above circumstances are met, the preferred
		# pack should change appropriately and clones should
		# (still) succeed.
		git clone --no-local . clone2
	)
'

test_expect_success 'tagged commits are selected for bitmapping' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
		cd repo &&

		test_commit --annotate base &&
		git repack -d &&

		# Remove refs/heads/main which points at the commit directly,
		# leaving only a reference to the annotated tag.
		git branch -M main &&
		git checkout base &&
		git branch -d main &&

		git multi-pack-index write --bitmap &&

		git rev-parse HEAD >want &&
		test-tool bitmap list-commits >actual &&
		grep $(cat want) actual
	)
'

test_expect_success 'do not follow replace objects for MIDX bitmap' '
	rm -fr repo &&
	git init repo &&
	test_when_finished "rm -fr repo" &&
	(
		cd repo &&

		test_commit A &&
		test_commit B &&
		git checkout --orphan=orphan A &&
		test_commit orphan &&

		git replace A HEAD &&
		git repack -ad --write-midx --write-bitmap-index &&

		# generating reachability bitmaps with replace refs
		# enabled will result in broken clones
		git clone --no-local --bare . clone.git
	)
'

corrupt_file () {
	chmod a+w "$1" &&
	printf "bogus" | dd of="$1" bs=1 seek="12" conv=notrunc
}

test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
	git init valid &&
	test_when_finished rm -rf valid &&

	test_commit_bulk 20 &&
	git repack -adbf &&

	# Move pack-bitmap aside so it is not deleted
	# in next repack.
	packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
	mv "$packbitmap" "$packbitmap.bak" &&

	test_commit_bulk 10 &&
	git repack -b --write-midx &&
	midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&

	# Copy MIDX bitmap to backup. Copy pack bitmap from backup.
	cp "$midxbitmap" "$midxbitmap.bak" &&
	cp "$packbitmap.bak" "$packbitmap" &&

	# fsck works at first
	git fsck 2>err &&
	test_must_be_empty err &&

	corrupt_file "$packbitmap" &&
	test_must_fail git fsck 2>err &&
	grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&

	cp "$packbitmap.bak" "$packbitmap" &&
	corrupt_file "$midxbitmap" &&
	test_must_fail git fsck 2>err &&
	grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&

	corrupt_file "$packbitmap" &&
	test_must_fail git fsck 2>err &&
	grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
	grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
'

test_expect_success 'corrupt MIDX with bitmap causes fallback' '
	git init corrupt-midx-bitmap &&
	(
		cd corrupt-midx-bitmap &&

		test_commit first &&
		git repack -d &&
		test_commit second &&
		git repack -d &&

		git multi-pack-index write --bitmap &&
		checksum=$(midx_checksum $objdir) &&
		for f in $midx $midx-$checksum.bitmap
		do
			mv $f $f.bak || return 1
		done &&

		# pack everything together, invalidating the MIDX
		git repack -ad &&
		# then restore the now-stale MIDX
		for f in $midx $midx-$checksum.bitmap
		do
			mv $f.bak $f || return 1
		done &&

		git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
		# should attempt opening the broken pack twice (once
		# from the attempt to load it via the stale bitmap, and
		# again when attempting to load it from the stale MIDX)
		# before falling back to the non-MIDX case
		test 2 -eq $(grep -c "could not open pack" err) &&
		test 6 -eq $(cat out)
	)
'

for allow_pack_reuse in single multi
do
	test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
		test_when_finished "rm -rf midx-without-btmp" &&
		git init midx-without-btmp &&
		(
			cd midx-without-btmp &&
			test_commit initial &&

			git repack -Adbl --write-bitmap-index --write-midx &&
			GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
				pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
			test_must_be_empty err
		)
	'
done

test_done
Commit	Line	Data
c51f5a64 TB	1	#!/bin/sh
	2
	3	test_description='exercise basic multi-pack bitmap functionality'
	4	. ./test-lib.sh
	5	. "${TEST_DIRECTORY}/lib-bitmap.sh"
	6
	7	# We'll be writing our own midx and bitmaps, so avoid getting confused by the
	8	# automatic ones.
	9	GIT_TEST_MULTI_PACK_INDEX=0
	10	GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
	11
7f514b7a TB	12	# This test exercise multi-pack bitmap functionality where the object order is
	13	# stored and read from a special chunk within the MIDX, so use the default
	14	# behavior here.
	15	sane_unset GIT_TEST_MIDX_WRITE_REV
	16	sane_unset GIT_TEST_MIDX_READ_RIDX
	17
c51f5a64 TB	18	bitmap_reuse_tests() {
	19	from=$1
	20	to=$2
76f14b77 AC	21	writeLookupTable=false
	22
	23	for i in $3-${$#}
	24	do
	25	case $i in
	26	"pack.writeBitmapLookupTable") writeLookupTable=true;;
	27	esac
	28	done
c51f5a64 TB	29
	30	test_expect_success "setup pack reuse tests ($from -> $to)" '
	31	rm -fr repo &&
	32	git init repo &&
	33	(
	34	cd repo &&
76f14b77	35	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64 TB	36	test_commit_bulk 16 &&
	37	git tag old-tip &&
	38
	39	git config core.multiPackIndex true &&
	40	if test "MIDX" = "$from"
	41	then
	42	git repack -Ad &&
	43	git multi-pack-index write --bitmap
	44	else
	45	git repack -Adb
	46	fi
	47	)
	48	'
	49
	50	test_expect_success "build bitmap from existing ($from -> $to)" '
	51	(
	52	cd repo &&
76f14b77	53	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64 TB	54	test_commit_bulk --id=further 16 &&
	55	git tag new-tip &&
	56
	57	if test "MIDX" = "$to"
	58	then
	59	git repack -d &&
	60	git multi-pack-index write --bitmap
	61	else
	62	git repack -Adb
	63	fi
	64	)
	65	'
	66
	67	test_expect_success "verify resulting bitmaps ($from -> $to)" '
	68	(
	69	cd repo &&
76f14b77	70	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64 TB	71	git for-each-ref &&
	72	git rev-list --test-bitmap refs/tags/old-tip &&
	73	git rev-list --test-bitmap refs/tags/new-tip
	74	)
	75	'
	76	}
	77
76f14b77 AC	78	test_midx_bitmap_cases () {
	79	writeLookupTable=false
	80	writeBitmapLookupTable=
	81
	82	for i in "$@"
	83	do
	84	case $i in
	85	"pack.writeBitmapLookupTable")
	86	writeLookupTable=true
	87	writeBitmapLookupTable="$i"
	88	;;
	89	esac
	90	done
	91
	92	test_expect_success 'setup test_repository' '
	93	rm -rf * .git &&
	94	git init &&
	95	git config pack.writeBitmapLookupTable '"$writeLookupTable"'
	96	'
c51f5a64	97
76f14b77	98	midx_bitmap_core
c51f5a64	99
76f14b77 AC	100	bitmap_reuse_tests 'pack' 'MIDX' "$writeBitmapLookupTable"
	101	bitmap_reuse_tests 'MIDX' 'pack' "$writeBitmapLookupTable"
	102	bitmap_reuse_tests 'MIDX' 'MIDX' "$writeBitmapLookupTable"
c51f5a64	103
76f14b77 AC	104	test_expect_success 'missing object closure fails gracefully' '
	105	rm -fr repo &&
	106	git init repo &&
	107	test_when_finished "rm -fr repo" &&
	108	(
	109	cd repo &&
	110	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64	111
76f14b77 AC	112	test_commit loose &&
76f14b77 AC	113	test_commit packed &&
c51f5a64	114
76f14b77 AC	115	# Do not pass "--revs"; we want a pack without the "loose"
	116	# commit.
	117	git pack-objects $objdir/pack/pack <<-EOF &&
	118	$(git rev-parse packed)
	119	EOF
c51f5a64	120
76f14b77 AC	121	test_must_fail git multi-pack-index write --bitmap 2>err &&
	122	grep "doesn.t have full closure" err &&
	123	test_path_is_missing $midx
	124	)
	125	'
c51f5a64	126
76f14b77	127	midx_bitmap_partial_tests
c51f5a64	128
76f14b77 AC	129	test_expect_success 'removing a MIDX clears stale bitmaps' '
	130	rm -fr repo &&
	131	git init repo &&
	132	test_when_finished "rm -fr repo" &&
	133	(
	134	cd repo &&
	135	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
	136	test_commit base &&
	137	git repack &&
	138	git multi-pack-index write --bitmap &&
	139
	140	# Write a MIDX and bitmap; remove the MIDX but leave the bitmap.
	141	stale_bitmap=$midx-$(midx_checksum $objdir).bitmap &&
	142	rm $midx &&
	143
	144	# Then write a new MIDX.
	145	test_commit new &&
	146	git repack &&
	147	git multi-pack-index write --bitmap &&
	148
	149	test_path_is_file $midx &&
	150	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	151	test_path_is_missing $stale_bitmap
	152	)
	153	'
c51f5a64	154
76f14b77 AC	155	test_expect_success 'pack.preferBitmapTips' '
	156	git init repo &&
	157	test_when_finished "rm -fr repo" &&
	158	(
	159	cd repo &&
	160	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64	161
76f14b77	162	test_commit_bulk --message="%s" 103 &&
c51f5a64	163
76f14b77 AC	164	git log --format="%H" >commits.raw &&
76f14b77 AC	165	sort <commits.raw >commits &&
c51f5a64	166
76f14b77 AC	167	git log --format="create refs/tags/%s %H" HEAD >refs &&
76f14b77 AC	168	git update-ref --stdin <refs &&
c51f5a64	169
76f14b77 AC	170	git multi-pack-index write --bitmap &&
	171	test_path_is_file $midx &&
	172	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
c51f5a64	173
76f14b77 AC	174	test-tool bitmap list-commits \| sort >bitmaps &&
	175	comm -13 bitmaps commits >before &&
	176	test_line_count = 1 before &&
c51f5a64	177
76f14b77 AC	178	perl -ne "printf(\"create refs/tags/include/%d \", $.); print" \
76f14b77 AC	179	<before \| git update-ref --stdin &&
c51f5a64	180
76f14b77 AC	181	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
76f14b77 AC	182	rm -fr $midx &&
c51f5a64	183
76f14b77 AC	184	git -c pack.preferBitmapTips=refs/tags/include \
	185	multi-pack-index write --bitmap &&
	186	test-tool bitmap list-commits \| sort >bitmaps &&
	187	comm -13 bitmaps commits >after &&
c51f5a64	188
76f14b77 AC	189	! test_cmp before after
	190	)
	191	'
c51f5a64	192
76f14b77 AC	193	test_expect_success 'writing a bitmap with --refs-snapshot' '
	194	git init repo &&
	195	test_when_finished "rm -fr repo" &&
	196	(
	197	cd repo &&
	198	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
c51f5a64	199
76f14b77 AC	200	test_commit one &&
76f14b77 AC	201	test_commit two &&
08944d1c	202
76f14b77	203	git rev-parse one >snapshot &&
08944d1c	204
76f14b77	205	git repack -ad &&
08944d1c	206
76f14b77 AC	207	# First, write a MIDX which see both refs/tags/one and
	208	# refs/tags/two (causing both of those commits to receive
	209	# bitmaps).
	210	git multi-pack-index write --bitmap &&
08944d1c	211
76f14b77 AC	212	test_path_is_file $midx &&
76f14b77 AC	213	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
08944d1c	214
76f14b77 AC	215	test-tool bitmap list-commits \| sort >bitmaps &&
	216	grep "$(git rev-parse one)" bitmaps &&
	217	grep "$(git rev-parse two)" bitmaps &&
08944d1c	218
76f14b77 AC	219	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
76f14b77 AC	220	rm -fr $midx &&
08944d1c	221
76f14b77 AC	222	# Then again, but with a refs snapshot which only sees
	223	# refs/tags/one.
	224	git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
08944d1c	225
76f14b77 AC	226	test_path_is_file $midx &&
76f14b77 AC	227	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
08944d1c	228
76f14b77 AC	229	test-tool bitmap list-commits \| sort >bitmaps &&
	230	grep "$(git rev-parse one)" bitmaps &&
	231	! grep "$(git rev-parse two)" bitmaps
	232	)
	233	'
08944d1c	234
76f14b77 AC	235	test_expect_success 'write a bitmap with --refs-snapshot (preferred tips)' '
	236	git init repo &&
	237	test_when_finished "rm -fr repo" &&
	238	(
	239	cd repo &&
	240	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
08944d1c	241
76f14b77	242	test_commit_bulk --message="%s" 103 &&
08944d1c	243
76f14b77 AC	244	git log --format="%H" >commits.raw &&
76f14b77 AC	245	sort <commits.raw >commits &&
08944d1c	246
76f14b77 AC	247	git log --format="create refs/tags/%s %H" HEAD >refs &&
76f14b77 AC	248	git update-ref --stdin <refs &&
08944d1c	249
76f14b77 AC	250	git multi-pack-index write --bitmap &&
	251	test_path_is_file $midx &&
	252	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
08944d1c	253
76f14b77 AC	254	test-tool bitmap list-commits \| sort >bitmaps &&
	255	comm -13 bitmaps commits >before &&
	256	test_line_count = 1 before &&
08944d1c	257
76f14b77 AC	258	(
	259	grep -vf before commits.raw &&
	260	# mark missing commits as preferred
	261	sed "s/^/+/" before
	262	) >snapshot &&
08944d1c	263
76f14b77 AC	264	rm -fr $midx-$(midx_checksum $objdir).bitmap &&
	265	rm -fr $midx &&
	266
	267	git multi-pack-index write --bitmap --refs-snapshot=snapshot &&
	268	test-tool bitmap list-commits \| sort >bitmaps &&
	269	comm -13 bitmaps commits >after &&
	270
	271	! test_cmp before after
	272	)
	273	'
	274
	275	test_expect_success 'hash-cache values are propagated from pack bitmaps' '
	276	rm -fr repo &&
	277	git init repo &&
	278	test_when_finished "rm -fr repo" &&
08944d1c	279	(
76f14b77 AC	280	cd repo &&
76f14b77 AC	281	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
08944d1c	282
76f14b77 AC	283	test_commit base &&
	284	test_commit base2 &&
	285	git repack -adb &&
08944d1c	286
76f14b77 AC	287	test-tool bitmap dump-hashes >pack.raw &&
	288	test_file_not_empty pack.raw &&
	289	sort pack.raw >pack.hashes &&
08944d1c	290
76f14b77 AC	291	test_commit new &&
	292	git repack &&
	293	git multi-pack-index write --bitmap &&
08944d1c	294
76f14b77 AC	295	test-tool bitmap dump-hashes >midx.raw &&
76f14b77 AC	296	sort midx.raw >midx.hashes &&
54156af0	297
76f14b77 AC	298	# ensure that every namehash in the pack bitmap can be found in
	299	# the midx bitmap (i.e., that there are no oid-namehash pairs
	300	# unique to the pack bitmap).
	301	comm -23 pack.hashes midx.hashes >dropped.hashes &&
	302	test_must_be_empty dropped.hashes
	303	)
	304	'
54156af0	305
76f14b77 AC	306	test_expect_success 'no .bitmap is written without any objects' '
	307	rm -fr repo &&
	308	git init repo &&
	309	test_when_finished "rm -fr repo" &&
	310	(
	311	cd repo &&
	312	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
54156af0	313
76f14b77 AC	314	empty="$(git pack-objects $objdir/pack/pack </dev/null)" &&
	315	cat >packs <<-EOF &&
	316	pack-$empty.idx
	317	EOF
54156af0	318
76f14b77 AC	319	git multi-pack-index write --bitmap --stdin-packs \
76f14b77 AC	320	<packs 2>err &&
54156af0	321
76f14b77	322	grep "bitmap without any objects" err &&
54156af0	323
76f14b77 AC	324	test_path_is_file $midx &&
	325	test_path_is_missing $midx-$(midx_checksum $objdir).bitmap
	326	)
	327	'
	328
	329	test_expect_success 'graceful fallback when missing reverse index' '
	330	rm -fr repo &&
	331	git init repo &&
	332	test_when_finished "rm -fr repo" &&
	333	(
	334	cd repo &&
	335	git config pack.writeBitmapLookupTable '"$writeLookupTable"' &&
eb57277b	336
76f14b77	337	test_commit base &&
eb57277b	338
76f14b77 AC	339	# write a pack and MIDX bitmap containing base
	340	git repack -adb &&
	341	git multi-pack-index write --bitmap &&
eb57277b	342
76f14b77 AC	343	GIT_TEST_MIDX_READ_RIDX=0 \
	344	git rev-list --use-bitmap-index HEAD 2>err &&
	345	! grep "ignoring extra bitmap file" err
	346	)
	347	'
	348	}
eb57277b	349
76f14b77 AC	350	test_midx_bitmap_cases
	351
	352	test_midx_bitmap_cases "pack.writeBitmapLookupTable"
eb57277b	353
76f14b77	354	test_expect_success 'multi-pack-index write writes lookup table if enabled' '
f8b60cf9 TB	355	rm -fr repo &&
	356	git init repo &&
	357	test_when_finished "rm -fr repo" &&
	358	(
	359	cd repo &&
f8b60cf9	360	test_commit base &&
76f14b77 AC	361	git config pack.writeBitmapLookupTable true &&
	362	git repack -ad &&
	363	GIT_TRACE2_EVENT="$(pwd)/trace" \
	364	git multi-pack-index write --bitmap &&
	365	grep "\"label\":\"writing_lookup_table\"" trace
f8b60cf9 TB	366	)
	367	'
	368
65168c42 TB	369	test_expect_success 'preferred pack change with existing MIDX bitmap' '
	370	git init preferred-pack-with-existing &&
	371	(
	372	cd preferred-pack-with-existing &&
	373
	374	test_commit base &&
	375	test_commit other &&
	376
	377	git rev-list --objects --no-object-names base >p1.objects &&
	378	git rev-list --objects --no-object-names other >p2.objects &&
	379
	380	p1="$(git pack-objects "$objdir/pack/pack" \
	381	--delta-base-offset <p1.objects)" &&
	382	p2="$(git pack-objects "$objdir/pack/pack" \
	383	--delta-base-offset <p2.objects)" &&
	384
	385	# Generate a MIDX containing the first two packs,
	386	# marking p1 as preferred, and ensure that it can be
	387	# successfully cloned.
	388	git multi-pack-index write --bitmap \
	389	--preferred-pack="pack-$p1.pack" &&
	390	test_path_is_file $midx &&
	391	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	392	git clone --no-local . clone1 &&
	393
	394	# Then generate a new pack which sorts ahead of any
	395	# existing pack (by tweaking the pack prefix).
	396	test_commit foo &&
	397	git pack-objects --all --unpacked $objdir/pack/pack0 &&
	398
	399	# Generate a new MIDX which changes the preferred pack
cdf517be	400	# to a pack contained in the existing MIDX.
65168c42 TB	401	git multi-pack-index write --bitmap \
	402	--preferred-pack="pack-$p2.pack" &&
	403	test_path_is_file $midx &&
	404	test_path_is_file $midx-$(midx_checksum $objdir).bitmap &&
	405
cdf517be TB	406	# When the above circumstances are met, the preferred
	407	# pack should change appropriately and clones should
	408	# (still) succeed.
	409	git clone --no-local . clone2
65168c42 TB	410	)
	411	'
	412
1dc4f1ef TB	413	test_expect_success 'tagged commits are selected for bitmapping' '
	414	rm -fr repo &&
	415	git init repo &&
	416	test_when_finished "rm -fr repo" &&
	417	(
	418	cd repo &&
	419
	420	test_commit --annotate base &&
	421	git repack -d &&
	422
	423	# Remove refs/heads/main which points at the commit directly,
	424	# leaving only a reference to the annotated tag.
	425	git branch -M main &&
	426	git checkout base &&
	427	git branch -d main &&
	428
	429	git multi-pack-index write --bitmap &&
	430
	431	git rev-parse HEAD >want &&
	432	test-tool bitmap list-commits >actual &&
	433	grep $(cat want) actual
	434	)
	435	'
	436
93e2ae1c XX	437	test_expect_success 'do not follow replace objects for MIDX bitmap' '
	438	rm -fr repo &&
	439	git init repo &&
	440	test_when_finished "rm -fr repo" &&
	441	(
	442	cd repo &&
	443
	444	test_commit A &&
	445	test_commit B &&
	446	git checkout --orphan=orphan A &&
	447	test_commit orphan &&
	448
	449	git replace A HEAD &&
	450	git repack -ad --write-midx --write-bitmap-index &&
	451
	452	# generating reachability bitmaps with replace refs
	453	# enabled will result in broken clones
	454	git clone --no-local --bare . clone.git
	455	)
	456	'
	457
756f1bcd DS	458	corrupt_file () {
	459	chmod a+w "$1" &&
	460	printf "bogus" \| dd of="$1" bs=1 seek="12" conv=notrunc
	461	}
	462
	463	test_expect_success 'git fsck correctly identifies good and bad bitmaps' '
	464	git init valid &&
	465	test_when_finished rm -rf valid &&
	466
	467	test_commit_bulk 20 &&
	468	git repack -adbf &&
	469
	470	# Move pack-bitmap aside so it is not deleted
	471	# in next repack.
	472	packbitmap=$(ls .git/objects/pack/pack-*.bitmap) &&
	473	mv "$packbitmap" "$packbitmap.bak" &&
	474
	475	test_commit_bulk 10 &&
	476	git repack -b --write-midx &&
	477	midxbitmap=$(ls .git/objects/pack/multi-pack-index-*.bitmap) &&
	478
	479	# Copy MIDX bitmap to backup. Copy pack bitmap from backup.
	480	cp "$midxbitmap" "$midxbitmap.bak" &&
	481	cp "$packbitmap.bak" "$packbitmap" &&
	482
	483	# fsck works at first
	484	git fsck 2>err &&
	485	test_must_be_empty err &&
	486
	487	corrupt_file "$packbitmap" &&
	488	test_must_fail git fsck 2>err &&
	489	grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err &&
	490
	491	cp "$packbitmap.bak" "$packbitmap" &&
	492	corrupt_file "$midxbitmap" &&
	493	test_must_fail git fsck 2>err &&
	494	grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
	495
	496	corrupt_file "$packbitmap" &&
	497	test_must_fail git fsck 2>err &&
	498	grep "bitmap file '\''$midxbitmap'\'' has invalid checksum" err &&
	499	grep "bitmap file '\''$packbitmap'\'' has invalid checksum" err
	500	'
	501
06f38678 TB	502	test_expect_success 'corrupt MIDX with bitmap causes fallback' '
	503	git init corrupt-midx-bitmap &&
	504	(
	505	cd corrupt-midx-bitmap &&
	506
	507	test_commit first &&
	508	git repack -d &&
	509	test_commit second &&
	510	git repack -d &&
	511
	512	git multi-pack-index write --bitmap &&
	513	checksum=$(midx_checksum $objdir) &&
	514	for f in $midx $midx-$checksum.bitmap
	515	do
	516	mv $f $f.bak \|\| return 1
	517	done &&
	518
	519	# pack everything together, invalidating the MIDX
	520	git repack -ad &&
	521	# then restore the now-stale MIDX
	522	for f in $midx $midx-$checksum.bitmap
	523	do
	524	mv $f.bak $f \|\| return 1
	525	done &&
	526
	527	git rev-list --count --objects --use-bitmap-index HEAD >out 2>err &&
	528	# should attempt opening the broken pack twice (once
	529	# from the attempt to load it via the stale bitmap, and
	530	# again when attempting to load it from the stale MIDX)
	531	# before falling back to the non-MIDX case
	532	test 2 -eq $(grep -c "could not open pack" err) &&
	533	test 6 -eq $(cat out)
	534	)
	535	'
	536
795006ff PS	537	for allow_pack_reuse in single multi
	538	do
	539	test_expect_success "reading MIDX without BTMP chunk does not complain with $allow_pack_reuse pack reuse" '
	540	test_when_finished "rm -rf midx-without-btmp" &&
	541	git init midx-without-btmp &&
	542	(
	543	cd midx-without-btmp &&
	544	test_commit initial &&
	545
	546	git repack -Adbl --write-bitmap-index --write-midx &&
	547	GIT_TEST_MIDX_READ_BTMP=false git -c pack.allowPackReuse=$allow_pack_reuse \
	548	pack-objects --all --use-bitmap-index --stdout </dev/null >/dev/null 2>err &&
	549	test_must_be_empty err
	550	)
	551	'
	552	done
	553
c51f5a64	554	test_done