[thirdparty/git.git] / t / t5302-pack-index.sh

#!/bin/sh
#
# Copyright (c) 2007 Nicolas Pitre
#

test_description='pack index with 64-bit offsets and object CRC'
. ./test-lib.sh

test_expect_success \
    'setup' \
    'rm -rf .git
     git init &&
     git config pack.threads 1 &&
     i=1 &&
     while test $i -le 100
     do
         iii=`printf '%03i' $i`
         test-genrandom "bar" 200 > wide_delta_$iii &&
         test-genrandom "baz $iii" 50 >> wide_delta_$iii &&
         test-genrandom "foo"$i 100 > deep_delta_$iii &&
         test-genrandom "foo"`expr $i + 1` 100 >> deep_delta_$iii &&
         test-genrandom "foo"`expr $i + 2` 100 >> deep_delta_$iii &&
         echo $iii >file_$iii &&
         test-genrandom "$iii" 8192 >>file_$iii &&
         git update-index --add file_$iii deep_delta_$iii wide_delta_$iii &&
         i=`expr $i + 1` || return 1
     done &&
     { echo 101 && test-genrandom 100 8192; } >file_101 &&
     git update-index --add file_101 &&
     tree=`git write-tree` &&
     commit=`git commit-tree $tree </dev/null` && {
	 echo $tree &&
	 git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)	.*/\\1/"
     } >obj-list &&
     git update-ref HEAD $commit'

test_expect_success \
    'pack-objects with index version 1' \
    'pack1=$(git pack-objects --index-version=1 test-1 <obj-list) &&
     git verify-pack -v "test-1-${pack1}.pack"'

test_expect_success \
    'pack-objects with index version 2' \
    'pack2=$(git pack-objects --index-version=2 test-2 <obj-list) &&
     git verify-pack -v "test-2-${pack2}.pack"'

test_expect_success \
    'both packs should be identical' \
    'cmp "test-1-${pack1}.pack" "test-2-${pack2}.pack"'

test_expect_success \
    'index v1 and index v2 should be different' \
    '! cmp "test-1-${pack1}.idx" "test-2-${pack2}.idx"'

test_expect_success \
    'index-pack with index version 1' \
    'git index-pack --index-version=1 -o 1.idx "test-1-${pack1}.pack"'

test_expect_success \
    'index-pack with index version 2' \
    'git index-pack --index-version=2 -o 2.idx "test-1-${pack1}.pack"'

test_expect_success \
    'index-pack results should match pack-objects ones' \
    'cmp "test-1-${pack1}.idx" "1.idx" &&
     cmp "test-2-${pack2}.idx" "2.idx"'

test_expect_success \
    'index v2: force some 64-bit offsets with pack-objects' \
    'pack3=$(git pack-objects --index-version=2,0x40000 test-3 <obj-list)'

if msg=$(git verify-pack -v "test-3-${pack3}.pack" 2>&1) ||
	! (echo "$msg" | grep "pack too large .* off_t")
then
	test_set_prereq OFF64_T
else
	say "# skipping tests concerning 64-bit offsets"
fi

test_expect_success OFF64_T \
    'index v2: verify a pack with some 64-bit offsets' \
    'git verify-pack -v "test-3-${pack3}.pack"'

test_expect_success OFF64_T \
    '64-bit offsets: should be different from previous index v2 results' \
    '! cmp "test-2-${pack2}.idx" "test-3-${pack3}.idx"'

test_expect_success OFF64_T \
    'index v2: force some 64-bit offsets with index-pack' \
    'git index-pack --index-version=2,0x40000 -o 3.idx "test-1-${pack1}.pack"'

test_expect_success OFF64_T \
    '64-bit offsets: index-pack result should match pack-objects one' \
    'cmp "test-3-${pack3}.idx" "3.idx"'

# returns the object number for given object in given pack index
index_obj_nr()
{
    idx_file=$1
    object_sha1=$2
    nr=0
    git show-index < $idx_file |
    while read offs sha1 extra
    do
      nr=$(($nr + 1))
      test "$sha1" = "$object_sha1" || continue
      echo "$(($nr - 1))"
      break
    done
}

# returns the pack offset for given object as found in given pack index
index_obj_offset()
{
    idx_file=$1
    object_sha1=$2
    git show-index < $idx_file | grep $object_sha1 |
    ( read offs extra && echo "$offs" )
}

test_expect_success \
    '[index v1] 1) stream pack to repository' \
    'git index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
     git prune-packed &&
     git count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
     cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
     cmp "test-1-${pack1}.idx"  ".git/objects/pack/pack-${pack1}.idx"'

test_expect_success \
    '[index v1] 2) create a stealth corruption in a delta base reference' \
    '# This test assumes file_101 is a delta smaller than 16 bytes.
     # It should be against file_100 but we substitute its base for file_099
     sha1_101=`git hash-object file_101` &&
     sha1_099=`git hash-object file_099` &&
     offs_101=`index_obj_offset 1.idx $sha1_101` &&
     nr_099=`index_obj_nr 1.idx $sha1_099` &&
     chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
     dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
        if=".git/objects/pack/pack-${pack1}.idx" \
        skip=$((4 + 256 * 4 + $nr_099 * 24)) \
        bs=1 count=20 conv=notrunc &&
     git cat-file blob $sha1_101 > file_101_foo1'

test_expect_success \
    '[index v1] 3) corrupted delta happily returned wrong data' \
    'test -f file_101_foo1 && ! cmp file_101 file_101_foo1'

test_expect_success \
    '[index v1] 4) confirm that the pack is actually corrupted' \
    'test_must_fail git fsck --full $commit'

test_expect_success \
    '[index v1] 5) pack-objects happily reuses corrupted data' \
    'pack4=$(git pack-objects test-4 <obj-list) &&
     test -f "test-4-${pack1}.pack"'

test_expect_success \
    '[index v1] 6) newly created pack is BAD !' \
    'test_must_fail git verify-pack -v "test-4-${pack1}.pack"'

test_expect_success \
    '[index v2] 1) stream pack to repository' \
    'rm -f .git/objects/pack/* &&
     git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
     git prune-packed &&
     git count-objects | ( read nr rest && test "$nr" -eq 1 ) &&
     cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
     cmp "test-2-${pack1}.idx"  ".git/objects/pack/pack-${pack1}.idx"'

test_expect_success \
    '[index v2] 2) create a stealth corruption in a delta base reference' \
    '# This test assumes file_101 is a delta smaller than 16 bytes.
     # It should be against file_100 but we substitute its base for file_099
     sha1_101=`git hash-object file_101` &&
     sha1_099=`git hash-object file_099` &&
     offs_101=`index_obj_offset 1.idx $sha1_101` &&
     nr_099=`index_obj_nr 1.idx $sha1_099` &&
     chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
     dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
        if=".git/objects/pack/pack-${pack1}.idx" \
        skip=$((8 + 256 * 4 + $nr_099 * 20)) \
        bs=1 count=20 conv=notrunc &&
     git cat-file blob $sha1_101 > file_101_foo2'

test_expect_success \
    '[index v2] 3) corrupted delta happily returned wrong data' \
    'test -f file_101_foo2 && ! cmp file_101 file_101_foo2'

test_expect_success \
    '[index v2] 4) confirm that the pack is actually corrupted' \
    'test_must_fail git fsck --full $commit'

test_expect_success \
    '[index v2] 5) pack-objects refuses to reuse corrupted data' \
    'test_must_fail git pack-objects test-5 <obj-list &&
     test_must_fail git pack-objects --no-reuse-object test-6 <obj-list'

test_expect_success \
    '[index v2] 6) verify-pack detects CRC mismatch' \
    'rm -f .git/objects/pack/* &&
     git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
     git verify-pack ".git/objects/pack/pack-${pack1}.pack" &&
     obj=`git hash-object file_001` &&
     nr=`index_obj_nr ".git/objects/pack/pack-${pack1}.idx" $obj` &&
     chmod +w ".git/objects/pack/pack-${pack1}.idx" &&
     printf xxxx | dd of=".git/objects/pack/pack-${pack1}.idx" conv=notrunc \
        bs=1 count=4 seek=$((8 + 256 * 4 + `wc -l <obj-list` * 20 + $nr * 4)) &&
     ( while read obj
       do git cat-file -p $obj >/dev/null || exit 1
       done <obj-list ) &&
     err=$(test_must_fail git verify-pack \
       ".git/objects/pack/pack-${pack1}.pack" 2>&1) &&
     echo "$err" | grep "CRC mismatch"'

test_expect_success 'running index-pack in the object store' '
    rm -f .git/objects/pack/* &&
    cp test-1-${pack1}.pack .git/objects/pack/pack-${pack1}.pack &&
    (
	cd .git/objects/pack
	git index-pack pack-${pack1}.pack
    ) &&
    test -f .git/objects/pack/pack-${pack1}.idx
'

test_done
Commit	Line	Data
6e541776 NP	1	#!/bin/sh
	2	#
	3	# Copyright (c) 2007 Nicolas Pitre
	4	#
	5
	6	test_description='pack index with 64-bit offsets and object CRC'
	7	. ./test-lib.sh
	8
	9	test_expect_success \
	10	'setup' \
	11	'rm -rf .git
5be60078	12	git init &&
1415be8f	13	git config pack.threads 1 &&
b3431bc6	14	i=1 &&
2b5c208f	15	while test $i -le 100
6e541776	16	do
2b5c208f NP	17	iii=`printf '%03i' $i`
	18	test-genrandom "bar" 200 > wide_delta_$iii &&
	19	test-genrandom "baz $iii" 50 >> wide_delta_$iii &&
	20	test-genrandom "foo"$i 100 > deep_delta_$iii &&
	21	test-genrandom "foo"`expr $i + 1` 100 >> deep_delta_$iii &&
	22	test-genrandom "foo"`expr $i + 2` 100 >> deep_delta_$iii &&
	23	echo $iii >file_$iii &&
	24	test-genrandom "$iii" 8192 >>file_$iii &&
	25	git update-index --add file_$iii deep_delta_$iii wide_delta_$iii &&
	26	i=`expr $i + 1` \|\| return 1
6e541776	27	done &&
bd4b0aeb	28	{ echo 101 && test-genrandom 100 8192; } >file_101 &&
5be60078 JH	29	git update-index --add file_101 &&
	30	tree=`git write-tree` &&
	31	commit=`git commit-tree $tree </dev/null` && {
6e541776	32	echo $tree &&
5be60078	33	git ls-tree $tree \| sed -e "s/.* \\([0-9a-f]\\) ./\\1/"
6e541776	34	} >obj-list &&
5be60078	35	git update-ref HEAD $commit'
6e541776 NP	36
	37	test_expect_success \
	38	'pack-objects with index version 1' \
5be60078 JH	39	'pack1=$(git pack-objects --index-version=1 test-1 <obj-list) &&
5be60078 JH	40	git verify-pack -v "test-1-${pack1}.pack"'
6e541776 NP	41
	42	test_expect_success \
	43	'pack-objects with index version 2' \
5be60078 JH	44	'pack2=$(git pack-objects --index-version=2 test-2 <obj-list) &&
5be60078 JH	45	git verify-pack -v "test-2-${pack2}.pack"'
6e541776 NP	46
	47	test_expect_success \
	48	'both packs should be identical' \
	49	'cmp "test-1-${pack1}.pack" "test-2-${pack2}.pack"'
	50
41ac414e	51	test_expect_success \
6e541776	52	'index v1 and index v2 should be different' \
41ac414e	53	'! cmp "test-1-${pack1}.idx" "test-2-${pack2}.idx"'
6e541776 NP	54
	55	test_expect_success \
	56	'index-pack with index version 1' \
3604e7c5	57	'git index-pack --index-version=1 -o 1.idx "test-1-${pack1}.pack"'
6e541776 NP	58
	59	test_expect_success \
	60	'index-pack with index version 2' \
3604e7c5	61	'git index-pack --index-version=2 -o 2.idx "test-1-${pack1}.pack"'
6e541776 NP	62
	63	test_expect_success \
	64	'index-pack results should match pack-objects ones' \
	65	'cmp "test-1-${pack1}.idx" "1.idx" &&
	66	cmp "test-2-${pack2}.idx" "2.idx"'
	67
	68	test_expect_success \
	69	'index v2: force some 64-bit offsets with pack-objects' \
8ed2fca4 JS	70	'pack3=$(git pack-objects --index-version=2,0x40000 test-3 <obj-list)'
8ed2fca4 JS	71
8ed2fca4	72	if msg=$(git verify-pack -v "test-3-${pack3}.pack" 2>&1) \|\|
bbf08124	73	! (echo "$msg" \| grep "pack too large .* off_t")
8ed2fca4	74	then
18bf8798	75	test_set_prereq OFF64_T
8ed2fca4	76	else
fadb5156	77	say "# skipping tests concerning 64-bit offsets"
8ed2fca4 JS	78	fi
8ed2fca4 JS	79
18bf8798	80	test_expect_success OFF64_T \
8ed2fca4 JS	81	'index v2: verify a pack with some 64-bit offsets' \
8ed2fca4 JS	82	'git verify-pack -v "test-3-${pack3}.pack"'
6e541776	83
18bf8798	84	test_expect_success OFF64_T \
6e541776	85	'64-bit offsets: should be different from previous index v2 results' \
41ac414e	86	'! cmp "test-2-${pack2}.idx" "test-3-${pack3}.idx"'
6e541776	87
18bf8798	88	test_expect_success OFF64_T \
6e541776	89	'index v2: force some 64-bit offsets with index-pack' \
3604e7c5	90	'git index-pack --index-version=2,0x40000 -o 3.idx "test-1-${pack1}.pack"'
6e541776	91
18bf8798	92	test_expect_success OFF64_T \
6e541776 NP	93	'64-bit offsets: index-pack result should match pack-objects one' \
	94	'cmp "test-3-${pack3}.idx" "3.idx"'
	95
2b5c208f NP	96	# returns the object number for given object in given pack index
	97	index_obj_nr()
	98	{
	99	idx_file=$1
	100	object_sha1=$2
	101	nr=0
	102	git show-index < $idx_file \|
	103	while read offs sha1 extra
	104	do
	105	nr=$(($nr + 1))
	106	test "$sha1" = "$object_sha1" \|\| continue
	107	echo "$(($nr - 1))"
	108	break
	109	done
	110	}
	111
	112	# returns the pack offset for given object as found in given pack index
	113	index_obj_offset()
	114	{
	115	idx_file=$1
	116	object_sha1=$2
	117	git show-index < $idx_file \| grep $object_sha1 \|
	118	( read offs extra && echo "$offs" )
	119	}
	120
6e541776 NP	121	test_expect_success \
6e541776 NP	122	'[index v1] 1) stream pack to repository' \
3604e7c5	123	'git index-pack --index-version=1 --stdin < "test-1-${pack1}.pack" &&
5be60078 JH	124	git prune-packed &&
5be60078 JH	125	git count-objects \| ( read nr rest && test "$nr" -eq 1 ) &&
6e541776 NP	126	cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
	127	cmp "test-1-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
	128
	129	test_expect_success \
	130	'[index v1] 2) create a stealth corruption in a delta base reference' \
2b5c208f NP	131	'# This test assumes file_101 is a delta smaller than 16 bytes.
	132	# It should be against file_100 but we substitute its base for file_099
	133	sha1_101=`git hash-object file_101` &&
	134	sha1_099=`git hash-object file_099` &&
	135	offs_101=`index_obj_offset 1.idx $sha1_101` &&
	136	nr_099=`index_obj_nr 1.idx $sha1_099` &&
	137	chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
	138	dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
	139	if=".git/objects/pack/pack-${pack1}.idx" \
	140	skip=$((4 + 256 * 4 + $nr_099 * 24)) \
	141	bs=1 count=20 conv=notrunc &&
	142	git cat-file blob $sha1_101 > file_101_foo1'
6e541776	143
41ac414e	144	test_expect_success \
6e541776	145	'[index v1] 3) corrupted delta happily returned wrong data' \
2b5c208f	146	'test -f file_101_foo1 && ! cmp file_101 file_101_foo1'
6e541776	147
41ac414e	148	test_expect_success \
6e541776	149	'[index v1] 4) confirm that the pack is actually corrupted' \
d492b31c	150	'test_must_fail git fsck --full $commit'
6e541776 NP	151
	152	test_expect_success \
	153	'[index v1] 5) pack-objects happily reuses corrupted data' \
5be60078	154	'pack4=$(git pack-objects test-4 <obj-list) &&
6e541776 NP	155	test -f "test-4-${pack1}.pack"'
6e541776 NP	156
41ac414e	157	test_expect_success \
6e541776	158	'[index v1] 6) newly created pack is BAD !' \
d492b31c	159	'test_must_fail git verify-pack -v "test-4-${pack1}.pack"'
6e541776 NP	160
	161	test_expect_success \
	162	'[index v2] 1) stream pack to repository' \
	163	'rm -f .git/objects/pack/* &&
3604e7c5	164	git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
5be60078 JH	165	git prune-packed &&
5be60078 JH	166	git count-objects \| ( read nr rest && test "$nr" -eq 1 ) &&
6e541776	167	cmp "test-1-${pack1}.pack" ".git/objects/pack/pack-${pack1}.pack" &&
5f9ffff3	168	cmp "test-2-${pack1}.idx" ".git/objects/pack/pack-${pack1}.idx"'
6e541776 NP	169
	170	test_expect_success \
	171	'[index v2] 2) create a stealth corruption in a delta base reference' \
2b5c208f NP	172	'# This test assumes file_101 is a delta smaller than 16 bytes.
	173	# It should be against file_100 but we substitute its base for file_099
	174	sha1_101=`git hash-object file_101` &&
	175	sha1_099=`git hash-object file_099` &&
	176	offs_101=`index_obj_offset 1.idx $sha1_101` &&
	177	nr_099=`index_obj_nr 1.idx $sha1_099` &&
	178	chmod +w ".git/objects/pack/pack-${pack1}.pack" &&
	179	dd of=".git/objects/pack/pack-${pack1}.pack" seek=$(($offs_101 + 1)) \
	180	if=".git/objects/pack/pack-${pack1}.idx" \
	181	skip=$((8 + 256 * 4 + $nr_099 * 20)) \
	182	bs=1 count=20 conv=notrunc &&
	183	git cat-file blob $sha1_101 > file_101_foo2'
6e541776	184
41ac414e	185	test_expect_success \
6e541776	186	'[index v2] 3) corrupted delta happily returned wrong data' \
2b5c208f	187	'test -f file_101_foo2 && ! cmp file_101 file_101_foo2'
6e541776	188
41ac414e	189	test_expect_success \
6e541776	190	'[index v2] 4) confirm that the pack is actually corrupted' \
d492b31c	191	'test_must_fail git fsck --full $commit'
6e541776	192
41ac414e	193	test_expect_success \
6e541776	194	'[index v2] 5) pack-objects refuses to reuse corrupted data' \
0e8189e2 NP	195	'test_must_fail git pack-objects test-5 <obj-list &&
0e8189e2 NP	196	test_must_fail git pack-objects --no-reuse-object test-6 <obj-list'
6e541776	197
85fe23ed NP	198	test_expect_success \
	199	'[index v2] 6) verify-pack detects CRC mismatch' \
	200	'rm -f .git/objects/pack/* &&
3604e7c5	201	git index-pack --index-version=2 --stdin < "test-1-${pack1}.pack" &&
85fe23ed	202	git verify-pack ".git/objects/pack/pack-${pack1}.pack" &&
2b5c208f NP	203	obj=`git hash-object file_001` &&
2b5c208f NP	204	nr=`index_obj_nr ".git/objects/pack/pack-${pack1}.idx" $obj` &&
85fe23ed	205	chmod +w ".git/objects/pack/pack-${pack1}.idx" &&
b689ccf6	206	printf xxxx \| dd of=".git/objects/pack/pack-${pack1}.idx" conv=notrunc \
2b5c208f	207	bs=1 count=4 seek=$((8 + 256 * 4 + `wc -l <obj-list` * 20 + $nr * 4)) &&
85fe23ed NP	208	( while read obj
	209	do git cat-file -p $obj >/dev/null \|\| exit 1
	210	done <obj-list ) &&
d492b31c SB	211	err=$(test_must_fail git verify-pack \
d492b31c SB	212	".git/objects/pack/pack-${pack1}.pack" 2>&1) &&
85fe23ed NP	213	echo "$err" \| grep "CRC mismatch"'
85fe23ed NP	214
a672ea6a NP	215	test_expect_success 'running index-pack in the object store' '
	216	rm -f .git/objects/pack/* &&
	217	cp test-1-${pack1}.pack .git/objects/pack/pack-${pack1}.pack &&
	218	(
	219	cd .git/objects/pack
	220	git index-pack pack-${pack1}.pack
	221	) &&
	222	test -f .git/objects/pack/pack-${pack1}.idx
	223	'
	224
6e541776	225	test_done