]> git.ipfire.org Git - thirdparty/git.git/blob - t/t5310-pack-bitmaps.sh
clone: allow "--bare" with "-o"
[thirdparty/git.git] / t / t5310-pack-bitmaps.sh
1 #!/bin/sh
2
3 test_description='exercise basic bitmap functionality'
4
5 . ./test-lib.sh
6 . "$TEST_DIRECTORY"/lib-bitmap.sh
7
8 # t5310 deals only with single-pack bitmaps, so don't write MIDX bitmaps in
9 # their place.
10 GIT_TEST_MULTI_PACK_INDEX_WRITE_BITMAP=0
11
12 objpath () {
13 echo ".git/objects/$(echo "$1" | sed -e 's|\(..\)|\1/|')"
14 }
15
16 # show objects present in pack ($1 should be associated *.idx)
17 list_packed_objects () {
18 git show-index <"$1" >object-list &&
19 cut -d' ' -f2 object-list
20 }
21
22 # has_any pattern-file content-file
23 # tests whether content-file has any entry from pattern-file with entries being
24 # whole lines.
25 has_any () {
26 grep -Ff "$1" "$2"
27 }
28
29 setup_bitmap_history
30
31 test_expect_success 'setup writing bitmaps during repack' '
32 git config repack.writeBitmaps true
33 '
34
35 test_expect_success 'full repack creates bitmaps' '
36 GIT_TRACE2_EVENT="$(pwd)/trace" \
37 git repack -ad &&
38 ls .git/objects/pack/ | grep bitmap >output &&
39 test_line_count = 1 output &&
40 grep "\"key\":\"num_selected_commits\",\"value\":\"106\"" trace &&
41 grep "\"key\":\"num_maximal_commits\",\"value\":\"107\"" trace
42 '
43
44 basic_bitmap_tests
45
46 test_expect_success 'incremental repack fails when bitmaps are requested' '
47 test_commit more-1 &&
48 test_must_fail git repack -d 2>err &&
49 test_i18ngrep "Incremental repacks are incompatible with bitmap" err
50 '
51
52 test_expect_success 'incremental repack can disable bitmaps' '
53 test_commit more-2 &&
54 git repack -d --no-write-bitmap-index
55 '
56
57 test_expect_success 'pack-objects respects --local (non-local loose)' '
58 git init --bare alt.git &&
59 echo $(pwd)/alt.git/objects >.git/objects/info/alternates &&
60 echo content1 >file1 &&
61 # non-local loose object which is not present in bitmapped pack
62 altblob=$(GIT_DIR=alt.git git hash-object -w file1) &&
63 # non-local loose object which is also present in bitmapped pack
64 git cat-file blob $blob | GIT_DIR=alt.git git hash-object -w --stdin &&
65 git add file1 &&
66 test_tick &&
67 git commit -m commit_file1 &&
68 echo HEAD | git pack-objects --local --stdout --revs >1.pack &&
69 git index-pack 1.pack &&
70 list_packed_objects 1.idx >1.objects &&
71 printf "%s\n" "$altblob" "$blob" >nonlocal-loose &&
72 ! has_any nonlocal-loose 1.objects
73 '
74
75 test_expect_success 'pack-objects respects --honor-pack-keep (local non-bitmapped pack)' '
76 echo content2 >file2 &&
77 blob2=$(git hash-object -w file2) &&
78 git add file2 &&
79 test_tick &&
80 git commit -m commit_file2 &&
81 printf "%s\n" "$blob2" "$bitmaptip" >keepobjects &&
82 pack2=$(git pack-objects pack2 <keepobjects) &&
83 mv pack2-$pack2.* .git/objects/pack/ &&
84 >.git/objects/pack/pack2-$pack2.keep &&
85 rm $(objpath $blob2) &&
86 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >2a.pack &&
87 git index-pack 2a.pack &&
88 list_packed_objects 2a.idx >2a.objects &&
89 ! has_any keepobjects 2a.objects
90 '
91
92 test_expect_success 'pack-objects respects --local (non-local pack)' '
93 mv .git/objects/pack/pack2-$pack2.* alt.git/objects/pack/ &&
94 echo HEAD | git pack-objects --local --stdout --revs >2b.pack &&
95 git index-pack 2b.pack &&
96 list_packed_objects 2b.idx >2b.objects &&
97 ! has_any keepobjects 2b.objects
98 '
99
100 test_expect_success 'pack-objects respects --honor-pack-keep (local bitmapped pack)' '
101 ls .git/objects/pack/ | grep bitmap >output &&
102 test_line_count = 1 output &&
103 packbitmap=$(basename $(cat output) .bitmap) &&
104 list_packed_objects .git/objects/pack/$packbitmap.idx >packbitmap.objects &&
105 test_when_finished "rm -f .git/objects/pack/$packbitmap.keep" &&
106 >.git/objects/pack/$packbitmap.keep &&
107 echo HEAD | git pack-objects --honor-pack-keep --stdout --revs >3a.pack &&
108 git index-pack 3a.pack &&
109 list_packed_objects 3a.idx >3a.objects &&
110 ! has_any packbitmap.objects 3a.objects
111 '
112
113 test_expect_success 'pack-objects respects --local (non-local bitmapped pack)' '
114 mv .git/objects/pack/$packbitmap.* alt.git/objects/pack/ &&
115 rm -f .git/objects/pack/multi-pack-index &&
116 test_when_finished "mv alt.git/objects/pack/$packbitmap.* .git/objects/pack/" &&
117 echo HEAD | git pack-objects --local --stdout --revs >3b.pack &&
118 git index-pack 3b.pack &&
119 list_packed_objects 3b.idx >3b.objects &&
120 ! has_any packbitmap.objects 3b.objects
121 '
122
123 test_expect_success 'pack-objects to file can use bitmap' '
124 # make sure we still have 1 bitmap index from previous tests
125 ls .git/objects/pack/ | grep bitmap >output &&
126 test_line_count = 1 output &&
127 # verify equivalent packs are generated with/without using bitmap index
128 packasha1=$(git pack-objects --no-use-bitmap-index --all packa </dev/null) &&
129 packbsha1=$(git pack-objects --use-bitmap-index --all packb </dev/null) &&
130 list_packed_objects packa-$packasha1.idx >packa.objects &&
131 list_packed_objects packb-$packbsha1.idx >packb.objects &&
132 test_cmp packa.objects packb.objects
133 '
134
135 test_expect_success 'full repack, reusing previous bitmaps' '
136 git repack -ad &&
137 ls .git/objects/pack/ | grep bitmap >output &&
138 test_line_count = 1 output
139 '
140
141 test_expect_success 'fetch (full bitmap)' '
142 git --git-dir=clone.git fetch origin second:second &&
143 git rev-parse HEAD >expect &&
144 git --git-dir=clone.git rev-parse HEAD >actual &&
145 test_cmp expect actual
146 '
147
148 test_expect_success 'create objects for missing-HAVE tests' '
149 blob=$(echo "missing have" | git hash-object -w --stdin) &&
150 tree=$(printf "100644 blob $blob\tfile\n" | git mktree) &&
151 parent=$(echo parent | git commit-tree $tree) &&
152 commit=$(echo commit | git commit-tree $tree -p $parent) &&
153 cat >revs <<-EOF
154 HEAD
155 ^HEAD^
156 ^$commit
157 EOF
158 '
159
160 test_expect_success 'pack-objects respects --incremental' '
161 cat >revs2 <<-EOF &&
162 HEAD
163 $commit
164 EOF
165 git pack-objects --incremental --stdout --revs <revs2 >4.pack &&
166 git index-pack 4.pack &&
167 list_packed_objects 4.idx >4.objects &&
168 test_line_count = 4 4.objects &&
169 git rev-list --objects $commit >revlist &&
170 cut -d" " -f1 revlist |sort >objects &&
171 test_cmp 4.objects objects
172 '
173
174 test_expect_success 'pack with missing blob' '
175 rm $(objpath $blob) &&
176 git pack-objects --stdout --revs <revs >/dev/null
177 '
178
179 test_expect_success 'pack with missing tree' '
180 rm $(objpath $tree) &&
181 git pack-objects --stdout --revs <revs >/dev/null
182 '
183
184 test_expect_success 'pack with missing parent' '
185 rm $(objpath $parent) &&
186 git pack-objects --stdout --revs <revs >/dev/null
187 '
188
189 test_expect_success JGIT,SHA1 'we can read jgit bitmaps' '
190 git clone --bare . compat-jgit.git &&
191 (
192 cd compat-jgit.git &&
193 rm -f objects/pack/*.bitmap &&
194 jgit gc &&
195 git rev-list --test-bitmap HEAD
196 )
197 '
198
199 test_expect_success JGIT,SHA1 'jgit can read our bitmaps' '
200 git clone --bare . compat-us.git &&
201 (
202 cd compat-us.git &&
203 git repack -adb &&
204 # jgit gc will barf if it does not like our bitmaps
205 jgit gc
206 )
207 '
208
209 test_expect_success 'splitting packs does not generate bogus bitmaps' '
210 test-tool genrandom foo $((1024 * 1024)) >rand &&
211 git add rand &&
212 git commit -m "commit with big file" &&
213 git -c pack.packSizeLimit=500k repack -adb &&
214 git init --bare no-bitmaps.git &&
215 git -C no-bitmaps.git fetch .. HEAD
216 '
217
218 test_expect_success 'set up reusable pack' '
219 rm -f .git/objects/pack/*.keep &&
220 git repack -adb &&
221 reusable_pack () {
222 git for-each-ref --format="%(objectname)" |
223 git pack-objects --delta-base-offset --revs --stdout "$@"
224 }
225 '
226
227 test_expect_success 'pack reuse respects --honor-pack-keep' '
228 test_when_finished "rm -f .git/objects/pack/*.keep" &&
229 for i in .git/objects/pack/*.pack
230 do
231 >${i%.pack}.keep || return 1
232 done &&
233 reusable_pack --honor-pack-keep >empty.pack &&
234 git index-pack empty.pack &&
235 git show-index <empty.idx >actual &&
236 test_must_be_empty actual
237 '
238
239 test_expect_success 'pack reuse respects --local' '
240 mv .git/objects/pack/* alt.git/objects/pack/ &&
241 test_when_finished "mv alt.git/objects/pack/* .git/objects/pack/" &&
242 reusable_pack --local >empty.pack &&
243 git index-pack empty.pack &&
244 git show-index <empty.idx >actual &&
245 test_must_be_empty actual
246 '
247
248 test_expect_success 'pack reuse respects --incremental' '
249 reusable_pack --incremental >empty.pack &&
250 git index-pack empty.pack &&
251 git show-index <empty.idx >actual &&
252 test_must_be_empty actual
253 '
254
255 test_expect_success 'truncated bitmap fails gracefully (ewah)' '
256 test_config pack.writebitmaphashcache false &&
257 git repack -ad &&
258 git rev-list --use-bitmap-index --count --all >expect &&
259 bitmap=$(ls .git/objects/pack/*.bitmap) &&
260 test_when_finished "rm -f $bitmap" &&
261 test_copy_bytes 256 <$bitmap >$bitmap.tmp &&
262 mv -f $bitmap.tmp $bitmap &&
263 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
264 test_cmp expect actual &&
265 test_i18ngrep corrupt.ewah.bitmap stderr
266 '
267
268 test_expect_success 'truncated bitmap fails gracefully (cache)' '
269 git repack -ad &&
270 git rev-list --use-bitmap-index --count --all >expect &&
271 bitmap=$(ls .git/objects/pack/*.bitmap) &&
272 test_when_finished "rm -f $bitmap" &&
273 test_copy_bytes 512 <$bitmap >$bitmap.tmp &&
274 mv -f $bitmap.tmp $bitmap &&
275 git rev-list --use-bitmap-index --count --all >actual 2>stderr &&
276 test_cmp expect actual &&
277 test_i18ngrep corrupted.bitmap.index stderr
278 '
279
280 # Create a state of history with these properties:
281 #
282 # - refs that allow a client to fetch some new history, while sharing some old
283 # history with the server; we use branches delta-reuse-old and
284 # delta-reuse-new here
285 #
286 # - the new history contains an object that is stored on the server as a delta
287 # against a base that is in the old history
288 #
289 # - the base object is not immediately reachable from the tip of the old
290 # history; finding it would involve digging down through history we know the
291 # other side has
292 #
293 # This should result in a state where fetching from old->new would not
294 # traditionally reuse the on-disk delta (because we'd have to dig to realize
295 # that the client has it), but we will do so if bitmaps can tell us cheaply
296 # that the other side has it.
297 test_expect_success 'set up thin delta-reuse parent' '
298 # This first commit contains the buried base object.
299 test-tool genrandom delta 16384 >file &&
300 git add file &&
301 git commit -m "delta base" &&
302 base=$(git rev-parse --verify HEAD:file) &&
303
304 # These intermediate commits bury the base back in history.
305 # This becomes the "old" state.
306 for i in 1 2 3 4 5
307 do
308 echo $i >file &&
309 git commit -am "intermediate $i" || return 1
310 done &&
311 git branch delta-reuse-old &&
312
313 # And now our new history has a delta against the buried base. Note
314 # that this must be smaller than the original file, since pack-objects
315 # prefers to create deltas from smaller objects to larger.
316 test-tool genrandom delta 16300 >file &&
317 git commit -am "delta result" &&
318 delta=$(git rev-parse --verify HEAD:file) &&
319 git branch delta-reuse-new &&
320
321 # Repack with bitmaps and double check that we have the expected delta
322 # relationship.
323 git repack -adb &&
324 have_delta $delta $base
325 '
326
327 # Now we can sanity-check the non-bitmap behavior (that the server is not able
328 # to reuse the delta). This isn't strictly something we care about, so this
329 # test could be scrapped in the future. But it makes sure that the next test is
330 # actually triggering the feature we want.
331 #
332 # Note that our tools for working with on-the-wire "thin" packs are limited. So
333 # we actually perform the fetch, retain the resulting pack, and inspect the
334 # result.
335 test_expect_success 'fetch without bitmaps ignores delta against old base' '
336 test_config pack.usebitmaps false &&
337 test_when_finished "rm -rf client.git" &&
338 git init --bare client.git &&
339 (
340 cd client.git &&
341 git config transfer.unpackLimit 1 &&
342 git fetch .. delta-reuse-old:delta-reuse-old &&
343 git fetch .. delta-reuse-new:delta-reuse-new &&
344 have_delta $delta $ZERO_OID
345 )
346 '
347
348 # And do the same for the bitmap case, where we do expect to find the delta.
349 test_expect_success 'fetch with bitmaps can reuse old base' '
350 test_config pack.usebitmaps true &&
351 test_when_finished "rm -rf client.git" &&
352 git init --bare client.git &&
353 (
354 cd client.git &&
355 git config transfer.unpackLimit 1 &&
356 git fetch .. delta-reuse-old:delta-reuse-old &&
357 git fetch .. delta-reuse-new:delta-reuse-new &&
358 have_delta $delta $base
359 )
360 '
361
362 test_expect_success 'pack.preferBitmapTips' '
363 git init repo &&
364 test_when_finished "rm -fr repo" &&
365 (
366 cd repo &&
367
368 # create enough commits that not all are receive bitmap
369 # coverage even if they are all at the tip of some reference.
370 test_commit_bulk --message="%s" 103 &&
371
372 git rev-list HEAD >commits.raw &&
373 sort <commits.raw >commits &&
374
375 git log --format="create refs/tags/%s %H" HEAD >refs &&
376 git update-ref --stdin <refs &&
377
378 git repack -adb &&
379 test-tool bitmap list-commits | sort >bitmaps &&
380
381 # remember which commits did not receive bitmaps
382 comm -13 bitmaps commits >before &&
383 test_file_not_empty before &&
384
385 # mark the commits which did not receive bitmaps as preferred,
386 # and generate the bitmap again
387 perl -pe "s{^}{create refs/tags/include/$. }" <before |
388 git update-ref --stdin &&
389 git -c pack.preferBitmapTips=refs/tags/include repack -adb &&
390
391 # finally, check that the commit(s) without bitmap coverage
392 # are not the same ones as before
393 test-tool bitmap list-commits | sort >bitmaps &&
394 comm -13 bitmaps commits >after &&
395
396 ! test_cmp before after
397 )
398 '
399
400 test_expect_success 'complains about multiple pack bitmaps' '
401 rm -fr repo &&
402 git init repo &&
403 test_when_finished "rm -fr repo" &&
404 (
405 cd repo &&
406
407 test_commit base &&
408
409 git repack -adb &&
410 bitmap="$(ls .git/objects/pack/pack-*.bitmap)" &&
411 mv "$bitmap" "$bitmap.bak" &&
412
413 test_commit other &&
414 git repack -ab &&
415
416 mv "$bitmap.bak" "$bitmap" &&
417
418 find .git/objects/pack -type f -name "*.pack" >packs &&
419 find .git/objects/pack -type f -name "*.bitmap" >bitmaps &&
420 test_line_count = 2 packs &&
421 test_line_count = 2 bitmaps &&
422
423 git rev-list --use-bitmap-index HEAD 2>err &&
424 grep "ignoring extra bitmap file" err
425 )
426 '
427
428 test_done