t/t5300-pack-object.sh

   1 #!/bin/sh
   2 #
   3 # Copyright (c) 2005 Junio C Hamano
   4 #
   5
   6 test_description='git pack-object
   7
   8 '
   9 . ./test-lib.sh
  10
  11 test_expect_success 'setup' '
  12         rm -f .git/index* &&
  13         perl -e "print \"a\" x 4096;" >a &&
  14         perl -e "print \"b\" x 4096;" >b &&
  15         perl -e "print \"c\" x 4096;" >c &&
  16         test-tool genrandom "seed a" 2097152 >a_big &&
  17         test-tool genrandom "seed b" 2097152 >b_big &&
  18         git update-index --add a a_big b b_big c &&
  19         cat c >d && echo foo >>d && git update-index --add d &&
  20         tree=$(git write-tree) &&
  21         commit=$(git commit-tree $tree </dev/null) &&
  22         {
  23                 echo $tree &&
  24                 echo $commit &&
  25                 git ls-tree $tree | sed -e "s/.* \\([0-9a-f]*\\)        .*/\\1/"
  26         } >obj-list &&
  27         {
  28                 git diff-tree --root -p $commit &&
  29                 while read object
  30                 do
  31                         t=$(git cat-file -t $object) &&
  32                         git cat-file $t $object || return 1
  33                 done <obj-list
  34         } >expect
  35 '
  36
  37 # usage: check_deltas <stderr_from_pack_objects> <cmp_op> <nr_deltas>
  38 # e.g.: check_deltas stderr -gt 0
  39 check_deltas() {
  40         deltas=$(perl -lne '/delta (\d+)/ and print $1' "$1") &&
  41         shift &&
  42         if ! test "$deltas" "$@"
  43         then
  44                 echo >&2 "unexpected number of deltas (compared $delta $*)"
  45                 return 1
  46         fi
  47 }
  48
  49 test_expect_success 'pack without delta' '
  50         packname_1=$(git pack-objects --progress --window=0 test-1 \
  51                         <obj-list 2>stderr) &&
  52         check_deltas stderr = 0
  53 '
  54
  55 test_expect_success 'pack-objects with bogus arguments' '
  56         test_must_fail git pack-objects --window=0 test-1 blah blah <obj-list
  57 '
  58
  59 check_unpack () {
  60         test_when_finished "rm -rf git2" &&
  61         git init --bare git2 &&
  62         git -C git2 unpack-objects -n <"$1".pack &&
  63         git -C git2 unpack-objects <"$1".pack &&
  64         (cd .git && find objects -type f -print) |
  65         while read path
  66         do
  67                 cmp git2/$path .git/$path || {
  68                         echo $path differs.
  69                         return 1
  70                 }
  71         done
  72 }
  73
  74 test_expect_success 'unpack without delta' '
  75         check_unpack test-1-${packname_1}
  76 '
  77
  78 test_expect_success 'pack with REF_DELTA' '
  79         packname_2=$(git pack-objects --progress test-2 <obj-list 2>stderr) &&
  80         check_deltas stderr -gt 0
  81 '
  82
  83 test_expect_success 'unpack with REF_DELTA' '
  84         check_unpack test-2-${packname_2}
  85 '
  86
  87 test_expect_success 'pack with OFS_DELTA' '
  88         packname_3=$(git pack-objects --progress --delta-base-offset test-3 \
  89                         <obj-list 2>stderr) &&
  90         check_deltas stderr -gt 0
  91 '
  92
  93 test_expect_success 'unpack with OFS_DELTA' '
  94         check_unpack test-3-${packname_3}
  95 '
  96
  97 test_expect_success 'compare delta flavors' '
  98         perl -e '\''
  99                 defined($_ = -s $_) or die for @ARGV;
 100                 exit 1 if $ARGV[0] <= $ARGV[1];
 101         '\'' test-2-$packname_2.pack test-3-$packname_3.pack
 102 '
 103
 104 check_use_objects () {
 105         test_when_finished "rm -rf git2" &&
 106         git init --bare git2 &&
 107         cp "$1".pack "$1".idx git2/objects/pack &&
 108         (
 109                 cd git2 &&
 110                 git diff-tree --root -p $commit &&
 111                 while read object
 112                 do
 113                         t=$(git cat-file -t $object) &&
 114                         git cat-file $t $object || exit 1
 115                 done
 116         ) <obj-list >current &&
 117         cmp expect current
 118 }
 119
 120 test_expect_success 'use packed objects' '
 121         check_use_objects test-1-${packname_1}
 122 '
 123
 124 test_expect_success 'use packed deltified (REF_DELTA) objects' '
 125         check_use_objects test-2-${packname_2}
 126 '
 127
 128 test_expect_success 'use packed deltified (OFS_DELTA) objects' '
 129         check_use_objects test-3-${packname_3}
 130 '
 131
 132 test_expect_success 'survive missing objects/pack directory' '
 133         (
 134                 rm -fr missing-pack &&
 135                 mkdir missing-pack &&
 136                 cd missing-pack &&
 137                 git init &&
 138                 GOP=.git/objects/pack &&
 139                 rm -fr $GOP &&
 140                 git index-pack --stdin --keep=test <../test-3-${packname_3}.pack &&
 141                 test -f $GOP/pack-${packname_3}.pack &&
 142                 cmp $GOP/pack-${packname_3}.pack ../test-3-${packname_3}.pack &&
 143                 test -f $GOP/pack-${packname_3}.idx &&
 144                 cmp $GOP/pack-${packname_3}.idx ../test-3-${packname_3}.idx &&
 145                 test -f $GOP/pack-${packname_3}.keep
 146         )
 147 '
 148
 149 test_expect_success \
 150     'verify pack' \
 151     'git verify-pack    test-1-${packname_1}.idx \
 152                         test-2-${packname_2}.idx \
 153                         test-3-${packname_3}.idx'
 154
 155 test_expect_success \
 156     'verify pack -v' \
 157     'git verify-pack -v test-1-${packname_1}.idx \
 158                         test-2-${packname_2}.idx \
 159                         test-3-${packname_3}.idx'
 160
 161 test_expect_success \
 162     'verify-pack catches mismatched .idx and .pack files' \
 163     'cat test-1-${packname_1}.idx >test-3.idx &&
 164      cat test-2-${packname_2}.pack >test-3.pack &&
 165      if git verify-pack test-3.idx
 166      then false
 167      else :;
 168      fi'
 169
 170 test_expect_success \
 171     'verify-pack catches a corrupted pack signature' \
 172     'cat test-1-${packname_1}.pack >test-3.pack &&
 173      echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=2 &&
 174      if git verify-pack test-3.idx
 175      then false
 176      else :;
 177      fi'
 178
 179 test_expect_success \
 180     'verify-pack catches a corrupted pack version' \
 181     'cat test-1-${packname_1}.pack >test-3.pack &&
 182      echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=7 &&
 183      if git verify-pack test-3.idx
 184      then false
 185      else :;
 186      fi'
 187
 188 test_expect_success \
 189     'verify-pack catches a corrupted type/size of the 1st packed object data' \
 190     'cat test-1-${packname_1}.pack >test-3.pack &&
 191      echo | dd of=test-3.pack count=1 bs=1 conv=notrunc seek=12 &&
 192      if git verify-pack test-3.idx
 193      then false
 194      else :;
 195      fi'
 196
 197 test_expect_success \
 198     'verify-pack catches a corrupted sum of the index file itself' \
 199     'l=$(wc -c <test-3.idx) &&
 200      l=$(expr $l - 20) &&
 201      cat test-1-${packname_1}.pack >test-3.pack &&
 202      printf "%20s" "" | dd of=test-3.idx count=20 bs=1 conv=notrunc seek=$l &&
 203      if git verify-pack test-3.pack
 204      then false
 205      else :;
 206      fi'
 207
 208 test_expect_success \
 209     'build pack index for an existing pack' \
 210     'cat test-1-${packname_1}.pack >test-3.pack &&
 211      git index-pack -o tmp.idx test-3.pack &&
 212      cmp tmp.idx test-1-${packname_1}.idx &&
 213
 214      git index-pack test-3.pack &&
 215      cmp test-3.idx test-1-${packname_1}.idx &&
 216
 217      cat test-2-${packname_2}.pack >test-3.pack &&
 218      git index-pack -o tmp.idx test-2-${packname_2}.pack &&
 219      cmp tmp.idx test-2-${packname_2}.idx &&
 220
 221      git index-pack test-3.pack &&
 222      cmp test-3.idx test-2-${packname_2}.idx &&
 223
 224      cat test-3-${packname_3}.pack >test-3.pack &&
 225      git index-pack -o tmp.idx test-3-${packname_3}.pack &&
 226      cmp tmp.idx test-3-${packname_3}.idx &&
 227
 228      git index-pack test-3.pack &&
 229      cmp test-3.idx test-3-${packname_3}.idx &&
 230
 231      cat test-1-${packname_1}.pack >test-4.pack &&
 232      rm -f test-4.keep &&
 233      git index-pack --keep=why test-4.pack &&
 234      cmp test-1-${packname_1}.idx test-4.idx &&
 235      test -f test-4.keep &&
 236
 237      :'
 238
 239 test_expect_success 'unpacking with --strict' '
 240
 241         for j in a b c d e f g
 242         do
 243                 for i in 0 1 2 3 4 5 6 7 8 9
 244                 do
 245                         o=$(echo $j$i | git hash-object -w --stdin) &&
 246                         echo "100644 $o 0 $j$i"
 247                 done
 248         done >LIST &&
 249         rm -f .git/index &&
 250         git update-index --index-info <LIST &&
 251         LIST=$(git write-tree) &&
 252         rm -f .git/index &&
 253         head -n 10 LIST | git update-index --index-info &&
 254         LI=$(git write-tree) &&
 255         rm -f .git/index &&
 256         tail -n 10 LIST | git update-index --index-info &&
 257         ST=$(git write-tree) &&
 258         git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
 259         PACK5=$( git pack-objects test-5 <actual ) &&
 260         PACK6=$( (
 261                         echo "$LIST"
 262                         echo "$LI"
 263                         echo "$ST"
 264                  ) | git pack-objects test-6 ) &&
 265         test_create_repo test-5 &&
 266         (
 267                 cd test-5 &&
 268                 git unpack-objects --strict <../test-5-$PACK5.pack &&
 269                 git ls-tree -r $LIST &&
 270                 git ls-tree -r $LI &&
 271                 git ls-tree -r $ST
 272         ) &&
 273         test_create_repo test-6 &&
 274         (
 275                 # tree-only into empty repo -- many unreachables
 276                 cd test-6 &&
 277                 test_must_fail git unpack-objects --strict <../test-6-$PACK6.pack
 278         ) &&
 279         (
 280                 # already populated -- no unreachables
 281                 cd test-5 &&
 282                 git unpack-objects --strict <../test-6-$PACK6.pack
 283         )
 284 '
 285
 286 test_expect_success 'index-pack with --strict' '
 287
 288         for j in a b c d e f g
 289         do
 290                 for i in 0 1 2 3 4 5 6 7 8 9
 291                 do
 292                         o=$(echo $j$i | git hash-object -w --stdin) &&
 293                         echo "100644 $o 0 $j$i"
 294                 done
 295         done >LIST &&
 296         rm -f .git/index &&
 297         git update-index --index-info <LIST &&
 298         LIST=$(git write-tree) &&
 299         rm -f .git/index &&
 300         head -n 10 LIST | git update-index --index-info &&
 301         LI=$(git write-tree) &&
 302         rm -f .git/index &&
 303         tail -n 10 LIST | git update-index --index-info &&
 304         ST=$(git write-tree) &&
 305         git rev-list --objects "$LIST" "$LI" "$ST" >actual &&
 306         PACK5=$( git pack-objects test-5 <actual ) &&
 307         PACK6=$( (
 308                         echo "$LIST"
 309                         echo "$LI"
 310                         echo "$ST"
 311                  ) | git pack-objects test-6 ) &&
 312         test_create_repo test-7 &&
 313         (
 314                 cd test-7 &&
 315                 git index-pack --strict --stdin <../test-5-$PACK5.pack &&
 316                 git ls-tree -r $LIST &&
 317                 git ls-tree -r $LI &&
 318                 git ls-tree -r $ST
 319         ) &&
 320         test_create_repo test-8 &&
 321         (
 322                 # tree-only into empty repo -- many unreachables
 323                 cd test-8 &&
 324                 test_must_fail git index-pack --strict --stdin <../test-6-$PACK6.pack
 325         ) &&
 326         (
 327                 # already populated -- no unreachables
 328                 cd test-7 &&
 329                 git index-pack --strict --stdin <../test-6-$PACK6.pack
 330         )
 331 '
 332
 333 test_expect_success 'honor pack.packSizeLimit' '
 334         git config pack.packSizeLimit 3m &&
 335         packname_10=$(git pack-objects test-10 <obj-list) &&
 336         test 2 = $(ls test-10-*.pack | wc -l)
 337 '
 338
 339 test_expect_success 'verify resulting packs' '
 340         git verify-pack test-10-*.pack
 341 '
 342
 343 test_expect_success 'tolerate packsizelimit smaller than biggest object' '
 344         git config pack.packSizeLimit 1 &&
 345         packname_11=$(git pack-objects test-11 <obj-list) &&
 346         test 5 = $(ls test-11-*.pack | wc -l)
 347 '
 348
 349 test_expect_success 'verify resulting packs' '
 350         git verify-pack test-11-*.pack
 351 '
 352
 353 test_expect_success 'set up pack for non-repo tests' '
 354         # make sure we have a pack with no matching index file
 355         cp test-1-*.pack foo.pack
 356 '
 357
 358 test_expect_success 'index-pack --stdin complains of non-repo' '
 359         nongit test_must_fail git index-pack --object-format=$(test_oid algo) --stdin <foo.pack &&
 360         test_path_is_missing non-repo/.git
 361 '
 362
 363 test_expect_success 'index-pack <pack> works in non-repo' '
 364         nongit git index-pack --object-format=$(test_oid algo) ../foo.pack &&
 365         test_path_is_file foo.idx
 366 '
 367
 368 test_expect_success 'index-pack --strict <pack> works in non-repo' '
 369         rm -f foo.idx &&
 370         nongit git index-pack --strict --object-format=$(test_oid algo) ../foo.pack &&
 371         test_path_is_file foo.idx
 372 '
 373
 374 test_expect_success !PTHREADS,!FAIL_PREREQS \
 375         'index-pack --threads=N or pack.threads=N warns when no pthreads' '
 376         test_must_fail git index-pack --threads=2 2>err &&
 377         grep ^warning: err >warnings &&
 378         test_line_count = 1 warnings &&
 379         grep -F "no threads support, ignoring --threads=2" err &&
 380
 381         test_must_fail git -c pack.threads=2 index-pack 2>err &&
 382         grep ^warning: err >warnings &&
 383         test_line_count = 1 warnings &&
 384         grep -F "no threads support, ignoring pack.threads" err &&
 385
 386         test_must_fail git -c pack.threads=2 index-pack --threads=4 2>err &&
 387         grep ^warning: err >warnings &&
 388         test_line_count = 2 warnings &&
 389         grep -F "no threads support, ignoring --threads=4" err &&
 390         grep -F "no threads support, ignoring pack.threads" err
 391 '
 392
 393 test_expect_success !PTHREADS,!FAIL_PREREQS \
 394         'pack-objects --threads=N or pack.threads=N warns when no pthreads' '
 395         git pack-objects --threads=2 --stdout --all </dev/null >/dev/null 2>err &&
 396         grep ^warning: err >warnings &&
 397         test_line_count = 1 warnings &&
 398         grep -F "no threads support, ignoring --threads" err &&
 399
 400         git -c pack.threads=2 pack-objects --stdout --all </dev/null >/dev/null 2>err &&
 401         grep ^warning: err >warnings &&
 402         test_line_count = 1 warnings &&
 403         grep -F "no threads support, ignoring pack.threads" err &&
 404
 405         git -c pack.threads=2 pack-objects --threads=4 --stdout --all </dev/null >/dev/null 2>err &&
 406         grep ^warning: err >warnings &&
 407         test_line_count = 2 warnings &&
 408         grep -F "no threads support, ignoring --threads" err &&
 409         grep -F "no threads support, ignoring pack.threads" err
 410 '
 411
 412 test_expect_success 'pack-objects in too-many-packs mode' '
 413         GIT_TEST_FULL_IN_PACK_ARRAY=1 git repack -ad &&
 414         git fsck
 415 '
 416
 417 test_expect_success 'setup: fake a SHA1 hash collision' '
 418         git init corrupt &&
 419         (
 420                 cd corrupt &&
 421                 long_a=$(git hash-object -w ../a | sed -e "s!^..!&/!") &&
 422                 long_b=$(git hash-object -w ../b | sed -e "s!^..!&/!") &&
 423                 test -f .git/objects/$long_b &&
 424                 cp -f   .git/objects/$long_a \
 425                         .git/objects/$long_b
 426         )
 427 '
 428
 429 test_expect_success 'make sure index-pack detects the SHA1 collision' '
 430         (
 431                 cd corrupt &&
 432                 test_must_fail git index-pack -o ../bad.idx ../test-3.pack 2>msg &&
 433                 test_i18ngrep "SHA1 COLLISION FOUND" msg
 434         )
 435 '
 436
 437 test_expect_success 'make sure index-pack detects the SHA1 collision (large blobs)' '
 438         (
 439                 cd corrupt &&
 440                 test_must_fail git -c core.bigfilethreshold=1 index-pack -o ../bad.idx ../test-3.pack 2>msg &&
 441                 test_i18ngrep "SHA1 COLLISION FOUND" msg
 442         )
 443 '
 444
 445 test_expect_success 'prefetch objects' '
 446         rm -rf server client &&
 447
 448         git init server &&
 449         test_config -C server uploadpack.allowanysha1inwant 1 &&
 450         test_config -C server uploadpack.allowfilter 1 &&
 451         test_config -C server protocol.version 2 &&
 452
 453         echo one >server/one &&
 454         git -C server add one &&
 455         git -C server commit -m one &&
 456         git -C server branch one_branch &&
 457
 458         echo two_a >server/two_a &&
 459         echo two_b >server/two_b &&
 460         git -C server add two_a two_b &&
 461         git -C server commit -m two &&
 462
 463         echo three >server/three &&
 464         git -C server add three &&
 465         git -C server commit -m three &&
 466         git -C server branch three_branch &&
 467
 468         # Clone, fetch "two" with blobs excluded, and re-push it. This requires
 469         # the client to have the blobs of "two" - verify that these are
 470         # prefetched in one batch.
 471         git clone --filter=blob:none --single-branch -b one_branch \
 472                 "file://$(pwd)/server" client &&
 473         test_config -C client protocol.version 2 &&
 474         TWO=$(git -C server rev-parse three_branch^) &&
 475         git -C client fetch --filter=blob:none origin "$TWO" &&
 476         GIT_TRACE_PACKET=$(pwd)/trace git -C client push origin "$TWO":refs/heads/two_branch &&
 477         grep "fetch> done" trace >donelines &&
 478         test_line_count = 1 donelines
 479 '
 480
 481 test_expect_success 'setup for --stdin-packs tests' '
 482         git init stdin-packs &&
 483         (
 484                 cd stdin-packs &&
 485
 486                 test_commit A &&
 487                 test_commit B &&
 488                 test_commit C &&
 489
 490                 for id in A B C
 491                 do
 492                         git pack-objects .git/objects/pack/pack-$id \
 493                                 --incremental --revs <<-EOF
 494                         refs/tags/$id
 495                         EOF
 496                 done &&
 497
 498                 ls -la .git/objects/pack
 499         )
 500 '
 501
 502 test_expect_success '--stdin-packs with excluded packs' '
 503         (
 504                 cd stdin-packs &&
 505
 506                 PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
 507                 PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
 508                 PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
 509
 510                 git pack-objects test --stdin-packs <<-EOF &&
 511                 $PACK_A
 512                 ^$PACK_B
 513                 $PACK_C
 514                 EOF
 515
 516                 (
 517                         git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
 518                         git show-index <$(ls .git/objects/pack/pack-C-*.idx)
 519                 ) >expect.raw &&
 520                 git show-index <$(ls test-*.idx) >actual.raw &&
 521
 522                 cut -d" " -f2 <expect.raw | sort >expect &&
 523                 cut -d" " -f2 <actual.raw | sort >actual &&
 524                 test_cmp expect actual
 525         )
 526 '
 527
 528 test_expect_success '--stdin-packs is incompatible with --filter' '
 529         (
 530                 cd stdin-packs &&
 531                 test_must_fail git pack-objects --stdin-packs --stdout \
 532                         --filter=blob:none </dev/null 2>err &&
 533                 test_i18ngrep "cannot use --filter with --stdin-packs" err
 534         )
 535 '
 536
 537 test_expect_success '--stdin-packs is incompatible with --revs' '
 538         (
 539                 cd stdin-packs &&
 540                 test_must_fail git pack-objects --stdin-packs --revs out \
 541                         </dev/null 2>err &&
 542                 test_i18ngrep "cannot use internal rev list with --stdin-packs" err
 543         )
 544 '
 545
 546 test_expect_success '--stdin-packs with loose objects' '
 547         (
 548                 cd stdin-packs &&
 549
 550                 PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
 551                 PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
 552                 PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
 553
 554                 test_commit D && # loose
 555
 556                 git pack-objects test2 --stdin-packs --unpacked <<-EOF &&
 557                 $PACK_A
 558                 ^$PACK_B
 559                 $PACK_C
 560                 EOF
 561
 562                 (
 563                         git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
 564                         git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
 565                         git rev-list --objects --no-object-names \
 566                                 refs/tags/C..refs/tags/D
 567
 568                 ) >expect.raw &&
 569                 ls -la . &&
 570                 git show-index <$(ls test2-*.idx) >actual.raw &&
 571
 572                 cut -d" " -f2 <expect.raw | sort >expect &&
 573                 cut -d" " -f2 <actual.raw | sort >actual &&
 574                 test_cmp expect actual
 575         )
 576 '
 577
 578 test_expect_success '--stdin-packs with broken links' '
 579         (
 580                 cd stdin-packs &&
 581
 582                 # make an unreachable object with a bogus parent
 583                 git cat-file -p HEAD >commit &&
 584                 sed "s/$(git rev-parse HEAD^)/$(test_oid zero)/" <commit |
 585                 git hash-object -w -t commit --stdin >in &&
 586
 587                 git pack-objects .git/objects/pack/pack-D <in &&
 588
 589                 PACK_A="$(basename .git/objects/pack/pack-A-*.pack)" &&
 590                 PACK_B="$(basename .git/objects/pack/pack-B-*.pack)" &&
 591                 PACK_C="$(basename .git/objects/pack/pack-C-*.pack)" &&
 592                 PACK_D="$(basename .git/objects/pack/pack-D-*.pack)" &&
 593
 594                 git pack-objects test3 --stdin-packs --unpacked <<-EOF &&
 595                 $PACK_A
 596                 ^$PACK_B
 597                 $PACK_C
 598                 $PACK_D
 599                 EOF
 600
 601                 (
 602                         git show-index <$(ls .git/objects/pack/pack-A-*.idx) &&
 603                         git show-index <$(ls .git/objects/pack/pack-C-*.idx) &&
 604                         git show-index <$(ls .git/objects/pack/pack-D-*.idx) &&
 605                         git rev-list --objects --no-object-names \
 606                                 refs/tags/C..refs/tags/D
 607                 ) >expect.raw &&
 608                 git show-index <$(ls test3-*.idx) >actual.raw &&
 609
 610                 cut -d" " -f2 <expect.raw | sort >expect &&
 611                 cut -d" " -f2 <actual.raw | sort >actual &&
 612                 test_cmp expect actual
 613         )
 614 '
 615
 616 test_expect_success 'negative window clamps to 0' '
 617         git pack-objects --progress --window=-1 neg-window <obj-list 2>stderr &&
 618         check_deltas stderr = 0
 619 '
 620
 621 test_done