t/t5317-pack-objects-filter-objects.sh

   1 #!/bin/sh
   2
   3 test_description='git pack-objects using object filtering'
   4
   5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   7
   8 TEST_PASSES_SANITIZE_LEAK=true
   9 . ./test-lib.sh
  10
  11 # Test blob:none filter.
  12
  13 test_expect_success 'setup r1' '
  14         git init r1 &&
  15         for n in 1 2 3 4 5
  16         do
  17                 echo "This is file: $n" > r1/file.$n &&
  18                 git -C r1 add file.$n &&
  19                 git -C r1 commit -m "$n" || return 1
  20         done
  21 '
  22
  23 parse_verify_pack_blob_oid () {
  24         awk '{print $1}' -
  25 }
  26
  27 test_expect_success 'verify blob count in normal packfile' '
  28         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
  29                 >ls_files_result &&
  30         test_parse_ls_files_stage_oids <ls_files_result |
  31         sort >expected &&
  32
  33         git -C r1 pack-objects --revs --stdout >all.pack <<-EOF &&
  34         HEAD
  35         EOF
  36         git -C r1 index-pack ../all.pack &&
  37
  38         git -C r1 verify-pack -v ../all.pack >verify_result &&
  39         grep blob verify_result |
  40         parse_verify_pack_blob_oid |
  41         sort >observed &&
  42
  43         test_cmp expected observed
  44 '
  45
  46 test_expect_success 'verify blob:none packfile has no blobs' '
  47         git -C r1 pack-objects --revs --stdout --filter=blob:none >filter.pack <<-EOF &&
  48         HEAD
  49         EOF
  50         git -C r1 index-pack ../filter.pack &&
  51
  52         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  53         ! grep blob verify_result
  54 '
  55
  56 test_expect_success 'verify normal and blob:none packfiles have same commits/trees' '
  57         git -C r1 verify-pack -v ../all.pack >verify_result &&
  58         grep -E "commit|tree" verify_result |
  59         parse_verify_pack_blob_oid |
  60         sort >expected &&
  61
  62         git -C r1 verify-pack -v ../filter.pack >verify_result &&
  63         grep -E "commit|tree" verify_result |
  64         parse_verify_pack_blob_oid |
  65         sort >observed &&
  66
  67         test_cmp expected observed
  68 '
  69
  70 test_expect_success 'get an error for missing tree object' '
  71         git init r5 &&
  72         echo foo >r5/foo &&
  73         git -C r5 add foo &&
  74         git -C r5 commit -m "foo" &&
  75         git -C r5 rev-parse HEAD^{tree} >tree &&
  76         del=$(sed "s|..|&/|" tree) &&
  77         rm r5/.git/objects/$del &&
  78         test_must_fail git -C r5 pack-objects --revs --stdout 2>bad_tree <<-EOF &&
  79         HEAD
  80         EOF
  81         grep "bad tree object" bad_tree
  82 '
  83
  84 test_expect_success 'setup for tests of tree:0' '
  85         mkdir r1/subtree &&
  86         echo "This is a file in a subtree" >r1/subtree/file &&
  87         git -C r1 add subtree/file &&
  88         git -C r1 commit -m subtree
  89 '
  90
  91 test_expect_success 'verify tree:0 packfile has no blobs or trees' '
  92         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
  93         HEAD
  94         EOF
  95         git -C r1 index-pack ../commitsonly.pack &&
  96         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
  97         ! grep -E "tree|blob" objs
  98 '
  99
 100 test_expect_success 'grab tree directly when using tree:0' '
 101         # We should get the tree specified directly but not its blobs or subtrees.
 102         git -C r1 pack-objects --revs --stdout --filter=tree:0 >commitsonly.pack <<-EOF &&
 103         HEAD:
 104         EOF
 105         git -C r1 index-pack ../commitsonly.pack &&
 106         git -C r1 verify-pack -v ../commitsonly.pack >objs &&
 107         awk "/tree|blob/{print \$1}" objs >trees_and_blobs &&
 108         git -C r1 rev-parse HEAD: >expected &&
 109         test_cmp expected trees_and_blobs
 110 '
 111
 112 # Test blob:limit=<n>[kmg] filter.
 113 # We boundary test around the size parameter.  The filter is strictly less than
 114 # the value, so size 500 and 1000 should have the same results, but 1001 should
 115 # filter more.
 116
 117 test_expect_success 'setup r2' '
 118         git init r2 &&
 119         for n in 1000 10000
 120         do
 121                 printf "%"$n"s" X > r2/large.$n &&
 122                 git -C r2 add large.$n &&
 123                 git -C r2 commit -m "$n" || return 1
 124         done
 125 '
 126
 127 test_expect_success 'verify blob count in normal packfile' '
 128         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 129         test_parse_ls_files_stage_oids <ls_files_result |
 130         sort >expected &&
 131
 132         git -C r2 pack-objects --revs --stdout >all.pack <<-EOF &&
 133         HEAD
 134         EOF
 135         git -C r2 index-pack ../all.pack &&
 136
 137         git -C r2 verify-pack -v ../all.pack >verify_result &&
 138         grep blob verify_result |
 139         parse_verify_pack_blob_oid |
 140         sort >observed &&
 141
 142         test_cmp expected observed
 143 '
 144
 145 test_expect_success 'verify blob:limit=500 omits all blobs' '
 146         git -C r2 pack-objects --revs --stdout --filter=blob:limit=500 >filter.pack <<-EOF &&
 147         HEAD
 148         EOF
 149         git -C r2 index-pack ../filter.pack &&
 150
 151         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 152         ! grep blob verify_result
 153 '
 154
 155 test_expect_success 'verify blob:limit=1000' '
 156         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1000 >filter.pack <<-EOF &&
 157         HEAD
 158         EOF
 159         git -C r2 index-pack ../filter.pack &&
 160
 161         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 162         ! grep blob verify_result
 163 '
 164
 165 test_expect_success 'verify blob:limit=1001' '
 166         git -C r2 ls-files -s large.1000 >ls_files_result &&
 167         test_parse_ls_files_stage_oids <ls_files_result |
 168         sort >expected &&
 169
 170         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 >filter.pack <<-EOF &&
 171         HEAD
 172         EOF
 173         git -C r2 index-pack ../filter.pack &&
 174
 175         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 176         grep blob verify_result |
 177         parse_verify_pack_blob_oid |
 178         sort >observed &&
 179
 180         test_cmp expected observed
 181 '
 182
 183 test_expect_success 'verify blob:limit=10001' '
 184         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 185         test_parse_ls_files_stage_oids <ls_files_result |
 186         sort >expected &&
 187
 188         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 >filter.pack <<-EOF &&
 189         HEAD
 190         EOF
 191         git -C r2 index-pack ../filter.pack &&
 192
 193         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 194         grep blob verify_result |
 195         parse_verify_pack_blob_oid |
 196         sort >observed &&
 197
 198         test_cmp expected observed
 199 '
 200
 201 test_expect_success 'verify blob:limit=1k' '
 202         git -C r2 ls-files -s large.1000 >ls_files_result &&
 203         test_parse_ls_files_stage_oids <ls_files_result |
 204         sort >expected &&
 205
 206         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k >filter.pack <<-EOF &&
 207         HEAD
 208         EOF
 209         git -C r2 index-pack ../filter.pack &&
 210
 211         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 212         grep blob verify_result |
 213         parse_verify_pack_blob_oid |
 214         sort >observed &&
 215
 216         test_cmp expected observed
 217 '
 218
 219 test_expect_success 'verify explicitly specifying oversized blob in input' '
 220         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 221         test_parse_ls_files_stage_oids <ls_files_result |
 222         sort >expected &&
 223
 224         echo HEAD >objects &&
 225         git -C r2 rev-parse HEAD:large.10000 >>objects &&
 226         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1k <objects >filter.pack &&
 227         git -C r2 index-pack ../filter.pack &&
 228
 229         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 230         grep blob verify_result |
 231         parse_verify_pack_blob_oid |
 232         sort >observed &&
 233
 234         test_cmp expected observed
 235 '
 236
 237 test_expect_success 'verify blob:limit=1m' '
 238         git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
 239         test_parse_ls_files_stage_oids <ls_files_result |
 240         sort >expected &&
 241
 242         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1m >filter.pack <<-EOF &&
 243         HEAD
 244         EOF
 245         git -C r2 index-pack ../filter.pack &&
 246
 247         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 248         grep blob verify_result |
 249         parse_verify_pack_blob_oid |
 250         sort >observed &&
 251
 252         test_cmp expected observed
 253 '
 254
 255 test_expect_success 'verify normal and blob:limit packfiles have same commits/trees' '
 256         git -C r2 verify-pack -v ../all.pack >verify_result &&
 257         grep -E "commit|tree" verify_result |
 258         parse_verify_pack_blob_oid |
 259         sort >expected &&
 260
 261         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 262         grep -E "commit|tree" verify_result |
 263         parse_verify_pack_blob_oid |
 264         sort >observed &&
 265
 266         test_cmp expected observed
 267 '
 268
 269 test_expect_success 'verify small limit and big limit results in small limit' '
 270         git -C r2 ls-files -s large.1000 >ls_files_result &&
 271         test_parse_ls_files_stage_oids <ls_files_result |
 272         sort >expected &&
 273
 274         git -C r2 pack-objects --revs --stdout --filter=blob:limit=1001 \
 275                 --filter=blob:limit=10001 >filter.pack <<-EOF &&
 276         HEAD
 277         EOF
 278         git -C r2 index-pack ../filter.pack &&
 279
 280         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 281         grep blob verify_result |
 282         parse_verify_pack_blob_oid |
 283         sort >observed &&
 284
 285         test_cmp expected observed
 286 '
 287
 288 test_expect_success 'verify big limit and small limit results in small limit' '
 289         git -C r2 ls-files -s large.1000 >ls_files_result &&
 290         test_parse_ls_files_stage_oids <ls_files_result |
 291         sort >expected &&
 292
 293         git -C r2 pack-objects --revs --stdout --filter=blob:limit=10001 \
 294                 --filter=blob:limit=1001 >filter.pack <<-EOF &&
 295         HEAD
 296         EOF
 297         git -C r2 index-pack ../filter.pack &&
 298
 299         git -C r2 verify-pack -v ../filter.pack >verify_result &&
 300         grep blob verify_result |
 301         parse_verify_pack_blob_oid |
 302         sort >observed &&
 303
 304         test_cmp expected observed
 305 '
 306
 307 # Test sparse:path=<path> filter.
 308 # !!!!
 309 # NOTE: sparse:path filter support has been dropped for security reasons,
 310 # so the tests have been changed to make sure that using it fails.
 311 # !!!!
 312 # Use a local file containing a sparse-checkout specification to filter
 313 # out blobs not required for the corresponding sparse-checkout.  We do not
 314 # require sparse-checkout to actually be enabled.
 315
 316 test_expect_success 'setup r3' '
 317         git init r3 &&
 318         mkdir r3/dir1 &&
 319         for n in sparse1 sparse2
 320         do
 321                 echo "This is file: $n" > r3/$n &&
 322                 git -C r3 add $n &&
 323                 echo "This is file: dir1/$n" > r3/dir1/$n &&
 324                 git -C r3 add dir1/$n || return 1
 325         done &&
 326         git -C r3 commit -m "sparse" &&
 327         echo dir1/ >pattern1 &&
 328         echo sparse1 >pattern2
 329 '
 330
 331 test_expect_success 'verify blob count in normal packfile' '
 332         git -C r3 ls-files -s sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 333                 >ls_files_result &&
 334         test_parse_ls_files_stage_oids <ls_files_result |
 335         sort >expected &&
 336
 337         git -C r3 pack-objects --revs --stdout >all.pack <<-EOF &&
 338         HEAD
 339         EOF
 340         git -C r3 index-pack ../all.pack &&
 341
 342         git -C r3 verify-pack -v ../all.pack >verify_result &&
 343         grep blob verify_result |
 344         parse_verify_pack_blob_oid |
 345         sort >observed &&
 346
 347         test_cmp expected observed
 348 '
 349
 350 test_expect_success 'verify sparse:path=pattern1 fails' '
 351         test_must_fail git -C r3 pack-objects --revs --stdout \
 352                 --filter=sparse:path=../pattern1 <<-EOF
 353         HEAD
 354         EOF
 355 '
 356
 357 test_expect_success 'verify sparse:path=pattern2 fails' '
 358         test_must_fail git -C r3 pack-objects --revs --stdout \
 359                 --filter=sparse:path=../pattern2 <<-EOF
 360         HEAD
 361         EOF
 362 '
 363
 364 # Test sparse:oid=<oid-ish> filter.
 365 # Use a blob containing a sparse-checkout specification to filter
 366 # out blobs not required for the corresponding sparse-checkout.  We do not
 367 # require sparse-checkout to actually be enabled.
 368
 369 test_expect_success 'setup r4' '
 370         git init r4 &&
 371         mkdir r4/dir1 &&
 372         for n in sparse1 sparse2
 373         do
 374                 echo "This is file: $n" > r4/$n &&
 375                 git -C r4 add $n &&
 376                 echo "This is file: dir1/$n" > r4/dir1/$n &&
 377                 git -C r4 add dir1/$n || return 1
 378         done &&
 379         echo dir1/ >r4/pattern &&
 380         git -C r4 add pattern &&
 381         git -C r4 commit -m "pattern"
 382 '
 383
 384 test_expect_success 'verify blob count in normal packfile' '
 385         git -C r4 ls-files -s pattern sparse1 sparse2 dir1/sparse1 dir1/sparse2 \
 386                 >ls_files_result &&
 387         test_parse_ls_files_stage_oids <ls_files_result |
 388         sort >expected &&
 389
 390         git -C r4 pack-objects --revs --stdout >all.pack <<-EOF &&
 391         HEAD
 392         EOF
 393         git -C r4 index-pack ../all.pack &&
 394
 395         git -C r4 verify-pack -v ../all.pack >verify_result &&
 396         grep blob verify_result |
 397         parse_verify_pack_blob_oid |
 398         sort >observed &&
 399
 400         test_cmp expected observed
 401 '
 402
 403 test_expect_success 'verify sparse:oid=OID' '
 404         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 405         test_parse_ls_files_stage_oids <ls_files_result |
 406         sort >expected &&
 407
 408         git -C r4 ls-files -s pattern >staged &&
 409         oid=$(test_parse_ls_files_stage_oids <staged) &&
 410         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=$oid >filter.pack <<-EOF &&
 411         HEAD
 412         EOF
 413         git -C r4 index-pack ../filter.pack &&
 414
 415         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 416         grep blob verify_result |
 417         parse_verify_pack_blob_oid |
 418         sort >observed &&
 419
 420         test_cmp expected observed
 421 '
 422
 423 test_expect_success 'verify sparse:oid=oid-ish' '
 424         git -C r4 ls-files -s dir1/sparse1 dir1/sparse2 >ls_files_result &&
 425         test_parse_ls_files_stage_oids <ls_files_result |
 426         sort >expected &&
 427
 428         git -C r4 pack-objects --revs --stdout --filter=sparse:oid=main:pattern >filter.pack <<-EOF &&
 429         HEAD
 430         EOF
 431         git -C r4 index-pack ../filter.pack &&
 432
 433         git -C r4 verify-pack -v ../filter.pack >verify_result &&
 434         grep blob verify_result |
 435         parse_verify_pack_blob_oid |
 436         sort >observed &&
 437
 438         test_cmp expected observed
 439 '
 440
 441 # Delete some loose objects and use pack-objects, but WITHOUT any filtering.
 442 # This models previously omitted objects that we did not receive.
 443
 444 test_expect_success 'setup r1 - delete loose blobs' '
 445         git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
 446                 >ls_files_result &&
 447         test_parse_ls_files_stage_oids <ls_files_result |
 448         sort >expected &&
 449
 450         for id in `cat expected | sed "s|..|&/|"`
 451         do
 452                 rm r1/.git/objects/$id || return 1
 453         done
 454 '
 455
 456 test_expect_success 'verify pack-objects fails w/ missing objects' '
 457         test_must_fail git -C r1 pack-objects --revs --stdout >miss.pack <<-EOF
 458         HEAD
 459         EOF
 460 '
 461
 462 test_expect_success 'verify pack-objects fails w/ --missing=error' '
 463         test_must_fail git -C r1 pack-objects --revs --stdout --missing=error >miss.pack <<-EOF
 464         HEAD
 465         EOF
 466 '
 467
 468 test_expect_success 'verify pack-objects w/ --missing=allow-any' '
 469         git -C r1 pack-objects --revs --stdout --missing=allow-any >miss.pack <<-EOF
 470         HEAD
 471         EOF
 472 '
 473
 474 test_done