t/perf/p5303-many-packs.sh

   1 #!/bin/sh
   2
   3 test_description='performance with large numbers of packs'
   4 . ./perf-lib.sh
   5
   6 test_perf_large_repo
   7
   8 # A real many-pack situation would probably come from having a lot of pushes
   9 # over time. We don't know how big each push would be, but we can fake it by
  10 # just walking the first-parent chain and having every 5 commits be their own
  11 # "push". This isn't _entirely_ accurate, as real pushes would have some
  12 # duplicate objects due to thin-pack fixing, but it's a reasonable
  13 # approximation.
  14 #
  15 # And then all of the rest of the objects can go in a single packfile that
  16 # represents the state before any of those pushes (actually, we'll generate
  17 # that first because in such a setup it would be the oldest pack, and we sort
  18 # the packs by reverse mtime inside git).
  19 repack_into_n () {
  20         rm -rf staging &&
  21         mkdir staging &&
  22
  23         git rev-list --first-parent HEAD |
  24         perl -e '
  25                 my $n = shift;
  26                 while (<>) {
  27                         last unless @commits < $n;
  28                         push @commits, $_ if $. % 5 == 1;
  29                 }
  30                 print reverse @commits;
  31         ' "$1" >pushes &&
  32
  33         # create base packfile
  34         base_pack=$(
  35                 head -n 1 pushes |
  36                 git pack-objects --delta-base-offset --revs staging/pack
  37         ) &&
  38         test_export base_pack &&
  39
  40         # create an empty packfile
  41         empty_pack=$(git pack-objects staging/pack </dev/null) &&
  42         test_export empty_pack &&
  43
  44         # and then incrementals between each pair of commits
  45         last= &&
  46         while read rev
  47         do
  48                 if test -n "$last"; then
  49                         {
  50                                 echo "$rev" &&
  51                                 echo "^$last"
  52                         } |
  53                         git pack-objects --delta-base-offset --revs \
  54                                 staging/pack || return 1
  55                 fi
  56                 last=$rev
  57         done <pushes &&
  58
  59         (
  60                 find staging -type f -name 'pack-*.pack' |
  61                         xargs -n 1 basename | grep -v "$base_pack" &&
  62                 printf "^pack-%s.pack\n" $base_pack
  63         ) >stdin.packs
  64
  65         # and install the whole thing
  66         rm -f .git/objects/pack/* &&
  67         mv staging/* .git/objects/pack/
  68 }
  69
  70 # Pretend we just have a single branch and no reflogs, and that everything is
  71 # in objects/pack; that makes our fake pack-building via repack_into_n()
  72 # much simpler.
  73 test_expect_success 'simplify reachability' '
  74         tip=$(git rev-parse --verify HEAD) &&
  75         git for-each-ref --format="option no-deref%0adelete %(refname)" |
  76         git update-ref --stdin &&
  77         rm -rf .git/logs &&
  78         git update-ref refs/heads/master $tip &&
  79         git symbolic-ref HEAD refs/heads/master &&
  80         git repack -ad
  81 '
  82
  83 for nr_packs in 1 50 1000
  84 do
  85         test_expect_success "create $nr_packs-pack scenario" '
  86                 repack_into_n $nr_packs
  87         '
  88
  89         test_perf "rev-list ($nr_packs)" '
  90                 git rev-list --objects --all >/dev/null
  91         '
  92
  93         test_perf "abbrev-commit ($nr_packs)" '
  94                 git rev-list --abbrev-commit HEAD >/dev/null
  95         '
  96
  97         # This simulates the interesting part of the repack, which is the
  98         # actual pack generation, without smudging the on-disk setup
  99         # between trials.
 100         test_perf "repack ($nr_packs)" '
 101                 GIT_TEST_FULL_IN_PACK_ARRAY=1 \
 102                 git pack-objects --keep-true-parents \
 103                   --honor-pack-keep --non-empty --all \
 104                   --reflog --indexed-objects --delta-base-offset \
 105                   --stdout </dev/null >/dev/null
 106         '
 107
 108         test_perf "repack with kept ($nr_packs)" '
 109                 git pack-objects --keep-true-parents \
 110                   --keep-pack=pack-$empty_pack.pack \
 111                   --honor-pack-keep --non-empty --all \
 112                   --reflog --indexed-objects --delta-base-offset \
 113                   --stdout </dev/null >/dev/null
 114         '
 115
 116         test_perf "repack with --stdin-packs ($nr_packs)" '
 117                 git pack-objects \
 118                   --keep-true-parents \
 119                   --stdin-packs \
 120                   --non-empty \
 121                   --delta-base-offset \
 122                   --stdout <stdin.packs >/dev/null
 123         '
 124 done
 125
 126 # Measure pack loading with 10,000 packs.
 127 test_expect_success 'generate lots of packs' '
 128         for i in $(test_seq 10000); do
 129                 echo "blob"
 130                 echo "data <<EOF"
 131                 echo "blob $i"
 132                 echo "EOF"
 133                 echo "checkpoint"
 134         done |
 135         git -c fastimport.unpackLimit=0 fast-import
 136 '
 137
 138 # The purpose of this test is to evaluate load time for a large number
 139 # of packs while doing as little other work as possible.
 140 test_perf "load 10,000 packs" '
 141         git rev-parse --verify "HEAD^{commit}"
 142 '
 143
 144 test_done