t/t4216-log-bloom.sh

   1 #!/bin/sh
   2
   3 test_description='git log for a path with Bloom filters'
   4 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
   5 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
   6
   7 . ./test-lib.sh
   8
   9 GIT_TEST_COMMIT_GRAPH=0
  10 GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=0
  11
  12 test_expect_success 'setup test - repo, commits, commit graph, log outputs' '
  13         git init &&
  14         mkdir A A/B A/B/C &&
  15         test_commit c1 A/file1 &&
  16         test_commit c2 A/B/file2 &&
  17         test_commit c3 A/B/C/file3 &&
  18         test_commit c4 A/file1 &&
  19         test_commit c5 A/B/file2 &&
  20         test_commit c6 A/B/C/file3 &&
  21         test_commit c7 A/file1 &&
  22         test_commit c8 A/B/file2 &&
  23         test_commit c9 A/B/C/file3 &&
  24         test_commit c10 file_to_be_deleted &&
  25         git checkout -b side HEAD~4 &&
  26         test_commit side-1 file4 &&
  27         git checkout main &&
  28         git merge side &&
  29         test_commit c11 file5 &&
  30         mv file5 file5_renamed &&
  31         git add file5_renamed &&
  32         git commit -m "rename" &&
  33         rm file_to_be_deleted &&
  34         git add . &&
  35         git commit -m "file removed" &&
  36         git commit --allow-empty -m "empty" &&
  37         git commit-graph write --reachable --changed-paths &&
  38
  39         test_oid_cache <<-EOF
  40         oid_version sha1:1
  41         oid_version sha256:2
  42         EOF
  43 '
  44
  45 graph_read_expect () {
  46         NUM_CHUNKS=6
  47         cat >expect <<- EOF
  48         header: 43475048 1 $(test_oid oid_version) $NUM_CHUNKS 0
  49         num_commits: $1
  50         chunks: oid_fanout oid_lookup commit_metadata generation_data bloom_indexes bloom_data
  51         options: bloom(1,10,7) read_generation_data
  52         EOF
  53         test-tool read-graph >actual &&
  54         test_cmp expect actual
  55 }
  56
  57 test_expect_success 'commit-graph write wrote out the bloom chunks' '
  58         graph_read_expect 16
  59 '
  60
  61 # Turn off any inherited trace2 settings for this test.
  62 sane_unset GIT_TRACE2 GIT_TRACE2_PERF GIT_TRACE2_EVENT
  63 sane_unset GIT_TRACE2_PERF_BRIEF
  64 sane_unset GIT_TRACE2_CONFIG_PARAMS
  65
  66 setup () {
  67         rm -f "$TRASH_DIRECTORY/trace.perf" &&
  68         git -c core.commitGraph=false log --pretty="format:%s" $1 >log_wo_bloom &&
  69         GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.perf" git -c core.commitGraph=true log --pretty="format:%s" $1 >log_w_bloom
  70 }
  71
  72 test_bloom_filters_used () {
  73         log_args=$1
  74         bloom_trace_prefix="statistics:{\"filter_not_present\":${2:-0},\"maybe\""
  75         setup "$log_args" &&
  76         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
  77         test_cmp log_wo_bloom log_w_bloom &&
  78     test_path_is_file "$TRASH_DIRECTORY/trace.perf"
  79 }
  80
  81 test_bloom_filters_not_used () {
  82         log_args=$1
  83         setup "$log_args" &&
  84         ! grep -q "statistics:{\"filter_not_present\":" "$TRASH_DIRECTORY/trace.perf" &&
  85         test_cmp log_wo_bloom log_w_bloom
  86 }
  87
  88 for path in A A/B A/B/C A/file1 A/B/file2 A/B/C/file3 file4 file5 file5_renamed file_to_be_deleted
  89 do
  90         for option in "" \
  91               "--all" \
  92                       "--full-history" \
  93                       "--full-history --simplify-merges" \
  94                       "--simplify-merges" \
  95                       "--simplify-by-decoration" \
  96                       "--follow" \
  97                       "--first-parent" \
  98                       "--topo-order" \
  99                       "--date-order" \
 100                       "--author-date-order" \
 101                       "--ancestry-path side..main"
 102         do
 103                 test_expect_success "git log option: $option for path: $path" '
 104                         test_bloom_filters_used "$option -- $path" &&
 105                         test_config commitgraph.readChangedPaths false &&
 106                         test_bloom_filters_not_used "$option -- $path"
 107                 '
 108         done
 109 done
 110
 111 test_expect_success 'git log -- folder works with and without the trailing slash' '
 112         test_bloom_filters_used "-- A" &&
 113         test_bloom_filters_used "-- A/"
 114 '
 115
 116 test_expect_success 'git log for path that does not exist. ' '
 117         test_bloom_filters_used "-- path_does_not_exist"
 118 '
 119
 120 test_expect_success 'git log with --walk-reflogs does not use Bloom filters' '
 121         test_bloom_filters_not_used "--walk-reflogs -- A"
 122 '
 123
 124 test_expect_success 'git log -- multiple path specs does not use Bloom filters' '
 125         test_bloom_filters_not_used "-- file4 A/file1"
 126 '
 127
 128 test_expect_success 'git log -- "." pathspec at root does not use Bloom filters' '
 129         test_bloom_filters_not_used "-- ."
 130 '
 131
 132 test_expect_success 'git log with wildcard that resolves to a single path uses Bloom filters' '
 133         test_bloom_filters_used "-- *4" &&
 134         test_bloom_filters_used "-- *renamed"
 135 '
 136
 137 test_expect_success 'git log with wildcard that resolves to a multiple paths does not uses Bloom filters' '
 138         test_bloom_filters_not_used "-- *" &&
 139         test_bloom_filters_not_used "-- file*"
 140 '
 141
 142 test_expect_success 'setup - add commit-graph to the chain without Bloom filters' '
 143         test_commit c14 A/anotherFile2 &&
 144         test_commit c15 A/B/anotherFile2 &&
 145         test_commit c16 A/B/C/anotherFile2 &&
 146         git commit-graph write --reachable --split --no-changed-paths &&
 147         test_line_count = 2 .git/objects/info/commit-graphs/commit-graph-chain
 148 '
 149
 150 test_expect_success 'use Bloom filters even if the latest graph does not have Bloom filters' '
 151         # Ensure that the number of empty filters is equal to the number of
 152         # filters in the latest graph layer to prove that they are loaded (and
 153         # ignored).
 154         test_bloom_filters_used "-- A/B" 3
 155 '
 156
 157 test_expect_success 'setup - add commit-graph to the chain with Bloom filters' '
 158         test_commit c17 A/anotherFile3 &&
 159         git commit-graph write --reachable --changed-paths --split &&
 160         test_line_count = 3 .git/objects/info/commit-graphs/commit-graph-chain
 161 '
 162
 163 test_bloom_filters_used_when_some_filters_are_missing () {
 164         log_args=$1
 165         bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":9"
 166         setup "$log_args" &&
 167         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
 168         test_cmp log_wo_bloom log_w_bloom
 169 }
 170
 171 test_expect_success 'Use Bloom filters if they exist in the latest but not all commit graphs in the chain.' '
 172         test_bloom_filters_used_when_some_filters_are_missing "-- A/B"
 173 '
 174
 175 test_expect_success 'persist filter settings' '
 176         test_when_finished rm -rf .git/objects/info/commit-graph* &&
 177         rm -rf .git/objects/info/commit-graph* &&
 178         GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
 179                 GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=9 \
 180                 GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY=15 \
 181                 git commit-graph write --reachable --changed-paths &&
 182         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2.txt &&
 183         GIT_TRACE2_EVENT="$(pwd)/trace2-auto.txt" \
 184                 git commit-graph write --reachable --changed-paths &&
 185         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2-auto.txt
 186 '
 187
 188 test_max_changed_paths () {
 189         grep "\"max_changed_paths\":$1" $2
 190 }
 191
 192 test_filter_not_computed () {
 193         grep "\"key\":\"filter-not-computed\",\"value\":\"$1\"" $2
 194 }
 195
 196 test_filter_computed () {
 197         grep "\"key\":\"filter-computed\",\"value\":\"$1\"" $2
 198 }
 199
 200 test_filter_trunc_empty () {
 201         grep "\"key\":\"filter-trunc-empty\",\"value\":\"$1\"" $2
 202 }
 203
 204 test_filter_trunc_large () {
 205         grep "\"key\":\"filter-trunc-large\",\"value\":\"$1\"" $2
 206 }
 207
 208 test_expect_success 'correctly report changes over limit' '
 209         git init limits &&
 210         (
 211                 cd limits &&
 212                 mkdir d &&
 213                 mkdir d/e &&
 214
 215                 for i in $(test_seq 1 2)
 216                 do
 217                         printf $i >d/file$i.txt &&
 218                         printf $i >d/e/file$i.txt || return 1
 219                 done &&
 220
 221                 mkdir mode &&
 222                 printf bash >mode/script.sh &&
 223
 224                 mkdir foo &&
 225                 touch foo/bar &&
 226                 touch foo.txt &&
 227
 228                 git add d foo foo.txt mode &&
 229                 git commit -m "files" &&
 230
 231                 # Commit has 7 file and 4 directory adds
 232                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
 233                         GIT_TRACE2_EVENT="$(pwd)/trace" \
 234                         git commit-graph write --reachable --changed-paths &&
 235                 test_max_changed_paths 10 trace &&
 236                 test_filter_computed 1 trace &&
 237                 test_filter_trunc_large 1 trace &&
 238
 239                 for path in $(git ls-tree -r --name-only HEAD)
 240                 do
 241                         git -c commitGraph.readChangedPaths=false log \
 242                                 -- $path >expect &&
 243                         git log -- $path >actual &&
 244                         test_cmp expect actual || return 1
 245                 done &&
 246
 247                 # Make a variety of path changes
 248                 printf new1 >d/e/file1.txt &&
 249                 printf new2 >d/file2.txt &&
 250                 rm d/e/file2.txt &&
 251                 rm -r foo &&
 252                 printf text >foo &&
 253                 mkdir f &&
 254                 printf new1 >f/file1.txt &&
 255
 256                 # including a mode-only change (counts as modified)
 257                 git update-index --chmod=+x mode/script.sh &&
 258
 259                 git add foo d f &&
 260                 git commit -m "complicated" &&
 261
 262                 # start from scratch and rebuild
 263                 rm -f .git/objects/info/commit-graph &&
 264                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
 265                         GIT_TRACE2_EVENT="$(pwd)/trace-edit" \
 266                         git commit-graph write --reachable --changed-paths &&
 267                 test_max_changed_paths 10 trace-edit &&
 268                 test_filter_computed 2 trace-edit &&
 269                 test_filter_trunc_large 2 trace-edit &&
 270
 271                 for path in $(git ls-tree -r --name-only HEAD)
 272                 do
 273                         git -c commitGraph.readChangedPaths=false log \
 274                                 -- $path >expect &&
 275                         git log -- $path >actual &&
 276                         test_cmp expect actual || return 1
 277                 done &&
 278
 279                 # start from scratch and rebuild
 280                 rm -f .git/objects/info/commit-graph &&
 281                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=11 \
 282                         GIT_TRACE2_EVENT="$(pwd)/trace-update" \
 283                         git commit-graph write --reachable --changed-paths &&
 284                 test_max_changed_paths 11 trace-update &&
 285                 test_filter_computed 2 trace-update &&
 286                 test_filter_trunc_large 0 trace-update &&
 287
 288                 for path in $(git ls-tree -r --name-only HEAD)
 289                 do
 290                         git -c commitGraph.readChangedPaths=false log \
 291                                 -- $path >expect &&
 292                         git log -- $path >actual &&
 293                         test_cmp expect actual || return 1
 294                 done
 295         )
 296 '
 297
 298 test_expect_success 'correctly report commits with no changed paths' '
 299         git init empty &&
 300         test_when_finished "rm -fr empty" &&
 301         (
 302                 cd empty &&
 303
 304                 git commit --allow-empty -m "initial commit" &&
 305
 306                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 307                         git commit-graph write --reachable --changed-paths &&
 308                 test_filter_computed 1 trace.event &&
 309                 test_filter_not_computed 0 trace.event &&
 310                 test_filter_trunc_empty 1 trace.event &&
 311                 test_filter_trunc_large 0 trace.event
 312         )
 313 '
 314
 315 test_expect_success 'Bloom generation is limited by --max-new-filters' '
 316         (
 317                 cd limits &&
 318                 test_commit c2 filter &&
 319                 test_commit c3 filter &&
 320                 test_commit c4 no-filter &&
 321
 322                 rm -f trace.event &&
 323                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 324                         git commit-graph write --reachable --split=replace \
 325                                 --changed-paths --max-new-filters=2 &&
 326
 327                 test_filter_computed 2 trace.event &&
 328                 test_filter_not_computed 3 trace.event &&
 329                 test_filter_trunc_empty 0 trace.event &&
 330                 test_filter_trunc_large 0 trace.event
 331         )
 332 '
 333
 334 test_expect_success 'Bloom generation backfills previously-skipped filters' '
 335         # Check specifying commitGraph.maxNewFilters over "git config" works.
 336         test_config -C limits commitGraph.maxNewFilters 1 &&
 337         (
 338                 cd limits &&
 339
 340                 rm -f trace.event &&
 341                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 342                         git commit-graph write --reachable --changed-paths \
 343                                 --split=replace &&
 344                 test_filter_computed 1 trace.event &&
 345                 test_filter_not_computed 4 trace.event &&
 346                 test_filter_trunc_empty 0 trace.event &&
 347                 test_filter_trunc_large 0 trace.event
 348         )
 349 '
 350
 351 test_expect_success '--max-new-filters overrides configuration' '
 352         git init override &&
 353         test_when_finished "rm -fr override" &&
 354         test_config -C override commitGraph.maxNewFilters 2 &&
 355         (
 356                 cd override &&
 357                 test_commit one &&
 358                 test_commit two &&
 359
 360                 rm -f trace.event &&
 361                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 362                         git commit-graph write --reachable --changed-paths \
 363                                 --max-new-filters=1 &&
 364                 test_filter_computed 1 trace.event &&
 365                 test_filter_not_computed 1 trace.event &&
 366                 test_filter_trunc_empty 0 trace.event &&
 367                 test_filter_trunc_large 0 trace.event
 368         )
 369 '
 370
 371 test_expect_success 'Bloom generation backfills empty commits' '
 372         git init empty &&
 373         test_when_finished "rm -fr empty" &&
 374         (
 375                 cd empty &&
 376                 for i in $(test_seq 1 6)
 377                 do
 378                         git commit --allow-empty -m "$i" || return 1
 379                 done &&
 380
 381                 # Generate Bloom filters for empty commits 1-6, two at a time.
 382                 for i in $(test_seq 1 3)
 383                 do
 384                         rm -f trace.event &&
 385                         GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 386                                 git commit-graph write --reachable \
 387                                         --changed-paths --max-new-filters=2 &&
 388                         test_filter_computed 2 trace.event &&
 389                         test_filter_not_computed 4 trace.event &&
 390                         test_filter_trunc_empty 2 trace.event &&
 391                         test_filter_trunc_large 0 trace.event || return 1
 392                 done &&
 393
 394                 # Finally, make sure that once all commits have filters, that
 395                 # none are subsequently recomputed.
 396                 rm -f trace.event &&
 397                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
 398                         git commit-graph write --reachable \
 399                                 --changed-paths --max-new-filters=2 &&
 400                 test_filter_computed 0 trace.event &&
 401                 test_filter_not_computed 6 trace.event &&
 402                 test_filter_trunc_empty 0 trace.event &&
 403                 test_filter_trunc_large 0 trace.event
 404         )
 405 '
 406
 407 test_done