Merge branch 'jh/partial-clone'

The machinery to clone & fetch, which in turn involves packing and
unpacking objects, have been told how to omit certain objects using
the filtering mechanism introduced by the jh/object-filtering
topic, and also mark the resulting pack as a promisor pack to
tolerate missing objects, taking advantage of the mechanism
introduced by the jh/fsck-promisors topic.

* jh/partial-clone:
  t5616: test bulk prefetch after partial fetch
  fetch: inherit filter-spec from partial clone
  t5616: end-to-end tests for partial clone
  fetch-pack: restore save_commit_buffer after use
  unpack-trees: batch fetching of missing blobs
  clone: partial clone
  partial-clone: define partial clone settings in config
  fetch: support filters
  fetch: refactor calculation of remote list
  fetch-pack: test support excluding large blobs
  fetch-pack: add --no-filter
  fetch-pack, index-pack, transport: partial clone
  upload-pack: add object filtering for partial clone

diff --cc Documentation/config.txt
Simple merge

diff --cc Documentation/technical/pack-protocol.txt
Simple merge

diff --cc builtin/clone.c
Simple merge

diff --cc builtin/fetch.c
Simple merge

diff --cc builtin/rev-list.c
Simple merge

diff --cc cache.h
Simple merge

diff --cc config.c
Simple merge

diff --cc connected.c
Simple merge

diff --cc environment.c
Simple merge

diff --cc fetch-pack.c
Simple merge

diff --cc t/t5601-clone.sh
index 8c437bf8721f929155fa2ddab1677c050ae5caca,13610b70f58a5e77c2141b6fc82e011a77f709d6..0b620377448bc3b97d39d2b6beceb4d185edb7ba
--- 1/t/t5601-clone.sh
--- 2/t/t5601-clone.sh
+++ b/t/t5601-clone.sh
@@@ -611,21 -571,105 +611,122 @@@ test_expect_success 'GIT_TRACE_PACKFIL
        git -C replay.git index-pack -v --stdin <tmp.pack
  '
  
 +hex2oct () {
 +      perl -ne 'printf "\\%03o", hex for /../g'
 +}
 +
 +test_expect_success 'clone on case-insensitive fs' '
 +      git init icasefs &&
 +      (
 +              cd icasefs
 +              o=$(git hash-object -w --stdin </dev/null | hex2oct) &&
 +              t=$(printf "100644 X\0${o}100644 x\0${o}" |
 +                      git hash-object -w -t tree --stdin) &&
 +              c=$(git commit-tree -m bogus $t) &&
 +              git update-ref refs/heads/bogus $c &&
 +              git clone -b bogus . bogus
 +      )
 +'
 +
+ partial_clone () {
+              SERVER="$1" &&
+              URL="$2" &&
+ 
+       rm -rf "$SERVER" client &&
+       test_create_repo "$SERVER" &&
+       test_commit -C "$SERVER" one &&
+       HASH1=$(git hash-object "$SERVER/one.t") &&
+       git -C "$SERVER" revert HEAD &&
+       test_commit -C "$SERVER" two &&
+       HASH2=$(git hash-object "$SERVER/two.t") &&
+       test_config -C "$SERVER" uploadpack.allowfilter 1 &&
+       test_config -C "$SERVER" uploadpack.allowanysha1inwant 1 &&
+ 
+       git clone --filter=blob:limit=0 "$URL" client &&
+ 
+       git -C client fsck &&
+ 
+       # Ensure that unneeded blobs are not inadvertently fetched.
+       test_config -C client extensions.partialclone "not a remote" &&
+       test_must_fail git -C client cat-file -e "$HASH1" &&
+ 
+       # But this blob was fetched, because clone performs an initial checkout
+       git -C client cat-file -e "$HASH2"
+ }
+ 
+ test_expect_success 'partial clone' '
+       partial_clone server "file://$(pwd)/server"
+ '
+ 
+ test_expect_success 'partial clone: warn if server does not support object filtering' '
+       rm -rf server client &&
+       test_create_repo server &&
+       test_commit -C server one &&
+ 
+       git clone --filter=blob:limit=0 "file://$(pwd)/server" client 2> err &&
+ 
+       test_i18ngrep "filtering not recognized by server" err
+ '
+ 
+ test_expect_success 'batch missing blob request during checkout' '
+       rm -rf server client &&
+ 
+       test_create_repo server &&
+       echo a >server/a &&
+       echo b >server/b &&
+       git -C server add a b &&
+ 
+       git -C server commit -m x &&
+       echo aa >server/a &&
+       echo bb >server/b &&
+       git -C server add a b &&
+       git -C server commit -m x &&
+ 
+       test_config -C server uploadpack.allowfilter 1 &&
+       test_config -C server uploadpack.allowanysha1inwant 1 &&
+ 
+       git clone --filter=blob:limit=0 "file://$(pwd)/server" client &&
+ 
+       # Ensure that there is only one negotiation by checking that there is
+       # only "done" line sent. ("done" marks the end of negotiation.)
+       GIT_TRACE_PACKET="$(pwd)/trace" git -C client checkout HEAD^ &&
+       grep "git> done" trace >done_lines &&
+       test_line_count = 1 done_lines
+ '
+ 
+ test_expect_success 'batch missing blob request does not inadvertently try to fetch gitlinks' '
+       rm -rf server client &&
+ 
+       test_create_repo repo_for_submodule &&
+       test_commit -C repo_for_submodule x &&
+ 
+       test_create_repo server &&
+       echo a >server/a &&
+       echo b >server/b &&
+       git -C server add a b &&
+       git -C server commit -m x &&
+ 
+       echo aa >server/a &&
+       echo bb >server/b &&
+       # Also add a gitlink pointing to an arbitrary repository
+       git -C server submodule add "$(pwd)/repo_for_submodule" c &&
+       git -C server add a b c &&
+       git -C server commit -m x &&
+ 
+       test_config -C server uploadpack.allowfilter 1 &&
+       test_config -C server uploadpack.allowanysha1inwant 1 &&
+ 
+       # Make sure that it succeeds
+       git clone --filter=blob:limit=0 "file://$(pwd)/server" client
+ '
+ 
+ . "$TEST_DIRECTORY"/lib-httpd.sh
+ start_httpd
+ 
+ test_expect_success 'partial clone using HTTP' '
+       partial_clone "$HTTPD_DOCUMENT_ROOT_PATH/server" "$HTTPD_URL/smart/server"
+ '
+ 
+ stop_httpd
+ 
  test_done

diff --cc transport-helper.c
Simple merge

diff --cc transport.c
Simple merge

diff --cc transport.h
Simple merge

diff --cc unpack-trees.c
index 96c3327f19de4b1060ed43ef291a57a76ff2b8cc,73a1cdb43e351a12c1f081344ff0d8a6e78f0793..e6a15bbe44f24555a8edb9461c2af553de70cba0
--- 1/unpack-trees.c
--- 2/unpack-trees.c
+++ b/unpack-trees.c
@@@ -14,7 -14,7 +14,8 @@@
  #include "dir.h"
  #include "submodule.h"
  #include "submodule-config.h"
 +#include "fsmonitor.h"
+ #include "fetch-object.h"
  
  /*
   * Error messages expected by scripts out of plumbing commands such as

diff --cc upload-pack.c
index d5de18127c63a9d017578e0969743392d39526d4,15b66051ce9a03aabb1b57f3839235f2ad9bdfb3..f51b6cfca9435d6b3b3f5b44b593a7c5cd2b1d3e
--- 1/upload-pack.c
--- 2/upload-pack.c
+++ b/upload-pack.c
@@@ -18,7 -20,7 +20,8 @@@
  #include "parse-options.h"
  #include "argv-array.h"
  #include "prio-queue.h"
 +#include "protocol.h"
+ #include "quote.h"
  
  static const char * const upload_pack_usage[] = {
        N_("git upload-pack [<options>] <dir>"),