From f2cba9299be45f8e027f7b45c6e4a3cae55576c6 Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 14 Mar 2011 16:48:38 -0700 Subject: [PATCH 1/2] fetch-pack: Finish negotation if remote replies "ACK %s ready" If multi_ack_detailed was selected in the protocol capabilities (both client and server are >= Git 1.6.6) the upload-pack side will send "ACK %s ready" when it knows how to safely cut the graph and produce a reasonable pack for the want list that was already sent on the connection. Upon receiving "ACK %s ready" there is no point in looking at the remaining commits inside of rev_list. Sending additional "have %s" lines to the remote will not construct a smaller pack. It is unlikely a commit older than the current cut point will have a better delta base than the cut point itself has. The original design of this code had fetch-pack empty rev_list by marking a commit and its transitive ancestors COMMON whenever the remote side said "ACK %s {continue,common}" and skipping over any already COMMON commits during get_rev(). This approach does not work when most of rev_list is actually COMMON_REF, commits that are pointed to by a reference on the remote, which exist locally, and which have not yet been sent to the remote as a "have %s" line. Most of the common references are tags in the ref/tags namespace, using points in the commit graph that are more than 1 commit apart. In git.git itself, this is currently 340 tags, 339 of which point to commits in the commit graph. fetch-pack pushes all of these into rev_list, but is unable to mark them COMMON and discard during a remote's "ACK %s {continue,common}" because it does not parse through the entire parent chain. Not parsing the entire parent chain is an optimization to avoid walking back to the roots of the repository. Assuming the client is only following the remote (and does not make its own local commits), the client needs 11 rounds to spin through the entire list of tags (32 commits per round, ceil(339/32) == 11). Unfortunately the server knows on the first "have %s" line that it can produce a good pack, and does not need to see the remaining 320 tags in the other 10 rounds. Over git:// and ssh:// this isn't as bad as it sounds, the client is only transmitting an extra 16,000 bytes that it doesn't need to send. Over smart HTTP, the client must do an additional 10 HTTP POST requests, each of which incurs round-trip latency, and must upload the entire state vector of all known common objects. On the final POST request, this is 16 KiB worth of data. Fix all of this by clearing rev_list as soon as the remote side says it can construct a pack. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index b99941393..5173dc9aa 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -379,6 +379,8 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; + if (ack == ACK_ready) + rev_list = NULL; break; } } From 761ecf0bc7b6cddf311f00877c59e6381cdbdeea Mon Sep 17 00:00:00 2001 From: "Shawn O. Pearce" Date: Mon, 14 Mar 2011 17:59:39 -0700 Subject: [PATCH 2/2] fetch-pack: Implement no-done capability If enabled on the connection "multi_ack_detailed no-done" as a pair allows the remote upload-pack process to send a PACK down to the client as soon as a "ACK %s ready" message was also sent. Over git:// and ssh:// where a bi-directional stream is in place this has very little difference over the classical version that waits for the client to send a "done\n" line by itself. It does slightly reduce the latency involved to start the pack stream as there is one less round-trip from client->server required. Over smart HTTP this avoids needing to send a final RPC that has all of the prior common objects. Instead the server is able to return a pack as soon as its ready to. For many common users the smart HTTP fetch is now just 2 requests: GET .../info/refs, and a POST .../git-upload-pack to not only negotiate but also receive the pack stream. Only users who have more than 32 local unshared commits with the remote will need additional requests to negotiate a common merge base. Signed-off-by: Shawn O. Pearce Signed-off-by: Junio C Hamano --- builtin/fetch-pack.c | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/builtin/fetch-pack.c b/builtin/fetch-pack.c index 5173dc9aa..59fbda522 100644 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@ -14,6 +14,7 @@ static int transfer_unpack_limit = -1; static int fetch_unpack_limit = -1; static int unpack_limit = 100; static int prefer_ofs_delta = 1; +static int no_done = 0; static struct fetch_pack_args args = { /* .uploadpack = */ "git-upload-pack", }; @@ -225,6 +226,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, const unsigned char *sha1; unsigned in_vain = 0; int got_continue = 0; + int got_ready = 0; struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; @@ -262,6 +264,7 @@ static int find_common(int fd[2], unsigned char *result_sha1, struct strbuf c = STRBUF_INIT; if (multi_ack == 2) strbuf_addstr(&c, " multi_ack_detailed"); if (multi_ack == 1) strbuf_addstr(&c, " multi_ack"); + if (no_done) strbuf_addstr(&c, " no-done"); if (use_sideband == 2) strbuf_addstr(&c, " side-band-64k"); if (use_sideband == 1) strbuf_addstr(&c, " side-band"); if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack"); @@ -379,8 +382,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, retval = 0; in_vain = 0; got_continue = 1; - if (ack == ACK_ready) + if (ack == ACK_ready) { rev_list = NULL; + got_ready = 1; + } break; } } @@ -394,8 +399,10 @@ static int find_common(int fd[2], unsigned char *result_sha1, } } done: - packet_buf_write(&req_buf, "done\n"); - send_request(fd[1], &req_buf); + if (!got_ready || !no_done) { + packet_buf_write(&req_buf, "done\n"); + send_request(fd[1], &req_buf); + } if (args.verbose) fprintf(stderr, "done\n"); if (retval != 0) { @@ -698,6 +705,11 @@ static struct ref *do_fetch_pack(int fd[2], if (args.verbose) fprintf(stderr, "Server supports multi_ack_detailed\n"); multi_ack = 2; + if (server_supports("no-done")) { + if (args.verbose) + fprintf(stderr, "Server supports no-done\n"); + no_done = 1; + } } else if (server_supports("multi_ack")) { if (args.verbose)