From: Brandon Williams <bmwill@google•com>
To: git@vger•kernel.org
Cc: git@jeffhostetler•com, gitster@pobox•com, jrnieder@gmail•com,
pclouds@gmail•com, peff@peff•net, sbeller@google•com,
stolee@gmail•com, jonathantanmy@google•com,
Brandon Williams <bmwill@google•com>
Subject: [PATCH v6 34/35] remote-curl: implement stateless-connect command
Date: Thu, 15 Mar 2018 10:31:41 -0700 [thread overview]
Message-ID: <20180315173142.176023-35-bmwill@google.com> (raw)
In-Reply-To: <20180315173142.176023-1-bmwill@google.com>
Teach remote-curl the 'stateless-connect' command which is used to
establish a stateless connection with servers which support protocol
version 2. This allows remote-curl to act as a proxy, allowing the git
client to communicate natively with a remote end, simply using
remote-curl as a pass through to convert requests to http.
Signed-off-by: Brandon Williams <bmwill@google•com>
---
remote-curl.c | 207 ++++++++++++++++++++++++++++++++++++++++-
t/t5702-protocol-v2.sh | 45 +++++++++
2 files changed, 251 insertions(+), 1 deletion(-)
diff --git a/remote-curl.c b/remote-curl.c
index 66a53f74bb..87f5b77b29 100644
--- a/remote-curl.c
+++ b/remote-curl.c
@@ -188,7 +188,12 @@ static struct ref *parse_git_refs(struct discovery *heads, int for_push)
heads->version = discover_version(&reader);
switch (heads->version) {
case protocol_v2:
- die("support for protocol v2 not implemented yet");
+ /*
+ * Do nothing. This isn't a list of refs but rather a
+ * capability advertisement. Client would have run
+ * 'stateless-connect' so we'll dump this capability listing
+ * and let them request the refs themselves.
+ */
break;
case protocol_v1:
case protocol_v0:
@@ -1085,6 +1090,202 @@ static void parse_push(struct strbuf *buf)
free(specs);
}
+/*
+ * Used to represent the state of a connection to an HTTP server when
+ * communicating using git's wire-protocol version 2.
+ */
+struct proxy_state {
+ char *service_name;
+ char *service_url;
+ struct curl_slist *headers;
+ struct strbuf request_buffer;
+ int in;
+ int out;
+ struct packet_reader reader;
+ size_t pos;
+ int seen_flush;
+};
+
+static void proxy_state_init(struct proxy_state *p, const char *service_name,
+ enum protocol_version version)
+{
+ struct strbuf buf = STRBUF_INIT;
+
+ memset(p, 0, sizeof(*p));
+ p->service_name = xstrdup(service_name);
+
+ p->in = 0;
+ p->out = 1;
+ strbuf_init(&p->request_buffer, 0);
+
+ strbuf_addf(&buf, "%s%s", url.buf, p->service_name);
+ p->service_url = strbuf_detach(&buf, NULL);
+
+ p->headers = http_copy_default_headers();
+
+ strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name);
+ p->headers = curl_slist_append(p->headers, buf.buf);
+ strbuf_reset(&buf);
+
+ strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name);
+ p->headers = curl_slist_append(p->headers, buf.buf);
+ strbuf_reset(&buf);
+
+ p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked");
+
+ /* Add the Git-Protocol header */
+ if (get_protocol_http_header(version, &buf))
+ p->headers = curl_slist_append(p->headers, buf.buf);
+
+ packet_reader_init(&p->reader, p->in, NULL, 0,
+ PACKET_READ_GENTLE_ON_EOF);
+
+ strbuf_release(&buf);
+}
+
+static void proxy_state_clear(struct proxy_state *p)
+{
+ free(p->service_name);
+ free(p->service_url);
+ curl_slist_free_all(p->headers);
+ strbuf_release(&p->request_buffer);
+}
+
+/*
+ * CURLOPT_READFUNCTION callback function.
+ * Attempts to copy over a single packet-line at a time into the
+ * curl provided buffer.
+ */
+static size_t proxy_in(char *buffer, size_t eltsize,
+ size_t nmemb, void *userdata)
+{
+ size_t max;
+ struct proxy_state *p = userdata;
+ size_t avail = p->request_buffer.len - p->pos;
+
+
+ if (eltsize != 1)
+ BUG("curl read callback called with size = %"PRIuMAX" != 1",
+ (uintmax_t)eltsize);
+ max = nmemb;
+
+ if (!avail) {
+ if (p->seen_flush) {
+ p->seen_flush = 0;
+ return 0;
+ }
+
+ strbuf_reset(&p->request_buffer);
+ switch (packet_reader_read(&p->reader)) {
+ case PACKET_READ_EOF:
+ die("unexpected EOF when reading from parent process");
+ case PACKET_READ_NORMAL:
+ packet_buf_write_len(&p->request_buffer, p->reader.line,
+ p->reader.pktlen);
+ break;
+ case PACKET_READ_DELIM:
+ packet_buf_delim(&p->request_buffer);
+ break;
+ case PACKET_READ_FLUSH:
+ packet_buf_flush(&p->request_buffer);
+ p->seen_flush = 1;
+ break;
+ }
+ p->pos = 0;
+ avail = p->request_buffer.len;
+ }
+
+ if (max < avail)
+ avail = max;
+ memcpy(buffer, p->request_buffer.buf + p->pos, avail);
+ p->pos += avail;
+ return avail;
+}
+
+static size_t proxy_out(char *buffer, size_t eltsize,
+ size_t nmemb, void *userdata)
+{
+ size_t size;
+ struct proxy_state *p = userdata;
+
+ if (eltsize != 1)
+ BUG("curl read callback called with size = %"PRIuMAX" != 1",
+ (uintmax_t)eltsize);
+ size = nmemb;
+
+ write_or_die(p->out, buffer, size);
+ return size;
+}
+
+/* Issues a request to the HTTP server configured in `p` */
+static int proxy_request(struct proxy_state *p)
+{
+ struct active_request_slot *slot;
+
+ slot = get_active_slot();
+
+ curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
+ curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
+ curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url);
+ curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers);
+
+ /* Setup function to read request from client */
+ curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in);
+ curl_easy_setopt(slot->curl, CURLOPT_READDATA, p);
+
+ /* Setup function to write server response to client */
+ curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out);
+ curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p);
+
+ if (run_slot(slot, NULL) != HTTP_OK)
+ return -1;
+
+ return 0;
+}
+
+static int stateless_connect(const char *service_name)
+{
+ struct discovery *discover;
+ struct proxy_state p;
+
+ /*
+ * Run the info/refs request and see if the server supports protocol
+ * v2. If and only if the server supports v2 can we successfully
+ * establish a stateless connection, otherwise we need to tell the
+ * client to fallback to using other transport helper functions to
+ * complete their request.
+ */
+ discover = discover_refs(service_name, 0);
+ if (discover->version != protocol_v2) {
+ printf("fallback\n");
+ fflush(stdout);
+ return -1;
+ } else {
+ /* Stateless Connection established */
+ printf("\n");
+ fflush(stdout);
+ }
+
+ proxy_state_init(&p, service_name, discover->version);
+
+ /*
+ * Dump the capability listing that we got from the server earlier
+ * during the info/refs request.
+ */
+ write_or_die(p.out, discover->buf, discover->len);
+
+ /* Peek the next packet line. Until we see EOF keep sending POSTs */
+ while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) {
+ if (proxy_request(&p)) {
+ /* We would have an err here */
+ break;
+ }
+ }
+
+ proxy_state_clear(&p);
+ return 0;
+}
+
int cmd_main(int argc, const char **argv)
{
struct strbuf buf = STRBUF_INIT;
@@ -1153,12 +1354,16 @@ int cmd_main(int argc, const char **argv)
fflush(stdout);
} else if (!strcmp(buf.buf, "capabilities")) {
+ printf("stateless-connect\n");
printf("fetch\n");
printf("option\n");
printf("push\n");
printf("check-connectivity\n");
printf("\n");
fflush(stdout);
+ } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) {
+ if (!stateless_connect(arg))
+ break;
} else {
error("remote-curl: unknown command '%s' from git", buf.buf);
return 1;
diff --git a/t/t5702-protocol-v2.sh b/t/t5702-protocol-v2.sh
index e3a7c09d4a..124063c2c4 100755
--- a/t/t5702-protocol-v2.sh
+++ b/t/t5702-protocol-v2.sh
@@ -201,4 +201,49 @@ test_expect_success 'ref advertisment is filtered during fetch using protocol v2
! grep "refs/tags/three" log
'
+# Test protocol v2 with 'http://' transport
+#
+. "$TEST_DIRECTORY"/lib-httpd.sh
+start_httpd
+
+test_expect_success 'create repo to be served by http:// transport' '
+ git init "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" &&
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" config http.receivepack true &&
+ test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" one
+'
+
+test_expect_success 'clone with http:// using protocol v2' '
+ test_when_finished "rm -f log" &&
+
+ GIT_TRACE_PACKET="$(pwd)/log" GIT_TRACE_CURL="$(pwd)/log" git -c protocol.version=2 \
+ clone "$HTTPD_URL/smart/http_parent" http_child &&
+
+ git -C http_child log -1 --format=%s >actual &&
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect &&
+ test_cmp expect actual &&
+
+ # Client requested to use protocol v2
+ grep "Git-Protocol: version=2" log &&
+ # Server responded using protocol v2
+ grep "git< version 2" log
+'
+
+test_expect_success 'fetch with http:// using protocol v2' '
+ test_when_finished "rm -f log" &&
+
+ test_commit -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" two &&
+
+ GIT_TRACE_PACKET="$(pwd)/log" git -C http_child -c protocol.version=2 \
+ fetch &&
+
+ git -C http_child log -1 --format=%s origin/master >actual &&
+ git -C "$HTTPD_DOCUMENT_ROOT_PATH/http_parent" log -1 --format=%s >expect &&
+ test_cmp expect actual &&
+
+ # Server responded using protocol v2
+ grep "git< version 2" log
+'
+
+stop_httpd
+
test_done
--
2.16.2.804.g6dcf76e118-goog
next prev parent reply other threads:[~2018-03-15 17:33 UTC|newest]
Thread overview: 85+ messages / expand[flat|nested] mbox.gz Atom feed top
2018-03-14 18:31 [PATCH v5 00/35] protocol version 2 Brandon Williams
2018-03-14 18:31 ` [PATCH v5 01/35] pkt-line: introduce packet_read_with_status Brandon Williams
2018-03-14 21:33 ` Junio C Hamano
2018-03-15 17:28 ` Brandon Williams
2018-03-15 18:37 ` Junio C Hamano
2018-03-14 18:31 ` [PATCH v5 02/35] pkt-line: allow peeking a packet line without consuming it Brandon Williams
2018-03-14 18:31 ` [PATCH v5 03/35] pkt-line: add delim packet support Brandon Williams
2018-03-14 18:31 ` [PATCH v5 04/35] upload-pack: convert to a builtin Brandon Williams
2018-03-14 18:31 ` [PATCH v5 05/35] upload-pack: factor out processing lines Brandon Williams
2018-03-14 18:31 ` [PATCH v5 06/35] transport: use get_refs_via_connect to get refs Brandon Williams
2018-03-14 18:31 ` [PATCH v5 07/35] connect: convert get_remote_heads to use struct packet_reader Brandon Williams
2018-03-14 18:31 ` [PATCH v5 08/35] connect: discover protocol version outside of get_remote_heads Brandon Williams
2018-03-14 18:31 ` [PATCH v5 09/35] transport: store protocol version Brandon Williams
2018-03-14 18:31 ` [PATCH v5 10/35] protocol: introduce enum protocol_version value protocol_v2 Brandon Williams
2018-03-14 18:31 ` [PATCH v5 11/35] test-pkt-line: introduce a packet-line test helper Brandon Williams
2018-03-14 18:31 ` [PATCH v5 12/35] serve: introduce git-serve Brandon Williams
2018-03-14 22:01 ` Junio C Hamano
2018-03-15 17:27 ` Brandon Williams
2018-03-14 18:31 ` [PATCH v5 13/35] ls-refs: introduce ls-refs server command Brandon Williams
2018-03-14 18:31 ` [PATCH v5 14/35] connect: request remote refs using v2 Brandon Williams
2018-03-14 18:31 ` [PATCH v5 15/35] transport: convert get_refs_list to take a list of ref prefixes Brandon Williams
2018-03-14 18:31 ` [PATCH v5 16/35] transport: convert transport_get_remote_refs " Brandon Williams
2018-03-14 18:31 ` [PATCH v5 17/35] ls-remote: pass ref prefixes when requesting a remote's refs Brandon Williams
2018-03-14 18:31 ` [PATCH v5 18/35] fetch: pass ref prefixes when fetching Brandon Williams
2018-03-14 18:31 ` [PATCH v5 19/35] push: pass ref prefixes when pushing Brandon Williams
2018-03-14 18:31 ` [PATCH v5 20/35] upload-pack: introduce fetch server command Brandon Williams
2018-03-14 18:31 ` [PATCH v5 21/35] fetch-pack: perform a fetch using v2 Brandon Williams
2018-03-14 18:31 ` [PATCH v5 22/35] fetch-pack: support shallow requests Brandon Williams
2018-03-14 18:32 ` [PATCH v5 23/35] connect: refactor git_connect to only get the protocol version once Brandon Williams
2018-03-14 18:32 ` [PATCH v5 24/35] connect: don't request v2 when pushing Brandon Williams
2018-03-14 18:32 ` [PATCH v5 25/35] transport-helper: remove name parameter Brandon Williams
2018-03-14 18:32 ` [PATCH v5 26/35] transport-helper: refactor process_connect_service Brandon Williams
2018-03-14 18:32 ` [PATCH v5 27/35] transport-helper: introduce stateless-connect Brandon Williams
2018-03-14 18:32 ` [PATCH v5 28/35] pkt-line: add packet_buf_write_len function Brandon Williams
2018-03-14 18:32 ` [PATCH v5 29/35] remote-curl: create copy of the service name Brandon Williams
2018-03-14 18:32 ` [PATCH v5 30/35] remote-curl: store the protocol version the server responded with Brandon Williams
2018-03-14 18:32 ` [PATCH v5 31/35] http: allow providing extra headers for http requests Brandon Williams
2018-03-14 18:32 ` [PATCH v5 32/35] http: don't always add Git-Protocol header Brandon Williams
2018-03-14 18:32 ` [PATCH v5 33/35] http: eliminate "# service" line when using protocol v2 Brandon Williams
2018-03-14 18:32 ` [PATCH v5 34/35] remote-curl: implement stateless-connect command Brandon Williams
2018-03-14 18:32 ` [PATCH v5 35/35] remote-curl: don't request v2 when pushing Brandon Williams
2018-03-15 17:31 ` [PATCH v6 00/35] protocol version 2 Brandon Williams
2018-03-15 17:31 ` [PATCH v6 01/35] pkt-line: introduce packet_read_with_status Brandon Williams
2018-03-15 17:31 ` [PATCH v6 02/35] pkt-line: allow peeking a packet line without consuming it Brandon Williams
2018-03-15 17:31 ` [PATCH v6 03/35] pkt-line: add delim packet support Brandon Williams
2018-03-15 17:31 ` [PATCH v6 04/35] upload-pack: convert to a builtin Brandon Williams
2018-03-15 17:31 ` [PATCH v6 05/35] upload-pack: factor out processing lines Brandon Williams
2018-03-15 17:31 ` [PATCH v6 06/35] transport: use get_refs_via_connect to get refs Brandon Williams
2018-03-15 17:31 ` [PATCH v6 07/35] connect: convert get_remote_heads to use struct packet_reader Brandon Williams
2018-03-27 15:27 ` Duy Nguyen
2018-03-27 16:11 ` Jeff King
2018-03-27 16:25 ` Duy Nguyen
2018-03-27 16:39 ` Duy Nguyen
2018-03-27 16:58 ` Brandon Williams
2018-03-15 17:31 ` [PATCH v6 08/35] connect: discover protocol version outside of get_remote_heads Brandon Williams
2018-03-15 17:31 ` [PATCH v6 09/35] transport: store protocol version Brandon Williams
2018-03-15 17:31 ` [PATCH v6 10/35] protocol: introduce enum protocol_version value protocol_v2 Brandon Williams
2018-03-15 17:31 ` [PATCH v6 11/35] test-pkt-line: introduce a packet-line test helper Brandon Williams
2018-03-15 17:31 ` [PATCH v6 12/35] serve: introduce git-serve Brandon Williams
2018-03-15 17:31 ` [PATCH v6 13/35] ls-refs: introduce ls-refs server command Brandon Williams
2018-03-15 17:31 ` [PATCH v6 14/35] connect: request remote refs using v2 Brandon Williams
2018-03-15 17:31 ` [PATCH v6 15/35] transport: convert get_refs_list to take a list of ref prefixes Brandon Williams
2018-03-15 17:31 ` [PATCH v6 16/35] transport: convert transport_get_remote_refs " Brandon Williams
2018-03-15 17:31 ` [PATCH v6 17/35] ls-remote: pass ref prefixes when requesting a remote's refs Brandon Williams
2018-03-15 17:31 ` [PATCH v6 18/35] fetch: pass ref prefixes when fetching Brandon Williams
2018-03-15 17:31 ` [PATCH v6 19/35] push: pass ref prefixes when pushing Brandon Williams
2018-03-15 17:31 ` [PATCH v6 20/35] upload-pack: introduce fetch server command Brandon Williams
2018-03-15 17:31 ` [PATCH v6 21/35] fetch-pack: perform a fetch using v2 Brandon Williams
2018-03-15 17:31 ` [PATCH v6 22/35] fetch-pack: support shallow requests Brandon Williams
2018-03-15 17:31 ` [PATCH v6 23/35] connect: refactor git_connect to only get the protocol version once Brandon Williams
2018-03-15 17:31 ` [PATCH v6 24/35] connect: don't request v2 when pushing Brandon Williams
2018-03-15 17:31 ` [PATCH v6 25/35] transport-helper: remove name parameter Brandon Williams
2018-03-15 17:31 ` [PATCH v6 26/35] transport-helper: refactor process_connect_service Brandon Williams
2018-03-15 17:31 ` [PATCH v6 27/35] transport-helper: introduce stateless-connect Brandon Williams
2018-03-15 17:31 ` [PATCH v6 28/35] pkt-line: add packet_buf_write_len function Brandon Williams
2018-03-15 17:31 ` [PATCH v6 29/35] remote-curl: create copy of the service name Brandon Williams
2018-03-15 17:31 ` [PATCH v6 30/35] remote-curl: store the protocol version the server responded with Brandon Williams
2018-03-15 17:31 ` [PATCH v6 31/35] http: allow providing extra headers for http requests Brandon Williams
2018-03-15 17:31 ` [PATCH v6 32/35] http: don't always add Git-Protocol header Brandon Williams
2018-03-15 17:31 ` [PATCH v6 33/35] http: eliminate "# service" line when using protocol v2 Brandon Williams
2018-03-15 17:31 ` Brandon Williams [this message]
2018-03-15 17:31 ` [PATCH v6 35/35] remote-curl: don't request v2 when pushing Brandon Williams
2018-03-30 9:29 ` [PATCH] {fetch,upload}-pack: clearly mark unreachable v2 code Ævar Arnfjörð Bjarmason
2018-03-30 19:03 ` Brandon Williams
2018-03-30 19:06 ` Stefan Beller
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20180315173142.176023-35-bmwill@google.com \
--to=bmwill@google$(echo .)com \
--cc=git@jeffhostetler$(echo .)com \
--cc=git@vger$(echo .)kernel.org \
--cc=gitster@pobox$(echo .)com \
--cc=jonathantanmy@google$(echo .)com \
--cc=jrnieder@gmail$(echo .)com \
--cc=pclouds@gmail$(echo .)com \
--cc=peff@peff$(echo .)net \
--cc=sbeller@google$(echo .)com \
--cc=stolee@gmail$(echo .)com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox