From: Junio C Hamano Date: Tue, 8 May 2018 06:59:15 +0000 (+0900) Subject: Merge branch 'bw/protocol-v2' X-Git-Tag: v2.18.0-rc0~105 X-Git-Url: https://granicus.if.org/sourcecode?a=commitdiff_plain;h=9bfa0f9be3e718f701200a242ea04259a4dc4dfc;p=git Merge branch 'bw/protocol-v2' The beginning of the next-gen transfer protocol. * bw/protocol-v2: (35 commits) remote-curl: don't request v2 when pushing remote-curl: implement stateless-connect command http: eliminate "# service" line when using protocol v2 http: don't always add Git-Protocol header http: allow providing extra headers for http requests remote-curl: store the protocol version the server responded with remote-curl: create copy of the service name pkt-line: add packet_buf_write_len function transport-helper: introduce stateless-connect transport-helper: refactor process_connect_service transport-helper: remove name parameter connect: don't request v2 when pushing connect: refactor git_connect to only get the protocol version once fetch-pack: support shallow requests fetch-pack: perform a fetch using v2 upload-pack: introduce fetch server command push: pass ref prefixes when pushing fetch: pass ref prefixes when fetching ls-remote: pass ref prefixes when requesting a remote's refs transport: convert transport_get_remote_refs to take a list of ref prefixes ... --- 9bfa0f9be3e718f701200a242ea04259a4dc4dfc diff --cc Makefile index 50da82b016,fb2175bcf1..d23f265636 --- a/Makefile +++ b/Makefile @@@ -660,49 -643,48 +659,50 @@@ X PROGRAMS += $(patsubst %.o,git-%$X,$(PROGRAM_OBJS)) -TEST_PROGRAMS_NEED_X += test-chmtime -TEST_PROGRAMS_NEED_X += test-ctype -TEST_PROGRAMS_NEED_X += test-config -TEST_PROGRAMS_NEED_X += test-date -TEST_PROGRAMS_NEED_X += test-delta -TEST_PROGRAMS_NEED_X += test-drop-caches -TEST_PROGRAMS_NEED_X += test-dump-cache-tree +TEST_BUILTINS_OBJS += test-chmtime.o +TEST_BUILTINS_OBJS += test-config.o +TEST_BUILTINS_OBJS += test-ctype.o +TEST_BUILTINS_OBJS += test-date.o +TEST_BUILTINS_OBJS += test-delta.o +TEST_BUILTINS_OBJS += test-drop-caches.o +TEST_BUILTINS_OBJS += test-dump-cache-tree.o +TEST_BUILTINS_OBJS += test-dump-split-index.o +TEST_BUILTINS_OBJS += test-example-decorate.o +TEST_BUILTINS_OBJS += test-genrandom.o +TEST_BUILTINS_OBJS += test-hashmap.o +TEST_BUILTINS_OBJS += test-index-version.o +TEST_BUILTINS_OBJS += test-lazy-init-name-hash.o +TEST_BUILTINS_OBJS += test-match-trees.o +TEST_BUILTINS_OBJS += test-mergesort.o +TEST_BUILTINS_OBJS += test-mktemp.o +TEST_BUILTINS_OBJS += test-online-cpus.o +TEST_BUILTINS_OBJS += test-path-utils.o +TEST_BUILTINS_OBJS += test-prio-queue.o +TEST_BUILTINS_OBJS += test-read-cache.o +TEST_BUILTINS_OBJS += test-ref-store.o +TEST_BUILTINS_OBJS += test-regex.o +TEST_BUILTINS_OBJS += test-revision-walking.o +TEST_BUILTINS_OBJS += test-run-command.o +TEST_BUILTINS_OBJS += test-scrap-cache-tree.o +TEST_BUILTINS_OBJS += test-sha1-array.o +TEST_BUILTINS_OBJS += test-sha1.o +TEST_BUILTINS_OBJS += test-sigchain.o +TEST_BUILTINS_OBJS += test-strcmp-offset.o +TEST_BUILTINS_OBJS += test-string-list.o +TEST_BUILTINS_OBJS += test-submodule-config.o +TEST_BUILTINS_OBJS += test-subprocess.o +TEST_BUILTINS_OBJS += test-urlmatch-normalization.o +TEST_BUILTINS_OBJS += test-wildmatch.o +TEST_BUILTINS_OBJS += test-write-cache.o + TEST_PROGRAMS_NEED_X += test-dump-fsmonitor -TEST_PROGRAMS_NEED_X += test-dump-split-index TEST_PROGRAMS_NEED_X += test-dump-untracked-cache -TEST_PROGRAMS_NEED_X += test-example-decorate TEST_PROGRAMS_NEED_X += test-fake-ssh -TEST_PROGRAMS_NEED_X += test-genrandom -TEST_PROGRAMS_NEED_X += test-hashmap -TEST_PROGRAMS_NEED_X += test-index-version -TEST_PROGRAMS_NEED_X += test-lazy-init-name-hash TEST_PROGRAMS_NEED_X += test-line-buffer -TEST_PROGRAMS_NEED_X += test-match-trees -TEST_PROGRAMS_NEED_X += test-mergesort -TEST_PROGRAMS_NEED_X += test-mktemp -TEST_PROGRAMS_NEED_X += test-online-cpus TEST_PROGRAMS_NEED_X += test-parse-options -TEST_PROGRAMS_NEED_X += test-path-utils + TEST_PROGRAMS_NEED_X += test-pkt-line -TEST_PROGRAMS_NEED_X += test-prio-queue -TEST_PROGRAMS_NEED_X += test-read-cache -TEST_PROGRAMS_NEED_X += test-write-cache -TEST_PROGRAMS_NEED_X += test-ref-store -TEST_PROGRAMS_NEED_X += test-regex -TEST_PROGRAMS_NEED_X += test-revision-walking -TEST_PROGRAMS_NEED_X += test-run-command -TEST_PROGRAMS_NEED_X += test-scrap-cache-tree -TEST_PROGRAMS_NEED_X += test-sha1 -TEST_PROGRAMS_NEED_X += test-sha1-array -TEST_PROGRAMS_NEED_X += test-sigchain -TEST_PROGRAMS_NEED_X += test-strcmp-offset -TEST_PROGRAMS_NEED_X += test-string-list -TEST_PROGRAMS_NEED_X += test-submodule-config -TEST_PROGRAMS_NEED_X += test-subprocess TEST_PROGRAMS_NEED_X += test-svn-fe -TEST_PROGRAMS_NEED_X += test-urlmatch-normalization -TEST_PROGRAMS_NEED_X += test-wildmatch +TEST_PROGRAMS_NEED_X += test-tool TEST_PROGRAMS = $(patsubst %,t/helper/%$X,$(TEST_PROGRAMS_NEED_X)) diff --cc builtin/clone.c index 7df5932b85,4db3079acc..84f1473d19 --- a/builtin/clone.c +++ b/builtin/clone.c @@@ -1126,16 -1101,10 +1126,16 @@@ int cmd_clone(int argc, const char **ar transport_set_option(transport, TRANS_OPT_UPLOADPACK, option_upload_pack); - if (transport->smart_options && !deepen) + if (filter_options.choice) { + transport_set_option(transport, TRANS_OPT_LIST_OBJECTS_FILTER, + filter_options.filter_spec); + transport_set_option(transport, TRANS_OPT_FROM_PROMISOR, "1"); + } + + if (transport->smart_options && !deepen && !filter_options.choice) transport->smart_options->check_self_contained_and_connected = 1; - refs = transport_get_remote_refs(transport); + refs = transport_get_remote_refs(transport, NULL); if (refs) { mapped_refs = wanted_peer_refs(refs, refspec); diff --cc builtin/fetch-pack.c index a7bc1366ab,f9d7d0b5a5..1a1bc63566 --- a/builtin/fetch-pack.c +++ b/builtin/fetch-pack.c @@@ -52,9 -53,8 +53,10 @@@ int cmd_fetch_pack(int argc, const cha struct fetch_pack_args args; struct oid_array shallow = OID_ARRAY_INIT; struct string_list deepen_not = STRING_LIST_INIT_DUP; + struct packet_reader reader; + fetch_if_missing = 0; + packet_trace_identity("fetch-pack"); memset(&args, 0, sizeof(args)); diff --cc builtin/upload-pack.c index 0000000000,a757df8da0..decde5a3b1 mode 000000,100644..100644 --- a/builtin/upload-pack.c +++ b/builtin/upload-pack.c @@@ -1,0 -1,74 +1,74 @@@ + #include "cache.h" + #include "builtin.h" -#include "exec_cmd.h" ++#include "exec-cmd.h" + #include "pkt-line.h" + #include "parse-options.h" + #include "protocol.h" + #include "upload-pack.h" + #include "serve.h" + + static const char * const upload_pack_usage[] = { + N_("git upload-pack [] "), + NULL + }; + + int cmd_upload_pack(int argc, const char **argv, const char *prefix) + { + const char *dir; + int strict = 0; + struct upload_pack_options opts = { 0 }; + struct serve_options serve_opts = SERVE_OPTIONS_INIT; + struct option options[] = { + OPT_BOOL(0, "stateless-rpc", &opts.stateless_rpc, + N_("quit after a single request/response exchange")), + OPT_BOOL(0, "advertise-refs", &opts.advertise_refs, + N_("exit immediately after initial ref advertisement")), + OPT_BOOL(0, "strict", &strict, + N_("do not try /.git/ if is no Git directory")), + OPT_INTEGER(0, "timeout", &opts.timeout, + N_("interrupt transfer after seconds of inactivity")), + OPT_END() + }; + + packet_trace_identity("upload-pack"); + check_replace_refs = 0; + + argc = parse_options(argc, argv, NULL, options, upload_pack_usage, 0); + + if (argc != 1) + usage_with_options(upload_pack_usage, options); + + if (opts.timeout) + opts.daemon_mode = 1; + + setup_path(); + + dir = argv[0]; + + if (!enter_repo(dir, strict)) + die("'%s' does not appear to be a git repository", dir); + + switch (determine_protocol_version_server()) { + case protocol_v2: + serve_opts.advertise_capabilities = opts.advertise_refs; + serve_opts.stateless_rpc = opts.stateless_rpc; + serve(&serve_opts); + break; + case protocol_v1: + /* + * v1 is just the original protocol with a version string, + * so just fall through after writing the version string. + */ + if (opts.advertise_refs || !opts.stateless_rpc) + packet_write_fmt(1, "version 1\n"); + + /* fallthrough */ + case protocol_v0: + upload_pack(&opts); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); + } + + return 0; + } diff --cc fetch-pack.c index 4a8bad8487,837e1fd21d..f93723fec4 --- a/fetch-pack.c +++ b/fetch-pack.c @@@ -1239,9 -1490,13 +1562,13 @@@ struct ref *fetch_pack(struct fetch_pac die(_("no matching remote head")); } prepare_shallow_info(&si, shallow); - ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, - &si, pack_lockfile); + if (version == protocol_v2) + ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought, + pack_lockfile); + else + ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought, + &si, pack_lockfile); - reprepare_packed_git(); + reprepare_packed_git(the_repository); update_shallow(args, sought, nr_sought, &si); clear_shallow_info(&si); return ref_cpy; diff --cc fetch-pack.h index 3e224a1822,7afca73056..6afa08b48b --- a/fetch-pack.h +++ b/fetch-pack.h @@@ -3,7 -3,7 +3,8 @@@ #include "string-list.h" #include "run-command.h" + #include "protocol.h" +#include "list-objects-filter-options.h" struct oid_array; diff --cc git.c index f598fae7b7,f85d682b62..158f2cbdb1 --- a/git.c +++ b/git.c @@@ -480,9 -477,10 +481,10 @@@ static struct cmd_struct commands[] = { "update-index", cmd_update_index, RUN_SETUP }, { "update-ref", cmd_update_ref, RUN_SETUP }, { "update-server-info", cmd_update_server_info, RUN_SETUP }, - { "upload-archive", cmd_upload_archive }, - { "upload-archive--writer", cmd_upload_archive_writer }, + { "upload-archive", cmd_upload_archive, NO_PARSEOPT }, + { "upload-archive--writer", cmd_upload_archive_writer, NO_PARSEOPT }, + { "upload-pack", cmd_upload_pack }, - { "var", cmd_var, RUN_SETUP_GENTLY }, + { "var", cmd_var, RUN_SETUP_GENTLY | NO_PARSEOPT }, { "verify-commit", cmd_verify_commit, RUN_SETUP }, { "verify-pack", cmd_verify_pack }, { "verify-tag", cmd_verify_tag, RUN_SETUP }, diff --cc http-backend.c index cc16cd04ad,5d241e9109..adaef16fad --- a/http-backend.c +++ b/http-backend.c @@@ -11,7 -10,7 +11,8 @@@ #include "url.h" #include "argv-array.h" #include "packfile.h" +#include "object-store.h" + #include "protocol.h" static const char content_type[] = "Content-Type"; static const char content_length[] = "Content-Length"; diff --cc remote-curl.c index 8d2ffaf8de,595447b16e..68e7b1cf58 --- a/remote-curl.c +++ b/remote-curl.c @@@ -13,7 -14,7 +14,8 @@@ #include "credential.h" #include "sha1-array.h" #include "send-pack.h" + #include "protocol.h" +#include "quote.h" static struct remote *remote; /* always ends with a trailing slash */ diff --cc transport.c index 94eccf29aa,342db492ca..16b2f54f22 --- a/transport.c +++ b/transport.c @@@ -18,7 -18,7 +18,8 @@@ #include "sha1-array.h" #include "sigchain.h" #include "transport-internal.h" + #include "protocol.h" +#include "object-store.h" static void set_upstreams(struct transport *transport, struct ref *refs, int pretend) @@@ -239,21 -252,29 +262,32 @@@ static int fetch_refs_via_pack(struct t data->options.check_self_contained_and_connected; args.cloning = transport->cloning; args.update_shallow = data->options.update_shallow; + args.from_promisor = data->options.from_promisor; + args.no_dependents = data->options.no_dependents; + args.filter_options = data->options.filter_options; + args.stateless_rpc = transport->stateless_rpc; - if (!data->got_remote_heads) { - connect_setup(transport, 0); - get_remote_heads(data->fd[0], NULL, 0, &refs_tmp, 0, - NULL, &data->shallow); - data->got_remote_heads = 1; + if (!data->got_remote_heads) + refs_tmp = get_refs_via_connect(transport, 0, NULL); + + switch (data->version) { + case protocol_v2: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); + break; + case protocol_v1: + case protocol_v0: + refs = fetch_pack(&args, data->fd, data->conn, + refs_tmp ? refs_tmp : transport->remote_refs, + dest, to_fetch, nr_heads, &data->shallow, + &transport->pack_lockfile, data->version); + break; + case protocol_unknown_version: + BUG("unknown protocol version"); } - refs = fetch_pack(&args, data->fd, data->conn, - refs_tmp ? refs_tmp : transport->remote_refs, - dest, to_fetch, nr_heads, &data->shallow, - &transport->pack_lockfile); close(data->fd[0]); close(data->fd[1]); if (finish_connect(data->conn)) diff --cc upload-pack.c index 6261d4fab3,4c9428c2db..87b4d32a6e --- a/upload-pack.c +++ b/upload-pack.c @@@ -20,13 -17,8 +19,9 @@@ #include "argv-array.h" #include "prio-queue.h" #include "protocol.h" +#include "quote.h" - - static const char * const upload_pack_usage[] = { - N_("git upload-pack [] "), - NULL - }; + #include "upload-pack.h" + #include "serve.h" /* Remember to update object flag allocation in object.h */ #define THEY_HAVE (1u << 11) @@@ -770,55 -860,15 +878,22 @@@ static void receive_needs(void if (!line) break; - if (skip_prefix(line, "shallow ", &arg)) { - struct object_id oid; - struct object *object; - if (get_oid_hex(arg, &oid)) - die("invalid shallow line: %s", line); - object = parse_object(&oid); - if (!object) - continue; - if (object->type != OBJ_COMMIT) - die("invalid shallow object %s", oid_to_hex(&oid)); - if (!(object->flags & CLIENT_SHALLOW)) { - object->flags |= CLIENT_SHALLOW; - add_object_array(object, NULL, &shallows); - } + if (process_shallow(line, &shallows)) continue; - } - if (skip_prefix(line, "deepen ", &arg)) { - char *end = NULL; - depth = strtol(arg, &end, 0); - if (!end || *end || depth <= 0) - die("Invalid deepen: %s", line); + if (process_deepen(line, &depth)) continue; - } - if (skip_prefix(line, "deepen-since ", &arg)) { - char *end = NULL; - deepen_since = parse_timestamp(arg, &end, 0); - if (!end || *end || !deepen_since || - /* revisions.c's max_age -1 is special */ - deepen_since == -1) - die("Invalid deepen-since: %s", line); - deepen_rev_list = 1; + if (process_deepen_since(line, &deepen_since, &deepen_rev_list)) continue; - } - if (skip_prefix(line, "deepen-not ", &arg)) { - char *ref = NULL; - struct object_id oid; - if (expand_ref(arg, strlen(arg), &oid, &ref) != 1) - die("git upload-pack: ambiguous deepen-not: %s", line); - string_list_append(&deepen_not, ref); - free(ref); - deepen_rev_list = 1; + if (process_deepen_not(line, &deepen_not, &deepen_rev_list)) continue; - } + + if (skip_prefix(line, "filter ", &arg)) { + if (!filter_capability_requested) + die("git upload-pack: filtering capability not negotiated"); + parse_list_objects_filter(&filter_options, arg); + continue; + } ++ if (!skip_prefix(line, "want ", &arg) || get_oid_hex(arg, &oid_buf)) die("git upload-pack: protocol error, "