[PATCH 0/3] http: authenticate packfile-URI downloads

Documentation/technical/packfile-uri.adoc

@@ -50,6 +50,12 @@ Client design
 The client has a config variable `fetch.uriprotocols` that determines which
 protocols the end user is willing to use. By default, this is empty.
 
+HTTP(S) packfile downloads use the HTTP authentication and redirect
+configuration used for other Git HTTP requests. Credentials are looked up
+for the pack URL, including its path when `credential.useHttpPath` is
+enabled, rather than copied from the fetch negotiation. A pack served by
+another host can use credentials configured for that host.
+
 When the client downloads the given URIs, it should store them with "keep"
 files, just like it does with the packfile in the `packfile` section. These
 additional "keep" files can only be removed after the refs have been updated -

http-fetch.c

@@ -56,22 +56,19 @@ static void fetch_single_packfile(struct object_id *packfile_hash,
 				  const char *url,
 				  const char **index_pack_args) {
 	struct http_pack_request *preq;
-	struct slot_results results;
 	int ret;
 
 	http_init(NULL, url, 0);
 
 	preq = new_direct_http_pack_request(packfile_hash->hash, xstrdup(url));
 	if (!preq)
 		die("couldn't create http pack request");
-	preq->slot->results = &results;
 	preq->index_pack_args = index_pack_args;
 	preq->preserve_index_pack_stdout = 1;
 
-	if (start_active_slot(preq->slot)) {
-		run_active_slot(preq->slot);
-		if (results.curl_result != CURLE_OK &&
-		    results.http_code != 416) {
+	ret = run_http_pack_request(preq);
+	if (ret != HTTP_START_FAILED) {
+		if (ret != HTTP_OK) {
 			struct url_info url;
 			char *nurl = url_normalize(preq->url, &url);
 			if (!nurl || !git_env_bool("GIT_TRACE_REDACT", 1)) {

http.c

@@ -2735,6 +2735,112 @@ struct http_pack_request *new_http_pack_request(
 					    strbuf_detach(&buf, NULL));
 }
 
+static void prepare_http_pack_request(struct http_pack_request *preq,
+				      off_t offset)
+{
+	preq->slot = get_active_slot();
+	curl_slist_free_all(preq->headers);
+	preq->headers = object_request_headers();
+	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
+	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
+	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
+	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
+	if (offset > 0)
+		http_opt_request_remainder(preq->slot->curl, offset);
+}
+
+static void update_http_pack_url(void *data)
+{
+	struct http_pack_request *preq = data;
+	struct urlmatch_config config = URLMATCH_CONFIG_INIT;
+	struct strvec wwwauth;
+	char *url;
+
+	if (preq->slot->http_code != 401 ||
+	    curl_easy_getinfo(preq->slot->curl, CURLINFO_EFFECTIVE_URL, &url) != CURLE_OK ||
+	    !url || !strcmp(preq->url, url))
+		return;
+
+	/* Change credential context before run_one_slot() handles the 401. */
+	wwwauth = http_auth.wwwauth_headers;
+	http_auth.wwwauth_headers = (struct strvec)STRVEC_INIT;
+	free(preq->url);
+	preq->url = xstrdup(url);
+	credential_from_url(&http_auth, preq->url);
+	http_auth.wwwauth_headers = wwwauth;
+
+	/* A direct retry must not reuse headers scoped to the redirect source. */
+	string_list_clear(&extra_http_headers, 0);
+	config.section = "http";
+	config.key = "extraheader";
+	config.collect_fn = http_options;
+	url_normalize(preq->url, &config.url);
+	repo_config(the_repository, urlmatch_config_entry, &config);
+	free(config.url.url);
+	urlmatch_config_release(&config);
+}
+
+static size_t fwrite_http_pack(char *ptr, size_t size, size_t nmemb, void *data)
+{
+	struct http_pack_request *preq = data;
+	long code;
+
+	if (curl_easy_getinfo(preq->slot->curl, CURLINFO_HTTP_CODE, &code) != CURLE_OK)
+		return 0;
+	/* Error bodies must not overwrite a partial pack shared with another fetch. */
+	if (code >= 300)
+		return size * nmemb;
+	return fwrite(ptr, size, nmemb, preq->packfile);
+}
+
+int run_http_pack_request(struct http_pack_request *preq)
+{
+	off_t offset = ftello(preq->packfile);
+	int attempts = 3;
+	int ret;
+
+	if (offset < 0)
+		return HTTP_START_FAILED;
+
+	for (;;) {
+		struct slot_results results = { .retry_after = -1 };
+
+		preq->headers = http_append_auth_header(&http_auth, preq->headers);
+		curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
+		curl_easy_setopt(preq->slot->curl, CURLOPT_HEADERFUNCTION, fwrite_wwwauth);
+		curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEHEADER, NULL);
+		curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq);
+		curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_http_pack);
+		/* Older curl versions omit challenge headers with FAILONERROR. */
+		curl_easy_setopt(preq->slot->curl, CURLOPT_FAILONERROR, 0L);
+		if (http_follow_config == HTTP_FOLLOW_INITIAL)
+			curl_easy_setopt(preq->slot->curl, CURLOPT_FOLLOWLOCATION, 1L);
+		preq->slot->callback_func = update_http_pack_url;
+		preq->slot->callback_data = preq;
+		ret = run_one_slot(preq->slot, &results);
+		preq->slot->results = NULL;
+		preq->slot->callback_func = NULL;
+		preq->slot->callback_data = NULL;
+
+		if (ret != HTTP_START_FAILED && results.http_code == 416) {
+			ret = HTTP_OK;
+			break;
+		}
+
+		if (ret != HTTP_REAUTH || !--attempts)
+			break;
+
+		/* Never truncate a partial pack to recover from an error response. */
+		if (ftello(preq->packfile) != offset) {
+			ret = HTTP_ERROR;
+			break;
+		}
+		http_reauth_prepare(1);
+		prepare_http_pack_request(preq, offset);
+	}
+	return ret;
+}
+
 struct http_pack_request *new_direct_http_pack_request(
 	const unsigned char *packed_git_hash, char *url)
 {
@@ -2778,20 +2884,14 @@ struct http_pack_request *new_direct_http_pack_request(
 	}
 	preq->packfile = xfdopen(fd, "w");
 
-	preq->slot = get_active_slot();
-	preq->headers = object_request_headers();
-	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEDATA, preq->packfile);
-	curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
-	curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
-	curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER, preq->headers);
+	prepare_http_pack_request(preq, prev_posn);
 
 	if (prev_posn > 0) {
 		if (http_is_verbose)
 			fprintf(stderr,
 				"Resuming fetch of pack %s at byte %"PRIuMAX"\n",
 				hash_to_hex(packed_git_hash),
 				(uintmax_t)prev_posn);
-		http_opt_request_remainder(preq->slot->curl, prev_posn);
 	}
 
 	return preq;

http.h

@@ -232,6 +232,14 @@ struct http_pack_request *new_http_pack_request(
 	const unsigned char *packed_git_hash, const char *base_url);
 struct http_pack_request *new_direct_http_pack_request(
 	const unsigned char *packed_git_hash, char *url);
+
+/*
+ * Run a direct pack request, retrying HTTP authentication challenges.
+ * Returns HTTP_OK (also for 416), or an HTTP error.
+ * The caller must initialize HTTP credentials from the pack URL, not the Git
+ * remote, before constructing the request.
+ */
+int run_http_pack_request(struct http_pack_request *preq);
 int finish_http_pack_request(struct http_pack_request *preq);
 void release_http_pack_request(struct http_pack_request *preq);