Serve Git repositories from a local cache

Dockerfile

@@ -18,7 +18,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o /
 
 FROM alpine:3.24.2
 
-RUN apk add --no-cache ca-certificates
+RUN apk add --no-cache ca-certificates git
 
 COPY --from=builder /proxy /usr/local/bin/proxy
 

README.md

@@ -68,6 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi
 | Arch | Arch Linux | | ✓ |
 | Chef | Chef | | ✗ |
 | Generic | Any | | ✓ |
+| Git | Any | | ✓ |
 | Helm | Kubernetes | Yes | ✓ |
 | Vagrant | Vagrant | | ✗ |
 
@@ -804,6 +805,27 @@ go.download_mirror = "http://localhost:8080/generic/golang"
 Other download hosts need their version-pinned paths declared in
 `upstream.generic_artifacts`.
 
+### Git repositories
+
+Configure named Git servers:
+
+```yaml
+upstream:
+  git:
+    github: "https://github.com"
+```
+
+Then point clones at the proxy:
+
+```bash
+git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/
+```
+
+Each repository is cached as a bare clone and refreshed whenever its branches
+or tags change upstream. See [configuration](docs/configuration.md#git-repositories)
+for setting the rewrite through environment variables, size limits, private
+repositories and outage behavior.
+
 ## Configuration
 
 The proxy can be configured via:
@@ -1125,6 +1147,7 @@ Recently cached:
 | `GET /apk/{repository}/*` | Alpine APK repository protocol |
 | `GET /arch/{repository}/*` | Arch Linux pacman repository protocol |
 | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) |
+| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only |
 | `GET /debian/*` | Debian/APT repository protocol (main archive) |
 | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) |
 | `GET /rpm/*` | RPM/Yum repository protocol |

config.example.yaml

@@ -304,6 +304,15 @@ upstream:
   #   apache:
   #     - 'dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)'
 
+  # Named Git servers (used by /git/{name}/{repo}), served read-only from
+  # cached bare repositories. repos limits which repositories may be served
+  # and is required when upstream.auth has credentials for the server.
+  # git:
+  #   github: "https://github.com"
+  #   wikimedia:
+  #     url: "https://gerrit.wikimedia.org/r"
+  #     repos: ["mediawiki/**"]
+
   # Authentication for upstream registries
   # Keys are absolute URL scopes. Scheme, host, effective port, and path
   # segment boundaries must match; the longest matching scope wins.
@@ -344,6 +353,14 @@ upstream:
     # "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com":
     #   type: ecr
 
+# Git repository cache, used when upstream.git is set
+# git:
+#   cache_dir: ./cache/git   # local disk, used by this process only
+#   max_size: ""             # evict least recently used repositories past this
+#   max_repo_size: 5GB       # refuse a repository that grows past this
+#   refresh_wait: 5s         # then serve the cached copy if the upstream is slow
+#   stale_max_age: 24h       # never serve a copy not checked for this long
+
 # Gradle HttpBuildCache configuration
 gradle:
   build_cache:

docs/architecture.md

@@ -301,6 +301,21 @@ HTTP protocol handlers for each registry type.
 - Proxies the Swift Package Registry v1 read endpoints
 - Rewrites release URLs and caches source archives
 
+**GitHandler:**
+- Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP
+- Validates repository paths and the `repos` allowlist, and maps cache errors to statuses
+
+### `internal/gitcache`
+
+Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed.
+
+- Each request for refs checks the upstream's branches and tags with protocol v2 `ls-refs` and fetches only when they changed; concurrent requests share one refresh
+- Clients are served by `git upload-pack --stateless-rpc`
+- Spawned `git` reaches the upstream only through a loopback relay that forwards with the proxy's upstream client, so address checks, `upstream.auth` and the access log apply
+- Commits a client asks for by ID that are on no cached branch or tag are fetched and kept under hidden `refs/proxy/wanted/{time}/{id}` refs until a branch or tag includes them or for 7 days; off for upstreams with credentials
+- Evicts least recently used repositories after each fetch once `git.max_size` is passed, and drops a repository that can no longer be refreshed within `git.max_repo_size` so it is cloned again; never one in use. A commit fetched by ID that does not fit is refused instead
+- Every fetched pack is kept as a pack (`transfer.unpackLimit=1`), so a transfer cut off at the size limit leaves nothing behind
+
 ### `internal/server`
 
 HTTP server setup, web UI, and API handlers.

docs/configuration.md

@@ -582,6 +582,123 @@ those names unless the shadowing is intended.
 This field has no environment variable, as with the other named upstream maps.
 `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`.
 
+### Git repositories
+
+Git servers configured under `upstream.git` are served read-only at
+`/git/{name}/{repo}` over Git's smart HTTP protocol, from bare repositories
+cached on local disk:
+
+```yaml
+upstream:
+  git:
+    github: "https://github.com"
+    wikimedia:
+      url: "https://gerrit.wikimedia.org/r"
+      repos: ["mediawiki/core", "mediawiki/extensions/**"]  # optional allowlist
+
+git:
+  cache_dir: /data/git   # local disk, used by this process only
+  max_size: 50GB         # evict least recently used repositories past this
+  max_repo_size: 5GB     # refuse a repository that grows past this
+  refresh_wait: 5s       # then serve the cached copy if the upstream is slow
+  stale_max_age: 24h     # never serve a copy not checked for this long
+```
+
+Clients reach the proxy through Git's `insteadOf` setting, which rewrites the
+start of a URL before Git connects. After
+
+```bash
+git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/
+```
+
+`git clone https://github.com/git-pkgs/purl` fetches from
+`http://localhost:8080/git/github/git-pkgs/purl`, while the clone's remote
+stays `https://github.com/git-pkgs/purl`.
+
+In CI, the same setting can come from environment variables instead of a
+config file (Git 2.31 or newer), next to the variables that point package
+managers at the proxy. Each `GIT_CONFIG_KEY_n` and `GIT_CONFIG_VALUE_n` pair is
+one `git config` setting, and `GIT_CONFIG_COUNT` says how many pairs there are.
+This sends both upstreams from the example above through the proxy:
+
+```bash
+export GIT_CONFIG_COUNT=2
+export GIT_CONFIG_KEY_0=url.http://localhost:8080/git/github/.insteadOf
+export GIT_CONFIG_VALUE_0=https://github.com/
+export GIT_CONFIG_KEY_1=url.http://localhost:8080/git/wikimedia/.insteadOf
+export GIT_CONFIG_VALUE_1=https://gerrit.wikimedia.org/r/
+```
+
+Every process the job starts inherits these variables, so the rewrite also
+applies when a package manager runs `git` for a Git dependency, as pip,
+Bundler, Composer, SwiftPM, and Go for modules it fetches directly do. Those
+fetches are cached too. npm downloads GitHub dependencies as tarballs over
+HTTPS where it can, without `git`, so the rewrite does not reach them.
+
+`repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a
+pattern ending in `/**` also matches every repository nested below its
+prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream
+without `repos` serves every repository on it, and a repository outside
+`repos` answers 404. When `upstream.auth` has credentials for an upstream,
+`repos` is required: everyone who can reach the proxy can read every
+repository it serves, so list only those they may read. Per-client
+permissions are not supported.
+
+Every request for a repository's refs checks the upstream's branches and tags,
+and fetches only when they changed, so a clone right after a push sees the new
+commit. Requests that arrive while a check or fetch is running share it. Only
+branches and tags are cached and served; other refs, such as Gerrit's
+`refs/changes/*`, are left out, and protocol v2 lets the server filter them
+before sending.
+
+A commit a client asks for by object ID that is on no cached branch or tag,
+such as a pull request head fetched by `actions/checkout`, is fetched from the
+upstream when the client asks for it, and kept until a cached branch or tag
+includes it or for 7 days. A commit the upstream could not provide, or could
+not be reached for, is not asked for again for a minute, or for an hour when
+it is larger than what `max_repo_size` leaves. This is off for upstreams that `upstream.auth` has
+credentials for: GitHub serves a commit from any repository in a fork network
+through any other, so a fetch by ID could reach repositories outside `repos`.
+Any commit the cache holds can be fetched by ID, including one a force-push
+left behind, as from the upstream.
+
+If the upstream fails or takes longer than `refresh_wait`, a repository checked
+successfully within `stale_max_age` is served from the cache while the refresh
+continues. A `stale_max_age` of `0` never serves a copy the upstream has not
+just confirmed. A repository the upstream reports missing (404), refuses (401, 403)
+or redirects is not served from the cache. Redirects are never followed, so
+point the upstream at a repository's current location.
+
+The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it).
+The `git` processes never reach the network themselves: their fetches go
+through a loopback endpoint in the proxy that forwards them with the same
+upstream client as every other request, so the address checks
+(`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log
+apply, and credentials are never passed to `git`.
+
+`max_repo_size` defaults to 5GB, and `max_size` to no limit. A clone or fetch
+that would pass `max_repo_size` is stopped, and nothing of it is kept. A new
+repository is then refused for an hour. When a refresh of one already cached
+does not fit, it is served as it is and dropped once no request is reading
+it, so the next request clones it again with only its branches and tags; if
+that is still too large, it is refused for an hour. A commit fetched by ID
+that does not fit is refused, and the repository kept.
+Least recently used repositories are evicted after each fetch that takes the
+cache past `max_size`, never one a request is reading. A transfer
+that receives nothing for 10 minutes is abandoned. Git's automatic garbage
+collection runs as part of fetches.
+
+Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so
+repointing an upstream starts a fresh cache, and the old URL's directory is
+removed at startup. Nothing else in `cache_dir` is touched, and the
+directories of upstreams no longer configured are only logged, so remove
+those yourself.
+
+Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings
+under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`,
+`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and
+`PROXY_GIT_STALE_MAX_AGE`.
+
 ## Authentication
 
 Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax.

internal/config/config.go

@@ -198,6 +198,9 @@ type Config struct {
 
 	// Health configures the /health endpoint behavior.
 	Health HealthConfig `json:"health" yaml:"health"`
+
+	// Git configures the Git repository cache served at /git/{name}/.
+	Git GitConfig `json:"git" yaml:"git"`
 }
 
 // CooldownConfig configures version cooldown periods.
@@ -741,6 +744,11 @@ type UpstreamConfig struct {
 	// Example: {"apache": ["dist/(?P<name>maven)/maven-3/(?P<version>[^/]+)/binaries/(?P<file>[^/]+)"]}.
 	GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"`
 
+	// Git maps names to Git servers, served read-only at /git/{name}/{repo}
+	// from a local cache of bare repositories. See GitUpstream.
+	// Example: {"github": {"url": "https://github.com"}}.
+	Git map[string]GitUpstream `json:"git" yaml:"git"`
+
 	// Auth configures authentication for upstream registries.
 	// Keys are absolute URL scopes matched by scheme, host, effective port,
 	// and path-segment prefix.
@@ -811,6 +819,9 @@ func (u *UpstreamConfig) Validate() error {
 	if err := u.validateGenericArtifacts(); err != nil {
 		return err
 	}
+	if err := u.validateGitUpstreams(); err != nil {
+		return err
+	}
 	if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil {
 		return err
 	}
@@ -1228,6 +1239,11 @@ func (c *Config) LoadFromEnv() {
 	setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE")
 	setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL")
 	setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL")
+	setEnvString(&c.Git.CacheDir, "PROXY_GIT_CACHE_DIR")
+	setEnvString(&c.Git.MaxSize, "PROXY_GIT_MAX_SIZE")
+	setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE")
+	setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT")
+	setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE")
 }
 
 // validateAbsoluteURL returns an error if value is not a parseable URL with
@@ -1378,6 +1394,10 @@ func (c *Config) validateComponents() error {
 		return err
 	}
 
+	if err := c.Git.Validate(); err != nil {
+		return err
+	}
+
 	return c.Gradle.BuildCache.Validate()
 }