diff --git a/Dockerfile b/Dockerfile index de784e82..41139993 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,7 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=${TARGETARCH} go build -ldflags="-s -w" -o / FROM alpine:3.24.2 -RUN apk add --no-cache ca-certificates +RUN apk add --no-cache ca-certificates git COPY --from=builder /proxy /usr/local/bin/proxy diff --git a/README.md b/README.md index 6fb3e8ab..8b74b9ec 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The proxy never uploads artifact bytes to a scanner. Each scanner is notified wi | Arch | Arch Linux | | ✓ | | Chef | Chef | | ✗ | | Generic | Any | | ✓ | +| Git | Any | | ✓ | | Helm | Kubernetes | Yes | ✓ | | Vagrant | Vagrant | | ✗ | @@ -804,6 +805,27 @@ go.download_mirror = "http://localhost:8080/generic/golang" Other download hosts need their version-pinned paths declared in `upstream.generic_artifacts`. +### Git repositories + +Configure named Git servers: + +```yaml +upstream: + git: + github: "https://github.com" +``` + +Then point clones at the proxy: + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +Each repository is cached as a bare clone and refreshed whenever its branches +or tags change upstream. See [configuration](docs/configuration.md#git-repositories) +for setting the rewrite through environment variables, size limits, private +repositories and outage behavior. + ## Configuration The proxy can be configured via: @@ -1125,6 +1147,7 @@ Recently cached: | `GET /apk/{repository}/*` | Alpine APK repository protocol | | `GET /arch/{repository}/*` | Arch Linux pacman repository protocol | | `GET /generic/{name}/*` | Generic HTTP download proxy (GitHub release assets, mise/aqua) | +| `GET /git/{name}/{repo}/info/refs`, `POST /git/{name}/{repo}/git-upload-pack` | Git smart HTTP protocol, read-only | | `GET /debian/*` | Debian/APT repository protocol (main archive) | | `GET /debian/{repository}/*` | Debian/APT repository protocol (named archive, e.g. security) | | `GET /rpm/*` | RPM/Yum repository protocol | diff --git a/config.example.yaml b/config.example.yaml index c41f30f3..597a3241 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -304,6 +304,15 @@ upstream: # apache: # - 'dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)' + # Named Git servers (used by /git/{name}/{repo}), served read-only from + # cached bare repositories. repos limits which repositories may be served + # and is required when upstream.auth has credentials for the server. + # git: + # github: "https://github.com" + # wikimedia: + # url: "https://gerrit.wikimedia.org/r" + # repos: ["mediawiki/**"] + # Authentication for upstream registries # Keys are absolute URL scopes. Scheme, host, effective port, and path # segment boundaries must match; the longest matching scope wins. @@ -344,6 +353,14 @@ upstream: # "https://123456789012.dkr.ecr.eu-west-1.amazonaws.com": # type: ecr +# Git repository cache, used when upstream.git is set +# git: +# cache_dir: ./cache/git # local disk, used by this process only +# max_size: "" # evict least recently used repositories past this +# max_repo_size: 5GB # refuse a repository that grows past this +# refresh_wait: 5s # then serve the cached copy if the upstream is slow +# stale_max_age: 24h # never serve a copy not checked for this long + # Gradle HttpBuildCache configuration gradle: build_cache: diff --git a/docs/architecture.md b/docs/architecture.md index a259e910..f9325b25 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -301,6 +301,21 @@ HTTP protocol handlers for each registry type. - Proxies the Swift Package Registry v1 read endpoints - Rewrites release URLs and caches source archives +**GitHandler:** +- Serves `/git/{name}/{repo}` from `internal/gitcache`, read-only over smart HTTP +- Validates repository paths and the `repos` allowlist, and maps cache errors to statuses + +### `internal/gitcache` + +Local bare clones of upstream Git repositories, outside the artifact cache and the database, at `{git.cache_dir}/{name}-{URL hash}/{repo}.git`. The hash of the upstream's URL means repointing an upstream starts a fresh cache; at startup only the configured upstreams' directories are loaded, and those of their old URLs removed. + +- Each request for refs checks the upstream's branches and tags with protocol v2 `ls-refs` and fetches only when they changed; concurrent requests share one refresh +- Clients are served by `git upload-pack --stateless-rpc` +- Spawned `git` reaches the upstream only through a loopback relay that forwards with the proxy's upstream client, so address checks, `upstream.auth` and the access log apply +- Commits a client asks for by ID that are on no cached branch or tag are fetched and kept under hidden `refs/proxy/wanted/{time}/{id}` refs until a branch or tag includes them or for 7 days; off for upstreams with credentials +- Evicts least recently used repositories after each fetch once `git.max_size` is passed, and drops a repository that can no longer be refreshed within `git.max_repo_size` so it is cloned again; never one in use. A commit fetched by ID that does not fit is refused instead +- Every fetched pack is kept as a pack (`transfer.unpackLimit=1`), so a transfer cut off at the size limit leaves nothing behind + ### `internal/server` HTTP server setup, web UI, and API handlers. diff --git a/docs/configuration.md b/docs/configuration.md index c735c34b..8463aa1e 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -582,6 +582,123 @@ those names unless the shadowing is intended. This field has no environment variable, as with the other named upstream maps. `PROXY_UPSTREAM_DEBIAN` still sets `upstream.debian`. +### Git repositories + +Git servers configured under `upstream.git` are served read-only at +`/git/{name}/{repo}` over Git's smart HTTP protocol, from bare repositories +cached on local disk: + +```yaml +upstream: + git: + github: "https://github.com" + wikimedia: + url: "https://gerrit.wikimedia.org/r" + repos: ["mediawiki/core", "mediawiki/extensions/**"] # optional allowlist + +git: + cache_dir: /data/git # local disk, used by this process only + max_size: 50GB # evict least recently used repositories past this + max_repo_size: 5GB # refuse a repository that grows past this + refresh_wait: 5s # then serve the cached copy if the upstream is slow + stale_max_age: 24h # never serve a copy not checked for this long +``` + +Clients reach the proxy through Git's `insteadOf` setting, which rewrites the +start of a URL before Git connects. After + +```bash +git config --global url."http://localhost:8080/git/github/".insteadOf https://github.com/ +``` + +`git clone https://github.com/git-pkgs/purl` fetches from +`http://localhost:8080/git/github/git-pkgs/purl`, while the clone's remote +stays `https://github.com/git-pkgs/purl`. + +In CI, the same setting can come from environment variables instead of a +config file (Git 2.31 or newer), next to the variables that point package +managers at the proxy. Each `GIT_CONFIG_KEY_n` and `GIT_CONFIG_VALUE_n` pair is +one `git config` setting, and `GIT_CONFIG_COUNT` says how many pairs there are. +This sends both upstreams from the example above through the proxy: + +```bash +export GIT_CONFIG_COUNT=2 +export GIT_CONFIG_KEY_0=url.http://localhost:8080/git/github/.insteadOf +export GIT_CONFIG_VALUE_0=https://github.com/ +export GIT_CONFIG_KEY_1=url.http://localhost:8080/git/wikimedia/.insteadOf +export GIT_CONFIG_VALUE_1=https://gerrit.wikimedia.org/r/ +``` + +Every process the job starts inherits these variables, so the rewrite also +applies when a package manager runs `git` for a Git dependency, as pip, +Bundler, Composer, SwiftPM, and Go for modules it fetches directly do. Those +fetches are cached too. npm downloads GitHub dependencies as tarballs over +HTTPS where it can, without `git`, so the rewrite does not reach them. + +`repos` patterns use `path.Match` syntax, where `*` does not cross a `/`; a +pattern ending in `/**` also matches every repository nested below its +prefix, so `mediawiki/**` matches `mediawiki/skins/Vector`. An upstream +without `repos` serves every repository on it, and a repository outside +`repos` answers 404. When `upstream.auth` has credentials for an upstream, +`repos` is required: everyone who can reach the proxy can read every +repository it serves, so list only those they may read. Per-client +permissions are not supported. + +Every request for a repository's refs checks the upstream's branches and tags, +and fetches only when they changed, so a clone right after a push sees the new +commit. Requests that arrive while a check or fetch is running share it. Only +branches and tags are cached and served; other refs, such as Gerrit's +`refs/changes/*`, are left out, and protocol v2 lets the server filter them +before sending. + +A commit a client asks for by object ID that is on no cached branch or tag, +such as a pull request head fetched by `actions/checkout`, is fetched from the +upstream when the client asks for it, and kept until a cached branch or tag +includes it or for 7 days. A commit the upstream could not provide, or could +not be reached for, is not asked for again for a minute, or for an hour when +it is larger than what `max_repo_size` leaves. This is off for upstreams that `upstream.auth` has +credentials for: GitHub serves a commit from any repository in a fork network +through any other, so a fetch by ID could reach repositories outside `repos`. +Any commit the cache holds can be fetched by ID, including one a force-push +left behind, as from the upstream. + +If the upstream fails or takes longer than `refresh_wait`, a repository checked +successfully within `stale_max_age` is served from the cache while the refresh +continues. A `stale_max_age` of `0` never serves a copy the upstream has not +just confirmed. A repository the upstream reports missing (404), refuses (401, 403) +or redirects is not served from the cache. Redirects are never followed, so +point the upstream at a repository's current location. + +The proxy runs `git` 2.31 or newer, which must be installed (the Docker image includes it). +The `git` processes never reach the network themselves: their fetches go +through a loopback endpoint in the proxy that forwards them with the same +upstream client as every other request, so the address checks +(`allow_loopback`, `allow_private_hosts`), `upstream.auth` and the access log +apply, and credentials are never passed to `git`. + +`max_repo_size` defaults to 5GB, and `max_size` to no limit. A clone or fetch +that would pass `max_repo_size` is stopped, and nothing of it is kept. A new +repository is then refused for an hour. When a refresh of one already cached +does not fit, it is served as it is and dropped once no request is reading +it, so the next request clones it again with only its branches and tags; if +that is still too large, it is refused for an hour. A commit fetched by ID +that does not fit is refused, and the repository kept. +Least recently used repositories are evicted after each fetch that takes the +cache past `max_size`, never one a request is reading. A transfer +that receives nothing for 10 minutes is abandoned. Git's automatic garbage +collection runs as part of fetches. + +Repositories are kept under `{cache_dir}/{name}-{hash of the URL}/`, so +repointing an upstream starts a fresh cache, and the old URL's directory is +removed at startup. Nothing else in `cache_dir` is touched, and the +directories of upstreams no longer configured are only logged, so remove +those yourself. + +Pushing, the dumb HTTP protocol and Git LFS are not supported. The settings +under `git` can also be set with `PROXY_GIT_CACHE_DIR`, `PROXY_GIT_MAX_SIZE`, +`PROXY_GIT_MAX_REPO_SIZE`, `PROXY_GIT_REFRESH_WAIT` and +`PROXY_GIT_STALE_MAX_AGE`. + ## Authentication Configure authentication for private upstream registries. The same authentication-aware client is used for metadata and artifact downloads, and credentials can reference environment variables using `${VAR_NAME}` syntax. diff --git a/internal/config/config.go b/internal/config/config.go index a9d7c212..c37ed4ad 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -198,6 +198,9 @@ type Config struct { // Health configures the /health endpoint behavior. Health HealthConfig `json:"health" yaml:"health"` + + // Git configures the Git repository cache served at /git/{name}/. + Git GitConfig `json:"git" yaml:"git"` } // CooldownConfig configures version cooldown periods. @@ -741,6 +744,11 @@ type UpstreamConfig struct { // Example: {"apache": ["dist/(?Pmaven)/maven-3/(?P[^/]+)/binaries/(?P[^/]+)"]}. GenericArtifacts map[string][]string `json:"generic_artifacts" yaml:"generic_artifacts"` + // Git maps names to Git servers, served read-only at /git/{name}/{repo} + // from a local cache of bare repositories. See GitUpstream. + // Example: {"github": {"url": "https://github.com"}}. + Git map[string]GitUpstream `json:"git" yaml:"git"` + // Auth configures authentication for upstream registries. // Keys are absolute URL scopes matched by scheme, host, effective port, // and path-segment prefix. @@ -811,6 +819,9 @@ func (u *UpstreamConfig) Validate() error { if err := u.validateGenericArtifacts(); err != nil { return err } + if err := u.validateGitUpstreams(); err != nil { + return err + } if err := validateNamedUpstreams("upstream.debian_repositories", u.DebianRepositories); err != nil { return err } @@ -1228,6 +1239,11 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Gradle.BuildCache.MaxSize, "PROXY_GRADLE_BUILD_CACHE_MAX_SIZE") setEnvString(&c.Gradle.BuildCache.SweepInterval, "PROXY_GRADLE_BUILD_CACHE_SWEEP_INTERVAL") setEnvString(&c.Health.StorageProbeInterval, "PROXY_HEALTH_STORAGE_PROBE_INTERVAL") + setEnvString(&c.Git.CacheDir, "PROXY_GIT_CACHE_DIR") + setEnvString(&c.Git.MaxSize, "PROXY_GIT_MAX_SIZE") + setEnvString(&c.Git.MaxRepoSize, "PROXY_GIT_MAX_REPO_SIZE") + setEnvString(&c.Git.RefreshWait, "PROXY_GIT_REFRESH_WAIT") + setEnvString(&c.Git.StaleMaxAge, "PROXY_GIT_STALE_MAX_AGE") } // validateAbsoluteURL returns an error if value is not a parseable URL with @@ -1378,6 +1394,10 @@ func (c *Config) validateComponents() error { return err } + if err := c.Git.Validate(); err != nil { + return err + } + return c.Gradle.BuildCache.Validate() } diff --git a/internal/config/git.go b/internal/config/git.go new file mode 100644 index 00000000..8a0028d2 --- /dev/null +++ b/internal/config/git.go @@ -0,0 +1,222 @@ +package config + +import ( + "encoding/json" + "fmt" + "path" + "strings" + "time" + + "gopkg.in/yaml.v3" +) + +const ( + defaultGitCacheDir = "./cache/git" + defaultGitMaxRepoSize = "5GB" + defaultGitRefreshWait = "5s" + defaultGitStaleMaxAge = "24h" +) + +// GitUpstream is an upstream.git entry: a Git server's base URL and, +// optionally, the repositories under it that may be served. In YAML or JSON +// it is either the URL alone or an object with url and repos. +type GitUpstream struct { + URL string `json:"url" yaml:"url"` + + // Repos lists path.Match patterns for repository paths under URL, such + // as "mediawiki/*". A pattern ending in "/**" also matches every + // repository nested below its prefix: "mediawiki/**" matches + // mediawiki/skins/Vector. Empty allows every repository. Required when + // upstream.auth has credentials for the URL, since everyone who can + // reach the proxy can read what it serves. + Repos []string `json:"repos" yaml:"repos"` +} + +// UnmarshalYAML accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalYAML(node *yaml.Node) error { + if node.Kind == yaml.ScalarNode { + return node.Decode(&g.URL) + } + type plain GitUpstream + return node.Decode((*plain)(g)) +} + +// UnmarshalJSON accepts a bare URL as well as the object form. +func (g *GitUpstream) UnmarshalJSON(data []byte) error { + if err := json.Unmarshal(data, &g.URL); err == nil { + return nil + } + type plain GitUpstream + return json.Unmarshal(data, (*plain)(g)) +} + +// AllowsRepo reports whether repo, a path under URL, may be served. +func (g GitUpstream) AllowsRepo(repo string) bool { + if len(g.Repos) == 0 { + return true + } + for _, pattern := range g.Repos { + if matchRepoPattern(pattern, repo) { + return true + } + } + return false +} + +func matchRepoPattern(pattern, repo string) bool { + prefix, nested := strings.CutSuffix(pattern, "/**") + if !nested { + ok, _ := path.Match(pattern, repo) + return ok + } + segments := strings.Count(prefix, "/") + 1 + parts := strings.SplitN(repo, "/", segments+1) + if len(parts) <= segments { + return false + } + ok, _ := path.Match(prefix, strings.Join(parts[:segments], "/")) + return ok +} + +// GitConfig configures the Git repository cache behind /git/{name}/. +type GitConfig struct { + // CacheDir holds the cached bare repositories. It must be local disk + // used by this process only. Default: ./cache/git. + CacheDir string `json:"cache_dir" yaml:"cache_dir"` + + // MaxSize evicts the least recently used repositories once their total + // size passes it. Empty or "0" disables the limit. + MaxSize string `json:"max_size" yaml:"max_size"` + + // MaxRepoSize is the most one repository may take; a clone or fetch that + // would pass it fails. "0" disables the limit. Default: 5GB. + MaxRepoSize string `json:"max_repo_size" yaml:"max_repo_size"` + + // RefreshWait is how long a request waits for a repository to be + // refreshed from upstream before it is served the cached copy. + // Default: 5s. + RefreshWait string `json:"refresh_wait" yaml:"refresh_wait"` + + // StaleMaxAge is how long after the last successful upstream check a + // cached copy may still be served when the upstream fails or is slow. + // Default: 24h. + StaleMaxAge string `json:"stale_max_age" yaml:"stale_max_age"` + + maxSize int64 + maxRepoSize int64 + refreshWait time.Duration + staleMaxAge time.Duration +} + +// MaxSizeBytes returns the parsed MaxSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxSizeBytes() int64 { return g.maxSize } + +// MaxRepoSizeBytes returns the parsed MaxRepoSize, 0 for no limit. Validate must run first. +func (g *GitConfig) MaxRepoSizeBytes() int64 { return g.maxRepoSize } + +// RefreshWaitDuration returns the parsed RefreshWait. Validate must run first. +func (g *GitConfig) RefreshWaitDuration() time.Duration { return g.refreshWait } + +// StaleMaxAgeDuration returns the parsed StaleMaxAge. Validate must run first. +func (g *GitConfig) StaleMaxAgeDuration() time.Duration { return g.staleMaxAge } + +// Validate checks the Git cache settings, applying defaults where unset. +func (g *GitConfig) Validate() error { + if g.CacheDir == "" { + g.CacheDir = defaultGitCacheDir + } + if g.MaxRepoSize == "" { + g.MaxRepoSize = defaultGitMaxRepoSize + } + if g.RefreshWait == "" { + g.RefreshWait = defaultGitRefreshWait + } + if g.StaleMaxAge == "" { + g.StaleMaxAge = defaultGitStaleMaxAge + } + + var err error + if g.maxSize, err = parseNonNegativeSize("git.max_size", g.MaxSize); err != nil { + return err + } + if g.maxRepoSize, err = parseNonNegativeSize("git.max_repo_size", g.MaxRepoSize); err != nil { + return err + } + if g.refreshWait, err = parseNonNegativeDuration("git.refresh_wait", g.RefreshWait); err != nil { + return err + } + if g.staleMaxAge, err = parseNonNegativeDuration("git.stale_max_age", g.StaleMaxAge); err != nil { + return err + } + return nil +} + +func parseNonNegativeSize(field, value string) (int64, error) { + size, err := ParseSize(value) + if err != nil { + return 0, fmt.Errorf("invalid %s: %w", field, err) + } + if size < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return size, nil +} + +func parseNonNegativeDuration(field, value string) (time.Duration, error) { + d, err := time.ParseDuration(value) + if err != nil { + return 0, fmt.Errorf("invalid %s %q: %w", field, value, err) + } + if d < 0 { + return 0, fmt.Errorf("invalid %s %q: must not be negative", field, value) + } + return d, nil +} + +// validateGitUpstreams checks upstream.git names, URLs and repository +// patterns. An upstream that upstream.auth has credentials for must list its +// repositories, so configuring a token does not open every repository it can +// read to everyone who can reach the proxy. +func (u *UpstreamConfig) validateGitUpstreams() error { + for name, upstream := range u.Git { + field := "upstream.git." + name + if name == "" || name == "." || name == ".." || name[0] == '.' || strings.ContainsAny(name, `/\\`) { + return fmt.Errorf("invalid upstream.git name %q", name) + } + if err := validateAbsoluteURL(field, upstream.URL); err != nil { + return err + } + if scheme, _, _ := strings.Cut(upstream.URL, "://"); !strings.EqualFold(scheme, "https") && !strings.EqualFold(scheme, "http") { + return fmt.Errorf("invalid %s %q: only http and https are supported", field, upstream.URL) + } + for _, pattern := range upstream.Repos { + if _, err := path.Match(strings.TrimSuffix(pattern, "/**"), ""); pattern == "" || err != nil { + return fmt.Errorf("invalid %s.repos pattern %q", field, pattern) + } + } + if len(upstream.Repos) == 0 && u.HasAuthWithin(upstream.URL) { + return fmt.Errorf("%s needs repos: upstream.auth has credentials for %s, "+ + "and everyone who can reach the proxy could read every repository they allow", field, upstream.URL) + } + } + return nil +} + +// HasAuthWithin reports whether an upstream.auth scope covers baseURL or a +// path under it. +func (u *UpstreamConfig) HasAuthWithin(baseURL string) bool { + base, err := parseAuthURL(baseURL) + if err != nil { + return false + } + for scope := range u.Auth { + configured, err := parseAuthURL(scope) + if err != nil { + continue + } + if authURLMatches(configured, base) || authURLMatches(base, configured) { + return true + } + } + return false +} diff --git a/internal/config/git_test.go b/internal/config/git_test.go new file mode 100644 index 00000000..6c9b47bd --- /dev/null +++ b/internal/config/git_test.go @@ -0,0 +1,172 @@ +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "slices" + "testing" + "time" + + "gopkg.in/yaml.v3" +) + +func TestGitUpstreamAcceptsURLOrObject(t *testing.T) { + want := map[string]GitUpstream{ + "github": {URL: "https://github.com"}, + "wikimedia": {URL: "https://gitlab.wikimedia.org", Repos: []string{"mediawiki/*"}}, + } + + var fromYAML struct { + Git map[string]GitUpstream `yaml:"git"` + } + err := yaml.Unmarshal([]byte(` +git: + github: "https://github.com" + wikimedia: + url: "https://gitlab.wikimedia.org" + repos: ["mediawiki/*"] +`), &fromYAML) + if err != nil { + t.Fatalf("YAML: %v", err) + } + + var fromJSON struct { + Git map[string]GitUpstream `json:"git"` + } + err = json.Unmarshal([]byte(`{"git": { + "github": "https://github.com", + "wikimedia": {"url": "https://gitlab.wikimedia.org", "repos": ["mediawiki/*"]} + }}`), &fromJSON) + if err != nil { + t.Fatalf("JSON: %v", err) + } + + for format, got := range map[string]map[string]GitUpstream{"YAML": fromYAML.Git, "JSON": fromJSON.Git} { + for name, w := range want { + if g := got[name]; g.URL != w.URL || !slices.Equal(g.Repos, w.Repos) { + t.Errorf("%s %s = %+v, want %+v", format, name, g, w) + } + } + } +} + +func TestGitUpstreamAllowsRepo(t *testing.T) { + open := GitUpstream{URL: "https://github.com"} + if !open.AllowsRepo("any/repo") { + t.Error("an upstream without repos refused a repository") + } + listed := GitUpstream{URL: "https://gitlab.example", Repos: []string{"mediawiki/*", "tools/one", "group/**"}} + for repo, want := range map[string]bool{ + "mediawiki/core": true, + "tools/one": true, + "tools/two": false, + "mediawiki/extensions/Echo": false, + "group/sub/project": true, + "group/project": true, + "group": false, + "groups/project": false, + } { + if got := listed.AllowsRepo(repo); got != want { + t.Errorf("AllowsRepo(%q) = %v, want %v", repo, got, want) + } + } +} + +func TestValidateGitUpstreams(t *testing.T) { + tests := []struct { + name string + upstream UpstreamConfig + wantErr bool + }{ + {name: "public", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}}}, + {name: "relative URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "github.com"}}}, wantErr: true}, + {name: "name with slash", upstream: UpstreamConfig{Git: map[string]GitUpstream{"a/b": {URL: "https://github.com"}}}, wantErr: true}, + {name: "name with leading dot", upstream: UpstreamConfig{Git: map[string]GitUpstream{".tmp": {URL: "https://github.com"}}}, wantErr: true}, + {name: "bad pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "https://github.com", Repos: []string{"["}}}}, wantErr: true}, + {name: "nested pattern", upstream: UpstreamConfig{Git: map[string]GitUpstream{"gerrit": {URL: "https://gerrit.example/r", Repos: []string{"mediawiki/**"}}}}}, + {name: "ssh URL", upstream: UpstreamConfig{Git: map[string]GitUpstream{"github": {URL: "ssh://git@github.com"}}}, wantErr: true}, + { + name: "credentials without repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials for a path under the upstream", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example"}}, + Auth: map[string]AuthConfig{"https://git.example/team": {Type: "bearer", Token: "t"}}, + }, + wantErr: true, + }, + { + name: "credentials with repos", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"private": {URL: "https://git.example", Repos: []string{"team/*"}}}, + Auth: map[string]AuthConfig{"https://git.example": {Type: "bearer", Token: "t"}}, + }, + }, + { + name: "credentials for another host", + upstream: UpstreamConfig{ + Git: map[string]GitUpstream{"github": {URL: "https://github.com"}}, + Auth: map[string]AuthConfig{"https://npm.example": {Type: "bearer", Token: "t"}}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := tt.upstream.Validate(); (err != nil) != tt.wantErr { + t.Errorf("Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestGitConfigDefaults(t *testing.T) { + var g GitConfig + if err := g.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if g.CacheDir != defaultGitCacheDir || g.MaxSizeBytes() != 0 || g.MaxRepoSizeBytes() != 5<<30 || + g.RefreshWaitDuration() != 5*time.Second || g.StaleMaxAgeDuration() != 24*time.Hour { + t.Errorf("defaults = %+v", g) + } + + for _, bad := range []GitConfig{{MaxSize: "lots"}, {RefreshWait: "soon"}, {StaleMaxAge: "-1h"}} { + if err := bad.Validate(); err == nil { + t.Errorf("Validate(%+v) accepted it", bad) + } + } +} + +func TestLoadGitConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yaml") + data := []byte(` +upstream: + git: + github: https://github.com +git: + cache_dir: /data/git + max_size: 50GB +`) + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatal(err) + } + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + t.Setenv("PROXY_GIT_REFRESH_WAIT", "2s") + cfg.LoadFromEnv() + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + if cfg.Upstream.Git["github"].URL != "https://github.com" || cfg.Git.CacheDir != "/data/git" || + cfg.Git.MaxSizeBytes() != 50<<30 || cfg.Git.RefreshWaitDuration() != 2*time.Second { + t.Errorf("loaded %+v and %+v", cfg.Upstream.Git, cfg.Git) + } +} diff --git a/internal/gitcache/cache.go b/internal/gitcache/cache.go new file mode 100644 index 00000000..eee4d34b --- /dev/null +++ b/internal/gitcache/cache.go @@ -0,0 +1,421 @@ +// Package gitcache serves Git repositories read-only over smart HTTP from a +// local cache of bare repositories, refreshed from configured upstreams. +package gitcache + +import ( + "cmp" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io/fs" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // tmpDirName holds clones in progress and evicted repositories being + // deleted. Upstream names cannot start with a dot, so it cannot clash + // with an upstream's directory, and the name is specific in case + // cache_dir is shared with other data. + tmpDirName = ".gitcache-tmp" + // usedStampInterval limits how often a repository's last use is written + // to disk. + usedStampInterval = time.Minute + // defaultLowSpeedTime is how many seconds a git transfer may receive + // nothing before it is abandoned. Fetches run with --quiet, so the + // upstream sends nothing while it counts objects; this is far longer. + defaultLowSpeedTime = 600 + ecosystem = "git" + upstreamHashLen = 8 + checkedStamp = "proxy-checked" + usedStamp = "proxy-used" + dirPermissions = 0o750 +) + +// Options configures a Cache. +type Options struct { + CacheDir string + MaxSize int64 + MaxRepoSize int64 + RefreshWait time.Duration + StaleMaxAge time.Duration + // Client makes every upstream request, including those of the git + // processes the cache runs. Redirects are never followed. + Client *http.Client + Logger *slog.Logger + // Upstreams maps each configured upstream's name to its directory in + // CacheDir, as UpstreamDir names it. Only those directories are loaded + // at startup. A directory UpstreamDir would name for one of these names + // at another URL, left by repointing the upstream, is removed. Nothing + // else in CacheDir is touched, so it may hold other data. + Upstreams map[string]string + + // refCheckTimeout and lowSpeedTime override their defaults in tests. + refCheckTimeout time.Duration + lowSpeedTime int +} + +// Cache is the Git repository cache. internal/handler serves it at /git. +type Cache struct { + opts Options + client *http.Client + logger *slog.Logger + gitPath string + dir string + relay *relay + fetches chan struct{} + refCheckTimeout time.Duration + lowSpeedTime int + + ctx context.Context + cancel context.CancelFunc + // refreshes counts the refreshes running, which Close waits for. + refreshes sync.WaitGroup + + mu sync.Mutex + // closed stops refreshes from starting once Close has begun. + closed bool + repos map[string]*repo + tooLarge map[string]time.Time + // failedWants records, by repository key and object ID, wants the + // upstream could not provide. + failedWants map[string]time.Time +} + +// repo is a cached repository, or one being cloned. users, refreshing and +// the map entry itself are guarded by Cache.mu. +type repo struct { + key string + dir string + + users int + refreshing *refreshCall + // writes serializes git fetches into the repository. + writes chan struct{} + + size atomic.Int64 + lastUsed atomic.Int64 + lastChecked atomic.Int64 + usedWritten atomic.Int64 +} + +// maxConcurrentFetches caps clones and fetches running at once. Refreshes +// outlive the requests that start them, so without a cap a client could +// start any number and hang up. +const maxConcurrentFetches = 16 + +// New creates the cache, loading repositories already in CacheDir. +func New(opts Options) (*Cache, error) { + gitPath, err := exec.LookPath("git") + if err != nil { + return nil, fmt.Errorf("upstream.git needs git installed: %w", err) + } + dir, err := filepath.Abs(opts.CacheDir) + if err != nil { + return nil, err + } + logger := opts.Logger + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + client := *opts.Client + client.Timeout = 0 + client.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + + c := &Cache{ + opts: opts, + client: &client, + logger: logger, + gitPath: gitPath, + dir: dir, + fetches: make(chan struct{}, maxConcurrentFetches), + refCheckTimeout: defaultRefCheckTimeout, + lowSpeedTime: defaultLowSpeedTime, + repos: make(map[string]*repo), + tooLarge: make(map[string]time.Time), + failedWants: make(map[string]time.Time), + } + if opts.refCheckTimeout > 0 { + c.refCheckTimeout = opts.refCheckTimeout + } + if opts.lowSpeedTime > 0 { + c.lowSpeedTime = opts.lowSpeedTime + } + if err := c.load(); err != nil { + return nil, err + } + if c.relay, err = newRelay(c.client, logger); err != nil { + return nil, fmt.Errorf("starting git relay: %w", err) + } + c.ctx, c.cancel = context.WithCancel(context.Background()) + // Applies a lowered git.max_size straight away. + c.evict() + return c, nil +} + +// Close stops the refreshes in progress and waits for them to finish, so +// none is still writing to the cache when it returns, then stops the relay. +// Fetches by ID run on request goroutines; stop the HTTP server first. +// Closing again does nothing. +func (c *Cache) Close() error { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil + } + c.closed = true + c.mu.Unlock() + c.cancel() + c.refreshes.Wait() + return c.relay.close() +} + +// UpstreamDir names the top-level cache directory of the upstream called +// name at baseURL. It includes a hash of the URL, so repointing an upstream +// starts a fresh cache rather than serving the old host's repositories. +func UpstreamDir(name, baseURL string) string { + digest := sha256.Sum256([]byte(strings.TrimSuffix(baseURL, "/"))) + return name + "-" + hex.EncodeToString(digest[:])[:upstreamHashLen] +} + +func (c *Cache) tmpDir() string { + return filepath.Join(c.dir, tmpDirName) +} + +// load clears what an earlier run left in the temporary directory and +// registers the repositories already cached. +func (c *Cache) load() error { + if err := os.RemoveAll(c.tmpDir()); err != nil { + return fmt.Errorf("clearing %s: %w", c.tmpDir(), err) + } + if err := os.MkdirAll(c.tmpDir(), dirPermissions); err != nil { + return err + } + c.removeRepointed() + for _, upstreamDir := range c.opts.Upstreams { + root := filepath.Join(c.dir, upstreamDir) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() || !strings.HasSuffix(d.Name(), ".git") || path == root { + return err + } + rel, _ := filepath.Rel(c.dir, path) + rp := newRepo(filepath.ToSlash(rel), path) + rp.size.Store(dirSize(path)) + rp.lastChecked.Store(stampTime(path, checkedStamp)) + rp.lastUsed.Store(stampTime(path, usedStamp)) + rp.usedWritten.Store(rp.lastUsed.Load()) + c.repos[rp.key] = rp + return filepath.SkipDir + }) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return err + } + } + return nil +} + +// upstreamDirPattern matches a directory UpstreamDir names, capturing the +// upstream's name. +var upstreamDirPattern = regexp.MustCompile(`^(.+)-[0-9a-f]{8}$`) + +// removeRepointed removes the directories of configured upstreams at URLs +// they no longer have. Each is moved aside first, so nothing can be read +// from it half deleted. Directories of upstreams no longer configured are +// only logged: nothing else says they are the cache's. +func (c *Cache) removeRepointed() { + entries, err := os.ReadDir(c.dir) + if err != nil { + c.logger.Warn("git cache: cannot list the cache directory", "error", err) + return + } + current := make(map[string]bool, len(c.opts.Upstreams)) + for _, upstreamDir := range c.opts.Upstreams { + current[upstreamDir] = true + } + var unconfigured []string + for _, entry := range entries { + name := entry.Name() + match := upstreamDirPattern.FindStringSubmatch(name) + if !entry.IsDir() || match == nil || current[name] { + continue + } + if _, configured := c.opts.Upstreams[match[1]]; !configured { + unconfigured = append(unconfigured, name) + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "repointed-") + if err == nil { + err = os.Rename(filepath.Join(c.dir, name), filepath.Join(trash, name)) + } + if err != nil { + c.logger.Warn("git cache: failed to remove the cache of a repointed upstream", "directory", name, "error", err) + continue + } + c.logger.Info("git cache: removed the cache of a repointed upstream", "directory", name) + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete a removed cache", "path", trash, "error", err) + } + } + if len(unconfigured) > 0 { + c.logger.Info("git cache: leaving directories that look like caches of upstreams no longer configured; delete them if they are", + "directories", unconfigured) + } +} + +// acquire returns the repository for key, counting the caller as a user +// until release, so it is not evicted while in use. +func (c *Cache) acquire(key string) *repo { + c.mu.Lock() + defer c.mu.Unlock() + rp := c.repos[key] + if rp == nil { + rp = newRepo(key, filepath.Join(c.dir, filepath.FromSlash(key))) + c.repos[key] = rp + } + rp.users++ + return rp +} + +func (c *Cache) release(rp *repo) { + now := time.Now() + rp.lastUsed.Store(now.UnixNano()) + exists := rp.exists() + if exists && now.Sub(time.Unix(0, rp.usedWritten.Load())) >= usedStampInterval { + rp.usedWritten.Store(now.UnixNano()) + touch(rp.dir, usedStamp, now) + } + + c.mu.Lock() + rp.users-- + idle := rp.users == 0 && c.repos[rp.key] == rp + // Forget repositories that never got cloned, such as ones missing upstream. + if idle && !exists { + delete(c.repos, rp.key) + } + _, tooLarge := c.tooLarge[rp.key] + c.mu.Unlock() + if idle && exists && tooLarge { + c.evict() + } +} + +func newRepo(key, dir string) *repo { + return &repo{key: key, dir: dir, writes: make(chan struct{}, 1)} +} + +// lockWrites waits for other fetches into the repository to finish. +func (rp *repo) lockWrites(ctx context.Context) (func(), error) { + select { + case rp.writes <- struct{}{}: + return func() { <-rp.writes }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +func (rp *repo) exists() bool { + _, err := os.Stat(rp.dir) + return err == nil +} + +// evict removes repositories nobody is using: the least recently used until +// the cache fits in MaxSize, and any that could not be refreshed within +// MaxRepoSize, so the next request clones them again with only their +// branches and tags. It runs after every fetch, since only fetches grow the +// cache, and when a repository too large to refresh is released. A +// repository is moved aside under the lock, so a request that comes after it +// clones it again instead of reading it while it is deleted. +func (c *Cache) evict() { + c.mu.Lock() + var total int64 + idle := make([]*repo, 0, len(c.repos)) + for _, rp := range c.repos { + total += rp.size.Load() + if rp.users == 0 { + idle = append(idle, rp) + } + } + slices.SortFunc(idle, func(a, b *repo) int { + return cmp.Compare(a.lastUsed.Load(), b.lastUsed.Load()) + }) + var removed []string + for _, rp := range idle { + _, marked := c.tooLarge[rp.key] + tooLarge := marked && rp.exists() + if !tooLarge && (c.opts.MaxSize <= 0 || total <= c.opts.MaxSize) { + continue + } + trash, err := os.MkdirTemp(c.tmpDir(), "evicted-") + if err == nil { + err = os.Rename(rp.dir, filepath.Join(trash, "repo")) + } + if err != nil && !errors.Is(err, fs.ErrNotExist) { + c.logger.Warn("git cache: failed to evict repository", "repository", rp.key, "error", err) + continue + } + delete(c.repos, rp.key) + total -= rp.size.Load() + removed = append(removed, trash) + reason := "lru" + if tooLarge { + // The clone that replaces it enforces the limit itself. + delete(c.tooLarge, rp.key) + reason = "too_large" + } + metrics.RecordArtifactEvicted(reason, ecosystem) + c.logger.Info("git cache: evicted repository", "repository", rp.key, "size", rp.size.Load(), "reason", reason) + } + c.mu.Unlock() + + for _, trash := range removed { + if err := os.RemoveAll(trash); err != nil { + c.logger.Warn("git cache: failed to delete evicted repository", "path", trash, "error", err) + } + } +} + +func dirSize(dir string) int64 { + var size int64 + _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error { + if err == nil && d.Type().IsRegular() { + if info, err := d.Info(); err == nil { + size += info.Size() + } + } + return nil + }) + return size +} + +// Stamps are empty files whose modification time records an event, so it +// survives restarts. +func stampTime(dir, name string) int64 { + info, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + return 0 + } + return info.ModTime().UnixNano() +} + +func touch(dir, name string, t time.Time) { + path := filepath.Join(dir, name) + if err := os.Chtimes(path, t, t); errors.Is(err, fs.ErrNotExist) { + if f, err := os.Create(path); err == nil { + _ = f.Close() + _ = os.Chtimes(path, t, t) + } + } +} diff --git a/internal/gitcache/cache_test.go b/internal/gitcache/cache_test.go new file mode 100644 index 00000000..f5d14b2b --- /dev/null +++ b/internal/gitcache/cache_test.go @@ -0,0 +1,659 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "maps" + "net/http" + "net/http/httptest" + "net/http/httputil" + "net/url" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/registries/safehttp" +) + +func TestCloneMatchesUpstream(t *testing.T) { + for _, version := range []string{"0", "1", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo.git", "-c", "protocol.version="+version) + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "-c", "protocol.version="+version, "ls-remote", "--heads", "--tags", tc.url("repo.git")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } + runGit(t, clone, "fsck", "--no-dangling", "--no-progress") + }) + } +} + +func TestUpstreamWithoutProtocolV2(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.v0Only = true + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + clone := tc.clone(t, "repo") + if got, want := runGit(t, clone, "rev-parse", "HEAD"), runGit(t, upstream.bare(), "rev-parse", "main"); got != want { + t.Errorf("HEAD = %s, want %s", got, want) + } +} + +func TestUnchangedRepositoryIsNotFetchedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + tc.clone(t, "repo") + fetches := upstream.gitRequests.Load() + tc.clone(t, "repo") + if got := upstream.gitRequests.Load(); got != fetches { + t.Errorf("second clone made %d upstream fetch requests, want none", got-fetches) + } +} + +func TestPushedCommitIsServedOnNextClone(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + existing := tc.clone(t, "repo") + + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("clone after push: HEAD = %s, want %s", got, head) + } + runGit(t, existing, "pull", "--quiet", "--ff-only") + if got := runGit(t, existing, "rev-parse", "HEAD"); got != head { + t.Errorf("pull after push: HEAD = %s, want %s", got, head) + } + if got := runGit(t, tc.clone(t, "repo", "--depth=1"), "rev-list", "--count", "HEAD"); got != "1" { + t.Errorf("shallow clone has %s commits, want 1", got) + } +} + +func TestOnlyBranchesAndTagsAreCachedAndPruned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, work, "push", "--quiet", "origin", "main:refs/changes/01/1/1", "main:refs/heads/gone", "main:refs/tags/gone") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + refs := tc.cachedRefs(t) + if strings.Contains(refs, "refs/changes/") { + t.Errorf("cached refs/changes:\n%s", refs) + } + if !strings.Contains(refs, "refs/heads/gone") || !strings.Contains(refs, "refs/tags/gone") { + t.Fatalf("branch and tag not cached:\n%s", refs) + } + + runGit(t, work, "push", "--quiet", "origin", ":refs/heads/gone", ":refs/tags/gone") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); strings.Contains(refs, "gone") { + t.Errorf("deleted branch or tag still cached:\n%s", refs) + } + + // Gerrit advertises a ref per patch set, so asking for every ref would + // list hundreds of thousands for a large repository. + bodies := upstream.lsRefsBodies() + if len(bodies) == 0 { + t.Fatal("no ls-refs request reached the upstream") + } + for _, body := range bodies { + if !strings.Contains(body, "ref-prefix refs/heads/") || !strings.Contains(body, "ref-prefix refs/tags/") { + t.Errorf("ls-refs asked for every ref, want branches and tags only:\n%q", body) + } + } +} + +func TestDefaultBranchChangeIsFollowed(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + runGit(t, work, "checkout", "--quiet", "-b", "trunk") + head := upstream.commit(work, "on trunk") + runGit(t, work, "push", "--quiet", "origin", "trunk") + runGit(t, upstream.bare(), "symbolic-ref", "HEAD", "refs/heads/trunk") + + clone := tc.clone(t, "repo") + if got := runGit(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want trunk's %s", got, head) + } +} + +func TestConcurrentColdClonesShareOneFetch(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + + release := upstream.holdRequests() + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + } + }) + } + time.Sleep(200 * time.Millisecond) + release() + wg.Wait() + + if got := upstream.gitSessions.Load(); got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} + +func TestMissingRepositoryLeavesNothingBehind(t *testing.T) { + upstream := newFakeUpstream(t) + tc := newTestCache(t, upstream.server.URL, nil) + + resp, err := http.Get(tc.url("missing") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("status %d, want 404", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if _, err := os.Stat(filepath.Join(tc.dir, "up", "missing.git")); !os.IsNotExist(err) { + t.Errorf("repository directory left behind: %v", err) + } +} + +func TestStaleCopyServedOnlyWithinStaleMaxAge(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + tc.clone(t, "repo") + + waitIdle(t, tc.Cache) + rp := tc.acquire("up/repo.git") + rp.lastChecked.Store(time.Now().Add(-2 * time.Hour).UnixNano()) + tc.release(rp) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served a copy older than stale_max_age:\n%s", out) + } +} + +func TestRefusedRefreshIsNotServedStale(t *testing.T) { + for _, status := range []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusMovedPermanently} { + t.Run(fmt.Sprint(status), func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(int32(status)) + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("served the cached copy after status %d:\n%s", status, out) + } + }) + } +} + +func TestSlowUpstreamServesCachedCopyAfterRefreshWait(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = 100 * time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + release := upstream.holdRequests() + defer release() + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != old { + t.Errorf("HEAD = %s, want the cached %s", got, old) + } + release() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +func TestUpstreamCredentialsReachEveryUpstreamRequest(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.requireAuth = "Bearer secret" + upstream.newRepo("repo") + client := &http.Client{Transport: headerTransport{"Authorization", "Bearer secret"}} + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + tc.clone(t, "repo") + if got := upstream.unauthed.Load(); got != 0 { + t.Errorf("%d upstream requests lacked credentials", got) + } + if upstream.gitRequests.Load() == 0 { + t.Error("no fetch went through the relay") + } +} + +type headerTransport struct{ name, value string } + +func (h headerTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r = r.Clone(r.Context()) + r.Header.Set(h.name, h.value) + return http.DefaultTransport.RoundTrip(r) +} + +// TestUpstreamAddressChecksApplyToGit uses a client that refuses loopback +// addresses, as the proxy's does by default. git must not reach the loopback +// upstream around it, whether the ref check or only git's fetch is refused. +func TestUpstreamAddressChecksApplyToGit(t *testing.T) { + blocked := safehttp.New(nil, safehttp.Options{}) + for name, client := range map[string]*http.Client{ + "every request": blocked, + "git's fetch": {Transport: gitOnlyTransport{blocked.Transport}}, + } { + t.Run(name, func(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.Client = client }) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("clone through a blocked upstream succeeded:\n%s", out) + } + if got := upstream.gitRequests.Load(); got != 0 { + t.Errorf("git reached the upstream with %d requests", got) + } + }) + } +} + +// gitOnlyTransport sends the cache's own ref checks with the default +// transport, and git's relayed requests with next. +type gitOnlyTransport struct{ next http.RoundTripper } + +func (g gitOnlyTransport) RoundTrip(r *http.Request) (*http.Response, error) { + if r.UserAgent() == gitUserAgent { + return http.DefaultTransport.RoundTrip(r) + } + return g.next.RoundTrip(r) +} + +// TestRedirectIsNotFollowed redirects the ref check, then only git's fetch. +func TestRedirectIsNotFollowed(t *testing.T) { + for _, redirectRefCheck := range []bool{true, false} { + t.Run(fmt.Sprint("ref check redirected: ", redirectRefCheck), func(t *testing.T) { + target := newFakeUpstream(t) + target.newRepo("repo") + targetURL, _ := url.Parse(target.server.URL) + passThrough := httputil.NewSingleHostReverseProxy(targetURL) + redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() == gitUserAgent && !redirectRefCheck { + passThrough.ServeHTTP(w, r) + return + } + http.Redirect(w, r, target.server.URL+r.URL.RequestURI(), http.StatusFound) + })) + defer redirect.Close() + tc := newTestCache(t, redirect.URL, nil) + + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("followed a redirect:\n%s", out) + } + if got := target.gitRequests.Load(); got != 0 { + t.Errorf("redirect target received %d requests from git", got) + } + }) + } +} + +func TestRepositoryOverMaxRepoSizeIsRefused(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + big := make([]byte, 256<<10) + _, _ = rand.Read(big) + if err := os.WriteFile(filepath.Join(work, "big.bin"), big, 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "big.bin") + runGit(t, work, "commit", "--quiet", "-m", "big") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = 64 << 10 }) + + resp, err := http.Get(tc.url("repo") + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusInsufficientStorage { + t.Errorf("status %d, want 507", resp.StatusCode) + } + waitForRepos(t, tc.Cache, 0) + if entries, _ := os.ReadDir(tc.tmpDir()); len(entries) != 0 { + t.Errorf("left %d entries in the temporary directory", len(entries)) + } + + requests := upstream.requests.Load() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Errorf("second attempt succeeded:\n%s", out) + } + if got := upstream.requests.Load(); got != requests { + t.Errorf("second attempt made %d upstream requests, want none", got-requests) + } +} + +// TestEvictionRemovesLeastRecentlyUsedIdleRepositories makes the least +// recently used repository one a request is reading, so the next one is +// evicted instead. +func TestEvictionRemovesLeastRecentlyUsedIdleRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + for _, name := range []string{"old", "busy", "new"} { + upstream.newRepo(name) + } + size := repoSize(t, upstream, "new") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = 2*size + size/2 }) + + tc.clone(t, "busy") + busy := tc.acquire("up/busy.git") + defer tc.release(busy) + busy.lastUsed.Store(0) + tc.clone(t, "old") + waitFor(t, func() bool { return tc.usersOf("up/old.git") == 0 }) + tc.repo("up/old.git").lastUsed.Store(1) + + // The third clone takes the cache past MaxSize. + tc.clone(t, "new") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "old.git")) }) + for _, name := range []string{"busy", "new"} { + if !fileExists(filepath.Join(tc.dir, "up", name+".git")) { + t.Errorf("%s was evicted", name) + } + } + tc.clone(t, "old") +} + +// repoSize is the size a cache gives repository name once cloned. +func repoSize(t *testing.T, upstream *fakeUpstream, name string) int64 { + t.Helper() + probe := newTestCache(t, upstream.server.URL, nil) + probe.clone(t, name) + return probe.repo("up/" + name + ".git").size.Load() +} + +func TestNewLoadsCachedRepositories(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := os.MkdirAll(filepath.Join(tc.tmpDir(), "clone-leftover"), 0o750); err != nil { + t.Fatal(err) + } + + reloaded, err := New(Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: tc.dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = reloaded.Close() }() + + rp := reloaded.repos["up/repo.git"] // nothing else uses reloaded yet + if rp == nil || rp.size.Load() == 0 || rp.lastChecked.Load() == 0 { + t.Fatalf("repository not loaded: %+v", rp) + } + if entries, _ := os.ReadDir(reloaded.tmpDir()); len(entries) != 0 { + t.Errorf("temporary directory not cleared: %v", entries) + } +} + +func waitForRepos(t *testing.T, c *Cache, n int) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.repos) == n + }) +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + for !cond() { + select { + case <-ctx.Done(): + t.Fatal("condition not met in time") + case <-time.After(20 * time.Millisecond): + } + } +} + +// TestRewoundCacheIsBroughtBackInLine changes the cache behind the upstream's +// back, so it only matches again if the next request refreshes branches and +// tags and prunes what the upstream does not have. +func TestRewoundCacheIsBroughtBackInLine(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + cached := filepath.Join(tc.dir, "up", "repo.git") + runGit(t, cached, "update-ref", "refs/heads/main", "main~1") + runGit(t, cached, "update-ref", "refs/heads/extra", "main") + runGit(t, cached, "tag", "extra", "main") + + want := runGit(t, "", "ls-remote", "--heads", "--tags", upstream.bare()) + if got := runGit(t, "", "ls-remote", "--heads", "--tags", tc.url("repo")); got != want { + t.Errorf("ls-remote through the cache:\n%s\nwant:\n%s", got, want) + } +} + +// TestConcurrentWarmClonesDuringRefresh serves clones from the cache while a +// fetch writes to it. +func TestConcurrentWarmClonesDuringRefresh(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.RefreshWait = time.Millisecond }) + old := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD") + head := upstream.commit(work, "second") + runGit(t, work, "push", "--quiet", "origin", "main") + + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + dir := filepath.Join(t.TempDir(), "clone") + if out, err := tryGit(t, "", "clone", "--quiet", tc.url("repo"), dir); err != nil { + t.Errorf("clone: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "HEAD"); got != old && got != head { + t.Errorf("HEAD = %s, want %s or %s", got, old, head) + } + runGit(t, dir, "fsck", "--no-dangling", "--no-progress") + }) + } + wg.Wait() + waitFor(t, func() bool { return strings.Contains(tc.cachedRefs(t), head) }) +} + +// TestStalledRefCheckFailsInsteadOfHanging has the upstream send headers and +// then nothing, as a broken load balancer might. +func TestStalledRefCheckFailsInsteadOfHanging(t *testing.T) { + stall := make(chan struct{}) + defer close(stall) + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", advertisementType) + w.WriteHeader(http.StatusOK) + _ = http.NewResponseController(w).Flush() + select { + case <-stall: + case <-r.Context().Done(): + } + })) + defer upstream.Close() + tc := newTestCache(t, upstream.URL, func(o *Options) { o.refCheckTimeout = 200 * time.Millisecond }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 10*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestStalledFetchFailsInsteadOfHanging has the upstream answer the ref check +// but never git's fetch, as a connection that stalls mid-pack would. +func TestStalledFetchFailsInsteadOfHanging(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.lowSpeedTime = 1 }) + + start := time.Now() + if out, err := tryGit(t, "", "ls-remote", tc.url("repo")); err == nil { + t.Fatalf("ls-remote succeeded:\n%s", out) + } + if elapsed := time.Since(start); elapsed > 20*time.Second { + t.Errorf("took %s to fail", elapsed) + } +} + +// TestFetchEvictsToMaxSize checks eviction runs on its own once a fetch has +// grown the cache past MaxSize. +func TestFetchEvictsToMaxSize(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("first") + upstream.newRepo("second") + size := repoSize(t, upstream, "first") + + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxSize = size + size/2 }) + tc.clone(t, "first") + waitIdle(t, tc.Cache) + tc.clone(t, "second") + waitFor(t, func() bool { return !fileExists(filepath.Join(tc.dir, "up", "first.git")) }) + if !fileExists(filepath.Join(tc.dir, "up", "second.git")) { + t.Error("evicted the repository just fetched") + } +} + +// TestNewRemovesRepointedUpstreams starts a cache whose directory holds an +// upstream's cache at its old URL, and one of an upstream no longer +// configured. Only the first is known to be the cache's to remove. +func TestNewRemovesRepointedUpstreams(t *testing.T) { + dir := t.TempDir() + current := UpstreamDir("up", "https://new.example") + repointed := UpstreamDir("up", "https://old.example") + unconfigured := UpstreamDir("gone", "https://gone.example") + // Named after a configured upstream's name, but another upstream's. + longer := UpstreamDir("up-api", "https://api.example") + for _, name := range []string{current, repointed, unconfigured, longer} { + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, name, "repo.git")) + } + c, err := New(Options{ + CacheDir: dir, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Upstreams: map[string]string{"up": current}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + + for name, want := range map[string]bool{current: true, repointed: false, unconfigured: true, longer: true} { + if got := fileExists(filepath.Join(dir, name)); got != want { + t.Errorf("%s present = %v, want %v", name, got, want) + } + } + if keys := slices.Collect(maps.Keys(c.repos)); len(keys) != 1 || keys[0] != current+"/repo.git" { + t.Errorf("loaded %v, want only %s/repo.git", keys, current) + } +} + +// TestCacheDirSharedWithOtherData points the cache at a directory that also +// holds an artifact store and a database, as git.cache_dir: ./cache would. +func TestCacheDirSharedWithOtherData(t *testing.T) { + dir := t.TempDir() + _ = os.MkdirAll(filepath.Join(dir, "artifacts", "npm"), 0o750) + _ = os.WriteFile(filepath.Join(dir, "artifacts", "npm", "pkg.tgz"), []byte("x"), 0o600) + _ = os.WriteFile(filepath.Join(dir, "proxy.db"), []byte("db"), 0o600) + runGit(t, "", "init", "--quiet", "--bare", filepath.Join(dir, "artifacts", "foo", "bar.git")) + c, err := New(Options{ + CacheDir: dir, + Client: &http.Client{}, + Upstreams: map[string]string{"github": UpstreamDir("github", "https://github.com")}, + }) + if err != nil { + t.Fatal(err) + } + defer func() { _ = c.Close() }() + for _, p := range []string{"artifacts/npm/pkg.tgz", "proxy.db", "artifacts/foo/bar.git"} { + if _, err := os.Stat(filepath.Join(dir, p)); err != nil { + t.Errorf("%s was deleted at startup: %v", p, err) + } + } + if len(c.repos) != 0 { + t.Errorf("loaded %v from outside the upstream directories", slices.Collect(maps.Keys(c.repos))) + } +} + +// TestCloseWaitsForRefreshes closes the cache while a clone is held up +// upstream. When Close returns, the clone must have finished, temporary +// directory included, so nothing is left writing to the cache. +func TestCloseWaitsForRefreshes(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + upstream.stallFetches.Store(true) + tc := newTestCache(t, upstream.server.URL, nil) + clones := func() []string { + names, _ := filepath.Glob(filepath.Join(tc.tmpDir(), "clone-*")) + return names + } + + go func() { _, _ = tryGit(t, "", "ls-remote", tc.url("repo")) }() + waitFor(t, func() bool { return len(clones()) > 0 }) + if err := tc.Close(); err != nil { + t.Fatal(err) + } + if left := clones(); len(left) != 0 { + t.Errorf("Close returned with the clone still running: %v", left) + } + + before := upstream.requests.Load() + if _, err := tc.Open(context.Background(), "up/other.git", upstream.server.URL+"/other", OpenOptions{}); !errors.Is(err, errClosed) { + t.Errorf("Open after Close: %v, want errClosed", err) + } + if got := upstream.requests.Load() - before; got != 0 { + t.Errorf("Open after Close made %d upstream requests", got) + } +} + +// TestCloseTwice checks closing again does nothing, as a cleanup that also +// closes the cache may. +func TestCloseTwice(t *testing.T) { + c, err := New(Options{CacheDir: t.TempDir(), Client: &http.Client{}}) + if err != nil { + t.Fatal(err) + } + if err := errors.Join(c.Close(), c.Close()); err != nil { + t.Errorf("closing twice: %v", err) + } +} diff --git a/internal/gitcache/git.go b/internal/gitcache/git.go new file mode 100644 index 00000000..736ee8a6 --- /dev/null +++ b/internal/gitcache/git.go @@ -0,0 +1,100 @@ +package gitcache + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "slices" + "strconv" + "strings" + "time" +) + +const ( + maxStderr = 4 << 10 + gitWaitDelay = 5 * time.Second +) + +// gitConfig applies to every git process the cache runs. +var gitConfig = [][2]string{ + // git would follow a redirect from the relay straight to the network. + {"http.followRedirects", "false"}, + {"protocol.version", "2"}, + // Auto gc stays on, as on any Git server, but runs inside the fetch + // rather than detaching, so no git process outlives the fetch that + // started it, and sizes are measured after it. + {"gc.autoDetach", "false"}, + {"maintenance.autoDetach", "false"}, + {"init.defaultBranch", "main"}, + // Keep every fetched pack as a pack. index-pack writes it to a + // temporary file and moves it into place only once complete, whereas a + // small pack is otherwise unpacked object by object, and a transfer cut + // off at max_repo_size would leave the objects before the cut, such as a + // commit without its tree's blobs. + {"transfer.unpackLimit", "1"}, + // Serve any commit the cache holds when a client asks for it by object + // ID, as the upstream would, including ones a force-push left behind, + // and do not advertise the refs that keep fetched ones. + {"uploadpack.allowAnySHA1InWant", "true"}, + {"uploadpack.hideRefs", wantedPrefix}, + // With http.lowSpeedTime, added per cache, gives up on a transfer that + // receives nothing for that long, so a connection that stalls mid-pack + // cannot hold a refresh for fetchTimeout. + {"http.lowSpeedLimit", "1"}, +} + +// command builds a git process with an environment of its own: none of the +// proxy's environment is passed on, so no system or user config, credential +// helper, HTTP proxy setting or GIT_ variable applies, and the only transport +// git may use is plain HTTP, which is what the relay speaks. +func (c *Cache) command(ctx context.Context, extraEnv []string, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, c.gitPath, args...) + env := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + c.tmpDir(), + "LC_ALL=C", + "GIT_CONFIG_NOSYSTEM=1", + "GIT_CONFIG_GLOBAL=" + os.DevNull, + "GIT_TERMINAL_PROMPT=0", + "GIT_ALLOW_PROTOCOL=http", + } + config := slices.Concat(gitConfig, [][2]string{{"http.lowSpeedTime", strconv.Itoa(c.lowSpeedTime)}}) + env = append(env, "GIT_CONFIG_COUNT="+strconv.Itoa(len(config))) + for i, kv := range config { + env = append(env, + "GIT_CONFIG_KEY_"+strconv.Itoa(i)+"="+kv[0], + "GIT_CONFIG_VALUE_"+strconv.Itoa(i)+"="+kv[1]) + } + env = append(env, extraEnv...) + cmd.Env = env + cmd.WaitDelay = gitWaitDelay + return cmd +} + +// run runs git in the repository at dir and returns its standard output. +func (c *Cache) run(ctx context.Context, dir string, args ...string) (string, error) { + cmd := c.command(ctx, nil, append([]string{"-C", dir}, args...)...) + var stdout bytes.Buffer + stderr := &limitedBuffer{max: maxStderr} + cmd.Stdout = &stdout + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("git %s: %w: %s", args[0], err, strings.TrimSpace(stderr.String())) + } + return stdout.String(), nil +} + +// limitedBuffer keeps the first max bytes written to it. +type limitedBuffer struct { + bytes.Buffer + max int +} + +func (b *limitedBuffer) Write(p []byte) (int, error) { + if room := b.max - b.Len(); room > 0 { + b.Buffer.Write(p[:min(len(p), room)]) + } + return len(p), nil +} diff --git a/internal/gitcache/helpers_test.go b/internal/gitcache/helpers_test.go new file mode 100644 index 00000000..ce6becae --- /dev/null +++ b/internal/gitcache/helpers_test.go @@ -0,0 +1,349 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "errors" + "io" + "log" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// gitEnv isolates the tests' own git commands from the machine's config. +func gitEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func runGit(t *testing.T, dir string, args ...string) string { + t.Helper() + out, err := tryGit(t, dir, args...) + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(out) +} + +func tryGit(t *testing.T, dir string, args ...string) (string, error) { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// httpBackendPath finds git http-backend, which serves the fake upstreams, +// and skips the test where git is installed without it. +func httpBackendPath(t *testing.T) string { + t.Helper() + dir := runGit(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(dir, name); fileExists(path) { + return path + } + } + t.Skip("git http-backend is not installed") + return "" +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// fakeUpstream serves bare repositories under root with git http-backend. +type fakeUpstream struct { + t *testing.T + root string + server *httptest.Server + + // status, when set, is returned instead of serving the request. + status atomic.Int32 + // requireAuth, when set, is the Authorization every request must carry. + requireAuth string + // v0Only drops Git-Protocol, so only protocol v0 is spoken. + v0Only bool + // hold, when set, blocks requests until it is closed. + holdMu sync.Mutex + hold chan struct{} + + requests atomic.Int32 + // gitRequests counts requests from git processes, which reach the + // upstream through the relay, and gitSessions the fetches they make. + gitRequests atomic.Int32 + gitSessions atomic.Int32 + unauthed atomic.Int32 + + // stallFetches makes git's own upload-pack requests hang. + stallFetches atomic.Bool + + bodiesMu sync.Mutex + bodies []string +} + +// recordBody keeps the decoded body of a POST. It also gives the request a +// Content-Length, since net/http/cgi refuses chunked bodies, which git sends +// once a request outgrows http.postBuffer. +func (u *fakeUpstream) recordBody(r *http.Request) { + raw, err := io.ReadAll(r.Body) + if err != nil { + u.t.Errorf("reading request body: %v", err) + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + r.TransferEncoding = nil + decoded := raw + if r.Header.Get("Content-Encoding") == "gzip" { + if zr, err := gzip.NewReader(bytes.NewReader(raw)); err == nil { + decoded, _ = io.ReadAll(zr) + } + } + u.bodiesMu.Lock() + u.bodies = append(u.bodies, string(decoded)) + u.bodiesMu.Unlock() +} + +// lsRefsBodies returns the ls-refs requests the upstream received. +func (u *fakeUpstream) lsRefsBodies() []string { + u.bodiesMu.Lock() + defer u.bodiesMu.Unlock() + var out []string + for _, body := range u.bodies { + if strings.Contains(body, "command=ls-refs") { + out = append(out, body) + } + } + return out +} + +func newFakeUpstream(t *testing.T) *fakeUpstream { + t.Helper() + u := &fakeUpstream{t: t, root: t.TempDir()} + backend := &cgi.Handler{ + Path: httpBackendPath(t), + Stderr: io.Discard, + Logger: log.New(io.Discard, "", 0), + Env: append(gitEnv(t.TempDir()), + "GIT_PROJECT_ROOT="+u.root, + "GIT_HTTP_EXPORT_ALL=1"), + } + u.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + u.requests.Add(1) + if r.UserAgent() != gitUserAgent { + u.gitRequests.Add(1) + if r.Method == http.MethodGet { + u.gitSessions.Add(1) + } + } + u.holdMu.Lock() + hold := u.hold + u.holdMu.Unlock() + if hold != nil { + <-hold + } + if status := u.status.Load(); status != 0 { + http.Error(w, "unavailable", int(status)) + return + } + if u.requireAuth != "" && r.Header.Get("Authorization") != u.requireAuth { + u.unauthed.Add(1) + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + if u.v0Only { + r.Header.Del("Git-Protocol") + } + if u.stallFetches.Load() && r.UserAgent() != gitUserAgent && r.Method == http.MethodPost { + // The server notices the client leaving only once the body is read. + _, _ = io.Copy(io.Discard, r.Body) + <-r.Context().Done() + return + } + if r.Method == http.MethodPost { + u.recordBody(r) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(u.server.Close) + return u +} + +func (u *fakeUpstream) holdRequests() func() { + u.holdMu.Lock() + defer u.holdMu.Unlock() + hold := make(chan struct{}) + u.hold = hold + var once sync.Once + return func() { + once.Do(func() { + u.holdMu.Lock() + u.hold = nil + u.holdMu.Unlock() + close(hold) + }) + } +} + +// newRepo creates {name}.git upstream with one commit on main and a tag, and +// returns a working clone to commit and push from. +func (u *fakeUpstream) newRepo(name string) string { + u.t.Helper() + bare := filepath.Join(u.root, name+".git") + runGit(u.t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := u.t.TempDir() + runGit(u.t, "", "clone", "--quiet", bare, work) + runGit(u.t, work, "checkout", "--quiet", "-b", "main") + u.commit(work, "first") + runGit(u.t, work, "tag", "-a", "-m", "v1", "v1") + runGit(u.t, work, "push", "--quiet", "origin", "main", "v1") + return work +} + +func (u *fakeUpstream) commit(work, msg string) string { + u.t.Helper() + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte(msg+"\n"), 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", "file.txt") + runGit(u.t, work, "commit", "--quiet", "-m", msg) + return runGit(u.t, work, "rev-parse", "HEAD") +} + +// waitIdle waits until no request or refresh is using a repository, so a +// test can inspect or change the cache without racing a request finishing. +func waitIdle(t *testing.T, c *Cache) { + t.Helper() + waitFor(t, func() bool { + c.mu.Lock() + defer c.mu.Unlock() + for _, rp := range c.repos { + if rp.users > 0 { + return false + } + } + return true + }) +} + +// bare is the upstream repository named "repo". +// repo returns the cache's entry for key, reading the map under the lock. +func (tc *testCache) repo(key string) *repo { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key] +} + +// usersOf is the number of requests and refreshes using the repository. +func (tc *testCache) usersOf(key string) int { + tc.mu.Lock() + defer tc.mu.Unlock() + return tc.repos[key].users +} + +func (u *fakeUpstream) bare() string { + return filepath.Join(u.root, "repo.git") +} + +type testCache struct { + *Cache + server *httptest.Server +} + +func newTestCache(t *testing.T, upstreamURL string, configure func(*Options)) *testCache { + t.Helper() + opts := Options{ + Upstreams: map[string]string{"up": "up"}, + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + } + if configure != nil { + configure(&opts) + } + c, err := New(opts) + if err != nil { + t.Fatalf("New: %v", err) + } + server := httptest.NewServer(testFront(c, upstreamURL, true)) + t.Cleanup(func() { + server.Close() + _ = c.Close() + }) + return &testCache{Cache: c, server: server} +} + +// testFront serves /git/up/{repo}/info/refs and git-upload-pack from c, as +// internal/handler does once it has validated the request, with repo +// fetched from upstreamURL/{repo}. fetchWants is OpenOptions.FetchWants. +func testFront(c *Cache, upstreamURL string, fetchWants bool) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := strings.TrimPrefix(r.URL.Path, "/git/up/") + repo, infoRefs := strings.CutSuffix(path, "/info/refs") + if !infoRefs { + var ok bool + if repo, ok = strings.CutSuffix(path, "/git-upload-pack"); !ok || r.Method != http.MethodPost { + http.Error(w, "not served", http.StatusForbidden) + return + } + } else if r.URL.RawQuery != "service=git-upload-pack" { + http.Error(w, "not served", http.StatusForbidden) + return + } + lease, err := c.Open(r.Context(), "up/"+strings.TrimSuffix(repo, ".git")+".git", upstreamURL+"/"+repo, + OpenOptions{Refresh: infoRefs, FetchWants: fetchWants}) + switch { + case errors.Is(err, ErrNotFound): + http.Error(w, err.Error(), http.StatusNotFound) + return + case errors.Is(err, ErrTooLarge): + http.Error(w, err.Error(), http.StatusInsufficientStorage) + return + case err != nil: + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer lease.Release() + if infoRefs { + lease.ServeInfoRefs(w, r) + } else { + lease.ServeUploadPack(w, r) + } + }) +} + +func (tc *testCache) url(repo string) string { + return tc.server.URL + "/git/up/" + repo +} + +// clone clones repo through the cache with the given protocol version and +// returns the working copy. +func (tc *testCache) clone(t *testing.T, repo string, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "clone") + runGit(t, "", append(append([]string{"clone", "--quiet"}, args...), tc.url(repo), dir)...) + return dir +} + +// cachedRefs lists the refs of the cached copy of "repo". +func (tc *testCache) cachedRefs(t *testing.T) string { + t.Helper() + return runGit(t, filepath.Join(tc.dir, "up", "repo.git"), "for-each-ref", "--format=%(objectname) %(refname)") +} diff --git a/internal/gitcache/pktline.go b/internal/gitcache/pktline.go new file mode 100644 index 00000000..5f6095ba --- /dev/null +++ b/internal/gitcache/pktline.go @@ -0,0 +1,65 @@ +package gitcache + +import ( + "bufio" + "fmt" + "io" + "strconv" + "strings" +) + +// pkt-line framing (gitprotocol-common): four hex digits giving the packet +// length including themselves, then the payload. The lengths 0000, 0001 and +// 0002 are the flush, delimiter and response-end packets. +const ( + pktFlush = "0000" + pktDelim = "0001" + pktHeaderLen = 4 + pktMaxLen = 65520 + pktSpecialLast = 2 +) + +type pktKind int + +const ( + pktData pktKind = iota + pktFlushKind + pktDelimKind + pktEndKind +) + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+pktHeaderLen, payload) +} + +type pktReader struct { + r *bufio.Reader +} + +func newPktReader(r io.Reader) *pktReader { + return &pktReader{r: bufio.NewReader(r)} +} + +// next returns the next packet. A data packet's payload has its trailing +// newline removed. +func (p *pktReader) next() (pktKind, string, error) { + var header [pktHeaderLen]byte + if _, err := io.ReadFull(p.r, header[:]); err != nil { + return 0, "", err + } + n, err := strconv.ParseUint(string(header[:]), 16, 16) + if err != nil { + return 0, "", fmt.Errorf("invalid pkt-line length %q", header) + } + if n <= pktSpecialLast { + return pktKind(n + 1), "", nil + } + if n < pktHeaderLen || n > pktMaxLen { + return 0, "", fmt.Errorf("invalid pkt-line length %d", n) + } + payload := make([]byte, n-pktHeaderLen) + if _, err := io.ReadFull(p.r, payload); err != nil { + return 0, "", err + } + return pktData, strings.TrimSuffix(string(payload), "\n"), nil +} diff --git a/internal/gitcache/refresh.go b/internal/gitcache/refresh.go new file mode 100644 index 00000000..801e8b3f --- /dev/null +++ b/internal/gitcache/refresh.go @@ -0,0 +1,336 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "maps" + "os" + "path/filepath" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + // fetchTimeout bounds one refresh, including a cold clone of a large + // repository. + fetchTimeout = time.Hour + // defaultRefCheckTimeout bounds the ref check, so an upstream that + // stalls mid-response does not hold a repository's refresh for + // fetchTimeout. + defaultRefCheckTimeout = 2 * time.Minute + // tooLargeRetryAfter is how long a repository that passed + // git.max_repo_size is left alone before it is tried again. + tooLargeRetryAfter = time.Hour +) + +// errUpstreamUnanswered marks a fetch the upstream never answered, such as +// one during an outage. +var errUpstreamUnanswered = errors.New("upstream did not answer the fetch") + +// errClosed is returned for a refresh asked for once the cache is closing. +var errClosed = errors.New("git cache is closed") + +// refreshCall is a refresh in progress, shared by every request for the +// repository that arrives while it runs. +type refreshCall struct { + done chan struct{} + err error +} + +// ensureFresh brings the repository up to date with its upstream before it is +// served. A repository checked successfully within StaleMaxAge is served +// from the cache if the upstream fails or takes longer than RefreshWait; the +// refresh carries on in the background either way. One never cloned, or not +// checked within StaleMaxAge, is served only once the refresh succeeds. +func (c *Cache) ensureFresh(ctx context.Context, rp *repo, repoURL string) error { + call := c.startRefresh(rp, repoURL) + checked := time.Unix(0, rp.lastChecked.Load()) + if !rp.exists() || time.Since(checked) > c.opts.StaleMaxAge { + select { + case <-call.done: + return call.err + case <-ctx.Done(): + return ctx.Err() + } + } + + timer := time.NewTimer(c.opts.RefreshWait) + defer timer.Stop() + select { + case <-call.done: + if call.err != nil && servesStale(call.err) { + c.logger.Warn("git cache: refresh failed, serving cached copy", + "repository", rp.key, "checked", checked, "error", call.err) + return nil + } + return call.err + case <-timer.C: + c.logger.Info("git cache: refresh still running, serving cached copy", + "repository", rp.key, "checked", checked) + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// servesStale reports whether the cached copy may be served after err: not +// when the upstream says the repository is gone or refuses access to it. +func servesStale(err error) bool { + return !errors.Is(err, ErrNotFound) && !errors.Is(err, ErrRefused) +} + +// startRefresh starts a refresh of the repository, or joins the one running. +// The refresh holds a use of the repository until it ends. +func (c *Cache) startRefresh(rp *repo, repoURL string) *refreshCall { + c.mu.Lock() + defer c.mu.Unlock() + if rp.refreshing != nil { + return rp.refreshing + } + call := &refreshCall{done: make(chan struct{})} + if c.closed { + call.err = errClosed + close(call.done) + return call + } + rp.refreshing = call + rp.users++ + c.refreshes.Add(1) + go func() { + defer c.refreshes.Done() + ctx, cancel := context.WithTimeout(c.ctx, fetchTimeout) + start := time.Now() + call.err = c.refresh(ctx, rp, repoURL) + cancel() + metrics.RecordUpstreamFetch(ecosystem, time.Since(start)) + // A refresh cut short by Close is not an upstream failure. + if call.err != nil && c.ctx.Err() == nil { + metrics.RecordUpstreamError(ecosystem, errorType(call.err), metrics.FailureDetail{URL: repoURL, Err: call.err}) + c.logger.Warn("git cache: refresh failed", "repository", rp.key, "error", call.err) + } + c.mu.Lock() + rp.refreshing = nil + c.mu.Unlock() + c.release(rp) + close(call.done) + if call.err == nil { + c.evict() + } + }() + return call +} + +// refresh checks the upstream's branches and tags and fetches if they differ +// from the cached ones, cloning the repository if it is not cached yet. +func (c *Cache) refresh(ctx context.Context, rp *repo, repoURL string) error { + if c.isTooLarge(rp.key) { + return ErrTooLarge + } + checkCtx, cancel := context.WithTimeout(ctx, c.refCheckTimeout) + remote, err := c.listUpstreamRefs(checkCtx, repoURL) + cancel() + if err != nil { + return err + } + if rp.exists() { + err = c.update(ctx, rp, repoURL, remote) + } else { + err = c.clone(ctx, rp, repoURL, remote) + } + if errors.Is(err, ErrTooLarge) { + c.markTooLarge(rp.key) + } + if err != nil { + return err + } + now := time.Now() + rp.lastChecked.Store(now.UnixNano()) + touch(rp.dir, checkedStamp, now) + return nil +} + +// clone builds the repository in a temporary directory and moves it into +// place, so no request ever reads a partial clone. +func (c *Cache) clone(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + tmp, err := os.MkdirTemp(c.tmpDir(), "clone-") + if err != nil { + return err + } + defer func() { _ = os.RemoveAll(tmp) }() + + if _, err := c.run(ctx, tmp, "init", "--quiet", "--bare"); err != nil { + return err + } + if err := c.fetch(ctx, tmp, repoURL, c.opts.MaxRepoSize, refreshFlags, branchesAndTags...); err != nil { + return err + } + if err := c.setHead(ctx, tmp, remote, ""); err != nil { + return err + } + size := dirSize(tmp) + if c.opts.MaxRepoSize > 0 && size > c.opts.MaxRepoSize { + return ErrTooLarge + } + if err := os.MkdirAll(filepath.Dir(rp.dir), dirPermissions); err != nil { + return err + } + if err := os.Rename(tmp, rp.dir); err != nil { + return fmt.Errorf("moving clone into place: %w", err) + } + rp.size.Store(size) + c.logger.Info("git cache: cloned repository", "repository", rp.key, "size", size) + return nil +} + +// update fetches when the upstream's branches or tags differ from the cached +// ones, and follows a change of the upstream's default branch. +func (c *Cache) update(ctx context.Context, rp *repo, repoURL string, remote remoteRefs) error { + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + local, wanted, err := c.localRefs(ctx, rp.dir) + if err != nil { + return err + } + fetched := !maps.Equal(local, remote.refs) + if fetched { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + err = c.fetch(ctx, rp.dir, repoURL, limit, refreshFlags, branchesAndTags...) + rp.size.Store(dirSize(rp.dir)) + if err != nil { + return err + } + } + if err := c.pruneWanted(ctx, rp.dir, wanted, fetched); err != nil { + c.logger.Warn("git cache: pruning wanted refs failed", "repository", rp.key, "error", err) + } + head, _ := c.run(ctx, rp.dir, "symbolic-ref", "--quiet", "HEAD") + return c.setHead(ctx, rp.dir, remote, strings.TrimSpace(head)) +} + +// branchesAndTags are the refspecs of a refresh. A bare clone has none of its +// own, and other refs, such as Gerrit's refs/changes, are left out. +var branchesAndTags = []string{"+refs/heads/*:refs/heads/*", "+refs/tags/*:refs/tags/*"} + +// refreshFlags delete the branches and tags the upstream no longer has, and +// update all refs together, so a client served while a refresh runs never +// sees only some of them moved. +var refreshFlags = []string{"--prune", "--atomic"} + +// fetchLimit returns how many more bytes the repository may take, 0 for no +// limit. +func (c *Cache) fetchLimit(rp *repo) (int64, error) { + if c.opts.MaxRepoSize <= 0 { + return 0, nil + } + limit := c.opts.MaxRepoSize - rp.size.Load() + if limit <= 0 { + return 0, ErrTooLarge + } + return limit, nil +} + +// fetch fetches refspecs from the upstream into the repository at dir, +// through the relay, adding flags to git fetch. limit caps the bytes +// fetched, 0 for no cap. +func (c *Cache) fetch(ctx context.Context, dir, repoURL string, limit int64, flags []string, refspecs ...string) error { + select { + case c.fetches <- struct{}{}: + defer func() { <-c.fetches }() + case <-ctx.Done(): + return ctx.Err() + } + + relayURL, route, unregister := c.relay.register(repoURL, limit) + defer unregister() + args := append([]string{"fetch", "--quiet", "--no-write-fetch-head"}, flags...) + _, err := c.run(ctx, dir, append(append(args, relayURL), refspecs...)...) + if err != nil { + // index-pack leaves its temporary pack behind when its input ends + // early. Fetches into a repository run one at a time, so any left + // now are this fetch's. + leftovers, _ := filepath.Glob(filepath.Join(dir, "objects", "pack", "tmp_*")) + for _, path := range leftovers { + _ = os.Remove(path) + } + } + if route.exceeded.Load() { + return ErrTooLarge + } + if err != nil && !route.answered.Load() { + return fmt.Errorf("%w: %w", errUpstreamUnanswered, err) + } + return err +} + +// setHead points HEAD at the upstream's default branch, when the upstream +// names one and it is among the cached branches. +func (c *Cache) setHead(ctx context.Context, dir string, remote remoteRefs, current string) error { + if remote.head == "" || remote.head == current { + return nil + } + if _, ok := remote.refs[remote.head]; !ok { + return nil + } + _, err := c.run(ctx, dir, "symbolic-ref", "HEAD", remote.head) + return err +} + +// localRefs lists the cached branches and tags, and the names of the wanted +// refs. +func (c *Cache) localRefs(ctx context.Context, dir string) (map[string]string, []string, error) { + out, err := c.run(ctx, dir, "for-each-ref", "--format=%(objectname) %(refname)", headsPrefix, tagsPrefix, wantedPrefix) + if err != nil { + return nil, nil, err + } + refs := make(map[string]string) + var wanted []string + for line := range strings.Lines(out) { + oid, name, ok := strings.Cut(strings.TrimSuffix(line, "\n"), " ") + switch { + case !ok: + case strings.HasPrefix(name, wantedPrefix): + wanted = append(wanted, name) + default: + refs[name] = oid + } + } + return refs, wanted, nil +} + +func errorType(err error) string { + switch { + case errors.Is(err, ErrNotFound): + return "not_found" + case errors.Is(err, ErrRefused): + return "refused" + case errors.Is(err, ErrTooLarge): + return "too_large" + } + return "fetch_failed" +} + +func (c *Cache) isTooLarge(key string) bool { + c.mu.Lock() + defer c.mu.Unlock() + since, ok := c.tooLarge[key] + if ok && time.Since(since) >= tooLargeRetryAfter { + delete(c.tooLarge, key) + return false + } + return ok +} + +func (c *Cache) markTooLarge(key string) { + c.mu.Lock() + defer c.mu.Unlock() + c.tooLarge[key] = time.Now() +} diff --git a/internal/gitcache/relay.go b/internal/gitcache/relay.go new file mode 100644 index 00000000..437d7b55 --- /dev/null +++ b/internal/gitcache/relay.go @@ -0,0 +1,175 @@ +package gitcache + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "io" + "log/slog" + "net" + "net/http" + "strings" + "sync" + "sync/atomic" + "time" +) + +const ( + relayIDBytes = 16 + relayReadHeaderTimeout = 30 * time.Second +) + +// relay is a loopback HTTP server that every git fetch the cache runs goes +// through. git never reaches the network itself: each fetch gets a route to +// one upstream repository, and the relay forwards its requests with the +// proxy's upstream client, so address checks, upstream.auth, the access log +// and the no-redirect rule apply as they do to the proxy's own requests, and +// git never sees a credential. +type relay struct { + client *http.Client + logger *slog.Logger + listener net.Listener + server *http.Server + base string + + mu sync.Mutex + routes map[string]*relayRoute +} + +type relayRoute struct { + repoURL string + // limit caps the response bytes the route relays, 0 for no cap. + limit int64 + relayed atomic.Int64 + exceeded atomic.Bool + // answered records that the upstream answered a git-upload-pack + // request, so a failed fetch was not an outage. + answered atomic.Bool +} + +func newRelay(client *http.Client, logger *slog.Logger) (*relay, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if listener, err = net.Listen("tcp", "[::1]:0"); err != nil { + return nil, err + } + } + rl := &relay{ + client: client, + logger: logger, + listener: listener, + base: "http://" + listener.Addr().String(), + routes: make(map[string]*relayRoute), + } + rl.server = &http.Server{Handler: rl, ReadHeaderTimeout: relayReadHeaderTimeout} + go func() { + if err := rl.server.Serve(listener); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("git relay stopped", "error", err) + } + }() + return rl, nil +} + +func (rl *relay) close() error { + return rl.server.Close() +} + +// register adds a route to repoURL and returns the URL git fetches from and +// a function that removes the route. +func (rl *relay) register(repoURL string, limit int64) (string, *relayRoute, func()) { + var b [relayIDBytes]byte + _, _ = rand.Read(b[:]) + id := hex.EncodeToString(b[:]) + route := &relayRoute{repoURL: repoURL, limit: limit} + + rl.mu.Lock() + rl.routes[id] = route + rl.mu.Unlock() + + return rl.base + "/" + id, route, func() { + rl.mu.Lock() + delete(rl.routes, id) + rl.mu.Unlock() + } +} + +func (rl *relay) ServeHTTP(w http.ResponseWriter, r *http.Request) { + id, suffix, _ := strings.Cut(strings.TrimPrefix(r.URL.Path, "/"), "/") + rl.mu.Lock() + route := rl.routes[id] + rl.mu.Unlock() + if route == nil { + http.NotFound(w, r) + return + } + + var target string + switch { + case suffix == "info/refs" && r.Method == http.MethodGet && r.URL.RawQuery == "service=git-upload-pack": + target = route.repoURL + "/info/refs?service=git-upload-pack" + case suffix == "git-upload-pack" && r.Method == http.MethodPost: + target = route.repoURL + "/git-upload-pack" + default: + http.NotFound(w, r) + return + } + + out, err := http.NewRequestWithContext(r.Context(), r.Method, target, r.Body) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + out.ContentLength = r.ContentLength + for _, name := range []string{"Git-Protocol", "Content-Type", "Content-Encoding", "Accept", "User-Agent"} { + if value := r.Header.Get(name); value != "" { + out.Header.Set(name, value) + } + } + + resp, err := rl.client.Do(out) + if err != nil { + rl.logger.Warn("git relay: upstream request failed", "upstream", route.repoURL, "error", err) + http.Error(w, "upstream request failed", http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + // A redirect passed on would send git to the Location itself. + if resp.StatusCode >= http.StatusMultipleChoices && resp.StatusCode < http.StatusBadRequest { + http.Error(w, "upstream redirect not followed", http.StatusBadGateway) + return + } + + if r.Method == http.MethodPost && resp.StatusCode == http.StatusOK { + route.answered.Store(true) + } + if ct := resp.Header.Get("Content-Type"); ct != "" { + w.Header().Set("Content-Type", ct) + } + w.WriteHeader(resp.StatusCode) + if _, err := io.Copy(w, &routeReader{r: resp.Body, route: route}); err != nil { + if route.exceeded.Load() { + // Cut the connection so git fails rather than keeping a pack + // that merely looks short. + panic(http.ErrAbortHandler) + } + if !errors.Is(err, context.Canceled) { + rl.logger.Warn("git relay: copying upstream response", "upstream", route.repoURL, "error", err) + } + } +} + +// routeReader counts relayed bytes against the route's limit. +type routeReader struct { + r io.Reader + route *relayRoute +} + +func (rr *routeReader) Read(p []byte) (int, error) { + n, err := rr.r.Read(p) + if total := rr.route.relayed.Add(int64(n)); rr.route.limit > 0 && total > rr.route.limit { + rr.route.exceeded.Store(true) + return 0, ErrTooLarge + } + return n, err +} diff --git a/internal/gitcache/relay_test.go b/internal/gitcache/relay_test.go new file mode 100644 index 00000000..45726bcb --- /dev/null +++ b/internal/gitcache/relay_test.go @@ -0,0 +1,96 @@ +package gitcache + +import ( + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" +) + +func newTestRelay(t *testing.T) *relay { + t.Helper() + rl, err := newRelay(&http.Client{}, slog.New(slog.DiscardHandler)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = rl.close() }) + return rl +} + +func TestRelayOnlyForwardsFetchRequests(t *testing.T) { + var forwarded atomic.Int32 + upstream := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + forwarded.Add(1) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, _, unregister := rl.register(upstream.URL+"/repo", 0) + + refused := []struct{ method, url string }{ + {http.MethodGet, rl.base + "/unknown/info/refs?service=git-upload-pack"}, + {http.MethodGet, base + "/info/refs?service=git-receive-pack"}, + {http.MethodPost, base + "/git-receive-pack"}, + {http.MethodGet, base + "/HEAD"}, + {http.MethodGet, base + "/git-upload-pack"}, + } + for _, r := range refused { + req, _ := http.NewRequest(r.method, r.url, nil) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("%s %s: status %d, want 404", r.method, r.url, resp.StatusCode) + } + } + if got := forwarded.Load(); got != 0 { + t.Errorf("forwarded %d refused requests", got) + } + + unregister() + resp, err := http.Get(base + "/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound || forwarded.Load() != 0 { + t.Errorf("unregistered route: status %d, forwarded %d", resp.StatusCode, forwarded.Load()) + } +} + +// TestRelayStopsAtLimit checks a fetch over git.max_repo_size is cut off +// rather than downloaded in full and thrown away. +func TestRelayStopsAtLimit(t *testing.T) { + const size, limit = 4 << 20, 64 << 10 + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.Copy(w, io.LimitReader(zeros{}, size)) + })) + defer upstream.Close() + rl := newTestRelay(t) + base, route, unregister := rl.register(upstream.URL+"/repo", limit) + defer unregister() + + resp, err := http.Post(base+"/git-upload-pack", requestType, strings.NewReader(pktFlush)) + if err != nil { + t.Fatal(err) + } + n, err := io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if err == nil { + t.Errorf("read %d bytes without an error", n) + } + if !route.exceeded.Load() || route.relayed.Load() > size/2 { + t.Errorf("exceeded %v after relaying %d of %d bytes", route.exceeded.Load(), route.relayed.Load(), size) + } +} + +type zeros struct{} + +func (zeros) Read(p []byte) (int, error) { + clear(p) + return len(p), nil +} diff --git a/internal/gitcache/serve.go b/internal/gitcache/serve.go new file mode 100644 index 00000000..b6fab6ea --- /dev/null +++ b/internal/gitcache/serve.go @@ -0,0 +1,151 @@ +package gitcache + +import ( + "compress/gzip" + "context" + "errors" + "io" + "net/http" + "path/filepath" + "regexp" + "strings" +) + +// gitProtocolPattern is what a Git-Protocol header may hold, such as +// "version=2". It is passed to upload-pack in its environment. +var gitProtocolPattern = regexp.MustCompile(`^[A-Za-z0-9=:._-]*$`) + +var errInvalidKey = errors.New("invalid repository key") + +// Lease is a use of a cached repository. The repository is not evicted +// until the lease is released. +type Lease struct { + cache *Cache + repo *repo + repoURL string + hit bool + fetchWants bool +} + +// OpenOptions says what Open does besides returning the repository. +type OpenOptions struct { + // Refresh brings a cached repository up to date with the upstream first, + // or leaves it as it is when the upstream fails or is slow and the copy + // is recent enough (see Options). + Refresh bool + // FetchWants lets ServeUploadPack fetch commits a client asks for by ID + // that the cache lacks. Leave it off where a fetch by ID could reach + // commits the client may not read through this repository, such as + // GitHub forks with credentials configured. + FetchWants bool +} + +// Open returns a lease on the repository cached under key, a relative path +// such as github-1a2b3c4d/acme/widgets.git, whose upstream is repoURL. A +// repository not cached yet is cloned first. +func (c *Cache) Open(ctx context.Context, key, repoURL string, opts OpenOptions) (*Lease, error) { + if !strings.HasSuffix(key, ".git") || !filepath.IsLocal(filepath.FromSlash(key)) { + return nil, errInvalidKey + } + rp := c.acquire(key) + hit := rp.exists() + if opts.Refresh || !hit { + if err := c.ensureFresh(ctx, rp, repoURL); err != nil { + c.release(rp) + return nil, err + } + } + return &Lease{cache: c, repo: rp, repoURL: repoURL, hit: hit, fetchWants: opts.FetchWants}, nil +} + +// Hit reports whether the repository was already cached when it was opened. +func (l *Lease) Hit() bool { + return l.hit +} + +// Release ends the lease. +func (l *Lease) Release() { + l.cache.release(l.repo) +} + +// ServeInfoRefs answers GET info/refs?service=git-upload-pack. +func (l *Lease) ServeInfoRefs(w http.ResponseWriter, r *http.Request) { + protocol := gitProtocol(r) + setNoCache(w) + w.Header().Set("Content-Type", advertisementType) + // Protocol v2 starts with its version line; earlier versions with the + // service name. + if !isProtocolV2(protocol) { + _, _ = io.WriteString(w, pktLine("# service=git-upload-pack\n")+pktFlush) + } + l.cache.uploadPack(w, r, l.repo, protocol, nil, "--advertise-refs") +} + +// ServeUploadPack answers POST git-upload-pack. With OpenOptions.FetchWants, +// commits the request wants that the cache lacks are fetched from the +// upstream first. +func (l *Lease) ServeUploadPack(w http.ResponseWriter, r *http.Request) { + c := l.cache + body := io.Reader(r.Body) + if r.Header.Get("Content-Encoding") == "gzip" { + gz, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, "invalid gzip body", http.StatusBadRequest) + return + } + defer func() { _ = gz.Close() }() + body = gz + } + if l.fetchWants { + var wants []string + wants, body = readWants(body) + if err := c.fetchMissingWants(r.Context(), l.repo, l.repoURL, wants); err != nil && r.Context().Err() == nil { + // upload-pack reports the missing objects to the client. + c.logger.Warn("git cache: fetching wanted objects failed", "repository", l.repo.key, "error", err) + } + } + setNoCache(w) + w.Header().Set("Content-Type", resultType) + c.uploadPack(w, r, l.repo, gitProtocol(r), body) +} + +// uploadPack runs git upload-pack on the cached repository in stateless mode, +// the way git http-backend does. +func (c *Cache) uploadPack(w http.ResponseWriter, r *http.Request, rp *repo, protocol string, stdin io.Reader, args ...string) { + var env []string + if protocol != "" { + env = append(env, "GIT_PROTOCOL="+protocol) + } + args = append(append([]string{"upload-pack", "--stateless-rpc"}, args...), rp.dir) + cmd := c.command(r.Context(), env, args...) + cmd.Stdin = stdin + cmd.Stdout = w + stderr := &limitedBuffer{max: maxStderr} + cmd.Stderr = stderr + if err := cmd.Run(); err != nil && r.Context().Err() == nil { + c.logger.Warn("git cache: upload-pack failed", + "repository", rp.key, "error", err, "stderr", strings.TrimSpace(stderr.String())) + } +} + +func gitProtocol(r *http.Request) string { + if protocol := r.Header.Get("Git-Protocol"); gitProtocolPattern.MatchString(protocol) { + return protocol + } + return "" +} + +func isProtocolV2(protocol string) bool { + for field := range strings.SplitSeq(protocol, ":") { + if field == "version=2" { + return true + } + } + return false +} + +func setNoCache(w http.ResponseWriter) { + w.Header().Set("Expires", "Fri, 01 Jan 1980 00:00:00 GMT") + w.Header().Set("Pragma", "no-cache") + w.Header().Set("Cache-Control", "no-cache, max-age=0, must-revalidate") +} diff --git a/internal/gitcache/serve_test.go b/internal/gitcache/serve_test.go new file mode 100644 index 00000000..b81370e4 --- /dev/null +++ b/internal/gitcache/serve_test.go @@ -0,0 +1,196 @@ +package gitcache + +import ( + "bytes" + "compress/gzip" + "fmt" + "io" + "net/http" + "os/exec" + "slices" + "strings" + "sync" + "testing" +) + +// TestRequestBodies sends the bodies git sends for large requests: gzipped, +// or chunked once one outgrows http.postBuffer. +func TestRequestBodies(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + lsRefs := pktLine("command=ls-refs\n") + pktDelim + pktLine("ref-prefix refs/heads/\n") + pktFlush + var gzipped bytes.Buffer + gz := gzip.NewWriter(&gzipped) + _, _ = io.WriteString(gz, lsRefs) + _ = gz.Close() + for _, tt := range []struct { + name string + body io.Reader + encoding string + }{ + // A reader of unknown length is sent chunked. + {"chunked", io.MultiReader(strings.NewReader(lsRefs)), ""}, + {"gzip", &gzipped, "gzip"}, + } { + req, _ := http.NewRequest(http.MethodPost, tc.url("repo")+"/git-upload-pack", tt.body) + req.Header.Set("Content-Type", requestType) + req.Header.Set("Git-Protocol", "version=2") + if tt.encoding != "" { + req.Header.Set("Content-Encoding", tt.encoding) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + out, _ := io.ReadAll(resp.Body) + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "refs/heads/main") { + t.Errorf("%s: status %d, body %q", tt.name, resp.StatusCode, out) + } + } +} + +// TestCloneWithChunkedRequests clones a repository with enough refs that +// git's requests, to the cache and from the cache to the upstream, outgrow +// http.postBuffer and are sent chunked. +func TestCloneWithChunkedRequests(t *testing.T) { + const tags = 2000 + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + var stream strings.Builder + for i := 1; i <= tags; i++ { + fmt.Fprintf(&stream, "commit refs/heads/main\nmark :%d\ncommitter test %d +0000\ndata 1\n\n", i, i) + if i == 1 { + stream.WriteString("from refs/heads/main^0\n") + } + fmt.Fprintf(&stream, "reset refs/tags/t%d\nfrom :%d\n\n", i, i) + } + importer := exec.Command("git", "fast-import", "--quiet") + importer.Dir, importer.Env, importer.Stdin = work, gitEnv(t.TempDir()), strings.NewReader(stream.String()) + if out, err := importer.CombinedOutput(); err != nil { + t.Fatalf("git fast-import: %v\n%s", err, out) + } + runGit(t, work, "push", "--quiet", "origin", "main", "--tags") + tc := newTestCache(t, upstream.server.URL, nil) + + // One want per tag is about 100KB, past a postBuffer just above git's + // minimum. + clone := tc.clone(t, "repo", "-c", "http.postBuffer=66000") + if got := strings.Count(runGit(t, clone, "tag", "--list"), "\n") + 1; got != tags+1 { + t.Errorf("cloned %d tags, want %d", got, tags+1) + } +} + +func TestParseV0Advertisement(t *testing.T) { + oid := strings.Repeat("a", 40) + peeled := strings.Repeat("b", 40) + adv := pktLine(oid+" HEAD\x00multi_ack symref=HEAD:refs/heads/trunk agent=git/2\n") + + pktLine(oid+" refs/heads/trunk\n") + + pktLine(oid+" refs/tags/v1\n") + + pktLine(peeled+" refs/tags/v1^{}\n") + + pktLine(oid+" refs/changes/01/1/1\n") + + pktLine(oid+" refs/pull/1/head\n") + + pktFlush + pkts := newPktReader(strings.NewReader(adv)) + kind, line, err := pkts.next() + if err != nil { + t.Fatal(err) + } + remote, err := parseV0Advertisement(pkts, kind, line) + if err != nil { + t.Fatal(err) + } + if remote.head != "refs/heads/trunk" { + t.Errorf("head = %q", remote.head) + } + want := map[string]string{"refs/heads/trunk": oid, "refs/tags/v1": oid} + if len(remote.refs) != len(want) || remote.refs["refs/heads/trunk"] != oid || remote.refs["refs/tags/v1"] != oid { + t.Errorf("refs = %v, want %v", remote.refs, want) + } +} + +// TestCommitOnNoBranchIsFetchedByID does what actions/checkout does: fetch +// one commit by object ID, here a pull request head that is on no branch. +func TestCommitOnNoBranchIsFetchedByID(t *testing.T) { + for _, version := range []string{"0", "2"} { + t.Run("protocol v"+version, func(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + runGit(t, work, "checkout", "--quiet", "-b", "pr") + sha := upstream.commit(work, "pull request") + runGit(t, work, "push", "--quiet", "origin", "pr:refs/pull/1/head") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + sessions := upstream.gitSessions.Load() + + var wg sync.WaitGroup + for range 4 { + wg.Go(func() { + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if out, err := tryGit(t, dir, "-c", "protocol.version="+version, "fetch", "--quiet", "--depth=1", tc.url("repo"), sha); err != nil { + t.Errorf("fetch by ID: %v\n%s", err, out) + return + } + if got := runGit(t, dir, "rev-parse", "FETCH_HEAD"); got != sha { + t.Errorf("FETCH_HEAD = %s, want %s", got, sha) + } + }) + } + wg.Wait() + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches for the commit, want 1", got) + } + if refs := runGit(t, "", "ls-remote", tc.url("repo")); strings.Contains(refs, wantedPrefix) { + t.Errorf("advertised %s refs:\n%s", wantedPrefix, refs) + } + + // A refresh that prunes branches keeps the commit. + runGit(t, work, "push", "--quiet", "origin", "main:refs/heads/new") + tc.clone(t, "repo") + if refs := tc.cachedRefs(t); !strings.Contains(refs, wantedPrefix) || !strings.Contains(refs, "/"+sha) { + t.Errorf("refresh dropped the fetched commit:\n%s", refs) + } + }) + } +} + +func TestReadWants(t *testing.T) { + a, b := strings.Repeat("a", 40), strings.Repeat("b", 40) + tests := map[string]struct { + body string + want []string + }{ + "v0": { + pktLine("want "+a+" multi_ack side-band-64k\n") + pktLine("want "+b+"\n") + pktLine("deepen 1\n") + pktFlush + pktLine("have "+a+"\n") + pktFlush, + []string{a, b}, + }, + "v2 fetch": { + pktLine("command=fetch\n") + pktLine("agent=git/2\n") + pktDelim + pktLine("thin-pack\n") + pktLine("want "+a+"\n") + pktLine("have "+b+"\n") + pktLine("want "+b+"\n") + pktLine("done\n") + pktFlush, + []string{a, b}, + }, + "v2 ls-refs": { + pktLine("command=ls-refs\n") + pktDelim + pktLine("want "+a+"\n") + pktFlush, + nil, + }, + "not an object ID": { + pktLine("want HEAD\n") + pktFlush, + nil, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + wants, replay := readWants(strings.NewReader(tt.body)) + if !slices.Equal(wants, tt.want) { + t.Errorf("wants = %v, want %v", wants, tt.want) + } + if got, _ := io.ReadAll(replay); string(got) != tt.body { + t.Errorf("replayed %q, want the whole body", got) + } + }) + } +} diff --git a/internal/gitcache/upstream.go b/internal/gitcache/upstream.go new file mode 100644 index 00000000..a21f39df --- /dev/null +++ b/internal/gitcache/upstream.go @@ -0,0 +1,210 @@ +package gitcache + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" +) + +const ( + advertisementType = "application/x-git-upload-pack-advertisement" + requestType = "application/x-git-upload-pack-request" + resultType = "application/x-git-upload-pack-result" + // Some servers only speak the smart protocol to clients that look like git. + gitUserAgent = "git/2 (git-pkgs-proxy)" + // maxRefsResponse bounds a ref listing; branches and tags of even large + // repositories take a few megabytes. + maxRefsResponse = 64 << 20 + + headsPrefix = "refs/heads/" + tagsPrefix = "refs/tags/" +) + +var ( + // ErrNotFound is returned when the upstream has no such repository. + ErrNotFound = errors.New("repository not found upstream") + // ErrRefused covers authentication failures and redirects. The cached + // copy is not served after one, since the upstream may have withdrawn + // access to it. + ErrRefused = errors.New("upstream refused the request") + // ErrTooLarge is returned for a repository larger than git.max_repo_size. + ErrTooLarge = errors.New("repository is larger than git.max_repo_size") +) + +// remoteRefs is what a repository advertises: its branch and tag tips, and +// the branch HEAD points at. +type remoteRefs struct { + refs map[string]string + head string +} + +// listUpstreamRefs asks the upstream for its branches, tags and HEAD. With +// protocol v2 the server filters by prefix, which matters for Gerrit, where +// every patch set is a ref. Servers without v2 send every ref, and the rest +// are dropped here. +func (c *Cache) listUpstreamRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + resp, err := c.upstreamRequest(ctx, http.MethodGet, repoURL+"/info/refs?service=git-upload-pack", "") + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + if ct := resp.Header.Get("Content-Type"); !strings.HasPrefix(ct, advertisementType) { + return remoteRefs{}, fmt.Errorf("upstream did not answer as a smart HTTP Git server (content type %q)", ct) + } + + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + kind, line, err := pkts.next() + if err == nil && kind == pktData && strings.HasPrefix(line, "# service=") { + if kind, _, err = pkts.next(); err == nil && kind != pktFlushKind { + err = errors.New("no flush after the service line") + } + if err == nil { + kind, line, err = pkts.next() + } + } + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + + if kind == pktData && line == "version 2" { + return c.lsRefs(ctx, repoURL) + } + if kind == pktData && line == "version 1" { + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return parseV0Advertisement(pkts, kind, line) +} + +// lsRefs runs the protocol v2 ls-refs command. +func (c *Cache) lsRefs(ctx context.Context, repoURL string) (remoteRefs, error) { + body := pktLine("command=ls-refs\n") + pktDelim + + pktLine("symrefs\n") + + pktLine("ref-prefix HEAD\n") + + pktLine("ref-prefix "+headsPrefix+"\n") + + pktLine("ref-prefix "+tagsPrefix+"\n") + + pktFlush + resp, err := c.upstreamRequest(ctx, http.MethodPost, repoURL+"/git-upload-pack", body) + if err != nil { + return remoteRefs{}, err + } + defer func() { _ = resp.Body.Close() }() + + remote := remoteRefs{refs: make(map[string]string)} + pkts := newPktReader(io.LimitReader(resp.Body, maxRefsResponse)) + for { + kind, line, err := pkts.next() + if err != nil { + return remoteRefs{}, fmt.Errorf("reading ls-refs response: %w", err) + } + if kind == pktFlushKind { + return remote, nil + } + if kind != pktData { + continue + } + fields := strings.Split(line, " ") + if len(fields) < 2 { //nolint:mnd // object id and ref name + return remoteRefs{}, fmt.Errorf("invalid ls-refs line %q", line) + } + if fields[1] == "HEAD" { + for _, attr := range fields[2:] { + if target, ok := strings.CutPrefix(attr, "symref-target:"); ok { + remote.head = target + } + } + continue + } + remote.add(fields[1], fields[0]) + } +} + +// parseV0Advertisement reads a protocol v0 ref advertisement whose first +// packet has already been read. The first ref carries the capabilities, +// including where HEAD points. +func parseV0Advertisement(pkts *pktReader, kind pktKind, line string) (remoteRefs, error) { + remote := remoteRefs{refs: make(map[string]string)} + first := true + for kind != pktFlushKind { + if kind == pktData { + ref, caps, _ := strings.Cut(line, "\x00") + if first { + remote.head = symrefTarget(caps, "HEAD") + first = false + } + oid, name, ok := strings.Cut(ref, " ") + if !ok { + return remoteRefs{}, fmt.Errorf("invalid ref advertisement line %q", ref) + } + remote.add(name, oid) + } + var err error + if kind, line, err = pkts.next(); err != nil { + return remoteRefs{}, fmt.Errorf("reading ref advertisement: %w", err) + } + } + return remote, nil +} + +// add keeps branches and tags, leaving out peeled tag entries. +func (r remoteRefs) add(name, oid string) { + if (strings.HasPrefix(name, headsPrefix) || strings.HasPrefix(name, tagsPrefix)) && !strings.HasSuffix(name, "^{}") { + r.refs[name] = oid + } +} + +func symrefTarget(caps, ref string) string { + for _, capability := range strings.Fields(caps) { + if value, ok := strings.CutPrefix(capability, "symref="+ref+":"); ok { + return value + } + } + return "" +} + +// upstreamRequest sends a smart HTTP request through the upstream client and +// classifies refusals. A non-empty body is sent as an upload-pack request. +func (c *Cache) upstreamRequest(ctx context.Context, method, target, body string) (*http.Response, error) { + var reader io.Reader + if body != "" { + reader = strings.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, method, target, reader) + if err != nil { + return nil, err + } + req.Header.Set("Git-Protocol", "version=2") + req.Header.Set("User-Agent", gitUserAgent) + if body != "" { + req.Header.Set("Content-Type", requestType) + req.Header.Set("Accept", resultType) + } + resp, err := c.client.Do(req) + if err != nil { + return nil, fmt.Errorf("upstream request: %w", err) + } + if err := upstreamStatusError(resp.StatusCode); err != nil { + _ = resp.Body.Close() + return nil, err + } + return resp, nil +} + +func upstreamStatusError(status int) error { + switch { + case status == http.StatusOK: + return nil + case status == http.StatusNotFound || status == http.StatusGone: + return fmt.Errorf("%w: status %d", ErrNotFound, status) + case status == http.StatusUnauthorized || status == http.StatusForbidden: + return fmt.Errorf("%w: status %d", ErrRefused, status) + case status >= http.StatusMultipleChoices && status < http.StatusBadRequest: + return fmt.Errorf("%w: redirect (status %d) not followed", ErrRefused, status) + default: + return fmt.Errorf("upstream returned status %d", status) + } +} diff --git a/internal/gitcache/wants.go b/internal/gitcache/wants.go new file mode 100644 index 00000000..eec8fc45 --- /dev/null +++ b/internal/gitcache/wants.go @@ -0,0 +1,264 @@ +package gitcache + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "maps" + "slices" + "strconv" + "strings" + "time" +) + +const ( + // wantedPrefix holds commits fetched because a client asked for them by + // object ID, such as a pull request head that is on no branch, as + // {wantedPrefix}{unix time}/{object ID}. The refs keep them from being + // garbage collected, and are hidden from clients. + wantedPrefix = "refs/proxy/wanted/" + // wantedMaxAge is how long a commit fetched by ID is kept while no + // cached branch or tag includes it. + wantedMaxAge = 7 * 24 * time.Hour + // wantRetryAfter is how long a want the upstream could not provide is + // not asked for again. Short, since a commit can reach a CI job before + // it reaches every upstream replica. A want too large for what + // max_repo_size leaves waits tooLargeRetryAfter instead. + wantRetryAfter = time.Minute + // maxFailedWants bounds the record of failed wants. + maxFailedWants = 4096 + // maxWantScan bounds how much of a request body is read for its wants. + maxWantScan = 1 << 20 + // maxWantFetch bounds the missing objects one request may have fetched. + maxWantFetch = 32 + // maxWantSplit bounds the wants of a failed fetch that are fetched again + // one at a time; a larger batch is refused as a whole. + maxWantSplit = 4 +) + +// readWants reads the object IDs a fetch request wants, from its first +// section, and returns them with a reader that replays the whole body. In +// protocol v0 that section holds the wants; in v2 it is the whole request. +func readWants(body io.Reader) ([]string, io.Reader) { + var consumed bytes.Buffer + pkts := newPktReader(io.TeeReader(io.LimitReader(body, maxWantScan), &consumed)) + var wants []string + for first := true; ; first = false { + kind, line, err := pkts.next() + if err != nil || kind == pktFlushKind { + break + } + if kind != pktData { + continue + } + if command, ok := strings.CutPrefix(line, "command="); ok && first && command != "fetch" { + break + } + if rest, ok := strings.CutPrefix(line, "want "); ok { + if oid, _, _ := strings.Cut(rest, " "); isObjectID(oid) { + wants = append(wants, oid) + } + } + } + return wants, io.MultiReader(&consumed, body) +} + +// fetchMissingWants fetches the wanted objects the repository lacks. Wants +// for branch and tag tips are already there after the refresh; this covers +// commits a client names directly, as actions/checkout does. +func (c *Cache) fetchMissingWants(ctx context.Context, rp *repo, repoURL string, wants []string) error { + missing, err := c.missingObjects(ctx, rp.dir, wants) + if err != nil { + return err + } + missing = c.withoutFailedWants(rp.key, missing) + if len(missing) == 0 || len(missing) > maxWantFetch { + return nil + } + unlock, err := rp.lockWrites(ctx) + if err != nil { + return err + } + defer unlock() + // Another request may have fetched them while this one waited. + if missing, err = c.missingObjects(ctx, rp.dir, missing); err != nil || len(missing) == 0 { + return err + } + defer c.evict() + + err = c.fetchWants(ctx, rp, repoURL, missing) + if err == nil || ctx.Err() != nil { + return err + } + if len(missing) == 1 || len(missing) > maxWantSplit || + errors.Is(err, ErrTooLarge) || errors.Is(err, errUpstreamUnanswered) { + // Fetched one at a time, wants too large for what max_repo_size + // leaves would exhaust the same budget, and an upstream that did + // not answer would be waited on once per want. + c.recordFailedWants(rp.key, missing, err) + return err + } + // Find which wants failed, so a good one asked for with a bad one is + // not refused too. These fetches hold the repository's write lock, so + // only a small batch is split. + var errs []error + for _, oid := range missing { + if err := c.fetchWants(ctx, rp, repoURL, []string{oid}); err != nil { + if ctx.Err() != nil { + return err + } + c.recordFailedWants(rp.key, []string{oid}, err) + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// fetchWants fetches oids from the upstream under wanted refs, within what +// max_repo_size leaves. A want that does not fit is refused, not the +// repository: transfer.unpackLimit makes a cut-off transfer leave nothing +// behind. +func (c *Cache) fetchWants(ctx context.Context, rp *repo, repoURL string, oids []string) error { + limit, err := c.fetchLimit(rp) + if err != nil { + return err + } + now := time.Now() + refspecs := make([]string, len(oids)) + for i, oid := range oids { + refspecs[i] = oid + ":" + wantedRef(oid, now) + } + err = c.fetch(ctx, rp.dir, repoURL, limit, []string{"--atomic"}, refspecs...) + rp.size.Store(dirSize(rp.dir)) + return err +} + +func wantedRef(oid string, at time.Time) string { + return wantedPrefix + strconv.FormatInt(at.Unix(), 10) + "/" + oid +} + +// pruneWanted deletes wanted refs that are no longer needed: those older +// than wantedMaxAge, and, after a fetch has moved branches or tags, those +// whose commit a cached branch or tag now includes. +func (c *Cache) pruneWanted(ctx context.Context, dir string, wanted []string, fetched bool) error { + if len(wanted) == 0 { + return nil + } + cutoff := time.Now().Add(-wantedMaxAge).Unix() + var drop []string + keep := make(map[string]string, len(wanted)) + for _, ref := range wanted { + created, oid, ok := strings.Cut(strings.TrimPrefix(ref, wantedPrefix), "/") + unix, err := strconv.ParseInt(created, 10, 64) + if !ok || err != nil || unix < cutoff || !isObjectID(oid) { + drop = append(drop, ref) + continue + } + keep[oid] = ref + } + var mergedErr error + if fetched && len(keep) > 0 { + // rev-list prints what the wanted commits, read from stdin, reach + // and no branch or tag does, so a wanted commit it leaves out is + // merged. + cmd := c.command(ctx, nil, "-C", dir, "rev-list", "--stdin", "--not", "--branches", "--tags") + cmd.Stdin = strings.NewReader(strings.Join(slices.Sorted(maps.Keys(keep)), "\n") + "\n") + out, err := cmd.Output() + if err != nil { + mergedErr = fmt.Errorf("listing unmerged wanted commits: %w", err) + } else { + unmerged := make(map[string]bool) + for line := range strings.Lines(string(out)) { + unmerged[strings.TrimSpace(line)] = true + } + for oid, ref := range keep { + if !unmerged[oid] { + drop = append(drop, ref) + } + } + } + } + if len(drop) == 0 { + return mergedErr + } + var stdin strings.Builder + for _, ref := range drop { + stdin.WriteString("delete " + ref + "\n") + } + cmd := c.command(ctx, nil, "-C", dir, "update-ref", "--stdin") + cmd.Stdin = strings.NewReader(stdin.String()) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("deleting wanted refs: %w: %s", err, strings.TrimSpace(string(out))) + } + return mergedErr +} + +// withoutFailedWants drops the object IDs a recent fetch for the +// repository could not get. +func (c *Cache) withoutFailedWants(key string, oids []string) []string { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + return slices.DeleteFunc(oids, func(oid string) bool { + until, ok := c.failedWants[key+" "+oid] + return ok && now.Before(until) + }) +} + +// recordFailedWants keeps oids from being fetched again for the repository +// for wantRetryAfter, or for tooLargeRetryAfter when they did not fit in +// max_repo_size. +func (c *Cache) recordFailedWants(key string, oids []string, err error) { + now := time.Now() + until := now.Add(wantRetryAfter) + if errors.Is(err, ErrTooLarge) { + until = now.Add(tooLargeRetryAfter) + } + c.mu.Lock() + defer c.mu.Unlock() + for k, expiry := range c.failedWants { + if !now.Before(expiry) { + delete(c.failedWants, k) + } + } + for _, oid := range oids { + if len(c.failedWants) >= maxFailedWants { + return + } + c.failedWants[key+" "+oid] = until + } +} + +// missingObjects returns the object IDs in oids the repository does not have. +func (c *Cache) missingObjects(ctx context.Context, dir string, oids []string) ([]string, error) { + if len(oids) == 0 { + return nil, nil + } + cmd := c.command(ctx, nil, "-C", dir, "cat-file", "--batch-check") + cmd.Stdin = strings.NewReader(strings.Join(oids, "\n") + "\n") + out, err := cmd.Output() + if err != nil { + return nil, err + } + var missing []string + for line := range strings.Lines(string(out)) { + if oid, ok := strings.CutSuffix(strings.TrimSuffix(line, "\n"), " missing"); ok { + missing = append(missing, oid) + } + } + return missing, nil +} + +func isObjectID(s string) bool { + if len(s) != 40 && len(s) != 64 { //nolint:mnd // SHA-1 and SHA-256 hex lengths + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') { + return false + } + } + return true +} diff --git a/internal/gitcache/wants_test.go b/internal/gitcache/wants_test.go new file mode 100644 index 00000000..f01df062 --- /dev/null +++ b/internal/gitcache/wants_test.go @@ -0,0 +1,319 @@ +package gitcache + +import ( + "crypto/rand" + "encoding/hex" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" +) + +// pushPullRequest commits on a new branch from main, with data in a file if +// given, and pushes it only as refs/pull/{n}/head, so the commit is on no +// branch or tag upstream. +func (u *fakeUpstream) pushPullRequest(work string, n int, data []byte) string { + u.t.Helper() + branch := "pr" + strconv.Itoa(n) + runGit(u.t, work, "checkout", "--quiet", "-b", branch, "main") + if data != nil { + if err := os.WriteFile(filepath.Join(work, branch+".bin"), data, 0o600); err != nil { + u.t.Fatal(err) + } + runGit(u.t, work, "add", branch+".bin") + } + runGit(u.t, work, "commit", "--quiet", "--allow-empty", "-m", branch) + runGit(u.t, work, "push", "--quiet", "origin", "HEAD:refs/pull/"+strconv.Itoa(n)+"/head") + runGit(u.t, work, "checkout", "--quiet", "main") + return runGit(u.t, work, "rev-parse", branch) +} + +// fetchByID fetches one commit by ID from url, as actions/checkout does. +func fetchByID(t *testing.T, url, sha string) error { + t.Helper() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + _, err := tryGit(t, dir, "fetch", "--quiet", "--depth=1", url, sha) + return err +} + +func randomBytes(n int) []byte { + b := make([]byte, n) + _, _ = rand.Read(b) + return b +} + +func (tc *testCache) wantedRefs(t *testing.T) []string { + t.Helper() + var refs []string + for line := range strings.Lines(tc.cachedRefs(t)) { + if _, name, _ := strings.Cut(strings.TrimSpace(line), " "); strings.HasPrefix(name, wantedPrefix) { + refs = append(refs, name) + } + } + return refs +} + +func TestMergedWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + if refs := tc.wantedRefs(t); len(refs) != 1 { + t.Fatalf("wanted refs %v, want one", refs) + } + + runGit(t, work, "push", "--quiet", "origin", "pr1:main") + tc.clone(t, "repo") + if refs := tc.wantedRefs(t); len(refs) != 0 { + t.Errorf("wanted refs %v after the commit was merged, want none", refs) + } +} + +func TestOldWantedCommitIsUnpinned(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + old := upstream.pushPullRequest(work, 1, nil) + recent := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + for _, sha := range []string{old, recent} { + if err := fetchByID(t, tc.url("repo"), sha); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + } + + cached := filepath.Join(tc.dir, "up", "repo.git") + for _, ref := range tc.wantedRefs(t) { + if strings.HasSuffix(ref, "/"+old) { + runGit(t, cached, "update-ref", "-d", ref) + } + } + runGit(t, cached, "update-ref", wantedRef(old, time.Now().Add(-wantedMaxAge-time.Hour)), old) + + tc.clone(t, "repo") + refs := tc.wantedRefs(t) + if len(refs) != 1 || !strings.HasSuffix(refs[0], "/"+recent) { + t.Errorf("wanted refs %v, want only the recent commit's", refs) + } +} + +// TestRepositoryTooLargeToRefreshIsClonedAgain fills a repository with a +// commit fetched by ID, then grows its branch upstream past what is left of +// max_repo_size. Rather than stop following the upstream, the repository is +// dropped once idle and cloned again with only its branches and tags, which +// fit. +func TestRepositoryTooLargeToRefreshIsClonedAgain(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + pinned := upstream.pushPullRequest(work, 1, randomBytes(100<<10)) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 150<<10 }) + tc.clone(t, "repo") + if err := fetchByID(t, tc.url("repo"), pinned); err != nil { + t.Fatalf("fetch by ID: %v", err) + } + + if err := os.WriteFile(filepath.Join(work, "grown.bin"), randomBytes(80<<10), 0o600); err != nil { + t.Fatal(err) + } + runGit(t, work, "add", "grown.bin") + head := upstream.commit(work, "grown") + runGit(t, work, "push", "--quiet", "origin", "main") + + // This clone's refresh does not fit. It is served the cached copy, or, + // if the repository is dropped between its requests, the new clone. + tc.clone(t, "repo") + if got := runGit(t, tc.clone(t, "repo"), "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s once cloned again", got, head) + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", pinned); err == nil { + t.Error("the repository still holds the commit fetched by ID, so it was not cloned again") + } +} + +// TestTooLargeWantKeepsRepository asks for a commit by ID that does not fit +// in what max_repo_size leaves. The commit is refused, for a while, and the +// repository is kept, with nothing of the cut-off transfer left in it. +func TestTooLargeWantKeepsRepository(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + large := upstream.pushPullRequest(work, 1, randomBytes(200<<10)) + small := upstream.pushPullRequest(work, 2, nil) + base := repoSize(t, upstream, "repo") + tc := newTestCache(t, upstream.server.URL, func(o *Options) { o.MaxRepoSize = base + 100<<10 }) + tc.clone(t, "repo") + + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched a commit larger than max_repo_size leaves") + } + waitIdle(t, tc.Cache) + cached := filepath.Join(tc.dir, "up", "repo.git") + if !fileExists(cached) { + t.Fatal("the repository was dropped") + } + if _, err := tryGit(t, cached, "cat-file", "-e", large); err == nil { + t.Error("the cut-off transfer left the commit behind") + } + if leftovers, _ := filepath.Glob(filepath.Join(cached, "objects", "pack", "tmp_*")); len(leftovers) != 0 { + t.Errorf("the cut-off transfer left %v", leftovers) + } + tc.mu.Lock() + until := tc.failedWants["up/repo.git "+large] + tc.mu.Unlock() + if time.Until(until) <= wantRetryAfter { + t.Errorf("the large commit is refused until %s, want for tooLargeRetryAfter", until) + } + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), large); err == nil { + t.Fatal("fetched the large commit on a second try") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second try made %d upstream fetch requests, want none", got) + } + if err := fetchByID(t, tc.url("repo"), small); err != nil { + t.Errorf("fetch of another commit by ID: %v", err) + } +} + +// TestWantsFetchedTogetherFailSeparately asks for a good and a bad commit +// in one request. The good one is still fetched, and only the bad one is +// refused for a while. +func TestWantsFetchedTogetherFailSeparately(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + good := upstream.pushPullRequest(work, 1, nil) + bad := hex.EncodeToString(randomBytes(20)) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), good, bad); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + waitIdle(t, tc.Cache) + if _, err := tryGit(t, filepath.Join(tc.dir, "up", "repo.git"), "cat-file", "-e", good); err != nil { + t.Error("the good commit was not fetched") + } + tc.mu.Lock() + _, goodRefused := tc.failedWants["up/repo.git "+good] + _, badRefused := tc.failedWants["up/repo.git "+bad] + tc.mu.Unlock() + if goodRefused || !badRefused { + t.Errorf("refused good=%v bad=%v, want only the bad commit", goodRefused, badRefused) + } +} + +func TestFailedWantIsNotRetriedAtOnce(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + missing := hex.EncodeToString(randomBytes(20)) + + before := upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if upstream.gitRequests.Load() == before { + t.Fatal("the first request did not ask the upstream") + } + before = upstream.gitRequests.Load() + if err := fetchByID(t, tc.url("repo"), missing); err == nil { + t.Fatal("fetched a commit the upstream does not have") + } + if got := upstream.gitRequests.Load() - before; got != 0 { + t.Errorf("the second request made %d upstream fetch requests, want none", got) + } +} + +func TestWantsAreNotFetchedWhenTurnedOff(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + sha := upstream.pushPullRequest(work, 1, nil) + tc := newTestCache(t, upstream.server.URL, nil) + server := httptest.NewServer(testFront(tc.Cache, upstream.server.URL, false)) + defer server.Close() + url := server.URL + "/git/up/repo" + runGit(t, "", "clone", "--quiet", url, filepath.Join(t.TempDir(), "clone")) + + sessions := upstream.gitSessions.Load() + if err := fetchByID(t, url, sha); err == nil { + t.Error("fetched a commit by ID with FetchWants off") + } + if got := upstream.gitSessions.Load() - sessions; got != 0 { + t.Errorf("upstream saw %d fetches, want none", got) + } +} + +// TestWantsDuringAnOutageAreNotFetchedOneByOne asks for two commits while +// the upstream is down. One failed fetch records both, rather than a fetch +// per commit, each waiting on the upstream. +func TestWantsDuringAnOutageAreNotFetchedOneByOne(t *testing.T) { + upstream := newFakeUpstream(t) + work := upstream.newRepo("repo") + runGit(t, upstream.bare(), "config", "uploadpack.allowAnySHA1InWant", "true") + first := upstream.pushPullRequest(work, 1, nil) + second := upstream.pushPullRequest(work, 2, nil) + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + upstream.status.Store(http.StatusServiceUnavailable) + sessions := upstream.gitSessions.Load() + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + if _, err := tryGit(t, dir, "fetch", "--quiet", tc.url("repo"), first, second); err == nil { + t.Fatal("fetched commits while the upstream was down") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } + tc.mu.Lock() + _, firstRefused := tc.failedWants["up/repo.git "+first] + _, secondRefused := tc.failedWants["up/repo.git "+second] + tc.mu.Unlock() + if !firstRefused || !secondRefused { + t.Errorf("refused first=%v second=%v, want both", firstRefused, secondRefused) + } +} + +// TestLargeBatchOfFailedWantsIsNotSplit asks for more unknown commits at +// once than are fetched again one by one, which would hold the repository's +// write lock for a fetch per commit. +func TestLargeBatchOfFailedWantsIsNotSplit(t *testing.T) { + upstream := newFakeUpstream(t) + upstream.newRepo("repo") + tc := newTestCache(t, upstream.server.URL, nil) + tc.clone(t, "repo") + + args := []string{"fetch", "--quiet", tc.url("repo")} + for range maxWantSplit + 1 { + args = append(args, hex.EncodeToString(randomBytes(20))) + } + dir := t.TempDir() + runGit(t, dir, "init", "--quiet") + sessions := upstream.gitSessions.Load() + if _, err := tryGit(t, dir, args...); err == nil { + t.Fatal("fetched commits the upstream does not have") + } + if got := upstream.gitSessions.Load() - sessions; got != 1 { + t.Errorf("upstream saw %d fetches, want 1", got) + } +} diff --git a/internal/handler/git.go b/internal/handler/git.go new file mode 100644 index 00000000..cdaeae2f --- /dev/null +++ b/internal/handler/git.go @@ -0,0 +1,213 @@ +package handler + +import ( + "errors" + "net/http" + "net/url" + "strings" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" + "github.com/git-pkgs/proxy/internal/metrics" +) + +const ( + gitEcosystem = "git" + + gitInfoRefs = "/info/refs" + gitUploadPack = "/git-upload-pack" + gitReceivePack = "/git-receive-pack" + gitRequestType = "application/x-git-upload-pack-request" + + // gitRequestTimeout replaces the server's read and write timeouts for Git + // requests: a cold clone of a large repository keeps the client waiting + // before the first byte, a large pack takes a while to send, and the rest + // of a request body is read only after any fetch it needs. + gitRequestTimeout = 2 * time.Hour +) + +// GitHandler serves Git repositories read-only over smart HTTP at +// /git/{name}/{repo}, from a local cache of the upstream.git hosts. Clients +// clone and fetch with ordinary git commands, usually through an insteadOf +// rewrite. Pushing and the dumb protocol are refused. +type GitHandler struct { + proxy *Proxy + cache *gitcache.Cache + upstreams map[string]gitUpstream +} + +type gitUpstream struct { + config.GitUpstream + // dir names the upstream's directory in the cache (gitcache.UpstreamDir). + dir string + // fetchWants is off when upstream.auth has credentials for the host. A + // fetch by ID there could reach commits outside the repos allowlist, + // as GitHub serves any commit in a repository's fork network. + fetchWants bool +} + +// NewGitHandler creates a Git handler serving upstream.git from cache. +func NewGitHandler(proxy *Proxy, cache *gitcache.Cache, upstream config.UpstreamConfig) *GitHandler { + h := &GitHandler{proxy: proxy, cache: cache, upstreams: make(map[string]gitUpstream, len(upstream.Git))} + for name, git := range upstream.Git { + git.URL = strings.TrimSuffix(git.URL, "/") + h.upstreams[name] = gitUpstream{ + GitUpstream: git, + dir: gitcache.UpstreamDir(name, git.URL), + fetchWants: !upstream.HasAuthWithin(git.URL), + } + } + return h +} + +// Routes returns the HTTP handler for Git requests. Mount this at /git. +func (h *GitHandler) Routes() http.Handler { + return http.HandlerFunc(h.serve) +} + +// gitRequest is a parsed /git/{name}/{repo}/{endpoint} request. +type gitRequest struct { + infoRefs bool + fetchWants bool + // key names the repository in the cache: {name}-{URL hash}/{repo}.git, + // with any .git the client gave removed from repo first, so both + // spellings share one copy. + key string + // repoURL is the upstream repository, spelled as the client spelled it. + repoURL string +} + +func (h *GitHandler) serve(w http.ResponseWriter, r *http.Request) { + req, status, msg := h.parse(r) + if status != 0 { + http.Error(w, msg, status) + return + } + deadline := time.Now().Add(gitRequestTimeout) + rc := http.NewResponseController(w) + if err := errors.Join(rc.SetReadDeadline(deadline), rc.SetWriteDeadline(deadline)); err != nil { + h.proxy.Logger.Warn("git: cannot extend the request deadlines", "error", err) + } + // upload-pack writes its response while its stdin is still attached to + // the request body. Without full duplex, an HTTP/1 server reads off and + // closes whatever body is left once the response is flushed. + if !req.infoRefs { + if err := rc.EnableFullDuplex(); err != nil { + h.proxy.Logger.Warn("git: cannot enable full duplex", "error", err) + } + } + + lease, err := h.cache.Open(r.Context(), req.key, req.repoURL, + gitcache.OpenOptions{Refresh: req.infoRefs, FetchWants: req.fetchWants}) + if err != nil { + h.serveError(w, r, err) + return + } + defer lease.Release() + if !req.infoRefs { + lease.ServeUploadPack(w, r) + return + } + if lease.Hit() { + metrics.RecordCacheHit(gitEcosystem) + } else { + metrics.RecordCacheMiss(gitEcosystem) + } + lease.ServeInfoRefs(w, r) +} + +func (h *GitHandler) parse(r *http.Request) (gitRequest, int, string) { + path := strings.TrimPrefix(r.URL.Path, "/") + var req gitRequest + switch { + case strings.HasSuffix(path, gitInfoRefs): + path = strings.TrimSuffix(path, gitInfoRefs) + req.infoRefs = true + switch r.URL.Query().Get("service") { + case "git-upload-pack": + case "git-receive-pack": + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusForbidden, "only the smart HTTP protocol is supported" + } + if r.Method != http.MethodGet { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + case strings.HasSuffix(path, gitUploadPack): + path = strings.TrimSuffix(path, gitUploadPack) + if r.Method != http.MethodPost { + return req, http.StatusMethodNotAllowed, "method not allowed" + } + if r.Header.Get("Content-Type") != gitRequestType { + return req, http.StatusUnsupportedMediaType, "unsupported content type" + } + case strings.HasSuffix(path, gitReceivePack): + return req, http.StatusForbidden, "pushing is not supported" + default: + return req, http.StatusNotFound, "not found" + } + + name, repoPath, _ := strings.Cut(path, "/") + upstream, ok := h.upstreams[name] + if !ok { + return req, http.StatusNotFound, "unknown upstream" + } + segments := strings.Split(repoPath, "/") + last := len(segments) - 1 + suffix := "" + if trimmed, ok := strings.CutSuffix(segments[last], ".git"); ok { + segments[last], suffix = trimmed, ".git" + } + escaped := make([]string, len(segments)) + for i, segment := range segments { + if !validGitSegment(segment, i == last) { + return req, http.StatusNotFound, "invalid repository path" + } + escaped[i] = url.PathEscape(segment) + } + repo := strings.Join(segments, "/") + if !upstream.AllowsRepo(repo) { + // As for a missing repository, so a prober learns nothing about + // the allowlist. + return req, http.StatusNotFound, "repository not found" + } + req.key = upstream.dir + "/" + repo + ".git" + req.repoURL = upstream.URL + "/" + strings.Join(escaped, "/") + suffix + req.fetchWants = upstream.fetchWants + return req, 0, "" +} + +// validGitSegment checks one segment of a repository path, which becomes both +// part of a directory under the cache and part of the upstream URL. Only the +// last segment may end in .git, so one repository's directory never sits +// inside another's. +func validGitSegment(segment string, last bool) bool { + if segment == "" || segment == "." || segment == ".." || (!last && strings.HasSuffix(segment, ".git")) { + return false + } + for _, r := range segment { + if r < ' ' || r == 0x7f || r == '\\' { + return false + } + } + return true +} + +// serveError answers a request whose repository could not be opened. The +// cache has already logged why. +func (h *GitHandler) serveError(w http.ResponseWriter, r *http.Request, err error) { + if r.Context().Err() != nil { + return + } + switch { + case errors.Is(err, gitcache.ErrNotFound): + http.Error(w, "repository not found", http.StatusNotFound) + case errors.Is(err, gitcache.ErrRefused): + http.Error(w, "upstream refused the request", http.StatusBadGateway) + case errors.Is(err, gitcache.ErrTooLarge): + http.Error(w, "repository is larger than git.max_repo_size", http.StatusInsufficientStorage) + default: + http.Error(w, "upstream unavailable", http.StatusBadGateway) + } +} diff --git a/internal/handler/git_test.go b/internal/handler/git_test.go new file mode 100644 index 00000000..ec0b6306 --- /dev/null +++ b/internal/handler/git_test.go @@ -0,0 +1,247 @@ +package handler + +import ( + "fmt" + "io" + "log/slog" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" + "github.com/git-pkgs/proxy/internal/gitcache" +) + +func TestParseGitRequest(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{ + "up": {URL: "https://git.example/r/"}, + "limited": {URL: "https://git.example", Repos: []string{"allowed/*"}}, + }}) + up, limited := h.upstreams["up"].dir, h.upstreams["limited"].dir + tests := []struct { + method, target string + status int + key, repoURL string + }{ + {"GET", "/up/repo.git/info/refs?service=git-upload-pack", 0, up + "/repo.git", "https://git.example/r/repo.git"}, + {"GET", "/up/a/b/info/refs?service=git-upload-pack", 0, up + "/a/b.git", "https://git.example/r/a/b"}, + {"POST", "/up/a/b%20c/git-upload-pack", 0, up + "/a/b c.git", "https://git.example/r/a/b%20c"}, + {"GET", "/up/a.git/b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a/../b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/a//b/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/.git/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + {"GET", "/up/repo/info/refs?service=git-receive-pack", http.StatusForbidden, "", ""}, + {"POST", "/up/repo/git-receive-pack", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/info/refs", http.StatusForbidden, "", ""}, + {"GET", "/up/repo/objects/info/packs", http.StatusNotFound, "", ""}, + {"POST", "/up/repo/info/refs?service=git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/up/repo/git-upload-pack", http.StatusMethodNotAllowed, "", ""}, + {"GET", "/limited/allowed/repo/info/refs?service=git-upload-pack", 0, limited + "/allowed/repo.git", "https://git.example/allowed/repo"}, + {"GET", "/limited/other/repo/info/refs?service=git-upload-pack", http.StatusNotFound, "", ""}, + } + for _, tt := range tests { + t.Run(tt.method+" "+tt.target, func(t *testing.T) { + r := httptest.NewRequest(tt.method, tt.target, nil) + r.Header.Set("Content-Type", gitRequestType) + req, status, _ := h.parse(r) + if status != tt.status { + t.Fatalf("status %d, want %d", status, tt.status) + } + if status == 0 && (req.key != tt.key || req.repoURL != tt.repoURL) { + t.Errorf("key %q, URL %q; want %q, %q", req.key, req.repoURL, tt.key, tt.repoURL) + } + }) + } + + r := httptest.NewRequest(http.MethodPost, "/up/repo/git-upload-pack", nil) + if _, status, _ := h.parse(r); status != http.StatusUnsupportedMediaType { + t.Errorf("POST without the request content type: status %d, want 415", status) + } +} + +// TestGitUpstreamCacheDirectory checks the cache directory follows the +// upstream's URL, so repointing an upstream does not serve the old host's +// repositories. +func TestGitUpstreamCacheDirectory(t *testing.T) { + dir := func(url string) string { + h := NewGitHandler(nil, nil, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: url}}}) + return h.upstreams["up"].dir + } + if a, b := dir("https://git.example"), dir("https://other.example"); a == b || !strings.HasPrefix(a, "up-") { + t.Errorf("directories %q and %q, want distinct ones named after the upstream", a, b) + } + if a, b := dir("https://git.example"), dir("https://git.example/"); a != b { + t.Errorf("a trailing slash changed the directory: %q, %q", a, b) + } +} + +// TestGitFetchByIDOffWithCredentials checks commits are not fetched by ID +// for an upstream that upstream.auth has credentials for: there a fetch by +// ID could reach repositories outside its repos allowlist. +func TestGitFetchByIDOffWithCredentials(t *testing.T) { + h := NewGitHandler(nil, nil, config.UpstreamConfig{ + Git: map[string]config.GitUpstream{ + "public": {URL: "https://github.com"}, + "private": {URL: "https://git.example", Repos: []string{"team/*"}}, + "scoped": {URL: "https://other.example", Repos: []string{"team/*"}}, + }, + Auth: map[string]config.AuthConfig{ + "https://git.example": {Type: "bearer", Token: "t"}, + "https://other.example/team": {Type: "bearer", Token: "t"}, + }, + }) + for name, want := range map[string]bool{"public": true, "private": false, "scoped": false} { + if got := h.upstreams[name].fetchWants; got != want { + t.Errorf("%s: fetchWants = %v, want %v", name, got, want) + } + } +} + +func runGitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), "HOME=" + t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitTestSetup is a GitHandler in front of a cache of one upstream +// repository, "repo", served by git http-backend. +type gitTestSetup struct { + work string + server *httptest.Server + logs *syncLog + // slow delays every upstream request by half a second. + slow atomic.Bool +} + +func newGitTestSetup(t *testing.T, readTimeout time.Duration) *gitTestSetup { + t.Helper() + execPath := runGitCommand(t, "", "--exec-path") + backendPath := "" + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if path := filepath.Join(execPath, name); fileExists(path) { + backendPath = path + } + } + if backendPath == "" { + t.Skip("git http-backend is not installed") + } + root := t.TempDir() + bare := filepath.Join(root, "repo.git") + runGitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + runGitCommand(t, bare, "config", "uploadpack.allowAnySHA1InWant", "true") + g := &gitTestSetup{work: t.TempDir(), logs: &syncLog{}} + runGitCommand(t, "", "clone", "--quiet", bare, g.work) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "first") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:main") + + backend := &cgi.Handler{ + Path: backendPath, + Env: []string{"PATH=" + os.Getenv("PATH"), "GIT_PROJECT_ROOT=" + root, "GIT_HTTP_EXPORT_ALL=1"}, + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if g.slow.Load() { + time.Sleep(500 * time.Millisecond) + } + backend.ServeHTTP(w, r) + })) + t.Cleanup(upstream.Close) + + logger := slog.New(slog.NewTextHandler(g.logs, nil)) + cache, err := gitcache.New(gitcache.Options{ + CacheDir: t.TempDir(), + RefreshWait: 5 * time.Second, + StaleMaxAge: time.Hour, + Client: &http.Client{}, + Logger: logger, + Upstreams: map[string]string{"up": gitcache.UpstreamDir("up", upstream.URL)}, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cache.Close() }) + h := NewGitHandler(&Proxy{Logger: logger}, cache, config.UpstreamConfig{Git: map[string]config.GitUpstream{"up": {URL: upstream.URL}}}) + g.server = httptest.NewUnstartedServer(http.StripPrefix("/git", h.Routes())) + g.server.Config.ReadTimeout = readTimeout + g.server.Start() + t.Cleanup(g.server.Close) + return g +} + +func (g *gitTestSetup) url() string { + return g.server.URL + "/git/up/repo" +} + +type syncLog struct { + mu sync.Mutex + buf strings.Builder +} + +func (l *syncLog) Write(p []byte) (int, error) { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.Write(p) +} + +func (l *syncLog) String() string { + l.mu.Lock() + defer l.mu.Unlock() + return l.buf.String() +} + +// TestGitRequestBodyOutlastsServerReadTimeout sends a protocol v0 request +// whose haves come after a want the cache must fetch first, so upload-pack +// reads them after the server's read timeout has passed. +func TestGitRequestBodyOutlastsServerReadTimeout(t *testing.T) { + g := newGitTestSetup(t, 200*time.Millisecond) + runGitCommand(t, g.work, "commit", "--quiet", "--allow-empty", "-m", "pull request") + runGitCommand(t, g.work, "push", "--quiet", "origin", "HEAD:refs/pull/1/head") + sha := runGitCommand(t, g.work, "rev-parse", "HEAD") + runGitCommand(t, "", "clone", "--quiet", g.url(), filepath.Join(t.TempDir(), "clone")) + + var body strings.Builder + body.WriteString(pktLine("want "+sha+" side-band-64k\n") + "0000") + for i := range 2000 { + body.WriteString(pktLine(fmt.Sprintf("have %040x\n", i+1))) + } + body.WriteString(pktLine("done\n")) + + g.slow.Store(true) + resp, err := http.Post(g.url()+"/git-upload-pack", gitRequestType, strings.NewReader(body.String())) + if err != nil { + t.Fatal(err) + } + defer func() { _ = resp.Body.Close() }() + out, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK || !strings.Contains(string(out), "PACK") { + t.Errorf("status %d, no pack in %d bytes: %.200q", resp.StatusCode, len(out), out) + } +} + +func pktLine(payload string) string { + return fmt.Sprintf("%04x%s", len(payload)+4, payload) +} + +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} diff --git a/internal/server/git_test.go b/internal/server/git_test.go new file mode 100644 index 00000000..d52f1a95 --- /dev/null +++ b/internal/server/git_test.go @@ -0,0 +1,160 @@ +package server + +import ( + "bytes" + "context" + "errors" + "log/slog" + "net" + "net/http" + "net/http/cgi" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/config" +) + +func gitTestEnv(home string) []string { + return []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + home, + "GIT_CONFIG_NOSYSTEM=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_AUTHOR_NAME=test", "GIT_AUTHOR_EMAIL=test@example.com", + "GIT_COMMITTER_NAME=test", "GIT_COMMITTER_EMAIL=test@example.com", + } +} + +func gitCommand(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = gitTestEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + return strings.TrimSpace(string(out)) +} + +// gitHTTPBackend finds git http-backend, which serves the test upstream, and +// skips the test where git is installed without it. +func gitHTTPBackend(t *testing.T) string { + t.Helper() + dir := gitCommand(t, "", "--exec-path") + for _, name := range []string{"git-http-backend", "git-http-backend.exe"} { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + return filepath.Join(dir, name) + } + } + t.Skip("git http-backend is not installed") + return "" +} + +// syncBuffer is a log destination the server and the test share. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// TestGitRouteServesClonesWithUpstreamCredentials clones through the whole +// server, with credentials from upstream.auth that only the proxy holds. +func TestGitRouteServesClonesWithUpstreamCredentials(t *testing.T) { + root := t.TempDir() + bare := filepath.Join(root, "team", "repo.git") + gitCommand(t, "", "init", "--quiet", "--bare", "--initial-branch=main", bare) + work := t.TempDir() + gitCommand(t, "", "clone", "--quiet", bare, work) + if err := os.WriteFile(filepath.Join(work, "file.txt"), []byte("hello\n"), 0o600); err != nil { + t.Fatal(err) + } + gitCommand(t, work, "add", "file.txt") + gitCommand(t, work, "commit", "--quiet", "-m", "first") + gitCommand(t, work, "push", "--quiet", "origin", "HEAD:main") + head := gitCommand(t, work, "rev-parse", "HEAD") + + backend := &cgi.Handler{ + Path: gitHTTPBackend(t), + Env: append(gitTestEnv(t.TempDir()), "GIT_PROJECT_ROOT="+root, "GIT_HTTP_EXPORT_ALL=1"), + } + upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer git-secret" { + http.Error(w, "unauthorized", http.StatusUnauthorized) + return + } + backend.ServeHTTP(w, r) + })) + defer upstream.Close() + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + tempDir := t.TempDir() + cfg := config.Default() + cfg.Listen = listener.Addr().String() + cfg.BaseURL = "http://" + cfg.Listen + cfg.Database.Path = filepath.Join(tempDir, "proxy.db") + cfg.Storage.URL = "file://" + filepath.Join(tempDir, "artifacts") + cfg.Upstream.AllowLoopback = true + cfg.Upstream.Git = map[string]config.GitUpstream{"private": {URL: upstream.URL, Repos: []string{"team/*"}}} + cfg.Upstream.Auth = map[string]config.AuthConfig{upstream.URL: {Type: "bearer", Token: "git-secret"}} + cfg.Git.CacheDir = filepath.Join(tempDir, "git") + if err := cfg.Validate(); err != nil { + t.Fatalf("validating config: %v", err) + } + + var logs syncBuffer + proxyServer, err := New(cfg, slog.New(slog.NewTextHandler(&logs, nil)), BuildInfo{Version: "test", Commit: "test"}) + if err != nil { + t.Fatalf("creating server: %v", err) + } + startErr := make(chan error, 1) + go func() { startErr <- proxyServer.Start(listener) }() + defer func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := proxyServer.Shutdown(ctx); err != nil { + t.Errorf("shutting down server: %v", err) + } + if err := <-startErr; !errors.Is(err, http.ErrServerClosed) { + t.Errorf("Start() error = %v, want %v", err, http.ErrServerClosed) + } + }() + + clone := filepath.Join(t.TempDir(), "clone") + gitCommand(t, "", "clone", "--quiet", cfg.BaseURL+"/git/private/team/repo.git", clone) + if got := gitCommand(t, clone, "rev-parse", "HEAD"); got != head { + t.Errorf("HEAD = %s, want %s", got, head) + } + if strings.Contains(logs.String(), "request deadlines") { + t.Errorf("request deadlines not extended:\n%s", logs.String()) + } + + resp, err := http.Get(cfg.BaseURL + "/git/private/other/repo.git/info/refs?service=git-upload-pack") + if err != nil { + t.Fatal(err) + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusNotFound { + t.Errorf("repository outside repos: status %d, want 404", resp.StatusCode) + } +} diff --git a/internal/server/middleware.go b/internal/server/middleware.go index f74590a9..c0f36628 100644 --- a/internal/server/middleware.go +++ b/internal/server/middleware.go @@ -107,7 +107,7 @@ func requestEcosystem(path string) string { switch segment { case "npm", "cargo", "hex", "pub", "pypi", "maven", "gradle", "nuget", "conan", "conda", "cran", "julia", "debian", "rpm", "arch", - "helm", "homebrew", "generic", "swift": + "helm", "homebrew", "generic", "swift", "git": return segment case "apk": return "alpine" diff --git a/internal/server/middleware_test.go b/internal/server/middleware_test.go index 875b152e..e924a823 100644 --- a/internal/server/middleware_test.go +++ b/internal/server/middleware_test.go @@ -314,7 +314,7 @@ func TestRequestEcosystemCoversEveryMountedRoute(t *testing.T) { "/conan/x": "conan", "/conda/x": "conda", "/cran/x": "cran", "/julia/x": "julia", "/swift/x": "swift", "/v2/x": "oci", "/apk/x": "alpine", "/arch/x": "arch", "/debian/x": "debian", "/rpm/x": "rpm", - "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", + "/helm/x": "helm", "/homebrew/x": "homebrew", "/generic/x": "generic", "/git/x": "git", } for path, want := range mounts { if got := requestEcosystem(path); got != want { diff --git a/internal/server/server.go b/internal/server/server.go index 2907f04c..6327a4e8 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -74,6 +74,7 @@ import ( "github.com/git-pkgs/proxy/internal/database" "github.com/git-pkgs/proxy/internal/denylist" "github.com/git-pkgs/proxy/internal/enrichment" + "github.com/git-pkgs/proxy/internal/gitcache" "github.com/git-pkgs/proxy/internal/handler" upstreamhttp "github.com/git-pkgs/proxy/internal/httpclient" "github.com/git-pkgs/proxy/internal/metrics" @@ -119,6 +120,7 @@ type Server struct { breakers *breakerMonitor ecoStats ecosystemStatsCache sources sourceTracker + git *gitcache.Cache // browseLimits overrides defaultBrowseLimits when set; tests use it. browseLimits archives.StreamOptions @@ -266,6 +268,27 @@ func (s *Server) serve(listener net.Listener) error { proxy.DirectServeBaseURL = s.cfg.Storage.DirectServeBaseURL proxy.StreamArtifacts = !s.cfg.Storage.CacheArtifacts + if len(s.cfg.Upstream.Git) > 0 { + upstreamDirs := make(map[string]string, len(s.cfg.Upstream.Git)) + for name, upstream := range s.cfg.Upstream.Git { + upstreamDirs[name] = gitcache.UpstreamDir(name, upstream.URL) + } + gitCache, err := gitcache.New(gitcache.Options{ + CacheDir: s.cfg.Git.CacheDir, + MaxSize: s.cfg.Git.MaxSizeBytes(), + MaxRepoSize: s.cfg.Git.MaxRepoSizeBytes(), + RefreshWait: s.cfg.Git.RefreshWaitDuration(), + StaleMaxAge: s.cfg.Git.StaleMaxAgeDuration(), + Client: artifactClient, + Logger: s.logger, + Upstreams: upstreamDirs, + }) + if err != nil { + return fmt.Errorf("configuring git cache: %w", err) + } + s.git = gitCache + } + // Create router with Chi r := chi.NewRouter() @@ -479,6 +502,10 @@ func (s *Server) mountProtocolHandlers(r chi.Router, proxy *handler.Proxy) { r.Mount("/debian", http.StripPrefix("/debian", debianHandler.Routes())) r.Mount("/rpm", http.StripPrefix("/rpm", rpmHandler.Routes())) r.Mount("/generic", http.StripPrefix("/generic", genericHandler.Routes())) + if s.git != nil { + gitHandler := handler.NewGitHandler(proxy, s.git, s.cfg.Upstream) + r.Mount("/git", http.StripPrefix("/git", gitHandler.Routes())) + } } // configureScanning builds the scanner group from cfg and wires it into @@ -623,6 +650,12 @@ func (s *Server) Shutdown(ctx context.Context) error { } } + if s.git != nil { + if err := s.git.Close(); err != nil { + errs = append(errs, fmt.Errorf("git cache close: %w", err)) + } + } + if s.storage != nil { if err := s.storage.Close(); err != nil { errs = append(errs, fmt.Errorf("storage close: %w", err))