diff --git a/README.md b/README.md index 6fb3e8ab..a9a3a3bf 100644 --- a/README.md +++ b/README.md @@ -1147,6 +1147,14 @@ Registered when `hosted.enabled` is set. Pushing and deleting require `Authoriza | `GET /api/hosted/{ecosystem}/{name}` | List the archives pushed for a package | | `DELETE /api/hosted/{ecosystem}/{name}/{version}` | Delete a pushed version | +### Cache API + +Registered when `cache_delete` is set. It has no authentication of its own. + +| Endpoint | Description | +|----------|-------------| +| `DELETE /api/cache/{ecosystem}/{name}/{version}` | Delete a version's cached files, or only `?filename=` | + ### Enrichment API The proxy provides REST endpoints for package metadata enrichment, vulnerability scanning, and outdated detection. @@ -1616,6 +1624,12 @@ cache/artifacts/ └── nginx-1.24.0-1.fc39.x86_64.rpm ``` +To delete one version's cached files, so the next request fetches them again, enable `cache_delete` and use the Delete buttons on the version page in the web UI, or the [cache API](docs/configuration.md#cache-delete-api): + +```bash +curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' +``` + Cache metadata is stored in SQLite (default) or PostgreSQL. To clear a local cache: ```bash diff --git a/config.example.yaml b/config.example.yaml index c41f30f3..99e684cc 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -14,6 +14,11 @@ base_url: "http://localhost:8080" # Set to "0" to disable the timeout. Default: "30s". # http_timeout: "30s" +# Register DELETE /api/cache and show Delete buttons on the UI's version page, +# for removing cached artifacts so they are fetched again. Has no +# authentication: restrict /api/cache and /ui at the ingress first. +# cache_delete: false + # Largest upstream artifact to fetch (e.g. "2GB"). Larger ones are refused # and not cached. Default: unlimited. # artifact_max_size: "" diff --git a/docs/configuration.md b/docs/configuration.md index c735c34b..0d54903c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1099,6 +1099,29 @@ curl -X POST http://localhost:8080/api/mirror \ -d '{"purls":["pkg:docker/library/alpine@3.20"],"platforms":["linux/amd64"]}' ``` +## Cache Delete API + +Deleting cached artifacts is disabled by default. Enable it to register `DELETE /api/cache` and show Delete buttons on the version page of the web UI: + +```yaml +cache_delete: true +``` + +Or via environment variable: `PROXY_CACHE_DELETE=true`. + +The endpoint has no authentication. Before enabling it, restrict `/api/cache` and `/ui` to trusted users at your ingress or reverse proxy. The UI sends its deletes to `/ui/api/cache`, so whatever protects `/ui` protects those too. + +Delete every cached file of a version, or only the one named by `filename`: + +```bash +curl -X DELETE http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre +curl -X DELETE 'http://localhost:8080/api/cache/maven/com.google.guava:guava/32.1.3-jre?filename=guava-32.1.3-jre.jar' +``` + +The name is the package name the UI shows, for example `group:artifact` for Maven. The response lists the deleted files. The next request for a deleted file fetches it from the upstream again. The stored bytes are removed after the same delay as a replaced artifact, so downloads already in progress can finish. + +Pushed archives are not touched: a whole-version delete skips them, and naming one answers 409. Delete those through the [hosted API](#hosted-packages). + ## Mirror Command The `proxy mirror` command pre-populates the cache from various sources. It accepts the same storage and database flags as `serve`. diff --git a/docs/swagger/docs.go b/docs/swagger/docs.go index 5e54d3c6..1ce6d5c8 100644 --- a/docs/swagger/docs.go +++ b/docs/swagger/docs.go @@ -115,6 +115,73 @@ const docTemplate = `{ } } }, + "/api/cache/{ecosystem}/{name}/{version}": { + "delete": { + "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", + "produces": [ + "application/json" + ], + "tags": [ + "cache" + ], + "summary": "Delete cached artifacts", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Delete only this file", + "name": "filename", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.CacheDeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/hosted/{ecosystem}/{name}": { "get": { "produces": [ @@ -753,6 +820,20 @@ const docTemplate = `{ } } }, + "server.CacheDeleteResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "array", + "items": { + "type": "string" + } + }, + "purl": { + "type": "string" + } + } + }, "server.EcosystemStatsEntry": { "type": "object", "properties": { diff --git a/docs/swagger/swagger.json b/docs/swagger/swagger.json index 65a271b2..77ef4766 100644 --- a/docs/swagger/swagger.json +++ b/docs/swagger/swagger.json @@ -108,6 +108,73 @@ } } }, + "/api/cache/{ecosystem}/{name}/{version}": { + "delete": { + "description": "Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted.", + "produces": [ + "application/json" + ], + "tags": [ + "cache" + ], + "summary": "Delete cached artifacts", + "parameters": [ + { + "type": "string", + "description": "Ecosystem", + "name": "ecosystem", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Package name as shown in the UI, e.g. com.google.guava:guava for Maven", + "name": "name", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Version", + "name": "version", + "in": "path", + "required": true + }, + { + "type": "string", + "description": "Delete only this file", + "name": "filename", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.CacheDeleteResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "404": { + "description": "Not Found", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + }, + "409": { + "description": "Conflict", + "schema": { + "$ref": "#/definitions/server.ErrorResponse" + } + } + } + } + }, "/api/hosted/{ecosystem}/{name}": { "get": { "produces": [ @@ -746,6 +813,20 @@ } } }, + "server.CacheDeleteResponse": { + "type": "object", + "properties": { + "deleted": { + "type": "array", + "items": { + "type": "string" + } + }, + "purl": { + "type": "string" + } + } + }, "server.EcosystemStatsEntry": { "type": "object", "properties": { diff --git a/internal/config/config.go b/internal/config/config.go index a9d7c212..1a973f8e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -190,6 +190,11 @@ type Config struct { // Disabled by default to prevent unauthenticated users from triggering downloads. MirrorAPI bool `json:"mirror_api" yaml:"mirror_api"` + // CacheDelete enables DELETE /api/cache for removing cached artifacts, and + // the delete buttons on the UI's version page. It has no authentication of + // its own, so restrict /api/cache and /ui at the ingress when enabling it. + CacheDelete bool `json:"cache_delete" yaml:"cache_delete"` + // Hosted configures pushing packages to the proxy (/api/hosted). Hosted HostedConfig `json:"hosted" yaml:"hosted"` @@ -1212,6 +1217,7 @@ func (c *Config) LoadFromEnv() { setEnvString(&c.Scanning.FetchBaseURL, "PROXY_SCANNING_FETCH_BASE_URL") setEnvBool(&c.CacheMetadata, "PROXY_CACHE_METADATA") setEnvBool(&c.MirrorAPI, "PROXY_MIRROR_API") + setEnvBool(&c.CacheDelete, "PROXY_CACHE_DELETE") setEnvBool(&c.Hosted.Enabled, "PROXY_HOSTED_ENABLED") setEnvString(&c.Hosted.Token, "PROXY_HOSTED_TOKEN") setEnvString(&c.Hosted.MaxUploadSize, "PROXY_HOSTED_MAX_UPLOAD_SIZE") diff --git a/internal/server/cache_api.go b/internal/server/cache_api.go new file mode 100644 index 00000000..771d333f --- /dev/null +++ b/internal/server/cache_api.go @@ -0,0 +1,86 @@ +package server + +import ( + "net/http" + "strings" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +// CacheDeleteResponse lists the cached files removed for a version. +type CacheDeleteResponse struct { + PURL string `json:"purl"` + Deleted []string `json:"deleted"` +} + +// handleCacheDelete handles DELETE /api/cache/{ecosystem}/{name}/{version} +// @Summary Delete cached artifacts +// @Description Removes the cached files of name@version, or only the one named by filename, so the next request fetches them from the upstream again. Pushed archives are not touched; delete those through /api/hosted. +// @Tags cache +// @Produce json +// @Param ecosystem path string true "Ecosystem" +// @Param name path string true "Package name as shown in the UI, e.g. com.google.guava:guava for Maven" +// @Param version path string true "Version" +// @Param filename query string false "Delete only this file" +// @Success 200 {object} CacheDeleteResponse +// @Failure 400 {object} ErrorResponse +// @Failure 404 {object} ErrorResponse +// @Failure 409 {object} ErrorResponse +// @Router /api/cache/{ecosystem}/{name}/{version} [delete] +func (s *Server) handleCacheDelete(w http.ResponseWriter, r *http.Request) { + segments, err := packagePathSegments(r) + if err != nil { + badRequest(w, err.Error()) + return + } + if len(segments) < minVersionPathSegments { + badRequest(w, "path must be /api/cache/{ecosystem}/{name}/{version}") + return + } + last := len(segments) - 1 + ecosystem, name, version := chi.URLParam(r, "ecosystem"), strings.Join(segments[:last], "/"), segments[last] + filename := r.URL.Query().Get("filename") + + versionPURL := s.cachedVersionPURL(ecosystem, name, version) + if versionPURL == "" { + notFound(w, "package not found") + return + } + artifacts, err := s.db.GetArtifactsByVersionPURL(versionPURL) + if err != nil { + s.logger.Error("failed to list artifacts", "purl", versionPURL, "error", err) + internalError(w, "failed to list artifacts") + return + } + + deleted := []string{} + for _, art := range artifacts { + if filename != "" && art.Filename != filename { + continue + } + if !art.StoragePath.Valid { + continue + } + if art.Source == database.SourceLocal { + if filename != "" { + writeError(w, http.StatusConflict, ErrCodeConflict, "pushed archives are deleted through /api/hosted") + return + } + continue + } + if err := s.db.DiscardArtifact(versionPURL, art.Filename, art.StoragePath.String); err != nil { + s.logger.Error("failed to delete cached artifact", "purl", versionPURL, "filename", art.Filename, "error", err) + internalError(w, "failed to delete cached artifact") + return + } + deleted = append(deleted, art.Filename) + } + if len(deleted) == 0 { + notFound(w, "no cached artifact matched") + return + } + + s.logger.Info("deleted cached artifacts", "purl", versionPURL, "files", deleted) + writeJSON(w, CacheDeleteResponse{PURL: versionPURL, Deleted: deleted}) +} diff --git a/internal/server/cache_api_test.go b/internal/server/cache_api_test.go new file mode 100644 index 00000000..ab3103a8 --- /dev/null +++ b/internal/server/cache_api_test.go @@ -0,0 +1,180 @@ +package server + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/git-pkgs/proxy/internal/database" + "github.com/go-chi/chi/v5" +) + +const ( + cacheTestPackage = "pkg:maven/com.example/lib" + cacheTestVersion = cacheTestPackage + "@1.0" +) + +// seedCacheDeleteVersion caches lib-1.0.jar and its .sha1 for +// com.example:lib 1.0, plus a pushed lib-1.0-local.jar when pushed is set. +func seedCacheDeleteVersion(t *testing.T, db *database.DB, pushed bool) { + t.Helper() + if err := db.UpsertPackage(&database.Package{PURL: cacheTestPackage, Ecosystem: "maven", Name: "com.example:lib"}); err != nil { + t.Fatalf("UpsertPackage: %v", err) + } + if err := db.UpsertVersion(&database.Version{PURL: cacheTestVersion, PackagePURL: cacheTestPackage}); err != nil { + t.Fatalf("UpsertVersion: %v", err) + } + files := map[string]string{"lib-1.0.jar": database.SourceUpstream, "lib-1.0.jar.sha1": database.SourceUpstream} + if pushed { + files["lib-1.0-local.jar"] = database.SourceLocal + } + for filename, source := range files { + if err := db.UpsertArtifact(&database.Artifact{ + VersionPURL: cacheTestVersion, + Filename: filename, + UpstreamURL: "https://repo1.maven.org/maven2/com/example/lib/1.0/" + filename, + StoragePath: sql.NullString{String: "objects/" + filename, Valid: true}, + FetchedAt: sql.NullTime{Time: time.Now(), Valid: true}, + Source: source, + }); err != nil { + t.Fatalf("UpsertArtifact %s: %v", filename, err) + } + } +} + +func cacheDelete(ts *testServer, target string) *httptest.ResponseRecorder { + r := chi.NewRouter() + r.Delete("/api/cache/{ecosystem}/*", ts.server.handleCacheDelete) + w := httptest.NewRecorder() + r.ServeHTTP(w, httptest.NewRequest(http.MethodDelete, target, nil)) + return w +} + +func cachedFiles(t *testing.T, db *database.DB) []string { + t.Helper() + artifacts, err := db.GetArtifactsByVersionPURL(cacheTestVersion) + if err != nil { + t.Fatalf("GetArtifactsByVersionPURL: %v", err) + } + var cached []string + for _, art := range artifacts { + if art.StoragePath.Valid { + cached = append(cached, art.Filename) + } + } + slices.Sort(cached) + return cached +} + +func queuedDeletes(t *testing.T, db *database.DB) []string { + t.Helper() + paths, err := db.GetDuePendingDeletes(time.Now().Add(time.Hour), 100) + if err != nil { + t.Fatalf("GetDuePendingDeletes: %v", err) + } + slices.Sort(paths) + return paths +} + +func TestCacheDelete_OneFile(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, false) + + w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0.jar") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + var resp CacheDeleteResponse + if err := json.NewDecoder(w.Body).Decode(&resp); err != nil { + t.Fatalf("decoding response: %v", err) + } + if resp.PURL != cacheTestVersion || !slices.Equal(resp.Deleted, []string{"lib-1.0.jar"}) { + t.Errorf("response = %+v, want lib-1.0.jar deleted from %s", resp, cacheTestVersion) + } + if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0.jar.sha1"}) { + t.Errorf("still cached = %v, want only the .sha1", got) + } + if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar"}) { + t.Errorf("queued for deletion = %v, want the jar's object", got) + } +} + +func TestCacheDelete_WholeVersionKeepsPushedFiles(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, true) + + w := cacheDelete(ts, "/api/cache/maven/com.example:lib/1.0") + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body: %s", w.Code, w.Body.String()) + } + if got := cachedFiles(t, ts.db); !slices.Equal(got, []string{"lib-1.0-local.jar"}) { + t.Errorf("still cached = %v, want only the pushed jar", got) + } + if got := queuedDeletes(t, ts.db); !slices.Equal(got, []string{"objects/lib-1.0.jar", "objects/lib-1.0.jar.sha1"}) { + t.Errorf("queued for deletion = %v, want both upstream objects", got) + } +} + +func TestCacheDelete_Refusals(t *testing.T) { + tests := []struct { + name string + target string + want int + }{ + {"pushed file", "/api/cache/maven/com.example:lib/1.0?filename=lib-1.0-local.jar", http.StatusConflict}, + {"unknown file", "/api/cache/maven/com.example:lib/1.0?filename=other.jar", http.StatusNotFound}, + {"unknown version", "/api/cache/maven/com.example:lib/2.0", http.StatusNotFound}, + {"unknown package", "/api/cache/maven/com.example:other/1.0", http.StatusNotFound}, + {"no version", "/api/cache/maven/lib", http.StatusBadRequest}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts := newTestServer(t) + defer ts.close() + seedCacheDeleteVersion(t, ts.db, true) + + w := cacheDelete(ts, tt.target) + if w.Code != tt.want { + t.Fatalf("status = %d, want %d; body: %s", w.Code, tt.want, w.Body.String()) + } + if got := cachedFiles(t, ts.db); len(got) != 3 { + t.Errorf("still cached = %v, want all three files untouched", got) + } + }) + } +} + +func TestVersionPage_CacheDeleteButtons(t *testing.T) { + for _, enabled := range []bool{false, true} { + ts := newTestServer(t) + seedCacheDeleteVersion(t, ts.db, true) + ts.server.cfg.CacheDelete = enabled + + w := httptest.NewRecorder() + ts.handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, "/ui/package/maven/com.example:lib/1.0", nil)) + ts.close() + if w.Code != http.StatusOK { + t.Fatalf("enabled=%v: status = %d, want 200", enabled, w.Code) + } + body := w.Body.String() + // One button per upstream file plus the version-wide one; the pushed + // jar gets none. + wantButtons := 0 + if enabled { + wantButtons = 3 + } + if got := strings.Count(body, `class="cache-delete `); got != wantButtons { + t.Errorf("enabled=%v: %d delete buttons, want %d", enabled, got, wantButtons) + } + if enabled && !strings.Contains(body, `data-cache-url="/ui/api/cache/maven/com.example:lib/1.0"`) { + t.Errorf("page has no data-cache-url for the version") + } + } +} diff --git a/internal/server/dashboard.go b/internal/server/dashboard.go index dbc3a69d..1ff6239d 100644 --- a/internal/server/dashboard.go +++ b/internal/server/dashboard.go @@ -80,6 +80,9 @@ type VersionShowData struct { IsOutdated bool LicenseCategory string HasCachedArtifact bool + // HasDeletableArtifact is true when a stored artifact was fetched from + // the upstream rather than pushed. + HasDeletableArtifact bool } // SearchPageData contains data for rendering the search results page. diff --git a/internal/server/errors.go b/internal/server/errors.go index 7cbe20ed..c856c0e4 100644 --- a/internal/server/errors.go +++ b/internal/server/errors.go @@ -11,6 +11,7 @@ const ( ErrCodeBadRequest = "BAD_REQUEST" ErrCodeNotFound = "NOT_FOUND" ErrCodeUnauthorized = "UNAUTHORIZED" + ErrCodeConflict = "CONFLICT" ErrCodeUpstream = "UPSTREAM_ERROR" ErrCodeInternal = "INTERNAL_ERROR" ErrCodeBusy = "BUSY" diff --git a/internal/server/layout.go b/internal/server/layout.go index 2da94699..3a518768 100644 --- a/internal/server/layout.go +++ b/internal/server/layout.go @@ -15,6 +15,8 @@ type Layout struct { BuildInfo BuildInfo UIBaseURL string CanonicalPath string + // CacheDelete shows the controls for deleting cached artifacts. + CacheDelete bool } func (s *Server) layoutFor(r *http.Request) Layout { @@ -22,5 +24,6 @@ func (s *Server) layoutFor(r *http.Request) Layout { BuildInfo: s.buildInfo, UIBaseURL: s.cfg.UIBaseURL, CanonicalPath: r.URL.Path, + CacheDelete: s.cfg.CacheDelete, } } diff --git a/internal/server/server.go b/internal/server/server.go index 2907f04c..3b5c2908 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -324,10 +324,16 @@ func (s *Server) serve(listener net.Listener) error { ui.Get("/package/{ecosystem}/*", s.handlePackagePath) ui.Get("/api/browse/{ecosystem}/*", s.handleBrowsePath) ui.Get("/api/compare/{ecosystem}/*", s.handleComparePath) + if s.cfg.CacheDelete { + ui.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) + } }) r.Get("/", func(w http.ResponseWriter, r *http.Request) { http.Redirect(w, r, "/ui/", http.StatusFound) }) + if s.cfg.CacheDelete { + r.Delete("/api/cache/{ecosystem}/*", s.handleCacheDelete) + } // API endpoints for enrichment data enrichSvc := enrichment.New(s.logger) @@ -1079,14 +1085,15 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, hasCached := firstBrowsableArtifact(artifacts) != nil data := VersionShowData{ - Layout: s.layoutFor(r), - Package: pkg, - Version: ver, - Artifacts: artifacts, - Vulnerabilities: vulns, - IsOutdated: isOutdated, - LicenseCategory: categorizeLicense(ver.License), - HasCachedArtifact: hasCached, + Layout: s.layoutFor(r), + Package: pkg, + Version: ver, + Artifacts: artifacts, + Vulnerabilities: vulns, + IsOutdated: isOutdated, + LicenseCategory: categorizeLicense(ver.License), + HasCachedArtifact: hasCached, + HasDeletableArtifact: hasDeletableArtifact(artifacts), } if err := s.templates.Render(w, "version_show", data); err != nil { @@ -1094,6 +1101,15 @@ func (s *Server) showVersion(w http.ResponseWriter, r *http.Request, ecosystem, } } +func hasDeletableArtifact(artifacts []database.Artifact) bool { + for _, art := range artifacts { + if art.StoragePath.Valid && art.Source != database.SourceLocal { + return true + } + } + return false +} + func (s *Server) cachedVersionPURL(ecosystem, name, version string) string { pkg, err := s.db.GetPackageByEcosystemName(ecosystem, name) if err != nil || pkg == nil { diff --git a/internal/server/templates/pages/version_show.html b/internal/server/templates/pages/version_show.html index 81bc067b..f25f2e95 100644 --- a/internal/server/templates/pages/version_show.html +++ b/internal/server/templates/pages/version_show.html @@ -69,10 +69,14 @@

Security V {{if .Artifacts}} -
-
+
+

Artifacts ({{len .Artifacts}})

+ {{if and .CacheDelete .HasDeletableArtifact}} + + {{end}}
+
{{range .Artifacts}}
@@ -81,6 +85,9 @@

Artifacts ({{len .Artifacts}})

{{if .Size.Valid}}{{.Size.Int64}} bytes{{end}} {{if gt .HitCount 0}}{{.HitCount}} cache hits{{end}} + {{if and $.CacheDelete .StoragePath.Valid (ne .Source "local")}} + + {{end}}
{{if .ContentHash.Valid}} @@ -100,6 +107,46 @@

Artifacts ({{len .Artifacts}})

{{end}}
+{{if .CacheDelete}} + +{{end}} {{else}}
No artifacts cached yet