diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 40d51ce18..7c11bdd1d 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -151,11 +151,13 @@ type ImagesConfig struct { // BuildConfig holds source-to-image build system settings. type BuildConfig struct { - MaxConcurrentSourceBuilds int `koanf:"max_concurrent_source_builds"` - BuilderImage string `koanf:"builder_image"` - Timeout int `koanf:"timeout"` - SecretsDir string `koanf:"secrets_dir"` - DockerSocket string `koanf:"docker_socket"` + MaxConcurrentSourceBuilds int `koanf:"max_concurrent_source_builds"` + BuilderImage string `koanf:"builder_image"` + Timeout int `koanf:"timeout"` + SecretsDir string `koanf:"secrets_dir"` + DockerSocket string `koanf:"docker_socket"` + Retention string `koanf:"retention"` // how long finished builds are kept + RetentionDuration time.Duration `koanf:"-"` // parsed by Validate } // BuildersConfig holds builder resource settings. @@ -420,6 +422,7 @@ func defaultConfig() *Config { Timeout: 600, SecretsDir: "", DockerSocket: "/var/run/docker.sock", + Retention: "2h", }, Builders: BuildersConfig{ @@ -705,6 +708,14 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + retention, err := time.ParseDuration(c.Build.Retention) + if err != nil { + return fmt.Errorf("build.retention must be a valid duration, got %q: %w", c.Build.Retention, err) + } + if retention <= 0 { + return fmt.Errorf("build.retention must be positive, got %q", c.Build.Retention) + } + c.Build.RetentionDuration = retention if c.Builders.MaxCount < 0 { return fmt.Errorf("builders.max_count must be >= 0, got %d", c.Builders.MaxCount) } diff --git a/config.example.darwin.yaml b/config.example.darwin.yaml index 6344eedaa..c436dcf56 100644 --- a/config.example.darwin.yaml +++ b/config.example.darwin.yaml @@ -136,6 +136,7 @@ build: docker_socket: /var/run/docker.sock max_concurrent_source_builds: 2 timeout: 600 + # retention: 2h # keep finished builds this long; sources are deleted when a build finishes # ============================================================================= # Resource Limits (same as Linux) diff --git a/config.example.yaml b/config.example.yaml index ac939ed26..be49ec897 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -175,6 +175,7 @@ data_dir: /var/lib/hypeman # docker_socket: /var/run/docker.sock # max_concurrent_source_builds: 2 # timeout: 600 +# retention: 2h # keep finished builds this long; sources are deleted when a build finishes # builders: # max_count: 0 # max number of builders (0 = unlimited) diff --git a/lib/builds/README.md b/lib/builds/README.md index f7ec2844f..f617a160a 100644 --- a/lib/builds/README.md +++ b/lib/builds/README.md @@ -81,6 +81,8 @@ builds/ └── build.log ``` +The source can carry build secrets, so it is readable only by hypeman and is deleted as soon as the build finishes. A background GC also deletes finished builds' sources, and removes the whole build directory once the build has been finished for `build.retention` (default `2h`). Queued and running builds are never collected, so startup recovery can re-run them. + ### Build Manager (`manager.go`) Orchestrates the build lifecycle: diff --git a/lib/builds/manager.go b/lib/builds/manager.go index 806ae5622..10664fdc4 100644 --- a/lib/builds/manager.go +++ b/lib/builds/manager.go @@ -5,6 +5,7 @@ import ( "context" _ "embed" "encoding/json" + "errors" "fmt" "log/slog" "net" @@ -113,6 +114,10 @@ type Config struct { // DockerSocket is the path to the Docker socket for building the builder image DockerSocket string + + // Retention is how long a finished build's metadata and logs are kept. + // Its source is deleted as soon as the build finishes. + Retention time.Duration } // DefaultConfig returns the default build manager configuration @@ -121,6 +126,7 @@ func DefaultConfig() Config { MaxConcurrentBuilds: 2, RegistryURL: "localhost:4973", DefaultTimeout: 600, // 10 minutes + Retention: 2 * time.Hour, } } @@ -175,6 +181,9 @@ func NewManager( if logger == nil { logger = slog.Default() } + if config.Retention <= 0 { + config.Retention = DefaultConfig().Retention + } m := &manager{ config: config, @@ -221,6 +230,7 @@ func (m *manager) Start(ctx context.Context) error { } } }() + go m.runBuildGC(ctx) m.logger.Info("build manager started") return nil } @@ -661,16 +671,64 @@ func (m *manager) BuilderHasBuilds(builderID string) bool { return false } -// storeSource stores the source tarball for a build +// storeSource stores the source tarball for a build. The source can carry +// build secrets, so it is readable only by the hypeman process. func (m *manager) storeSource(buildID string, data []byte) error { sourceDir := m.paths.BuildSourceDir(buildID) - if err := ensureDir(sourceDir); err != nil { + if err := os.MkdirAll(sourceDir, 0700); err != nil { return err } + return os.WriteFile(sourceDir+"/source.tar.gz", data, 0600) +} + +// removeSource deletes a build's source once the build no longer needs it. +func (m *manager) removeSource(buildID string) { + if err := os.RemoveAll(m.paths.BuildSourceDir(buildID)); err != nil { + m.logger.Warn("failed to remove build source", "id", buildID, "error", err) + } +} + +const buildGCInterval = 10 * time.Minute + +// runBuildGC periodically cleans up finished builds until ctx is done. +func (m *manager) runBuildGC(ctx context.Context) { + ticker := time.NewTicker(buildGCInterval) + defer ticker.Stop() + for { + m.gcFinishedBuilds() + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} - // Write source tarball - sourcePath := sourceDir + "/source.tar.gz" - return writeFile(sourcePath, data) +// gcFinishedBuilds deletes the source of every finished build, and the whole +// build directory once its metadata has not changed for the retention window. +// Queued and running builds are left alone so startup recovery can re-run them. +func (m *manager) gcFinishedBuilds() { + all, err := listAllBuilds(m.paths) + if err != nil { + m.logger.Error("list builds for gc", "error", err) + return + } + for _, meta := range all { + if meta.Status != StatusReady && meta.Status != StatusFailed && meta.Status != StatusCancelled { + continue + } + info, err := os.Stat(m.paths.BuildMetadata(meta.ID)) + if err != nil { + continue + } + if time.Since(info.ModTime()) < m.config.Retention { + m.removeSource(meta.ID) + continue + } + if err := deleteBuild(m.paths, meta.ID); err != nil && !errors.Is(err, ErrNotFound) { + m.logger.Warn("failed to delete expired build", "id", meta.ID, "error", err) + } + } } // runBuild executes a build in a builder VM @@ -1106,6 +1164,8 @@ func (m *manager) updateStatus(id string, status string, err error) { // updateBuildComplete updates the build with final results func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) { + defer m.removeSource(id) + meta, readErr := readMetadata(m.paths, id) if readErr != nil { m.logger.Error("read metadata for completion", "id", id, "error", readErr) @@ -1247,6 +1307,7 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { // Remove from queue if m.queue.Cancel(id) { m.updateStatus(id, StatusCancelled, nil) + m.removeSource(id) return nil } return ErrBuildInProgress // Was already picked up @@ -1258,6 +1319,7 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error { m.instanceManager.DeleteInstance(ctx, *meta.BuilderInstance) } m.updateStatus(id, StatusCancelled, nil) + m.removeSource(id) return nil case StatusReady, StatusFailed, StatusCancelled: @@ -1589,14 +1651,6 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err // Helper functions -func ensureDir(path string) error { - return os.MkdirAll(path, 0755) -} - -func writeFile(path string, data []byte) error { - return os.WriteFile(path, data, 0644) -} - func readFile(path string) ([]byte, error) { return os.ReadFile(path) } diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index 05015bb98..38c4afbe6 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -576,11 +576,17 @@ func TestCreateBuild_Success(t *testing.T) { assert.Equal(t, StatusQueued, build.Status) assert.NotNil(t, build.CreatedAt) - // Verify source was stored + // Verify source was stored readable only by hypeman sourcePath := filepath.Join(tempDir, "builds", build.ID, "source", "source.tar.gz") data, err := os.ReadFile(sourcePath) require.NoError(t, err) assert.Equal(t, sourceData, data) + info, err := os.Stat(sourcePath) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0600), info.Mode().Perm()) + info, err = os.Stat(filepath.Dir(sourcePath)) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0700), info.Mode().Perm()) // Verify config was written configPath := filepath.Join(tempDir, "builds", build.ID, "config.json") @@ -757,6 +763,49 @@ func TestCancelBuild_AlreadyCompleted(t *testing.T) { assert.Contains(t, err.Error(), "already completed") } +// writeTestBuild persists a build with a source holding a synthetic secret. +func writeTestBuild(t *testing.T, mgr *manager, id, status string) { + t.Helper() + require.NoError(t, writeMetadata(mgr.paths, &buildMetadata{ID: id, Status: status, CreatedAt: time.Now()})) + require.NoError(t, mgr.storeSource(id, []byte("API_KEY='synthetic-secret'"))) +} + +func TestFinishedBuildSourceIsRemoved(t *testing.T) { + mgr, _, _, tempDir := setupTestManager(t) + defer os.RemoveAll(tempDir) + + writeTestBuild(t, mgr, "completed", StatusBuilding) + mgr.updateBuildComplete("completed", StatusFailed, nil, nil, nil, nil) + assert.NoDirExists(t, mgr.paths.BuildSourceDir("completed")) + assert.FileExists(t, mgr.paths.BuildMetadata("completed")) + + writeTestBuild(t, mgr, "cancelled", StatusBuilding) + require.NoError(t, mgr.CancelBuild(context.Background(), "cancelled")) + assert.NoDirExists(t, mgr.paths.BuildSourceDir("cancelled")) + assert.FileExists(t, mgr.paths.BuildMetadata("cancelled")) +} + +func TestGCFinishedBuilds(t *testing.T) { + mgr, _, _, tempDir := setupTestManager(t) + defer os.RemoveAll(tempDir) + mgr.config.Retention = time.Hour + expired := time.Now().Add(-2 * time.Hour) + + writeTestBuild(t, mgr, "pending", StatusBuilding) + require.NoError(t, os.Chtimes(mgr.paths.BuildMetadata("pending"), expired, expired)) + writeTestBuild(t, mgr, "recent", StatusReady) + writeTestBuild(t, mgr, "expired", StatusFailed) + require.NoError(t, os.Chtimes(mgr.paths.BuildMetadata("expired"), expired, expired)) + + mgr.gcFinishedBuilds() + + // Pending builds keep their source so recovery can re-run them. + assert.FileExists(t, filepath.Join(mgr.paths.BuildSourceDir("pending"), "source.tar.gz")) + assert.NoDirExists(t, mgr.paths.BuildSourceDir("recent")) + assert.FileExists(t, mgr.paths.BuildMetadata("recent")) + assert.NoDirExists(t, mgr.paths.BuildDir("expired")) +} + func TestGetBuildLogs_Empty(t *testing.T) { mgr, _, _, tempDir := setupTestManager(t) defer os.RemoveAll(tempDir) diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 8de7c33d3..136657abb 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -442,6 +442,7 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins DefaultTimeout: cfg.Build.Timeout, RegistrySecret: cfg.JwtSecret, // Use same secret for registry tokens DockerSocket: cfg.Build.DockerSocket, + Retention: cfg.Build.RetentionDuration, } // Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics)