Delete build sources when builds finish and expire old builds

cmd/api/config/config.go

@@ -151,11 +151,13 @@ type ImagesConfig struct {
 
 // BuildConfig holds source-to-image build system settings.
 type BuildConfig struct {
-	MaxConcurrentSourceBuilds int    `koanf:"max_concurrent_source_builds"`
-	BuilderImage              string `koanf:"builder_image"`
-	Timeout                   int    `koanf:"timeout"`
-	SecretsDir                string `koanf:"secrets_dir"`
-	DockerSocket              string `koanf:"docker_socket"`
+	MaxConcurrentSourceBuilds int           `koanf:"max_concurrent_source_builds"`
+	BuilderImage              string        `koanf:"builder_image"`
+	Timeout                   int           `koanf:"timeout"`
+	SecretsDir                string        `koanf:"secrets_dir"`
+	DockerSocket              string        `koanf:"docker_socket"`
+	Retention                 string        `koanf:"retention"` // how long finished builds are kept
+	RetentionDuration         time.Duration `koanf:"-"`         // parsed by Validate
 }
 
 // BuildersConfig holds builder resource settings.
@@ -420,6 +422,7 @@ func defaultConfig() *Config {
 			Timeout:                   600,
 			SecretsDir:                "",
 			DockerSocket:              "/var/run/docker.sock",
+			Retention:                 "2h",
 		},
 
 		Builders: BuildersConfig{
@@ -705,6 +708,14 @@ func (c *Config) Validate() error {
 	if c.Build.Timeout <= 0 {
 		return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout)
 	}
+	retention, err := time.ParseDuration(c.Build.Retention)
+	if err != nil {
+		return fmt.Errorf("build.retention must be a valid duration, got %q: %w", c.Build.Retention, err)
+	}
+	if retention <= 0 {
+		return fmt.Errorf("build.retention must be positive, got %q", c.Build.Retention)
+	}
+	c.Build.RetentionDuration = retention
 	if c.Builders.MaxCount < 0 {
 		return fmt.Errorf("builders.max_count must be >= 0, got %d", c.Builders.MaxCount)
 	}

config.example.darwin.yaml

@@ -136,6 +136,7 @@ build:
   docker_socket: /var/run/docker.sock
   max_concurrent_source_builds: 2
   timeout: 600
+  # retention: 2h  # keep finished builds this long; sources are deleted when a build finishes
 
 # =============================================================================
 # Resource Limits (same as Linux)

config.example.yaml

@@ -175,6 +175,7 @@ data_dir: /var/lib/hypeman
 #   docker_socket: /var/run/docker.sock
 #   max_concurrent_source_builds: 2
 #   timeout: 600
+#   retention: 2h                   # keep finished builds this long; sources are deleted when a build finishes
 
 # builders:
 #   max_count: 0                    # max number of builders (0 = unlimited)

lib/builds/README.md

@@ -81,6 +81,8 @@ builds/
         └── build.log
 ```
 
+The source can carry build secrets, so it is readable only by hypeman and is deleted as soon as the build finishes. A background GC also deletes finished builds' sources, and removes the whole build directory once the build has been finished for `build.retention` (default `2h`). Queued and running builds are never collected, so startup recovery can re-run them.
+
 ### Build Manager (`manager.go`)
 
 Orchestrates the build lifecycle:

lib/builds/manager.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	_ "embed"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
@@ -113,6 +114,10 @@ type Config struct {
 
 	// DockerSocket is the path to the Docker socket for building the builder image
 	DockerSocket string
+
+	// Retention is how long a finished build's metadata and logs are kept.
+	// Its source is deleted as soon as the build finishes.
+	Retention time.Duration
 }
 
 // DefaultConfig returns the default build manager configuration
@@ -121,6 +126,7 @@ func DefaultConfig() Config {
 		MaxConcurrentBuilds: 2,
 		RegistryURL:         "localhost:4973",
 		DefaultTimeout:      600, // 10 minutes
+		Retention:           2 * time.Hour,
 	}
 }
 
@@ -175,6 +181,9 @@ func NewManager(
 	if logger == nil {
 		logger = slog.Default()
 	}
+	if config.Retention <= 0 {
+		config.Retention = DefaultConfig().Retention
+	}
 
 	m := &manager{
 		config:            config,
@@ -221,6 +230,7 @@ func (m *manager) Start(ctx context.Context) error {
 			}
 		}
 	}()
+	go m.runBuildGC(ctx)
 	m.logger.Info("build manager started")
 	return nil
 }
@@ -661,16 +671,64 @@ func (m *manager) BuilderHasBuilds(builderID string) bool {
 	return false
 }
 
-// storeSource stores the source tarball for a build
+// storeSource stores the source tarball for a build. The source can carry
+// build secrets, so it is readable only by the hypeman process.
 func (m *manager) storeSource(buildID string, data []byte) error {
 	sourceDir := m.paths.BuildSourceDir(buildID)
-	if err := ensureDir(sourceDir); err != nil {
+	if err := os.MkdirAll(sourceDir, 0700); err != nil {
 		return err
 	}
+	return os.WriteFile(sourceDir+"/source.tar.gz", data, 0600)
+}
+
+// removeSource deletes a build's source once the build no longer needs it.
+func (m *manager) removeSource(buildID string) {
+	if err := os.RemoveAll(m.paths.BuildSourceDir(buildID)); err != nil {
+		m.logger.Warn("failed to remove build source", "id", buildID, "error", err)
+	}
+}
+
+const buildGCInterval = 10 * time.Minute
+
+// runBuildGC periodically cleans up finished builds until ctx is done.
+func (m *manager) runBuildGC(ctx context.Context) {
+	ticker := time.NewTicker(buildGCInterval)
+	defer ticker.Stop()
+	for {
+		m.gcFinishedBuilds()
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+		}
+	}
+}
 
-	// Write source tarball
-	sourcePath := sourceDir + "/source.tar.gz"
-	return writeFile(sourcePath, data)
+// gcFinishedBuilds deletes the source of every finished build, and the whole
+// build directory once its metadata has not changed for the retention window.
+// Queued and running builds are left alone so startup recovery can re-run them.
+func (m *manager) gcFinishedBuilds() {
+	all, err := listAllBuilds(m.paths)
+	if err != nil {
+		m.logger.Error("list builds for gc", "error", err)
+		return
+	}
+	for _, meta := range all {
+		if meta.Status != StatusReady && meta.Status != StatusFailed && meta.Status != StatusCancelled {
+			continue
+		}
+		info, err := os.Stat(m.paths.BuildMetadata(meta.ID))
+		if err != nil {
+			continue
+		}
+		if time.Since(info.ModTime()) < m.config.Retention {
+			m.removeSource(meta.ID)
+			continue
+		}
+		if err := deleteBuild(m.paths, meta.ID); err != nil && !errors.Is(err, ErrNotFound) {
+			m.logger.Warn("failed to delete expired build", "id", meta.ID, "error", err)
+		}
+	}
 }
 
 // runBuild executes a build in a builder VM
@@ -1106,6 +1164,8 @@ func (m *manager) updateStatus(id string, status string, err error) {
 
 // updateBuildComplete updates the build with final results
 func (m *manager) updateBuildComplete(id string, status string, digest *string, errMsg *string, provenance *BuildProvenance, durationMS *int64) {
+	defer m.removeSource(id)
+
 	meta, readErr := readMetadata(m.paths, id)
 	if readErr != nil {
 		m.logger.Error("read metadata for completion", "id", id, "error", readErr)
@@ -1247,6 +1307,7 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error {
 		// Remove from queue
 		if m.queue.Cancel(id) {
 			m.updateStatus(id, StatusCancelled, nil)
+			m.removeSource(id)
 			return nil
 		}
 		return ErrBuildInProgress // Was already picked up
@@ -1258,6 +1319,7 @@ func (m *manager) CancelBuild(ctx context.Context, id string) error {
 			m.instanceManager.DeleteInstance(ctx, *meta.BuilderInstance)
 		}
 		m.updateStatus(id, StatusCancelled, nil)
+		m.removeSource(id)
 		return nil
 
 	case StatusReady, StatusFailed, StatusCancelled:
@@ -1589,14 +1651,6 @@ func (m *manager) refreshBuildToken(buildID string, req *CreateBuildRequest) err
 
 // Helper functions
 
-func ensureDir(path string) error {
-	return os.MkdirAll(path, 0755)
-}
-
-func writeFile(path string, data []byte) error {
-	return os.WriteFile(path, data, 0644)
-}
-
 func readFile(path string) ([]byte, error) {
 	return os.ReadFile(path)
 }

lib/builds/manager_test.go

@@ -576,11 +576,17 @@ func TestCreateBuild_Success(t *testing.T) {
 	assert.Equal(t, StatusQueued, build.Status)
 	assert.NotNil(t, build.CreatedAt)
 
-	// Verify source was stored
+	// Verify source was stored readable only by hypeman
 	sourcePath := filepath.Join(tempDir, "builds", build.ID, "source", "source.tar.gz")
 	data, err := os.ReadFile(sourcePath)
 	require.NoError(t, err)
 	assert.Equal(t, sourceData, data)
+	info, err := os.Stat(sourcePath)
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+	info, err = os.Stat(filepath.Dir(sourcePath))
+	require.NoError(t, err)
+	assert.Equal(t, os.FileMode(0700), info.Mode().Perm())
 
 	// Verify config was written
 	configPath := filepath.Join(tempDir, "builds", build.ID, "config.json")
@@ -757,6 +763,49 @@ func TestCancelBuild_AlreadyCompleted(t *testing.T) {
 	assert.Contains(t, err.Error(), "already completed")
 }
 
+// writeTestBuild persists a build with a source holding a synthetic secret.
+func writeTestBuild(t *testing.T, mgr *manager, id, status string) {
+	t.Helper()
+	require.NoError(t, writeMetadata(mgr.paths, &buildMetadata{ID: id, Status: status, CreatedAt: time.Now()}))
+	require.NoError(t, mgr.storeSource(id, []byte("API_KEY='synthetic-secret'")))
+}
+
+func TestFinishedBuildSourceIsRemoved(t *testing.T) {
+	mgr, _, _, tempDir := setupTestManager(t)
+	defer os.RemoveAll(tempDir)
+
+	writeTestBuild(t, mgr, "completed", StatusBuilding)
+	mgr.updateBuildComplete("completed", StatusFailed, nil, nil, nil, nil)
+	assert.NoDirExists(t, mgr.paths.BuildSourceDir("completed"))
+	assert.FileExists(t, mgr.paths.BuildMetadata("completed"))
+
+	writeTestBuild(t, mgr, "cancelled", StatusBuilding)
+	require.NoError(t, mgr.CancelBuild(context.Background(), "cancelled"))
+	assert.NoDirExists(t, mgr.paths.BuildSourceDir("cancelled"))
+	assert.FileExists(t, mgr.paths.BuildMetadata("cancelled"))
+}
+
+func TestGCFinishedBuilds(t *testing.T) {
+	mgr, _, _, tempDir := setupTestManager(t)
+	defer os.RemoveAll(tempDir)
+	mgr.config.Retention = time.Hour
+	expired := time.Now().Add(-2 * time.Hour)
+
+	writeTestBuild(t, mgr, "pending", StatusBuilding)
+	require.NoError(t, os.Chtimes(mgr.paths.BuildMetadata("pending"), expired, expired))
+	writeTestBuild(t, mgr, "recent", StatusReady)
+	writeTestBuild(t, mgr, "expired", StatusFailed)
+	require.NoError(t, os.Chtimes(mgr.paths.BuildMetadata("expired"), expired, expired))
+
+	mgr.gcFinishedBuilds()
+
+	// Pending builds keep their source so recovery can re-run them.
+	assert.FileExists(t, filepath.Join(mgr.paths.BuildSourceDir("pending"), "source.tar.gz"))
+	assert.NoDirExists(t, mgr.paths.BuildSourceDir("recent"))
+	assert.FileExists(t, mgr.paths.BuildMetadata("recent"))
+	assert.NoDirExists(t, mgr.paths.BuildDir("expired"))
+}
+
 func TestGetBuildLogs_Empty(t *testing.T) {
 	mgr, _, _, tempDir := setupTestManager(t)
 	defer os.RemoveAll(tempDir)

lib/providers/providers.go

@@ -442,6 +442,7 @@ func ProvideBuildManager(p *paths.Paths, cfg *config.Config, instanceManager ins
 		DefaultTimeout:      cfg.Build.Timeout,
 		RegistrySecret:      cfg.JwtSecret, // Use same secret for registry tokens
 		DockerSocket:        cfg.Build.DockerSocket,
+		Retention:           cfg.Build.RetentionDuration,
 	}
 
 	// Configure secret provider (use NoOpSecretProvider as fallback to avoid nil panics)