From 6c50f4235c3d385059591eb9c39db8656271af55 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:14:11 +0200 Subject: [PATCH] refac(gcp): resolve the container registry once for all data centers The registry was written straight into the single install config, so which registry the nodes pull from was decided per config rather than per project. It now resolves onto the environment (ContainerRegistryURL plus credentials) and updateInstallConfig applies it to every data center's config and vault. All three registry types go through the same field, which also fixes the artifact registry never recording its URI on the create path. EnsureLocalContainerRegistry is split in two, because it early-returned when the registry was already running and thereby skipped distributing the registry certificate. A re-run that adds a data center hits exactly that path, and its nodes would then fail every pull with "certificate signed by unknown authority". Starting the registry stays conditional; distributing the certificate now always runs, over every data center's cluster nodes, which is safe because it is idempotent. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- internal/bootstrap/gcp/gcp_test.go | 70 +++++++++++++++--- internal/bootstrap/gcp/install_config.go | 14 +++- internal/bootstrap/gcp/registry.go | 94 ++++++++++++++++-------- 3 files changed, 136 insertions(+), 42 deletions(-) diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index b37564107..5c6db7562 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -910,8 +910,7 @@ var _ = Describe("GCP Bootstrapper", func() { Describe("Valid EnsureLocalContainerRegistry", func() { It("installs local registry", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) + icg.EXPECT().GetVault().Return(&files.InstallVault{}) // Setup mocked node // Check if running - return error to simulate not running @@ -927,7 +926,57 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureLocalContainerRegistry() Expect(err).NotTo(HaveOccurred()) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + Expect(bs.Env.RegistryUsername).To(Equal("custom-registry")) + Expect(bs.Env.RegistryPassword).NotTo(BeEmpty()) + Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.Jumpbox.GetInternalIP() + ":5000")) + }) + + // A re-run that adds a data center finds the registry already up. Its nodes still + // need the registry's self-signed certificate, or every image pull fails. + It("distributes the registry certificate even when the registry is already running", func() { + vault := &files.InstallVault{} + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}}) + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}}) + icg.EXPECT().GetVault().Return(vault) + + bs.Env.MultiDC = true + bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)} + bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)} + secondary := &datacenter.DataCenter{ID: 2, Suffix: "-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + // Registry is already running with credentials in the vault. + nodeClient.EXPECT().RunCommand(bs.Env.Jumpbox, "root", mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "podman ps") + })).Return(nil) + + scpTargets := []string{} + + nodeClient.EXPECT().RunCommand(bs.Env.Jumpbox, "root", mock.MatchedBy(func(cmd string) bool { + return strings.HasPrefix(cmd, "scp ") + })).RunAndReturn(func(_ *node.Node, _ string, cmd string) error { + scpTargets = append(scpTargets, cmd) + return nil + }).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4) + + // Register the second data center only after ensureDataCenters would have run, + // mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap. + bs.Env.DataCenters = []*datacenter.DataCenter{ + { + ID: 1, + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + bs.Env.DataCenters[0].ConfigManager = icg + + Expect(bs.EnsureLocalContainerRegistry()).To(Succeed()) + Expect(scpTargets).To(HaveLen(4)) + Expect(bs.Env.RegistryPassword).To(Equal("existing-password")) }) }) @@ -1049,17 +1098,14 @@ var _ = Describe("GCP Bootstrapper", func() { csEnv.GitHubPAT = "fake-pat" csEnv.RegistryUser = "custom-registry" }) - It("sets configuration options in installconfig", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) - + // The resolved registry and its credentials live on the environment; every data center's + // config picks them up in updateInstallConfig. + It("resolves ghcr.io as the registry for all data centers", func() { err := bs.EnsureGitHubAccessConfigured() Expect(err).NotTo(HaveOccurred()) - Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io")) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser)) - Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT)) - Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse()) - Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse()) + Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io")) + Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser)) + Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT)) }) Context("When GitHub PAT is missing", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 69b91b0de..e8391706e 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -145,7 +145,19 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir - if b.Env.RegistryType != RegistryTypeGitHub { + if b.Env.ContainerRegistryURL != "" { + dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL + } + + if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" { + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}}) + } + + if b.Env.RegistryType == RegistryTypeGitHub { + dc.InstallConfig.Registry.ReplaceImagesInBom = false + dc.InstallConfig.Registry.LoadContainerImages = false + } else { dc.InstallConfig.Registry.ReplaceImagesInBom = true dc.InstallConfig.Registry.LoadContainerImages = true } diff --git a/internal/bootstrap/gcp/registry.go b/internal/bootstrap/gcp/registry.go index 4ccb67b1f..b60955809 100644 --- a/internal/bootstrap/gcp/registry.go +++ b/internal/bootstrap/gcp/registry.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" "github.com/codesphere-cloud/oms/internal/util" "github.com/lithammer/shortuuid" ) @@ -66,13 +67,13 @@ func (b *GCPBootstrapper) validateRegistryParams() error { } // EnsureArtifactRegistry ensures the project's GCP Artifact Registry repository exists and -// points the install config's registry server at it +// resolves it as the registry all data centers pull their images from. func (b *GCPBootstrapper) EnsureArtifactRegistry() error { repoName := "codesphere-registry" repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName) if err == nil && repo != nil { - b.Env.InstallConfig.EnsureRegistry().Server = repo.GetRegistryUri() + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -81,21 +82,54 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo) } + b.Env.ContainerRegistryURL = repo.GetRegistryUri() + return nil } -// EnsureLocalContainerRegistry installs a docker registry on the jumpbox to speed up image loading time +// EnsureLocalContainerRegistry installs a container registry on the jumpbox to speed up image +// loading time, and makes every cluster node of every data center trust its certificate. func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + registryNode, err := b.registryNode() + if err != nil { + return err + } + + if err := b.ensureDataCenters(); err != nil { + return err + } + + registryServer, err := b.ensureRegistryRunning(registryNode) + if err != nil { + return err + } + + b.Env.ContainerRegistryURL = registryServer + + // The certificate must be distributed on every run, not only when the registry was just + // created: a re-run that adds a data center finds the registry already up, and that data + // center's nodes would otherwise not trust it. + return b.distributeRegistryCert(registryNode, b.clusterNodes()) +} + +// registryNode returns the jumpbox the local container registry runs on. It is shared by all +// data centers, so a single registry serves every cluster. +func (b *GCPBootstrapper) registryNode() (*node.Node, error) { registryNode := b.Env.Jumpbox if registryNode == nil { - return fmt.Errorf("jumpbox not found in bootstrap environment") + return nil, fmt.Errorf("jumpbox not found in bootstrap environment") } if registryNode.GetInternalIP() == "" { - return fmt.Errorf("jumpbox has no internal IP") + return nil, fmt.Errorf("jumpbox has no internal IP") } - registry := b.Env.InstallConfig.EnsureRegistry() + return registryNode, nil +} + +// ensureRegistryRunning starts the container registry on the registry node and generates its +// credentials when it is not already serving. Returns the registry server address. +func (b *GCPBootstrapper) ensureRegistryRunning(registryNode *node.Node) (string, error) { localRegistryServer := registryNode.GetInternalIP() + ":5000" // Figure out if registry is already running @@ -103,29 +137,30 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := registryNode.RunSSHCommand("root", checkCommand) + vault := b.primaryDC().ConfigManager.GetVault() registryUsername := "" registryPassword := "" - if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { registryUsername = s.Fields.Password } - if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { registryPassword = s.Fields.Password } - if err == nil && registry.Server == localRegistryServer && - registryUsername != "" && registryPassword != "" { + if err == nil && registryUsername != "" && registryPassword != "" { b.stlog.Logf("Local container registry already running on the jumpbox") - return nil + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword + + return localRegistryServer, nil } - registry.Server = localRegistryServer registryUsername = "custom-registry" registryPassword = shortuuid.New() - - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}}) + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword commands := []string{ "apt-get update", @@ -145,20 +180,25 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { -v /root/registry.crt:/certs/registry.crt \ -v /root/registry.key:/certs/registry.key \ registry:3`, - `mkdir -p /etc/docker/certs.d/` + registry.Server, - `cp /root/registry.crt /etc/docker/certs.d/` + registry.Server + `/ca.crt`, + `mkdir -p /etc/docker/certs.d/` + localRegistryServer, + `cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`, } for _, cmd := range commands { b.stlog.Logf("Running command on the jumpbox: %s", util.Truncate(cmd, 12)) err := registryNode.RunSSHCommand("root", cmd) if err != nil { - return fmt.Errorf("failed to run command on the jumpbox: %w", err) + return "", fmt.Errorf("failed to run command on the jumpbox: %w", err) } } - allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...) - for _, node := range allNodes { + return localRegistryServer, nil +} + +// distributeRegistryCert installs the local registry's self-signed certificate on the given +// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center. +func (b *GCPBootstrapper) distributeRegistryCert(registryNode *node.Node, nodes []*node.Node) error { + for _, node := range nodes { b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName()) err := registryNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt") @@ -180,20 +220,16 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { return nil } -// EnsureGitHubAccessConfigured points the install config at ghcr.io and stores the GitHub -// credentials in the vault. The cluster pulls images from GHCR directly +// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from. The +// credentials are written into every data center's vault by updateInstallConfig. func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { if b.Env.GitHubPAT == "" { return fmt.Errorf("GitHub PAT is not set") } - registry := b.Env.InstallConfig.EnsureRegistry() - registry.Server = "ghcr.io" - registry.ReplaceImagesInBom = false - registry.LoadContainerImages = false - - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}}) + b.Env.ContainerRegistryURL = "ghcr.io" + b.Env.RegistryUsername = b.Env.RegistryUser + b.Env.RegistryPassword = b.Env.GitHubPAT return nil }