diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index b3756410..5c6db756 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -910,8 +910,7 @@ var _ = Describe("GCP Bootstrapper", func() { Describe("Valid EnsureLocalContainerRegistry", func() { It("installs local registry", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) + icg.EXPECT().GetVault().Return(&files.InstallVault{}) // Setup mocked node // Check if running - return error to simulate not running @@ -927,7 +926,57 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.EnsureLocalContainerRegistry() Expect(err).NotTo(HaveOccurred()) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + Expect(bs.Env.RegistryUsername).To(Equal("custom-registry")) + Expect(bs.Env.RegistryPassword).NotTo(BeEmpty()) + Expect(bs.Env.ContainerRegistryURL).To(Equal(bs.Env.Jumpbox.GetInternalIP() + ":5000")) + }) + + // A re-run that adds a data center finds the registry already up. Its nodes still + // need the registry's self-signed certificate, or every image pull fails. + It("distributes the registry certificate even when the registry is already running", func() { + vault := &files.InstallVault{} + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: "custom-registry"}}) + vault.SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: "existing-password"}}) + icg.EXPECT().GetVault().Return(vault) + + bs.Env.MultiDC = true + bs.Env.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1", nodeClient)} + bs.Env.CephNodes = []*node.Node{fakeNode("ceph-1", nodeClient)} + secondary := &datacenter.DataCenter{ID: 2, Suffix: "-dc2"} + secondary.ControlPlaneNodes = []*node.Node{fakeNode("k0s-1-dc2", nodeClient)} + secondary.CephNodes = []*node.Node{fakeNode("ceph-1-dc2", nodeClient)} + + // Registry is already running with credentials in the vault. + nodeClient.EXPECT().RunCommand(bs.Env.Jumpbox, "root", mock.MatchedBy(func(cmd string) bool { + return strings.Contains(cmd, "podman ps") + })).Return(nil) + + scpTargets := []string{} + + nodeClient.EXPECT().RunCommand(bs.Env.Jumpbox, "root", mock.MatchedBy(func(cmd string) bool { + return strings.HasPrefix(cmd, "scp ") + })).RunAndReturn(func(_ *node.Node, _ string, cmd string) error { + scpTargets = append(scpTargets, cmd) + return nil + }).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "update-ca-certificates").Return(nil).Times(4) + nodeClient.EXPECT().RunCommand(mock.Anything, "root", "systemctl restart docker.service || true").Return(nil).Times(4) + + // Register the second data center only after ensureDataCenters would have run, + // mirroring what EnsureComputeInstances produces for a --multi-dc bootstrap. + bs.Env.DataCenters = []*datacenter.DataCenter{ + { + ID: 1, + ControlPlaneNodes: bs.Env.ControlPlaneNodes, + CephNodes: bs.Env.CephNodes, + }, + secondary, + } + bs.Env.DataCenters[0].ConfigManager = icg + + Expect(bs.EnsureLocalContainerRegistry()).To(Succeed()) + Expect(scpTargets).To(HaveLen(4)) + Expect(bs.Env.RegistryPassword).To(Equal("existing-password")) }) }) @@ -1049,17 +1098,14 @@ var _ = Describe("GCP Bootstrapper", func() { csEnv.GitHubPAT = "fake-pat" csEnv.RegistryUser = "custom-registry" }) - It("sets configuration options in installconfig", func() { - vault := &files.InstallVault{} - icg.EXPECT().GetVault().Return(vault) - + // The resolved registry and its credentials live on the environment; every data center's + // config picks them up in updateInstallConfig. + It("resolves ghcr.io as the registry for all data centers", func() { err := bs.EnsureGitHubAccessConfigured() Expect(err).NotTo(HaveOccurred()) - Expect(bs.Env.InstallConfig.Registry.Server).To(Equal("ghcr.io")) - Expect(vault.GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal(csEnv.RegistryUser)) - Expect(vault.GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal(csEnv.GitHubPAT)) - Expect(bs.Env.InstallConfig.Registry.LoadContainerImages).To(BeFalse()) - Expect(bs.Env.InstallConfig.Registry.ReplaceImagesInBom).To(BeFalse()) + Expect(bs.Env.ContainerRegistryURL).To(Equal("ghcr.io")) + Expect(bs.Env.RegistryUsername).To(Equal(csEnv.RegistryUser)) + Expect(bs.Env.RegistryPassword).To(Equal(csEnv.GitHubPAT)) }) Context("When GitHub PAT is missing", func() { diff --git a/internal/bootstrap/gcp/install_config.go b/internal/bootstrap/gcp/install_config.go index 69b91b0d..e8391706 100644 --- a/internal/bootstrap/gcp/install_config.go +++ b/internal/bootstrap/gcp/install_config.go @@ -145,7 +145,19 @@ func (b *GCPBootstrapper) updateInstallConfig(dc *datacenter.DataCenter) error { // secrets.baseDir, so sharing a directory would let one data center's ceph and kubernetes // steps overwrite another's credentials. dc.InstallConfig.Secrets.BaseDir = dc.SecretsDir - if b.Env.RegistryType != RegistryTypeGitHub { + if b.Env.ContainerRegistryURL != "" { + dc.InstallConfig.Registry.Server = b.Env.ContainerRegistryURL + } + + if b.Env.RegistryUsername != "" || b.Env.RegistryPassword != "" { + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUsername}}) + dc.ConfigManager.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.RegistryPassword}}) + } + + if b.Env.RegistryType == RegistryTypeGitHub { + dc.InstallConfig.Registry.ReplaceImagesInBom = false + dc.InstallConfig.Registry.LoadContainerImages = false + } else { dc.InstallConfig.Registry.ReplaceImagesInBom = true dc.InstallConfig.Registry.LoadContainerImages = true } diff --git a/internal/bootstrap/gcp/registry.go b/internal/bootstrap/gcp/registry.go index 4ccb67b1..b6095580 100644 --- a/internal/bootstrap/gcp/registry.go +++ b/internal/bootstrap/gcp/registry.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" "github.com/codesphere-cloud/oms/internal/util" "github.com/lithammer/shortuuid" ) @@ -66,13 +67,13 @@ func (b *GCPBootstrapper) validateRegistryParams() error { } // EnsureArtifactRegistry ensures the project's GCP Artifact Registry repository exists and -// points the install config's registry server at it +// resolves it as the registry all data centers pull their images from. func (b *GCPBootstrapper) EnsureArtifactRegistry() error { repoName := "codesphere-registry" repo, err := b.GCPClient.GetArtifactRegistry(b.Env.ProjectID, b.Env.Region, repoName) if err == nil && repo != nil { - b.Env.InstallConfig.EnsureRegistry().Server = repo.GetRegistryUri() + b.Env.ContainerRegistryURL = repo.GetRegistryUri() return nil } @@ -81,21 +82,54 @@ func (b *GCPBootstrapper) EnsureArtifactRegistry() error { return fmt.Errorf("failed to create artifact registry: %w, repo: %v", err, repo) } + b.Env.ContainerRegistryURL = repo.GetRegistryUri() + return nil } -// EnsureLocalContainerRegistry installs a docker registry on the jumpbox to speed up image loading time +// EnsureLocalContainerRegistry installs a container registry on the jumpbox to speed up image +// loading time, and makes every cluster node of every data center trust its certificate. func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { + registryNode, err := b.registryNode() + if err != nil { + return err + } + + if err := b.ensureDataCenters(); err != nil { + return err + } + + registryServer, err := b.ensureRegistryRunning(registryNode) + if err != nil { + return err + } + + b.Env.ContainerRegistryURL = registryServer + + // The certificate must be distributed on every run, not only when the registry was just + // created: a re-run that adds a data center finds the registry already up, and that data + // center's nodes would otherwise not trust it. + return b.distributeRegistryCert(registryNode, b.clusterNodes()) +} + +// registryNode returns the jumpbox the local container registry runs on. It is shared by all +// data centers, so a single registry serves every cluster. +func (b *GCPBootstrapper) registryNode() (*node.Node, error) { registryNode := b.Env.Jumpbox if registryNode == nil { - return fmt.Errorf("jumpbox not found in bootstrap environment") + return nil, fmt.Errorf("jumpbox not found in bootstrap environment") } if registryNode.GetInternalIP() == "" { - return fmt.Errorf("jumpbox has no internal IP") + return nil, fmt.Errorf("jumpbox has no internal IP") } - registry := b.Env.InstallConfig.EnsureRegistry() + return registryNode, nil +} + +// ensureRegistryRunning starts the container registry on the registry node and generates its +// credentials when it is not already serving. Returns the registry server address. +func (b *GCPBootstrapper) ensureRegistryRunning(registryNode *node.Node) (string, error) { localRegistryServer := registryNode.GetInternalIP() + ":5000" // Figure out if registry is already running @@ -103,29 +137,30 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { checkCommand := `test "$(podman ps --filter 'name=registry' --format '{{.Names}}' | wc -l)" -eq "1"` err := registryNode.RunSSHCommand("root", checkCommand) + vault := b.primaryDC().ConfigManager.GetVault() registryUsername := "" registryPassword := "" - if s := b.icg.GetVault().GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryUsername); s != nil && s.Fields != nil { registryUsername = s.Fields.Password } - if s := b.icg.GetVault().GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { + if s := vault.GetSecret(files.SecretRegistryPassword); s != nil && s.Fields != nil { registryPassword = s.Fields.Password } - if err == nil && registry.Server == localRegistryServer && - registryUsername != "" && registryPassword != "" { + if err == nil && registryUsername != "" && registryPassword != "" { b.stlog.Logf("Local container registry already running on the jumpbox") - return nil + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword + + return localRegistryServer, nil } - registry.Server = localRegistryServer registryUsername = "custom-registry" registryPassword = shortuuid.New() - - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: registryUsername}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: registryPassword}}) + b.Env.RegistryUsername = registryUsername + b.Env.RegistryPassword = registryPassword commands := []string{ "apt-get update", @@ -145,20 +180,25 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { -v /root/registry.crt:/certs/registry.crt \ -v /root/registry.key:/certs/registry.key \ registry:3`, - `mkdir -p /etc/docker/certs.d/` + registry.Server, - `cp /root/registry.crt /etc/docker/certs.d/` + registry.Server + `/ca.crt`, + `mkdir -p /etc/docker/certs.d/` + localRegistryServer, + `cp /root/registry.crt /etc/docker/certs.d/` + localRegistryServer + `/ca.crt`, } for _, cmd := range commands { b.stlog.Logf("Running command on the jumpbox: %s", util.Truncate(cmd, 12)) err := registryNode.RunSSHCommand("root", cmd) if err != nil { - return fmt.Errorf("failed to run command on the jumpbox: %w", err) + return "", fmt.Errorf("failed to run command on the jumpbox: %w", err) } } - allNodes := append(b.Env.ControlPlaneNodes, b.Env.CephNodes...) - for _, node := range allNodes { + return localRegistryServer, nil +} + +// distributeRegistryCert installs the local registry's self-signed certificate on the given +// nodes. It is idempotent, so it is safe — and required — to re-run for an additional data center. +func (b *GCPBootstrapper) distributeRegistryCert(registryNode *node.Node, nodes []*node.Node) error { + for _, node := range nodes { b.stlog.Logf("Configuring node '%s' to trust local registry certificate", node.GetName()) err := registryNode.RunSSHCommand("root", "scp -o StrictHostKeyChecking=no /root/registry.crt root@"+node.GetInternalIP()+":/usr/local/share/ca-certificates/registry.crt") @@ -180,20 +220,16 @@ func (b *GCPBootstrapper) EnsureLocalContainerRegistry() error { return nil } -// EnsureGitHubAccessConfigured points the install config at ghcr.io and stores the GitHub -// credentials in the vault. The cluster pulls images from GHCR directly +// EnsureGitHubAccessConfigured resolves ghcr.io as the registry all data centers pull from. The +// credentials are written into every data center's vault by updateInstallConfig. func (b *GCPBootstrapper) EnsureGitHubAccessConfigured() error { if b.Env.GitHubPAT == "" { return fmt.Errorf("GitHub PAT is not set") } - registry := b.Env.InstallConfig.EnsureRegistry() - registry.Server = "ghcr.io" - registry.ReplaceImagesInBom = false - registry.LoadContainerImages = false - - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryUsername, Fields: &files.SecretFields{Password: b.Env.RegistryUser}}) - b.icg.GetVault().SetSecret(files.SecretEntry{Name: files.SecretRegistryPassword, Fields: &files.SecretFields{Password: b.Env.GitHubPAT}}) + b.Env.ContainerRegistryURL = "ghcr.io" + b.Env.RegistryUsername = b.Env.RegistryUser + b.Env.RegistryPassword = b.Env.GitHubPAT return nil }