From 9d645c31676c6def266fb2c73bc8924256b454f7 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 31 Jul 2026 15:20:07 +0200 Subject: [PATCH 1/4] test(gcp): end-to-end multi-data-center bootstrap coverage and docs Covers a full --multi-dc bootstrap in one spec: two data centers with their own nodes, IPs, configs and vaults, the second one pointed at the first's PostgreSQL server, both told about the topology, and the secrets partitioned the way a shared database and separate clusters require. This is the test that pins the invariants the individual commits only enforce locally. Also documents multi-DC on GCP in the installation guide: the quota footprint, the install order, the two config/vault/secrets-dir paths, the topology keys and the DNS scheme, including the per-data-center platform records the frontend depends on. Co-Authored-By: Claude Opus 5 (1M context) Signed-off-by: Jona Neef --- ...nstall-codesphere-on-any-infrastructure.md | 109 +++++ internal/bootstrap/gcp/multi_dc_test.go | 420 ++++++++++++++++++ 2 files changed, 529 insertions(+) create mode 100644 internal/bootstrap/gcp/multi_dc_test.go diff --git a/docs/install-codesphere-on-any-infrastructure.md b/docs/install-codesphere-on-any-infrastructure.md index 6ff6b46c2..597776b87 100644 --- a/docs/install-codesphere-on-any-infrastructure.md +++ b/docs/install-codesphere-on-any-infrastructure.md @@ -463,3 +463,112 @@ Run the Codesphere smoke test when an API key for the environment is available: ```bash oms smoketest codesphere --help ``` + +## Appendix: multiple data centers + +A Codesphere instance can span several data centers. They share one PostgreSQL server and one +platform domain, but each runs its own Kubernetes and Ceph cluster and hosts its own workspaces. +The steps above describe one data center; repeat them per data center with these differences. + +**Configuration.** All data centers keep the same `codesphere.domain`, `codesphere.plans`, +`codesphere.deployConfig`, `codesphere.managedServices`, feature flags and Git provider settings — +that data is stored in the shared database, so it must not diverge. Per data center: + +| Setting | Per data center | +| --- | --- | +| `dataCenter.id` / `dataCenter.name` | Distinct. The k0s cluster is named `codesphere-`, so the names must differ. | +| `codesphere.workspaceHostingBaseDomain` | `.`, resolving to that data center's workspace gateway. | +| `codesphere.publicIp` | That data center's public address. | +| `secrets.baseDir` | A directory of its own. The installer resolves the vault from `secrets.baseDir` and writes `kubeConfig` and the Ceph credentials back into it, so sharing a directory lets one data center overwrite another's. | +| `ceph.*`, `kubernetes.*`, gateway annotations | That data center's own hosts and addresses. | +| `cluster.monitoring.prometheus.remoteWrite.clusterName` | That data center's name. | + +**Topology.** Every data center's config lists all of them, so the platform knows which data centers +a user can pick from: + +```yaml +dataCenters: + - id: 1 + name: multidc + city: Karlsruhe + countryCode: DE + - id: 2 + name: multidc-dc2 + city: Karlsruhe + countryCode: DE +defaultDataCenterId: 1 +``` + +The list is identical in every data center's config, while `dataCenter` stays the local one. +Leaving `dataCenters` out defaults it to the local data center alone — correct for a single data +center, but it makes a multi-data-center instance render as single. + +**Shared PostgreSQL.** The first data center installs the server. Every other one sets +`postgres.mode: external` with `postgres.serverAddress` pointing at it, omits `postgres.primary` +and `postgres.replica`, copies `postgres.caCertPem` from the first data center's config, and adds +`postgres` to `operations.skip`. Use the server's IP address, not its hostname: the generated +server certificate carries an IP SAN only. + +**Secrets.** Derive each additional data center's vault from the first one's, keeping everything +that both must agree on and regenerating only what belongs to a single cluster: + +| Keep identical | Why | +| --- | --- | +| `postgresPassword`, `postgresReplicaPassword`, `postgresCaKeyPem`, and every `postgresUser*` / `postgresPassword*` pair | The roles live on the shared server. Divergent values make one data center's pods fail authentication, or let its install rotate credentials the other is using. | +| `tokenPrivateKey`, `tokenPublicKey` | A session token minted in one data center is presented to services in the other. | +| `domainAuthPrivateKey` / `PublicKey`, `mounterHmacSecret`, `mongoDbPasswordEncryptionKey` | They sign or encrypt rows in the shared database. | +| Registry, OAuth, OIDC and OpenBao credentials | The same external services. | + +| Regenerate per data center | Paired config field | +| --- | --- | +| `selfSignedCaKeyPem` | `cluster.certificates.ca.certPem` | +| `cephSshPrivateKey` | `ceph.cephAdmSshKey.publicKey` | +| `acmeEabMacKey` | `codesphere.certIssuer.acme.eabKeyId` | +| `kubeConfig`, and everything prefixed `ceph`, `csi` or `rgw` | written by the installer's `ceph` and `kubernetes` steps | + +Always clear the paired config field together with the vault secret. The generators are gated on +the vault entry, so a config field left in place keeps a stale value next to a fresh key. + +**DNS.** The platform name stays shared; the per-data-center platform, workspace and SSH names are +not: + +| Record | Target | +| --- | --- | +| ``, `*.` | The first data center's platform gateway | +| `.`, `*..` | **That** data center's platform gateway | +| `.`, `*..` | That data center's workspace gateway | +| `*..ssh.` | That data center's workspace SSH proxy | + +The second row is easy to miss. The platform builds each data center's service endpoint as +`.` and the browser calls it directly — before rendering any UI, it asks +the endpoint of the data center a workspace lives in for its configuration. Left to the +`*.` wildcard, that name resolves to the first data center's gateway, which has +no route for it, and the whole UI fails to load with a connection error. + +**Install order.** Install the first data center to completion before starting the next. Its +install creates the database, roles and schema that the others reuse. Every data center runs the +`ceph`, `kubernetes`, `set-up-cluster`, `codesphere` and `ms-backends` steps — the `codesphere` +step is what registers the data center in the configmap. + +### On GCP, for testing + +`oms beta bootstrap-gcp --multi-dc=true` builds a two-data-center instance in one GCP project, +applying everything above automatically. It shares the project's VPC, jumpbox and PostgreSQL VM, +and gives each data center three Ceph nodes, three k0s nodes and three static IPs of its own: + +```bash +oms beta bootstrap-gcp \ + --project-name multidc-test --billing-account "$BILLING" \ + --base-domain oms-testing.example.com \ + --multi-dc=true --datacenter-name multidc \ + --install-version +``` + +The second data center's resources are suffixed `-dc2`: VMs `ceph-1-dc2` … `k0s-3-dc2`, static IPs +`gateway-dc2`, `public-gateway-dc2`, `ssh-proxy-dc2`, local files `config-dc2.yaml` and +`prod-dc2.vault.yaml`, and `/etc/codesphere/config-dc2.yaml` plus +`/etc/codesphere/secrets-dc2/` on the shared jumpbox. Workspaces resolve under +`1.ws.` and `2.ws.`. + +Two data centers mean 14 VMs — roughly 100 vCPUs — and 6 regional static addresses. `--multi-dc` cannot be combined with +`--datacenter-id`, since the IDs are derived, and requires `--write-config`. diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go new file mode 100644 index 000000000..7fd595a8b --- /dev/null +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -0,0 +1,420 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package gcp_test + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "cloud.google.com/go/compute/apiv1/computepb" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/stretchr/testify/mock" + "google.golang.org/api/cloudbilling/v1" + + "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" + "github.com/codesphere-cloud/oms/internal/codesphere" + "github.com/codesphere-cloud/oms/internal/env" + "github.com/codesphere-cloud/oms/internal/github" + "github.com/codesphere-cloud/oms/internal/installer" + "github.com/codesphere-cloud/oms/internal/installer/files" + "github.com/codesphere-cloud/oms/internal/installer/node" + "github.com/codesphere-cloud/oms/internal/portal" + "github.com/codesphere-cloud/oms/internal/util" +) + +// realConfigManager returns an install config manager that keeps its files in memory, so tests +// can inspect the config and vault a data center would actually be installed with. +func realConfigManager(writes map[string][]byte) installer.InstallConfigManager { + icm, err := installer.NewInstallConfigManager("plain", "") + Expect(err).NotTo(HaveOccurred()) + icm.(*installer.InstallConfig).SetFileIO(&recordingFileIO{writes: writes}) + + return icm +} + +// recordingFileIO records written files and reports every other path as missing, so a bootstrap +// run behaves as if nothing existed locally beforehand. +type recordingFileIO struct { + util.FilesystemWriter + writes map[string][]byte + temps int +} + +func (f *recordingFileIO) CreateAndWrite(path string, content []byte, _ string) error { + f.writes[path] = content + return nil +} + +func (f *recordingFileIO) WriteFile(path string, content []byte, _ os.FileMode) error { + f.writes[path] = content + return nil +} + +func (f *recordingFileIO) ReadFile(path string) ([]byte, error) { + if content, ok := f.writes[path]; ok { + return content, nil + } + + return nil, fs.ErrNotExist +} + +func (f *recordingFileIO) Exists(path string) bool { + _, ok := f.writes[path] + + return ok +} + +func (f *recordingFileIO) MkdirAll(string, os.FileMode) error { + return nil +} + +// CreateTemp, Rename and Remove keep the vault's atomic write in memory: it writes a temporary +// file next to the target and renames it over it. +func (f *recordingFileIO) CreateTemp(dir, pattern string) (string, error) { + f.temps++ + + return filepath.Join(dir, fmt.Sprintf("%s.%d", pattern, f.temps)), nil +} + +func (f *recordingFileIO) Rename(oldPath, newPath string) error { + content, ok := f.writes[oldPath] + if !ok { + return fs.ErrNotExist + } + + f.writes[newPath] = content + delete(f.writes, oldPath) + + return nil +} + +func (f *recordingFileIO) Remove(path string) error { + delete(f.writes, path) + + return nil +} + +var _ = Describe("Multi-DC bootstrap", func() { + var ( + nodeClient *node.MockNodeClient + csEnv *gcp.CodesphereEnvironment + gc *gcp.MockGCPClientManager + fw *util.MockFileIO + writes map[string][]byte + primaryICG installer.InstallConfigManager + + bs *gcp.GCPBootstrapper + ) + + BeforeEach(func() { + nodeClient = node.NewMockNodeClient(GinkgoT()) + gc = gcp.NewMockGCPClientManager(GinkgoT()) + fw = util.NewMockFileIO(GinkgoT()) + writes = map[string][]byte{} + primaryICG = realConfigManager(writes) + + csEnv = &gcp.CodesphereEnvironment{ + MultiDC: true, + ProjectName: "test-project", + BillingAccount: "test-billing-account", + BaseDomain: "example.com", + Region: "us-central1", + Zone: "us-central1-a", + DNSProjectID: "dns-project", + DNSZoneName: "test-zone", + ProjectTTL: "1h", + SecretsDir: "/etc/codesphere/secrets", + DatacenterName: "multidc", + InstallConfigPath: "config.yaml", + SecretsFilePath: "prod.vault.yaml", + WriteConfig: true, + RegistryType: gcp.RegistryTypeGitHub, + RegistryUser: "registry-user", + GitHubPAT: "fake-pat", + SSHPublicKeyPath: "key.pub", + RootDiskSize: 50, + InternalFlags: gcp.DefaultInternalFlags, + PreviewFlags: gcp.DefaultPreviewFlags, + FeatureFlags: gcp.DefaultFeatureFlags, + } + }) + + JustBeforeEach(func() { + var err error + + bs, err = gcp.NewGCPBootstrapper( + context.Background(), + env.NewEnv(), + bootstrap.NewStepLogger(false), + csEnv, + primaryICG, + gc, + fw, + nodeClient, + portal.NewMockPortal(GinkgoT()), + util.NewFakeTime(), + github.NewMockGitHubClient(GinkgoT()), + ) + Expect(err).NotTo(HaveOccurred()) + + // The primary data center reuses the bootstrapper's config manager; every other one + // needs its own, so seed the layout before Bootstrap derives it. + bs.Env.DataCenters = gcp.BuildDataCenters(bs.Env) + for _, dc := range bs.Env.DataCenters[1:] { + dc.ConfigManager = realConfigManager(writes) + } + }) + + // expectBootstrapMocks sets up the GCP, SSH and file mocks for a full two-data-center run. + expectBootstrapMocks := func(projectID string) { + const vmCount = 14 + + fw.EXPECT().Exists(mock.Anything).Return(false) + fw.EXPECT().MkdirAll(mock.Anything, os.FileMode(0755)).Return(nil) + fw.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + // The SSH keys are read once and handed to every VM. + fw.EXPECT().ReadFile(mock.Anything).Return([]byte("ssh-rsa AAA..."), nil).Once() + + // EnsureProject only creates a project when the lookup fails with this exact message. + gc.EXPECT().GetProjectByName(mock.Anything, "test-project").Return(nil, fmt.Errorf("project not found: test-project")) + gc.EXPECT().CreateProjectID("test-project").Return(projectID) + gc.EXPECT().CreateProject(mock.Anything, mock.Anything, "test-project", mock.Anything).Return(mock.Anything, nil) + gc.EXPECT().GetBillingInfo(projectID).Return(&cloudbilling.ProjectBillingInfo{BillingEnabled: false}, nil) + gc.EXPECT().EnableBilling(projectID, "test-billing-account").Return(nil) + gc.EXPECT().EnableAPIs(projectID, mock.Anything).Return(nil) + gc.EXPECT().CreateServiceAccount(projectID, "cloud-controller", "cloud-controller").Return("cc@p.iam.gserviceaccount.com", false, nil) + gc.EXPECT().AssignIAMRole(projectID, "cloud-controller", projectID, []string{"roles/compute.admin"}).Return(nil) + gc.EXPECT().AssignIAMRole("dns-project", "cloud-controller", projectID, []string{"roles/dns.admin"}).Return(nil) + // OpenFGA database backups: service account, storage role, bucket and HMAC key. + openfgaSA := "openfga-backup@" + projectID + ".iam.gserviceaccount.com" + gc.EXPECT().CreateServiceAccount(projectID, "openfga-backup", "openfga-backup").Return(openfgaSA, true, nil) + gc.EXPECT().AssignIAMRole(projectID, "openfga-backup", projectID, []string{"roles/storage.objectAdmin"}).Return(nil) + gc.EXPECT().EnsureStorageBucket(projectID, projectID+"-openfga-backup", "us-central1").Return(nil) + gc.EXPECT().CreateHMACKey(projectID, openfgaSA).Return("fake-access-id", "fake-secret", nil) + gc.EXPECT().CreateVPC(projectID, "us-central1", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil) + gc.EXPECT().CreateFirewallRule(projectID, mock.Anything).Return(nil).Times(5) + + mockGetInstanceNotFoundThenRunning(gc, projectID, "us-central1-a", makeRunningInstance("10.10.0.2", "1.2.3.4"), vmCount) + gc.EXPECT().CreateInstance(projectID, "us-central1-a", mock.Anything).Return(nil).Times(vmCount) + + // Two data centers reserve three addresses each, suffixed for the second one. + for _, name := range []string{ + "gateway", "public-gateway", "ssh-proxy", + "gateway-dc2", "public-gateway-dc2", "ssh-proxy-dc2", + } { + ip := fmt.Sprintf("203.0.113.%d", len(name)) + gc.EXPECT().GetAddress(projectID, "us-central1", name).Return(nil, fmt.Errorf("not found")).Once() + gc.EXPECT().CreateAddress(projectID, "us-central1", mock.MatchedBy(func(addr *computepb.Address) bool { + return addr.GetName() == name + })).Return(ip, nil).Once() + } + + gc.EXPECT().EnsureDNSManagedZone("dns-project", "test-zone", "example.com.", mock.Anything).Return(nil) + gc.EXPECT().EnsureDNSRecordSets("dns-project", "test-zone", mock.Anything).Return(nil) + + nodeClient.EXPECT().WaitReady(mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().HasFile(mock.Anything, mock.Anything).Return(false) + nodeClient.EXPECT().RunCommand(mock.Anything, mock.Anything, mock.Anything).Return(nil) + nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + } + + It("bootstraps two data centers sharing one database", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + Expect(bs.Env.DataCenters).To(HaveLen(2)) + primary, secondary := bs.Env.DataCenters[0], bs.Env.DataCenters[1] + + By("giving each data center its own ceph and k0s nodes") + Expect(vmNamesOfNodes(primary.CephNodes)).To(Equal([]string{"ceph-1", "ceph-2", "ceph-3"})) + Expect(vmNamesOfNodes(primary.ControlPlaneNodes)).To(Equal([]string{"k0s-1", "k0s-2", "k0s-3"})) + Expect(vmNamesOfNodes(secondary.CephNodes)).To(Equal([]string{"ceph-1-dc2", "ceph-2-dc2", "ceph-3-dc2"})) + Expect(vmNamesOfNodes(secondary.ControlPlaneNodes)).To(Equal([]string{"k0s-1-dc2", "k0s-2-dc2", "k0s-3-dc2"})) + + By("sharing the platform domain and scoping the workspace domain") + Expect(primary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(secondary.InstallConfig.Codesphere.Domain).To(Equal("cs.example.com")) + Expect(primary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("1.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.WorkspaceHostingBaseDomain).To(Equal("2.ws.example.com")) + Expect(secondary.InstallConfig.Codesphere.CustomDomains.CNameBaseDomain).To(Equal("2.ws.example.com")) + + By("giving each data center a distinct ID and name") + Expect(primary.InstallConfig.Datacenter.ID).To(Equal(1)) + Expect(secondary.InstallConfig.Datacenter.ID).To(Equal(2)) + // The k0s cluster is named codesphere-, so the names must differ. + Expect(primary.InstallConfig.Datacenter.Name).To(Equal("multidc")) + Expect(secondary.InstallConfig.Datacenter.Name).To(Equal("multidc-dc2")) + + By("installing postgres in the primary data center and reusing it in the secondary") + Expect(primary.InstallConfig.Postgres.Mode).To(Equal("install")) + Expect(primary.InstallConfig.Postgres.Primary.IP).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Mode).To(Equal("external")) + // The server certificate carries only an IP SAN, so the address must be the IP. + Expect(secondary.InstallConfig.Postgres.ServerAddress).To(Equal("10.10.0.2")) + Expect(secondary.InstallConfig.Postgres.Primary).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.Replica).To(BeNil()) + Expect(secondary.InstallConfig.Postgres.CACertPem).To(Equal(primary.InstallConfig.Postgres.CACertPem)) + Expect(secondary.InstallConfig.Postgres.CACertPem).NotTo(BeEmpty()) + + By("skipping the postgres install step in the secondary data center only") + Expect(secondary.InstallConfig.Operations.Skip).To(ContainElement("postgres")) + // Both data centers install their own ceph and kubernetes. + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("ceph")) + Expect(secondary.InstallConfig.Operations.Skip).NotTo(ContainElement("kubernetes")) + + if primary.InstallConfig.Operations != nil { + Expect(primary.InstallConfig.Operations.Skip).NotTo(ContainElement("postgres")) + } + + By("telling both data centers about every data center of the installation") + // The installer defaults dataCenters to the local one, so without the list each data + // center renders as a single-data-center instance. + topology := []files.DatacenterConfig{ + {ID: 1, Name: "multidc", City: "Karlsruhe", CountryCode: "DE"}, + {ID: 2, Name: "multidc-dc2", City: "Karlsruhe", CountryCode: "DE"}, + } + for _, dc := range bs.Env.DataCenters { + Expect(dc.InstallConfig.DataCenters).To(Equal(topology), "data center %d", dc.ID) + Expect(dc.InstallConfig.DefaultDataCenterID).To(Equal(1), "data center %d", dc.ID) + // The local data center is still the one this config installs. + Expect(dc.InstallConfig.Datacenter.ID).To(Equal(dc.ID)) + } + + By("giving each data center its own vault directory on the shared jumpbox") + Expect(primary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets")) + Expect(secondary.InstallConfig.Secrets.BaseDir).To(Equal("/etc/codesphere/secrets-dc2")) + Expect(writes).To(HaveKey("config.yaml")) + Expect(writes).To(HaveKey("config-dc2.yaml")) + Expect(writes).To(HaveKey("prod.vault.yaml")) + Expect(writes).To(HaveKey("prod-dc2.vault.yaml")) + + primaryVault := primary.ConfigManager.GetVault() + secondaryVault := secondary.ConfigManager.GetVault() + + By("sharing every secret both data centers need to agree on") + + shared := []string{ + files.SecretPostgresPassword, + files.SecretPostgresReplicaPassword, + files.SecretPostgresCaKeyPem, + files.SecretTokenPrivateKey, + files.SecretTokenPublicKey, + files.SecretDomainAuthPrivateKey, + files.SecretMounterHmacSecret, + files.SecretMongoDbPasswordEncryptionKey, + } + for _, svc := range codesphere.PostgresServices { + shared = append(shared, files.PostgresUserSecretName(svc.Name), files.PostgresPasswordSecretName(svc.Name)) + } + + for _, name := range shared { + Expect(primaryVault.GetSecret(name)).NotTo(BeNil(), "primary should have %s", name) + Expect(secondaryVault.GetSecret(name)).To(Equal(primaryVault.GetSecret(name)), "%s must be shared", name) + } + + By("regenerating the per-cluster secrets for the secondary data center") + + for _, name := range []string{files.SecretSelfSignedCaKeyPem, files.SecretCephSshPrivateKey} { + Expect(secondaryVault.GetSecret(name)).NotTo(BeNil(), "secondary should have %s", name) + Expect(secondaryVault.GetSecret(name).File.Content). + NotTo(Equal(primaryVault.GetSecret(name).File.Content), "%s must be per data center", name) + } + + Expect(secondary.InstallConfig.Cluster.Certificates.CA.CertPem). + NotTo(Equal(primary.InstallConfig.Cluster.Certificates.CA.CertPem)) + Expect(secondary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey). + NotTo(Equal(primary.InstallConfig.Ceph.CephAdmSSHKey.PublicKey)) + + By("recording the DNS records it created so cleanup can delete them") + Expect(bs.Env.DNSRecords).To(Equal(gcp.DataCenterDNSRecordNames("example.com", bs.Env.DataCenters))) + + By("pointing each data center's install command at its own config and vault") + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config.yaml")) + Expect(bs.InstallCommand(primary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets/prod.vault.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("-c /etc/codesphere/config-dc2.yaml")) + Expect(bs.InstallCommand(secondary, "pkg.tar.gz")).To(ContainSubstring("--vault /etc/codesphere/secrets-dc2/prod.vault.yaml")) + + By("producing postgres blocks the installer's validation accepts") + + for _, dc := range bs.Env.DataCenters { + for _, problem := range dc.ConfigManager.ValidateInstallConfig() { + Expect(problem).NotTo(ContainSubstring("postgres"), "data center %d", dc.ID) + } + } + }) + + It("installs a separate k0s cluster per data center", func() { + csEnv.Jumpbox = &node.Node{NodeClient: nodeClient, FileIO: fw} + for _, dc := range []struct{ config, secretsDir string }{ + {"/etc/codesphere/config.yaml", "/etc/codesphere/secrets"}, + {"/etc/codesphere/config-dc2.yaml", "/etc/codesphere/secrets-dc2"}, + } { + nodeClient.EXPECT().RunCommand(csEnv.Jumpbox, "root", mock.MatchedBy(func(command string) bool { + return strings.HasPrefix(command, "oms install k0s ") && + strings.Contains(command, "--install-config "+dc.config+" ") && + strings.Contains(command, "--vault "+dc.secretsDir+"/prod.vault.yaml ") && + strings.Contains(command, "--vault-priv-key "+dc.secretsDir+"/age_key.txt") + })).Return(nil).Once() + } + + Expect(bs.InstallK0s()).To(Succeed()) + }) + + It("mirrors the primary data center onto the legacy infra file fields", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + primary := bs.Env.DataCenters[0] + Expect(bs.Env.ControlPlaneNodes).To(Equal(primary.ControlPlaneNodes)) + Expect(bs.Env.CephNodes).To(Equal(primary.CephNodes)) + Expect(bs.Env.GatewayIP).To(Equal(primary.GatewayIP)) + Expect(bs.Env.PublicGatewayIP).To(Equal(primary.PublicGatewayIP)) + Expect(bs.Env.SshProxyIP).To(Equal(primary.SSHProxyIP)) + }) + + Describe("validateMultiDC", func() { + DescribeTable("rejects flag combinations it cannot satisfy", + func(mutate func(), wantErr string) { + mutate() + Expect(bs.ValidateInput()).To(MatchError(ContainSubstring(wantErr))) + }, + Entry("without write-config", func() { csEnv.WriteConfig = false }, "multi-dc requires write-config"), + Entry("with an explicit datacenter ID", func() { csEnv.DatacenterIDExplicit = true }, "datacenter-id cannot be combined with multi-dc"), + Entry("without a datacenter name", func() { csEnv.DatacenterName = "" }, "datacenter-name is required with multi-dc"), + Entry("without a config path", func() { csEnv.InstallConfigPath = "" }, "cannot derive a per-data-center path"), + Entry("without a secrets path", func() { csEnv.SecretsFilePath = "" }, "cannot derive a per-data-center path"), + ) + + It("accepts the default multi-dc flags", func() { + Expect(bs.ValidateInput()).To(Succeed()) + }) + + It("does not constrain a single-data-center bootstrap", func() { + csEnv.MultiDC = false + csEnv.WriteConfig = false + csEnv.DatacenterName = "" + csEnv.DatacenterIDExplicit = true + + Expect(bs.ValidateInput()).To(Succeed()) + }) + }) +}) + +// vmNamesOfNodes returns the names of the given nodes, in order. +func vmNamesOfNodes(nodes []*node.Node) []string { + names := make([]string, len(nodes)) + for i, n := range nodes { + names[i] = n.GetName() + } + + return names +} From fe4ff62504e80b3f28bbddea8d36a32fb38c191b Mon Sep 17 00:00:00 2001 From: NJona <25478046+NJona@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:18:07 +0000 Subject: [PATCH 2/4] chore(docs): Auto-update docs and licenses Signed-off-by: NJona <25478046+NJona@users.noreply.github.com> --- ...nstall-codesphere-on-any-infrastructure.md | 109 ------------------ 1 file changed, 109 deletions(-) diff --git a/docs/install-codesphere-on-any-infrastructure.md b/docs/install-codesphere-on-any-infrastructure.md index 597776b87..6ff6b46c2 100644 --- a/docs/install-codesphere-on-any-infrastructure.md +++ b/docs/install-codesphere-on-any-infrastructure.md @@ -463,112 +463,3 @@ Run the Codesphere smoke test when an API key for the environment is available: ```bash oms smoketest codesphere --help ``` - -## Appendix: multiple data centers - -A Codesphere instance can span several data centers. They share one PostgreSQL server and one -platform domain, but each runs its own Kubernetes and Ceph cluster and hosts its own workspaces. -The steps above describe one data center; repeat them per data center with these differences. - -**Configuration.** All data centers keep the same `codesphere.domain`, `codesphere.plans`, -`codesphere.deployConfig`, `codesphere.managedServices`, feature flags and Git provider settings — -that data is stored in the shared database, so it must not diverge. Per data center: - -| Setting | Per data center | -| --- | --- | -| `dataCenter.id` / `dataCenter.name` | Distinct. The k0s cluster is named `codesphere-`, so the names must differ. | -| `codesphere.workspaceHostingBaseDomain` | `.`, resolving to that data center's workspace gateway. | -| `codesphere.publicIp` | That data center's public address. | -| `secrets.baseDir` | A directory of its own. The installer resolves the vault from `secrets.baseDir` and writes `kubeConfig` and the Ceph credentials back into it, so sharing a directory lets one data center overwrite another's. | -| `ceph.*`, `kubernetes.*`, gateway annotations | That data center's own hosts and addresses. | -| `cluster.monitoring.prometheus.remoteWrite.clusterName` | That data center's name. | - -**Topology.** Every data center's config lists all of them, so the platform knows which data centers -a user can pick from: - -```yaml -dataCenters: - - id: 1 - name: multidc - city: Karlsruhe - countryCode: DE - - id: 2 - name: multidc-dc2 - city: Karlsruhe - countryCode: DE -defaultDataCenterId: 1 -``` - -The list is identical in every data center's config, while `dataCenter` stays the local one. -Leaving `dataCenters` out defaults it to the local data center alone — correct for a single data -center, but it makes a multi-data-center instance render as single. - -**Shared PostgreSQL.** The first data center installs the server. Every other one sets -`postgres.mode: external` with `postgres.serverAddress` pointing at it, omits `postgres.primary` -and `postgres.replica`, copies `postgres.caCertPem` from the first data center's config, and adds -`postgres` to `operations.skip`. Use the server's IP address, not its hostname: the generated -server certificate carries an IP SAN only. - -**Secrets.** Derive each additional data center's vault from the first one's, keeping everything -that both must agree on and regenerating only what belongs to a single cluster: - -| Keep identical | Why | -| --- | --- | -| `postgresPassword`, `postgresReplicaPassword`, `postgresCaKeyPem`, and every `postgresUser*` / `postgresPassword*` pair | The roles live on the shared server. Divergent values make one data center's pods fail authentication, or let its install rotate credentials the other is using. | -| `tokenPrivateKey`, `tokenPublicKey` | A session token minted in one data center is presented to services in the other. | -| `domainAuthPrivateKey` / `PublicKey`, `mounterHmacSecret`, `mongoDbPasswordEncryptionKey` | They sign or encrypt rows in the shared database. | -| Registry, OAuth, OIDC and OpenBao credentials | The same external services. | - -| Regenerate per data center | Paired config field | -| --- | --- | -| `selfSignedCaKeyPem` | `cluster.certificates.ca.certPem` | -| `cephSshPrivateKey` | `ceph.cephAdmSshKey.publicKey` | -| `acmeEabMacKey` | `codesphere.certIssuer.acme.eabKeyId` | -| `kubeConfig`, and everything prefixed `ceph`, `csi` or `rgw` | written by the installer's `ceph` and `kubernetes` steps | - -Always clear the paired config field together with the vault secret. The generators are gated on -the vault entry, so a config field left in place keeps a stale value next to a fresh key. - -**DNS.** The platform name stays shared; the per-data-center platform, workspace and SSH names are -not: - -| Record | Target | -| --- | --- | -| ``, `*.` | The first data center's platform gateway | -| `.`, `*..` | **That** data center's platform gateway | -| `.`, `*..` | That data center's workspace gateway | -| `*..ssh.` | That data center's workspace SSH proxy | - -The second row is easy to miss. The platform builds each data center's service endpoint as -`.` and the browser calls it directly — before rendering any UI, it asks -the endpoint of the data center a workspace lives in for its configuration. Left to the -`*.` wildcard, that name resolves to the first data center's gateway, which has -no route for it, and the whole UI fails to load with a connection error. - -**Install order.** Install the first data center to completion before starting the next. Its -install creates the database, roles and schema that the others reuse. Every data center runs the -`ceph`, `kubernetes`, `set-up-cluster`, `codesphere` and `ms-backends` steps — the `codesphere` -step is what registers the data center in the configmap. - -### On GCP, for testing - -`oms beta bootstrap-gcp --multi-dc=true` builds a two-data-center instance in one GCP project, -applying everything above automatically. It shares the project's VPC, jumpbox and PostgreSQL VM, -and gives each data center three Ceph nodes, three k0s nodes and three static IPs of its own: - -```bash -oms beta bootstrap-gcp \ - --project-name multidc-test --billing-account "$BILLING" \ - --base-domain oms-testing.example.com \ - --multi-dc=true --datacenter-name multidc \ - --install-version -``` - -The second data center's resources are suffixed `-dc2`: VMs `ceph-1-dc2` … `k0s-3-dc2`, static IPs -`gateway-dc2`, `public-gateway-dc2`, `ssh-proxy-dc2`, local files `config-dc2.yaml` and -`prod-dc2.vault.yaml`, and `/etc/codesphere/config-dc2.yaml` plus -`/etc/codesphere/secrets-dc2/` on the shared jumpbox. Workspaces resolve under -`1.ws.` and `2.ws.`. - -Two data centers mean 14 VMs — roughly 100 vCPUs — and 6 regional static addresses. `--multi-dc` cannot be combined with -`--datacenter-id`, since the IDs are derived, and requires `--write-config`. From 22998e52766e9f5dcb3d7edda0252c25b4e5f476 Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Tue, 29 Sep 2026 17:08:53 +0200 Subject: [PATCH 3/4] test(gcp): cover the local and Artifact Registry paths of a multi-DC bootstrap The end-to-end spec only exercised GHCR; these check that both data centers share one registry, its credentials, and the jumpbox certificate. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jona Neef --- internal/bootstrap/gcp/multi_dc_test.go | 99 +++++++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go index 7fd595a8b..d038f0e65 100644 --- a/internal/bootstrap/gcp/multi_dc_test.go +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" + "cloud.google.com/go/artifactregistry/apiv1/artifactregistrypb" "cloud.google.com/go/compute/apiv1/computepb" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -18,6 +19,7 @@ import ( "google.golang.org/api/cloudbilling/v1" "github.com/codesphere-cloud/oms/internal/bootstrap" + "github.com/codesphere-cloud/oms/internal/bootstrap/datacenter" "github.com/codesphere-cloud/oms/internal/bootstrap/gcp" "github.com/codesphere-cloud/oms/internal/codesphere" "github.com/codesphere-cloud/oms/internal/env" @@ -225,6 +227,103 @@ var _ = Describe("Multi-DC bootstrap", func() { nodeClient.EXPECT().CopyFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) } + // registryCertTargets returns the names of the nodes the jumpbox copied the local + // registry's certificate to. + registryCertTargets := func() []string { + targets := []string{} + + for _, call := range nodeClient.Calls { + if call.Method != "RunCommand" { + continue + } + + if cmd, _ := call.Arguments.Get(2).(string); strings.HasPrefix(cmd, "scp ") && strings.Contains(cmd, "registry.crt") { + targets = append(targets, call.Arguments.Get(0).(*node.Node).GetName()+" -> "+cmd[strings.LastIndex(cmd, "@")+1:]) + } + } + + return targets + } + + Context("with the local container registry", func() { + BeforeEach(func() { + csEnv.RegistryType = gcp.RegistryTypeLocalContainer + csEnv.GitHubPAT = "" + csEnv.RegistryUser = "" + }) + + It("serves both data centers from one registry on the jumpbox", func() { + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + server := bs.Env.Jumpbox.GetInternalIP() + ":5000" + + primary, secondary := bs.Env.DataCenters[0], bs.Env.DataCenters[1] + for _, dc := range []*datacenter.DataCenter{primary, secondary} { + Expect(dc.InstallConfig.Registry.Server).To(Equal(server), "data center %d", dc.ID) + Expect(dc.InstallConfig.Registry.LoadContainerImages).To(BeTrue(), "data center %d", dc.ID) + Expect(dc.InstallConfig.Registry.ReplaceImagesInBom).To(BeTrue(), "data center %d", dc.ID) + Expect(dc.ConfigManager.GetVault().GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("custom-registry")) + } + + Expect(secondary.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password). + To(Equal(primary.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password)) + Expect(primary.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password).NotTo(BeEmpty()) + + // Every cluster node of both data centers must trust the jumpbox registry. + targets := registryCertTargets() + GinkgoWriter.Printf("registry certificate copied: %v\n", targets) + Expect(targets).To(HaveLen(len(primary.CephNodes) + len(primary.ControlPlaneNodes) + + len(secondary.CephNodes) + len(secondary.ControlPlaneNodes))) + + for _, target := range targets { + Expect(target).To(HavePrefix("jumpbox -> ")) + } + + Expect(len(secondary.CephNodes) + len(secondary.ControlPlaneNodes)).To(BeNumerically(">", 0)) + + for _, dc := range []*datacenter.DataCenter{primary, secondary} { + Expect(bs.InstallCommand(dc, "pkg.tar.gz")).NotTo(ContainSubstring("load-container-images")) + } + }) + }) + + Context("with a GCP Artifact Registry", func() { + BeforeEach(func() { + csEnv.RegistryType = gcp.RegistryTypeArtifactRegistry + csEnv.GitHubPAT = "" + csEnv.RegistryUser = "" + }) + + It("points both data centers at the created repository", func() { + const uri = "us-central1-docker.pkg.dev/test-project-12345/codesphere-registry" + + gc.EXPECT().GetArtifactRegistry("test-project-12345", "us-central1", "codesphere-registry").Return(nil, fmt.Errorf("not found")) + gc.EXPECT().CreateArtifactRegistry("test-project-12345", "us-central1", "codesphere-registry"). + Return(&artifactregistrypb.Repository{RegistryUri: uri}, nil) + + const writer = "artifact-registry-writer@test-project-12345.iam.gserviceaccount.com" + gc.EXPECT().CreateServiceAccount("test-project-12345", "artifact-registry-writer", "artifact-registry-writer").Return(writer, true, nil) + gc.EXPECT().CreateServiceAccountKey("test-project-12345", writer).Return("writer-key", nil) + gc.EXPECT().AssignIAMRole("test-project-12345", "artifact-registry-writer", "test-project-12345", []string{"roles/artifactregistry.writer"}).Return(nil) + expectBootstrapMocks("test-project-12345") + + Expect(bs.Bootstrap()).To(Succeed()) + + for _, dc := range bs.Env.DataCenters { + Expect(dc.InstallConfig.Registry.Server).To(Equal(uri), "data center %d", dc.ID) + Expect(dc.InstallConfig.Registry.LoadContainerImages).To(BeTrue(), "data center %d", dc.ID) + // Only the primary vault is written when the key is created; the others must inherit it. + Expect(dc.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword)).NotTo(BeNil(), "data center %d", dc.ID) + Expect(dc.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password).To(Equal("writer-key"), "data center %d", dc.ID) + Expect(dc.ConfigManager.GetVault().GetSecret(files.SecretRegistryUsername).Fields.Password).To(Equal("_json_key_base64"), "data center %d", dc.ID) + } + + Expect(registryCertTargets()).To(BeEmpty()) + }) + }) + It("bootstraps two data centers sharing one database", func() { expectBootstrapMocks("test-project-12345") From 042b68ed92ed13723bf4955b170479b955bfeb3a Mon Sep 17 00:00:00 2001 From: Jona Neef Date: Fri, 2 Oct 2026 10:28:17 +0200 Subject: [PATCH 4/4] test(gcp): expect the Postgres node among the multi-DC registry certificate targets Co-Authored-By: Claude Opus 5.5 Signed-off-by: Jona Neef --- internal/bootstrap/gcp/multi_dc_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/bootstrap/gcp/multi_dc_test.go b/internal/bootstrap/gcp/multi_dc_test.go index d038f0e65..40c745b3d 100644 --- a/internal/bootstrap/gcp/multi_dc_test.go +++ b/internal/bootstrap/gcp/multi_dc_test.go @@ -271,11 +271,12 @@ var _ = Describe("Multi-DC bootstrap", func() { To(Equal(primary.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password)) Expect(primary.ConfigManager.GetVault().GetSecret(files.SecretRegistryPassword).Fields.Password).NotTo(BeEmpty()) - // Every cluster node of both data centers must trust the jumpbox registry. + // Every cluster node of both data centers and the shared Postgres node must trust the + // jumpbox registry. targets := registryCertTargets() GinkgoWriter.Printf("registry certificate copied: %v\n", targets) Expect(targets).To(HaveLen(len(primary.CephNodes) + len(primary.ControlPlaneNodes) + - len(secondary.CephNodes) + len(secondary.ControlPlaneNodes))) + len(secondary.CephNodes) + len(secondary.ControlPlaneNodes) + 1)) for _, target := range targets { Expect(target).To(HavePrefix("jumpbox -> "))