fix(gcp): give every data center its own DNS records

cli/cmd/bootstrap_gcp_cleanup_test.go

@@ -328,7 +328,38 @@ var _ = Describe("BootstrapGcpCleanupCmd", func() {
 
 				mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true)
 				mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil)
-				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", "example.com").Return(nil)
+				// An infra file without a recorded record list predates multi-DC support, so
+				// cleanup falls back to the single-data-center record names.
+				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", gcp.GetDNSRecordNames("example.com")).Return(nil)
+				mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil)
+				mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil)
+
+				err := cleanupCmd.ExecuteCleanup(deps)
+				Expect(err).NotTo(HaveOccurred())
+			})
+		})
+
+		Context("when the infra file recorded the DNS records it created", func() {
+			It("should delete exactly those records", func() {
+				cleanupCmd.Opts.ProjectID = "test-project"
+				cleanupCmd.Opts.Force = true
+
+				recorded := []gcp.DNSRecordName{
+					{Name: "cs.example.com.", Rtype: "A"},
+					{Name: "2.ws.example.com.", Rtype: "A"},
+				}
+				validEnv := gcp.CodesphereEnvironment{
+					ProjectID:   "test-project",
+					BaseDomain:  "example.com",
+					DNSZoneName: "test-zone",
+					MultiDC:     true,
+					DNSRecords:  recorded,
+				}
+				envData, _ := json.Marshal(validEnv)
+
+				mockFileIO.EXPECT().Exists("/tmp/test-infra.json").Return(true)
+				mockFileIO.EXPECT().ReadFile("/tmp/test-infra.json").Return(envData, nil)
+				mockGCPClient.EXPECT().DeleteDNSRecordSets("test-project", "test-zone", recorded).Return(nil)
 				mockGCPClient.EXPECT().DeleteProject("test-project").Return(nil)
 				mockFileIO.EXPECT().Remove("/tmp/test-infra.json").Return(nil)
 

docs/oms_install_k0s.md

@@ -48,7 +48,7 @@ $ oms install k0s --no-download
   -f, --force                   Force new download and installation
   -h, --help                    help for k0s
       --install-config string   Path to Codesphere install-config file (required)
-      --k0sctl-version string   Version of k0sctl to use (default "v0.31.1")
+      --k0sctl-version string   Version of k0sctl to use (default "v0.33.1")
       --no-download             Skip downloading k0s binary
   -p, --package string          Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from
       --ssh-key-path string     SSH private key path for remote installation

internal/bootstrap/datacenter/datacenter.go

@@ -79,6 +79,16 @@ func (dc *DataCenter) RemoteAgeKeyPath() string {
 	return filepath.Join(dc.SecretsDir, "age_key.txt")
 }
 
+// PlatformDomain returns the host this data center serves its own platform services on. The
+// platform derives it from the data center ID and codesphere.domain, which OMS sets to
+// cs.<base-domain>, and the frontend calls it directly — the browser asks
+// <dc-id>.cs.<base-domain> for the config of the data center a workspace lives in. So it has to
+// resolve to this data center's platform gateway, not to the primary's, which is what
+// cs.<base-domain> and its wildcard point at.
+func (dc *DataCenter) PlatformDomain(baseDomain string) string {
+	return fmt.Sprintf("%d.cs.%s", dc.ID, baseDomain)
+}
+
 // K0sConfigScriptPath returns the local filename of this data center's k0s configuration script.
 func (dc *DataCenter) K0sConfigScriptPath() string {
 	return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix)

internal/bootstrap/gcp/cleanup.go

@@ -199,7 +199,26 @@ func (e *CleanupExecutor) CleanupDNSRecords() error {
 		return nil
 	}
 
-	return e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.BaseDomain)
+	if err := e.Deps.GCPClient.DeleteDNSRecordSets(e.DNSProjectID, e.DNSZoneName, e.dnsRecords()); err != nil {
+		return fmt.Errorf("failed to delete DNS record sets: %w", err)
+	}
+
+	return nil
+}
+
+// dnsRecords returns the DNS records to delete. The bootstrap records what it created in the
+// infra file, which is authoritative. Older infra files predate that, and a cleanup driven only
+// by --project-id has no infra file at all, so both fall back to deriving the names.
+func (e *CleanupExecutor) dnsRecords() []DNSRecordName {
+	if len(e.InfraEnv.DNSRecords) > 0 {
+		return e.InfraEnv.DNSRecords
+	}
+
+	if len(e.InfraEnv.DataCenters) > 0 {
+		return DataCenterDNSRecordNames(e.BaseDomain, e.InfraEnv.DataCenters)
+	}
+
+	return GetDNSRecordNames(e.BaseDomain)
 }
 
 // RemoveDNSIAMBinding removes the cloud-controller service account's IAM binding

internal/bootstrap/gcp/dns.go

@@ -0,0 +1,138 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package gcp
+
+import (
+	"fmt"
+
+	"github.com/codesphere-cloud/oms/internal/bootstrap/datacenter"
+	"google.golang.org/api/dns/v1"
+)
+
+// DNSRecordName identifies a DNS record set that OMS manages.
+type DNSRecordName struct {
+	Name  string `json:"name"`
+	Rtype string `json:"rtype"`
+}
+
+// GetDNSRecordNames returns the DNS record names a single-data-center bootstrap creates for a
+// given base domain. It is the fallback for infra files written before multi-DC support, which
+// do not record the created records.
+func GetDNSRecordNames(baseDomain string) []DNSRecordName {
+	return []DNSRecordName{
+		{fmt.Sprintf("cs.%s.", baseDomain), "A"},
+		{fmt.Sprintf("*.cs.%s.", baseDomain), "A"},
+		{fmt.Sprintf("ws.%s.", baseDomain), "A"},
+		{fmt.Sprintf("*.ws.%s.", baseDomain), "A"},
+		{fmt.Sprintf("*.ssh.cs.%s.", baseDomain), "A"},
+	}
+}
+
+// DataCenterDNSRecordNames returns every DNS record OMS creates for the given data center
+// layout: the shared platform gateway names plus each data center's workspace and SSH names.
+func DataCenterDNSRecordNames(baseDomain string, dcs []*datacenter.DataCenter) []DNSRecordName {
+	records := []DNSRecordName{
+		{fmt.Sprintf("cs.%s.", baseDomain), "A"},
+		{fmt.Sprintf("*.cs.%s.", baseDomain), "A"},
+	}
+	for _, dc := range dcs {
+		records = append(records,
+			DNSRecordName{fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), "A"},
+			DNSRecordName{fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), "A"},
+			DNSRecordName{fmt.Sprintf("*.%s.", dc.SSHBaseDomain), "A"},
+		)
+		if len(dcs) > 1 {
+			records = append(records,
+				DNSRecordName{fmt.Sprintf("%s.", dc.PlatformDomain(baseDomain)), "A"},
+				DNSRecordName{fmt.Sprintf("*.%s.", dc.PlatformDomain(baseDomain)), "A"},
+			)
+		}
+	}
+
+	return records
+}
+
+// ensureDNSPermissions grants the cloud-controller service account DNS admin on the project that
+// hosts the managed zone, which is the DNS project when one is configured.
+func (b *GCPBootstrapper) ensureDNSPermissions() error {
+	dnsProject := b.Env.DNSProjectID
+	if b.Env.DNSProjectID == "" {
+		dnsProject = b.Env.ProjectID
+	}
+
+	err := b.ensureIAMRoleWithRetry(dnsProject, "cloud-controller", b.Env.ProjectID, []string{"roles/dns.admin"})
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// EnsureDNSRecords creates the managed zone and the A records for the platform, every data
+// center's workspace and SSH names and, with more than one data center, each one's own platform
+// host. It records the created names in the environment so cleanup deletes exactly those.
+func (b *GCPBootstrapper) EnsureDNSRecords() error {
+	if err := b.ensureDataCenters(); err != nil {
+		return err
+	}
+
+	gcpProject := b.Env.DNSProjectID
+	if b.Env.DNSProjectID == "" {
+		gcpProject = b.Env.ProjectID
+	}
+
+	zoneName := b.Env.DNSZoneName
+
+	err := b.GCPClient.EnsureDNSManagedZone(gcpProject, zoneName, b.Env.BaseDomain+".", "Codesphere DNS zone")
+	if err != nil {
+		return fmt.Errorf("failed to ensure DNS managed zone: %w", err)
+	}
+
+	// The platform is served from one domain shared by all data centers, pointing at the
+	// primary data center's gateway.
+	records := []*dns.ResourceRecordSet{
+		dnsARecord(fmt.Sprintf("cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP),
+		dnsARecord(fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain), b.primaryDC().GatewayIP),
+	}
+	// Workspaces and their SSH endpoints resolve per data center, so each one gets its own
+	// names pointing at its own public gateway and SSH proxy.
+	for _, dc := range b.Env.DataCenters {
+		records = append(records,
+			dnsARecord(fmt.Sprintf("%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP),
+			dnsARecord(fmt.Sprintf("*.%s.", dc.WorkspaceHostingBaseDomain), dc.PublicGatewayIP),
+			dnsARecord(fmt.Sprintf("*.%s.", dc.SSHBaseDomain), dc.SSHProxyIP),
+		)
+		// The platform calls each data center's own services at <dc-id>.cs.<base-domain>, which
+		// the wildcard above would send to the primary data center's gateway. A single data
+		// center is that primary, so it needs no record of its own.
+		if len(b.Env.DataCenters) > 1 {
+			platformDomain := dc.PlatformDomain(b.Env.BaseDomain)
+			records = append(records,
+				dnsARecord(fmt.Sprintf("%s.", platformDomain), dc.GatewayIP),
+				dnsARecord(fmt.Sprintf("*.%s.", platformDomain), dc.GatewayIP),
+			)
+		}
+	}
+
+	err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records)
+	if err != nil {
+		return fmt.Errorf("failed to ensure DNS record sets: %w", err)
+	}
+
+	// Record what was created so cleanup deletes exactly these records instead of recomputing
+	// the list from the base domain.
+	b.Env.DNSRecords = DataCenterDNSRecordNames(b.Env.BaseDomain, b.Env.DataCenters)
+
+	return nil
+}
+
+// dnsARecord builds a short-TTL A record set, as used during initial setup.
+func dnsARecord(name, ip string) *dns.ResourceRecordSet {
+	return &dns.ResourceRecordSet{
+		Name:    name,
+		Type:    "A",
+		Ttl:     300,
+		Rrdatas: []string{ip},
+	}
+}

internal/bootstrap/gcp/dns_test.go

@@ -0,0 +1,38 @@
+// Copyright (c) Codesphere Inc.
+// SPDX-License-Identifier: Apache-2.0
+
+package gcp_test
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/codesphere-cloud/oms/internal/bootstrap/gcp"
+)
+
+var _ = Describe("DataCenterDNSRecordNames", func() {
+	It("returns the single-DC records for one data center", func() {
+		dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{BaseDomain: "example.com"})
+
+		Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(ConsistOf(gcp.GetDNSRecordNames("example.com")))
+	})
+
+	It("shares the platform names and scopes the workspace names per data center", func() {
+		dcs := gcp.BuildDataCenters(&gcp.CodesphereEnvironment{MultiDC: true, BaseDomain: "example.com"})
+
+		Expect(gcp.DataCenterDNSRecordNames("example.com", dcs)).To(Equal([]gcp.DNSRecordName{
+			{Name: "cs.example.com.", Rtype: "A"},
+			{Name: "*.cs.example.com.", Rtype: "A"},
+			{Name: "1.ws.example.com.", Rtype: "A"},
+			{Name: "*.1.ws.example.com.", Rtype: "A"},
+			{Name: "*.1.ssh.cs.example.com.", Rtype: "A"},
+			{Name: "1.cs.example.com.", Rtype: "A"},
+			{Name: "*.1.cs.example.com.", Rtype: "A"},
+			{Name: "2.ws.example.com.", Rtype: "A"},
+			{Name: "*.2.ws.example.com.", Rtype: "A"},
+			{Name: "*.2.ssh.cs.example.com.", Rtype: "A"},
+			{Name: "2.cs.example.com.", Rtype: "A"},
+			{Name: "*.2.cs.example.com.", Rtype: "A"},
+		}))
+	})
+})

internal/bootstrap/gcp/gcp.go

@@ -24,7 +24,6 @@ import (
 	"github.com/codesphere-cloud/oms/internal/portal"
 	"github.com/codesphere-cloud/oms/internal/testuser"
 	"github.com/codesphere-cloud/oms/internal/util"
-	"google.golang.org/api/dns/v1"
 )
 
 // InstallerArchiveName is the package artifact bootstrapping downloads and installs.
@@ -60,23 +59,6 @@ func CheckOMSManagedLabel(labels map[string]string) bool {
 	return exists && value == "true"
 }
 
-// GetDNSRecordNames returns the DNS record names that OMS creates for a given base domain.
-func GetDNSRecordNames(baseDomain string) []struct {
-	Name  string
-	Rtype string
-} {
-	return []struct {
-		Name  string
-		Rtype string
-	}{
-		{fmt.Sprintf("cs.%s.", baseDomain), "A"},
-		{fmt.Sprintf("*.cs.%s.", baseDomain), "A"},
-		{fmt.Sprintf("ws.%s.", baseDomain), "A"},
-		{fmt.Sprintf("*.ws.%s.", baseDomain), "A"},
-		{fmt.Sprintf("*.ssh.cs.%s.", baseDomain), "A"},
-	}
-}
-
 // This should ALWAYS be empty. Internal flags are for internal feature
 // development and not intended for customer use.
 // Atm. it's not empty as the internal flags below are likely preview or
@@ -152,6 +134,9 @@ type CodesphereEnvironment struct {
 	MultiDC bool `json:"multi_dc"`
 	// DataCenters holds the per-data-center state. It always has at least one entry.
 	DataCenters []*datacenter.DataCenter `json:"datacenters"`
+	// DNSRecords records the DNS records the bootstrap created, so cleanup deletes exactly
+	// those instead of recomputing the list.
+	DNSRecords []DNSRecordName `json:"dns_records,omitempty"`
 	// ControlPlaneNodes and CephNodes are where the primary data center's nodes lived before
 	// multi-DC support. The steps that have not been migrated to DataCenters yet still use
 	// them, and infra files written by an earlier OMS carry the nodes here.
@@ -680,20 +665,6 @@ func (b *GCPBootstrapper) validateTelemetryExportParams() error {
 	return nil
 }
 
-func (b *GCPBootstrapper) ensureDnsPermissions() error {
-	dnsProject := b.Env.DNSProjectID
-	if b.Env.DNSProjectID == "" {
-		dnsProject = b.Env.ProjectID
-	}
-
-	err := b.ensureIAMRoleWithRetry(dnsProject, "cloud-controller", b.Env.ProjectID, []string{"roles/dns.admin"})
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 func (b *GCPBootstrapper) EnsureVPC() error {
 	networkName := fmt.Sprintf("%s-vpc", b.Env.ProjectID)
 	subnetName := fmt.Sprintf("%s-%s-subnet", b.Env.ProjectID, b.Env.Region)
@@ -1025,60 +996,6 @@ func (b *GCPBootstrapper) EnsureHostsConfigured() error {
 	return nil
 }
 
-func (b *GCPBootstrapper) EnsureDNSRecords() error {
-	gcpProject := b.Env.DNSProjectID
-	if b.Env.DNSProjectID == "" {
-		gcpProject = b.Env.ProjectID
-	}
-
-	zoneName := b.Env.DNSZoneName
-
-	err := b.GCPClient.EnsureDNSManagedZone(gcpProject, zoneName, b.Env.BaseDomain+".", "Codesphere DNS zone")
-	if err != nil {
-		return fmt.Errorf("failed to ensure DNS managed zone: %w", err)
-	}
-
-	records := []*dns.ResourceRecordSet{
-		{
-			Name:    fmt.Sprintf("cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.GatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.GatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.ws.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.PublicGatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("ws.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.PublicGatewayIP},
-		},
-		{
-			Name:    fmt.Sprintf("*.ssh.cs.%s.", b.Env.BaseDomain),
-			Type:    "A",
-			Ttl:     300,
-			Rrdatas: []string{b.Env.SshProxyIP},
-		},
-	}
-
-	err = b.GCPClient.EnsureDNSRecordSets(gcpProject, zoneName, records)
-	if err != nil {
-		return fmt.Errorf("failed to ensure DNS record sets: %w", err)
-	}
-
-	return nil
-}
-
 // InstallCodesphere installs Codesphere into every data center from the shared jumpbox, in
 // ascending data center order. The order matters: the primary data center's install creates the
 // database, roles and schema that the secondary ones reuse.