diff --git a/NOTICE b/NOTICE index 377368eb1..8e9dfb6fd 100644 --- a/NOTICE +++ b/NOTICE @@ -199,7 +199,7 @@ License URL: https://github.com/beorn7/perks/blob/v1.0.1/LICENSE Module: github.com/blang/semver/v4 Version: v4.0.0 License: MIT -License URL: https://github.com/blang/semver/blob/v4.0.0/v4/LICENSE +License URL: https://github.com/blang/semver/blob/v4.0.0/LICENSE ---------- Module: github.com/bmatcuk/doublestar/v4 @@ -397,7 +397,7 @@ License URL: https://github.com/emirpasic/gods/blob/v1.18.1/LICENSE Module: github.com/evanphx/json-patch/v5 Version: v5.9.11 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v5.9.11/v5/LICENSE +License URL: https://github.com/evanphx/json-patch/blob/v5.9.11/LICENSE ---------- Module: github.com/exponent-io/jsonpath @@ -721,7 +721,7 @@ License URL: https://github.com/googleapis/enterprise-certificate-proxy/blob/v0. Module: github.com/googleapis/gax-go/v2 Version: v2.26.2 License: BSD-3-Clause -License URL: https://github.com/googleapis/gax-go/blob/v2.26.2/v2/LICENSE +License URL: https://github.com/googleapis/gax-go/blob/v2.26.2/LICENSE ---------- Module: github.com/gorilla/websocket @@ -1507,7 +1507,7 @@ License URL: https://cs.opensource.google/go/x/time/+/v0.16.0:LICENSE Module: gomodules.xyz/jsonpatch/v2 Version: v2.5.0 License: Apache-2.0 -License URL: https://github.com/gomodules/jsonpatch/blob/v2.5.0/v2/LICENSE +License URL: https://github.com/gomodules/jsonpatch/blob/v2.5.0/LICENSE ---------- Module: google.golang.org/api diff --git a/cli/cmd/bootstrap_gcp.go b/cli/cmd/bootstrap_gcp.go index d59d940b8..4e15e01f6 100644 --- a/cli/cmd/bootstrap_gcp.go +++ b/cli/cmd/bootstrap_gcp.go @@ -97,6 +97,7 @@ func AddBootstrapGcpCmd(parent *cobra.Command, opts *util.GlobalOptions) { flags.StringVar(&bootstrapGcpCmd.CodesphereEnv.DNSZoneName, "dns-zone-name", "oms-testing", "Cloud DNS Zone Name (optional)") flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.Preemptible, "preemptible", false, "Use preemptible VMs for Codesphere infrastructure. Mutually exclusive with --spot-vms (default: false)") flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.SpotVMs, "spot-vms", false, "Use Spot VMs for Codesphere infrastructure. Falls back to standard VMs if spot capacity unavailable. Mutually exclusive with --preemptible (default: false)") + flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.Airgapped, "airgapped", false, "Install k0s from the k0s airgap image bundle, so the cluster nodes pull no k0s images from the internet (default: false)") flags.Int64Var(&bootstrapGcpCmd.CodesphereEnv.RootDiskSize, "root-disk-size", 50, "Instance root disk size in GB (default: 50)") flags.BoolVar(&bootstrapGcpCmd.CodesphereEnv.WriteConfig, "write-config", true, "Write generated install config to file (default: true)") diff --git a/cli/cmd/k0s/download_k0s.go b/cli/cmd/k0s/download_k0s.go index 0d291de9b..4edcebd2d 100644 --- a/cli/cmd/k0s/download_k0s.go +++ b/cli/cmd/k0s/download_k0s.go @@ -29,6 +29,7 @@ type DownloadK0sOpts struct { *util.GlobalOptions Version string Force bool + Airgap bool } func (c *DownloadK0sCmd) RunE(_ *cobra.Command, args []string) error { @@ -36,12 +37,7 @@ func (c *DownloadK0sCmd) RunE(_ *cobra.Command, args []string) error { env := c.Env k0s := installer.NewK0s(hw, env, c.FileWriter) - err := c.DownloadK0s(k0s) - if err != nil { - return fmt.Errorf("failed to download k0s: %w", err) - } - - return nil + return c.DownloadK0s(k0s) } func AddDownloadCmd(download *cobra.Command, opts *util.GlobalOptions) { @@ -55,6 +51,7 @@ func AddDownloadCmd(download *cobra.Command, opts *util.GlobalOptions) { {Cmd: "", Desc: "Download k0s using the Go-native implementation"}, {Cmd: "--version 1.22.0", Desc: "Download a specific version of k0s"}, {Cmd: "--force", Desc: "Force download even if k0s binary exists"}, + {Cmd: "--airgapped", Desc: "Also download the airgap image bundle for that version"}, }), }, Opts: DownloadK0sOpts{GlobalOptions: opts}, @@ -63,6 +60,7 @@ func AddDownloadCmd(download *cobra.Command, opts *util.GlobalOptions) { } k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", "", "Version of k0s to download") k0s.cmd.Flags().BoolVarP(&k0s.Opts.Force, "force", "f", false, "Force download even if k0s binary exists") + k0s.cmd.Flags().BoolVarP(&k0s.Opts.Airgap, "airgapped", "a", false, "Downloads the airgapped bundle for that version") util.AddCmd(download, k0s.cmd) @@ -70,21 +68,28 @@ func AddDownloadCmd(download *cobra.Command, opts *util.GlobalOptions) { } func (c *DownloadK0sCmd) DownloadK0s(k0s installer.K0sManager) error { - version := c.Opts.Version - var err error - if version == "" { - version, err = k0s.GetLatestVersion() - if err != nil { - return fmt.Errorf("failed to get latest k0s version: %w", err) - } + version, err := resolveK0sVersion(k0s, c.Opts.Version) + if err != nil { + return err } - k0sPath, err := k0s.Download(version, c.Opts.Force, !c.Opts.Verbose) + opts := installer.DownloadOptions{Force: c.Opts.Force, Quiet: !c.Opts.Verbose} + + k0sPath, err := k0s.Download(version, opts) if err != nil { return fmt.Errorf("failed to download k0s: %w", err) } - log.Printf("k0s binary downloaded successfully at '%s'", k0sPath) + if c.Opts.Airgap { + bundlePath, err := k0s.EnsureAirgapBundle(version, opts) + if err != nil { + return fmt.Errorf("failed to download k0s airgap bundle: %w", err) + } + + log.Printf("k0s airgap bundle downloaded to '%s'", bundlePath) + } + + log.Printf("k0s binary downloaded successfully to '%s'", k0sPath) return nil } diff --git a/cli/cmd/k0s/download_k0s_test.go b/cli/cmd/k0s/download_k0s_test.go index 33dde3222..feec588c1 100644 --- a/cli/cmd/k0s/download_k0s_test.go +++ b/cli/cmd/k0s/download_k0s_test.go @@ -8,6 +8,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/spf13/cobra" "github.com/codesphere-cloud/oms/cli/cmd/k0s" "github.com/codesphere-cloud/oms/cli/cmd/util" @@ -23,11 +24,13 @@ var _ = Describe("DownloadK0sk0s", func() { globalOpts *util.GlobalOptions mockEnv *env.MockEnv mockFileWriter *intutil.MockFileIO + mockK0sManager *installer.MockK0sManager ) BeforeEach(func() { mockEnv = env.NewMockEnv(GinkgoT()) mockFileWriter = intutil.NewMockFileIO(GinkgoT()) + mockK0sManager = installer.NewMockK0sManager(GinkgoT()) globalOpts = &util.GlobalOptions{} opts = &k0s.DownloadK0sOpts{ GlobalOptions: globalOpts, @@ -48,9 +51,8 @@ var _ = Describe("DownloadK0sk0s", func() { Context("DownloadK0s method", func() { It("fails when k0s manager fails to get latest version", func() { - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) - c.Opts.Version = "" // Test auto-version detection + mockK0sManager.EXPECT().GetLatestVersion().Return("", errors.New("network error")) err := c.DownloadK0s(mockK0sManager) @@ -60,11 +62,9 @@ var _ = Describe("DownloadK0sk0s", func() { }) It("fails when k0s manager fails to download", func() { - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) - c.Opts.Version = "v1.29.1+k0s.0" - mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", false, true).Return("", errors.New("download failed")) + mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", installer.DownloadOptions{Quiet: true}).Return("", errors.New("download failed")) err := c.DownloadK0s(mockK0sManager) Expect(err).To(HaveOccurred()) @@ -73,27 +73,65 @@ var _ = Describe("DownloadK0sk0s", func() { }) It("succeeds when version is specified and download works", func() { - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + c.Opts.Version = "v1.29.1+k0s.0" + mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", installer.DownloadOptions{Quiet: true}).Return("/test/workdir/k0s", nil) + + err := c.DownloadK0s(mockK0sManager) + Expect(err).ToNot(HaveOccurred()) + }) + + It("requests the airgap bundle when --airgapped is set", func() { c.Opts.Version = "v1.29.1+k0s.0" + c.Opts.Airgap = true - mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", false, true).Return("/test/workdir/k0s", nil) + opts := installer.DownloadOptions{Quiet: true} + mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", opts).Return("/test/workdir/k0s", nil) + mockK0sManager.EXPECT().EnsureAirgapBundle("v1.29.1+k0s.0", opts).Return("/cache/k0s-airgap-bundle-amd64", nil) err := c.DownloadK0s(mockK0sManager) Expect(err).ToNot(HaveOccurred()) }) - It("succeeds when version is auto-detected and download works", func() { - mockK0sManager := installer.NewMockK0sManager(GinkgoT()) + It("reports airgap bundle download failures", func() { + c.Opts.Version = "v1.29.1+k0s.0" + c.Opts.Airgap = true + + opts := installer.DownloadOptions{Quiet: true} + mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", opts).Return("/test/workdir/k0s", nil) + mockK0sManager.EXPECT().EnsureAirgapBundle("v1.29.1+k0s.0", opts).Return("", errors.New("network error")) + err := c.DownloadK0s(mockK0sManager) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to download k0s airgap bundle")) + Expect(err.Error()).To(ContainSubstring("network error")) + }) + + It("succeeds when version is auto-detected and download works", func() { c.Opts.Version = "" // Test auto-version detection c.Opts.Force = true c.Opts.Verbose = true + mockK0sManager.EXPECT().GetLatestVersion().Return("v1.29.1+k0s.0", nil) - mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", true, false).Return("/test/workdir/k0s", nil) + mockK0sManager.EXPECT().Download("v1.29.1+k0s.0", installer.DownloadOptions{Force: true, Quiet: false}).Return("/test/workdir/k0s", nil) err := c.DownloadK0s(mockK0sManager) Expect(err).ToNot(HaveOccurred()) }) }) + + Context("AddDownloadCmd", func() { + It("registers --airgapped with the -a shorthand and default false", func() { + download := &cobra.Command{Use: "download"} + + k0s.AddDownloadCmd(download, &util.GlobalOptions{}) + + Expect(download.Commands()).To(HaveLen(1)) + + flag := download.Commands()[0].Flags().Lookup("airgapped") + Expect(flag).ToNot(BeNil()) + Expect(flag.Shorthand).To(Equal("a")) + Expect(flag.DefValue).To(Equal("false")) + }) + }) }) diff --git a/cli/cmd/k0s/install_k0s.go b/cli/cmd/k0s/install_k0s.go index e519a4cf1..4d189b590 100644 --- a/cli/cmd/k0s/install_k0s.go +++ b/cli/cmd/k0s/install_k0s.go @@ -21,7 +21,7 @@ import ( intutil "github.com/codesphere-cloud/oms/internal/util" ) -// InstallK0sCmd represents the k0s download command +// InstallK0sCmd represents the k0s command type InstallK0sCmd struct { cmd *cobra.Command Opts InstallK0sOpts @@ -29,20 +29,43 @@ type InstallK0sCmd struct { FileWriter intutil.FileIO } +// InstallK0sOpts holds the flags of the k0s install command type InstallK0sOpts struct { *util.GlobalOptions - Version string - K0sctlVersion string - Package string - InstallConfig string - SSHKeyPath string - Force bool - NoDownload bool - Vault string - VaultPrivKey string - VaultType string + Version string + K0sctlVersion string + Package string + InstallConfig string + SSHKeyPath string + Force bool + NoDownload bool + Airgap bool + AirgapBundlePath string + ConfigOnly bool + K0sctlConfig string + Vault string + VaultPrivKey string + VaultType string } +// resolveK0sVersion returns the requested k0s version, or the latest version when +// none was requested. +func resolveK0sVersion(k0s installer.K0sManager, version string) (string, error) { + if version != "" { + return version, nil + } + + latestVersion, err := k0s.GetLatestVersion() + if err != nil { + return "", fmt.Errorf("failed to get latest k0s version: %w", err) + } + + log.Printf("Using latest k0s version: %s", latestVersion) + + return latestVersion, nil +} + +// RunE runs the k0s install command. func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { hw := portal.NewHttpWrapper() env := c.Env @@ -53,6 +76,7 @@ func (c *InstallK0sCmd) RunE(_ *cobra.Command, args []string) error { return c.InstallK0s(pm, k0s, k0sctl) } +// AddInstallCmd registers the k0s install command in the parent command func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { k0s := InstallK0sCmd{ cmd: &cobra.Command{ @@ -73,26 +97,33 @@ func AddInstallCmd(install *cobra.Command, opts *util.GlobalOptions) { {Cmd: "--ssh-key-path ", Desc: "SSH private key path for remote installation"}, {Cmd: "--force", Desc: "Force new download and installation"}, {Cmd: "--no-download", Desc: "Skip downloading k0s binary (expects it to be on remote nodes)"}, + {Cmd: "--airgapped", Desc: "Install k0s without internet access using an airgap image bundle"}, + {Cmd: "--airgapped --airgap-bundle ", Desc: "Install k0s airgapped from a local airgap image bundle"}, + {Cmd: "--install-config --config-only --k0sctl-config ", Desc: "Only generate the k0sctl config without installing k0s"}, + {Cmd: "--k0sctl-config ", Desc: "Install k0s from a previously generated k0sctl config"}, }), }, Opts: InstallK0sOpts{GlobalOptions: opts}, Env: env.NewEnv(), FileWriter: intutil.NewFilesystemWriter(), } + k0s.cmd.Flags().StringVarP(&k0s.Opts.Version, "version", "v", installer.DefaultK0sVersion, "Version of k0s to install") k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlVersion, "k0sctl-version", installer.DefaultK0sctlVersion, "Version of k0sctl to use") k0s.cmd.Flags().StringVarP(&k0s.Opts.Package, "package", "p", "", "Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from") - k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required)") + k0s.cmd.Flags().StringVar(&k0s.Opts.InstallConfig, "install-config", "", "Path to Codesphere install-config file (required unless --k0sctl-config is installed)") k0s.cmd.Flags().StringVar(&k0s.Opts.SSHKeyPath, "ssh-key-path", "", "SSH private key path for remote installation") k0s.cmd.Flags().BoolVarP(&k0s.Opts.Force, "force", "f", false, "Force new download and installation") k0s.cmd.Flags().BoolVar(&k0s.Opts.NoDownload, "no-download", false, "Skip downloading k0s binary") + k0s.cmd.Flags().BoolVar(&k0s.Opts.Airgap, "airgapped", false, "Install k0s without internet access by uploading the airgap image bundle to the workers") + k0s.cmd.Flags().StringVar(&k0s.Opts.AirgapBundlePath, "airgap-bundle", "", "Path to the k0s airgap image bundle to install from (requires --airgapped)") + k0s.cmd.Flags().BoolVar(&k0s.Opts.ConfigOnly, "config-only", false, "Only generate the k0sctl config, without installing k0s") + k0s.cmd.Flags().StringVar(&k0s.Opts.K0sctlConfig, "k0sctl-config", "", "With --config-only, where to write the generated k0sctl config; otherwise an existing k0sctl config to install k0s from instead of generating one") k0s.cmd.Flags().StringVar(&k0s.Opts.Vault, "vault", "", "Path to prod.vault.yaml to save the kubeconfig into (optional)") k0s.cmd.Flags().StringVar(&k0s.Opts.VaultPrivKey, "vault-priv-key", "", "Path to the age private key to decrypt the vault (optional, for SOPS-encrypted vaults)") k0s.cmd.Flags().StringVar(&k0s.Opts.VaultType, "vault-type", "sops", "Vault storage type (sops or plain)") - _ = k0s.cmd.MarkFlagRequired("install-config") - util.AddCmd(install, k0s.cmd) k0s.cmd.RunE = k0s.RunE @@ -103,32 +134,32 @@ const ( vaultSecretNameKubeconfig = "kubeConfig" ) +// InstallK0s generates a k0sctl config from the command options and installs k0s. +// With ConfigOnly it stops after generating the config, and with an existing +// K0sctlConfig it installs that config instead of generating one. func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0sManager, k0sctl installer.K0sctlManager) error { if err := c.FileWriter.MkdirAll(c.Env.GetOmsWorkdir(), 0755); err != nil { return fmt.Errorf("failed to create oms workdir: %w", err) } - config, err := c.loadInstallConfig() - if err != nil { + if err := c.validateOptions(); err != nil { return err } - k0sVersion, err := c.determineK0sVersion(k0s) - if err != nil { - return err - } + c.warnAboutNetworkAccess() - k0sBinaryPath, err := c.getK0sBinaryPath(pm, k0s, k0sVersion) + k0sctlConfigPath, err := c.prepareK0sctlConfig(pm, k0s) if err != nil { return err } - k0sctlPath, err := c.downloadK0sctl(k0sctl) - if err != nil { - return err + if c.Opts.ConfigOnly { + log.Printf("Skipping k0s installation, install the generated config with 'oms install k0s --k0sctl-config %s'", k0sctlConfigPath) + + return nil } - k0sctlConfigPath, err := c.generateK0sctlConfig(config, k0sVersion, k0sBinaryPath) + k0sctlPath, err := c.downloadK0sctl(k0sctl) if err != nil { return err } @@ -146,6 +177,95 @@ func (c *InstallK0sCmd) InstallK0s(pm installer.PackageManager, k0s installer.K0 return nil } +func (c *InstallK0sCmd) installsExistingConfig() bool { + return c.Opts.K0sctlConfig != "" && !c.Opts.ConfigOnly +} + +func (c *InstallK0sCmd) validateOptions() error { + if c.Opts.AirgapBundlePath != "" && !c.Opts.Airgap { + return fmt.Errorf("--airgap-bundle requires --airgapped") + } + + if c.Opts.NoDownload && c.Opts.Airgap { + return fmt.Errorf("--no-download cannot be combined with --airgapped, as the nodes would download k0s from the internet") + } + + if !c.installsExistingConfig() { + if c.Opts.InstallConfig == "" { + return fmt.Errorf("--install-config is required unless an existing --k0sctl-config is installed") + } + + return nil + } + + if c.Opts.InstallConfig != "" || c.Opts.Package != "" || c.Opts.NoDownload || c.Opts.Airgap { + return fmt.Errorf("--k0sctl-config without --config-only installs an existing k0sctl config and cannot be combined with --install-config, --package, --no-download or --airgapped") + } + + if !c.FileWriter.Exists(c.Opts.K0sctlConfig) { + return fmt.Errorf("k0sctl config '%s' does not exist", c.Opts.K0sctlConfig) + } + + return nil +} + +func (c *InstallK0sCmd) prepareK0sctlConfig(pm installer.PackageManager, k0s installer.K0sManager) (string, error) { + if c.installsExistingConfig() { + log.Printf("Using existing k0sctl configuration at %s", c.Opts.K0sctlConfig) + + return c.Opts.K0sctlConfig, nil + } + + config, err := c.loadInstallConfig() + if err != nil { + return "", err + } + + k0sVersion, err := resolveK0sVersion(k0s, c.Opts.Version) + if err != nil { + return "", err + } + + k0sBinaryPath, err := c.getK0sBinaryPath(pm, k0s, k0sVersion) + if err != nil { + return "", err + } + + airgapBundlePath, err := c.getAirgapBundlePath(k0s, k0sVersion) + if err != nil { + return "", err + } + + k0sctlOptions := installer.K0sctlOptions{ + K0sVersion: k0sVersion, + SSHKeyPath: c.Opts.SSHKeyPath, + K0sBinaryPath: k0sBinaryPath, + AirgapBundlePath: airgapBundlePath, + } + + return c.generateK0sctlConfig(config, k0sctlOptions) +} + +func (c *InstallK0sCmd) downloadOptions() installer.DownloadOptions { + return installer.DownloadOptions{Force: c.Opts.Force} +} + +// warnAboutNetworkAccess logs the steps of an airgapped installation that still +// require internet access, so they can be prepared before going offline. +func (c *InstallK0sCmd) warnAboutNetworkAccess() { + if !c.Opts.Airgap { + return + } + + if c.Opts.AirgapBundlePath == "" { + log.Println("Warning: --airgapped without --airgap-bundle uses the cached airgap bundle or downloads it from the internet; pre-download it with 'oms download k0s --airgapped' or pass --airgap-bundle for a truly offline installation") + } + + if !c.Opts.ConfigOnly && (c.Opts.K0sctlVersion == "" || c.Opts.K0sctlVersion == installer.DefaultK0sctlVersion) { + log.Println("Warning: k0sctl is downloaded from the internet unless it is already cached; pass --k0sctl-version with a pre-cached version for a truly offline installation") + } +} + func (c *InstallK0sCmd) loadInstallConfig() (*files.RootConfig, error) { config, err := installer.NewConfig().ParseConfigYaml(c.Opts.InstallConfig) if err != nil { @@ -159,19 +279,6 @@ func (c *InstallK0sCmd) loadInstallConfig() (*files.RootConfig, error) { return &config, nil } -func (c *InstallK0sCmd) determineK0sVersion(k0s installer.K0sManager) (string, error) { - k0sVersion := c.Opts.Version - if k0sVersion == "" { - var err error - k0sVersion, err = k0s.GetLatestVersion() - if err != nil { - return "", fmt.Errorf("failed to get latest k0s version: %w", err) - } - log.Printf("Using latest k0s version: %s", k0sVersion) - } - return k0sVersion, nil -} - func (c *InstallK0sCmd) getK0sBinaryPath(pm installer.PackageManager, k0s installer.K0sManager, k0sVersion string) (string, error) { if c.Opts.NoDownload { return "", nil @@ -181,28 +288,56 @@ func (c *InstallK0sCmd) getK0sBinaryPath(pm installer.PackageManager, k0s instal if err := pm.ExtractDependency(defaultK0sPath, c.Opts.Force, c.Opts.Verbose); err != nil { return "", fmt.Errorf("failed to extract k0s from package: %w", err) } + return pm.GetDependencyPath(defaultK0sPath), nil } - k0sBinaryPath, err := k0s.Download(k0sVersion, c.Opts.Force, false) + k0sBinaryPath, err := k0s.Download(k0sVersion, c.downloadOptions()) if err != nil { return "", fmt.Errorf("failed to download k0s: %w", err) } + return k0sBinaryPath, nil } +// getAirgapBundlePath returns the local airgap image bundle that k0sctl uploads to the +// worker nodes. It is empty for installations with internet access. +func (c *InstallK0sCmd) getAirgapBundlePath(k0s installer.K0sManager, k0sVersion string) (string, error) { + if !c.Opts.Airgap { + return "", nil + } + + if c.Opts.AirgapBundlePath != "" { + if !c.FileWriter.Exists(c.Opts.AirgapBundlePath) { + return "", fmt.Errorf("airgap bundle '%s' does not exist", c.Opts.AirgapBundlePath) + } + + return c.Opts.AirgapBundlePath, nil + } + + bundlePath, err := k0s.EnsureAirgapBundle(k0sVersion, c.downloadOptions()) + if err != nil { + return "", fmt.Errorf("failed to download k0s airgap bundle: %w", err) + } + + return bundlePath, nil +} + func (c *InstallK0sCmd) downloadK0sctl(k0sctl installer.K0sctlManager) (string, error) { - log.Println("Downloading k0sctl...") - k0sctlPath, err := k0sctl.Download(c.Opts.K0sctlVersion, c.Opts.Force, false) + log.Println("Preparing k0sctl...") + + k0sctlPath, err := k0sctl.Download(c.Opts.K0sctlVersion, c.downloadOptions()) if err != nil { return "", fmt.Errorf("failed to download k0sctl: %w", err) } + return k0sctlPath, nil } -func (c *InstallK0sCmd) generateK0sctlConfig(config *files.RootConfig, k0sVersion string, k0sBinaryPath string) (string, error) { +func (c *InstallK0sCmd) generateK0sctlConfig(config *files.RootConfig, options installer.K0sctlOptions) (string, error) { log.Println("Generating k0sctl configuration from install-config...") - k0sctlConfig, err := installer.GenerateK0sctlConfig(config, k0sVersion, c.Opts.SSHKeyPath, k0sBinaryPath) + + k0sctlConfig, err := installer.GenerateK0sctlConfig(config, options) if err != nil { return "", fmt.Errorf("failed to generate k0sctl config: %w", err) } @@ -212,17 +347,23 @@ func (c *InstallK0sCmd) generateK0sctlConfig(config *files.RootConfig, k0sVersio return "", fmt.Errorf("failed to marshal k0sctl config: %w", err) } - k0sctlConfigPath := filepath.Join(c.Env.GetOmsWorkdir(), fmt.Sprintf("k0sctl-config-%s.yaml", config.Datacenter.Name)) + k0sctlConfigPath := c.Opts.K0sctlConfig + if k0sctlConfigPath == "" { + k0sctlConfigPath = filepath.Join(c.Env.GetOmsWorkdir(), fmt.Sprintf("k0sctl-config-%s.yaml", config.Datacenter.Name)) + } + if err := c.FileWriter.WriteFile(k0sctlConfigPath, k0sctlConfigData, 0644); err != nil { return "", fmt.Errorf("failed to write k0sctl config: %w", err) } log.Printf("Generated k0sctl configuration at %s", k0sctlConfigPath) + return k0sctlConfigPath, nil } func (c *InstallK0sCmd) deployK0sCluster(k0sctl installer.K0sctlManager, k0sctlPath string, k0sctlConfigPath string) error { log.Println("Applying k0sctl configuration to deploy k0s cluster...") + if err := k0sctl.Apply(k0sctlConfigPath, k0sctlPath, c.Opts.Force); err != nil { return fmt.Errorf("failed to apply k0sctl config: %w", err) } @@ -235,25 +376,29 @@ func (c *InstallK0sCmd) deployK0sCluster(k0sctl installer.K0sctlManager, k0sctlP func (c *InstallK0sCmd) saveKubeconfigToVault(k0sctl installer.K0sctlManager, k0sctlConfigPath, k0sctlPath string) error { log.Println("Retrieving kubeconfig from k0sctl for vault...") + kubeconfigContent, err := k0sctl.GetKubeconfig(k0sctlConfigPath, k0sctlPath) if err != nil { return fmt.Errorf("failed to retrieve kubeconfig from k0sctl: %w", err) } + kubeconfigContent = strings.TrimRight(kubeconfigContent, "\n\r") - vault, err := c.loadOrCreateVault() + store, err := c.vaultStore() + if err != nil { + return err + } + + vaultData, err := store.LoadOrCreate() if err != nil { return fmt.Errorf("failed to load vault: %w", err) } - for _, s := range vault.Secrets { - if s.Name == vaultSecretNameKubeconfig { - log.Printf("Updating existing %s secret in vault", vaultSecretNameKubeconfig) - break - } + if vaultData.GetSecret(vaultSecretNameKubeconfig) != nil { + log.Printf("Updating existing %s secret in vault", vaultSecretNameKubeconfig) } - vault.SetSecret(files.SecretEntry{ + vaultData.SetSecret(files.SecretEntry{ Name: vaultSecretNameKubeconfig, File: &files.SecretFile{ Name: vaultSecretNameKubeconfig, @@ -261,37 +406,20 @@ func (c *InstallK0sCmd) saveKubeconfigToVault(k0sctl installer.K0sctlManager, k0 }, }) - store, err := c.vaultStore() - if err != nil { - return err - } - - if err := store.Save(vault); err != nil { + if err := store.Save(vaultData); err != nil { return err } log.Printf("Saved kubeconfig to %s", c.Opts.Vault) - return nil -} - -func (c *InstallK0sCmd) loadOrCreateVault() (*files.InstallVault, error) { - store, err := c.vaultStore() - if err != nil { - return nil, err - } - data, err := store.LoadOrCreate() - if err != nil { - return nil, fmt.Errorf("failed to load vault: %w", err) - } - - return data, nil + return nil } func (c *InstallK0sCmd) vaultStore() (vault.Vault, error) { - vault, err := vault.NewFromString(c.Opts.VaultType, vault.Options{Path: c.Opts.Vault, AgeKey: c.Opts.VaultPrivKey}) + store, err := vault.NewFromString(c.Opts.VaultType, vault.Options{Path: c.Opts.Vault, AgeKey: c.Opts.VaultPrivKey}) if err != nil { return nil, fmt.Errorf("failed to load vault: %w", err) } - return vault, nil + + return store, nil } diff --git a/cli/cmd/k0s/install_k0s_test.go b/cli/cmd/k0s/install_k0s_test.go index cf23d6b96..8b55cdf6d 100644 --- a/cli/cmd/k0s/install_k0s_test.go +++ b/cli/cmd/k0s/install_k0s_test.go @@ -64,6 +64,7 @@ var _ = Describe("InstallK0sCmd", func() { Context("RunE method", func() { It("fails when install-config is not provided", func() { c.Opts.InstallConfig = "" + mockEnv.EXPECT().GetOmsWorkdir().Return("/test/workdir").Times(2) mockFileWriter.EXPECT().MkdirAll("/test/workdir", os.FileMode(0755)).Return(nil) @@ -85,7 +86,9 @@ var _ = Describe("InstallK0sCmd", func() { mockPM = installer.NewMockPackageManager(GinkgoT()) mockK0s = installer.NewMockK0sManager(GinkgoT()) mockK0sctl = installer.NewMockK0sctlManager(GinkgoT()) + var err error + tempDir, err = os.MkdirTemp("", "install-k0s-test-*") Expect(err).NotTo(HaveOccurred()) }) @@ -94,6 +97,7 @@ var _ = Describe("InstallK0sCmd", func() { mockPM.AssertExpectations(GinkgoT()) mockK0s.AssertExpectations(GinkgoT()) mockK0sctl.AssertExpectations(GinkgoT()) + if tempDir != "" { _ = os.RemoveAll(tempDir) } @@ -134,14 +138,19 @@ var _ = Describe("InstallK0sCmd", func() { Expect(err).NotTo(HaveOccurred()) err = os.WriteFile(configPath, configData, 0644) Expect(err).NotTo(HaveOccurred()) + return configPath } + setupWorkdirMocks := func() { + mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) + mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + } + It("fails when install-config file does not exist", func() { c.Opts.InstallConfig = "/nonexistent/install-config.yaml" - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) @@ -151,8 +160,7 @@ var _ = Describe("InstallK0sCmd", func() { It("fails when install-config specifies external Kubernetes", func() { c.Opts.InstallConfig = writeTestConfig(createTestConfig(false)) - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) @@ -165,11 +173,10 @@ var _ = Describe("InstallK0sCmd", func() { c.Opts.Version = "v1.30.0+k0s.0" c.Opts.Force = true - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", true, false).Return(nil) mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") - mockK0sctl.EXPECT().Download("", true, false).Return("/tmp/k0sctl", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{Force: true}).Return("/tmp/k0sctl", nil) mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", true).Return(nil) @@ -182,10 +189,9 @@ var _ = Describe("InstallK0sCmd", func() { c.Opts.Package = "" c.Opts.Version = "v1.29.0+k0s.0" - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) - mockK0s.EXPECT().Download("v1.29.0+k0s.0", false, false).Return("/downloaded/k0s", nil) - mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.29.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) @@ -197,26 +203,225 @@ var _ = Describe("InstallK0sCmd", func() { c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) c.Opts.Package = "" - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() mockK0s.EXPECT().GetLatestVersion().Return("v1.30.0+k0s.0", nil) - mockK0s.EXPECT().Download("v1.30.0+k0s.0", false, false).Return("", os.ErrNotExist) + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("", os.ErrNotExist) err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to download k0s")) }) + It("installs k0s airgapped from a local bundle", func() { + config := createTestConfig(true) + config.Kubernetes.Workers = []files.K8sNode{{IPAddress: "192.168.1.101"}} + c.Opts.InstallConfig = writeTestConfig(config) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + c.Opts.AirgapBundlePath = filepath.Join(tempDir, "k0s-airgap-bundle-amd64") + + err := os.WriteFile(c.Opts.AirgapBundlePath, []byte("bundle"), 0644) + Expect(err).NotTo(HaveOccurred()) + + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockFileWriter.EXPECT().Exists(c.Opts.AirgapBundlePath).Return(true) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) + // The generated k0sctl config must upload the bundle to the workers and + // the embedded k0s config must stop pulling images. + mockFileWriter.EXPECT().WriteFile( + mock.Anything, + mock.MatchedBy(func(data []byte) bool { + return strings.Contains(string(data), "dstDir: /var/lib/k0s/images") && + strings.Contains(string(data), "default_pull_policy: Never") + }), + mock.Anything, + ).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) + + err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("resolves the airgap bundle from the cache when no path is given", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockK0s.EXPECT().EnsureAirgapBundle("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/cache/k0s-airgap-bundle-amd64", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails when the airgap bundle path does not exist", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + c.Opts.AirgapBundlePath = "/nonexistent/bundle.tar" + + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockFileWriter.EXPECT().Exists("/nonexistent/bundle.tar").Return(false) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("does not exist")) + }) + + It("fails when an airgap bundle is given without --airgapped", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.AirgapBundlePath = "/cache/k0s-airgap-bundle-amd64" + + setupWorkdirMocks() + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("--airgap-bundle requires --airgapped")) + }) + + It("fails when --no-download is combined with --airgapped", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Airgap = true + c.Opts.NoDownload = true + + setupWorkdirMocks() + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(MatchError(ContainSubstring("--no-download cannot be combined with --airgapped"))) + }) + + It("fails without install-config unless an existing k0sctl config is installed", func() { + c.Opts.ConfigOnly = true + c.Opts.K0sctlConfig = filepath.Join(tempDir, "k0sctl.yaml") + + setupWorkdirMocks() + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(MatchError(ContainSubstring("--install-config is required"))) + }) + + It("only generates the airgap k0sctl config with --config-only", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + c.Opts.ConfigOnly = true + c.Opts.K0sctlConfig = "/etc/codesphere/k0sctl-config.yaml" + + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockK0s.EXPECT().EnsureAirgapBundle("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/cache/bundle", nil) + mockFileWriter.EXPECT().WriteFile( + "/etc/codesphere/k0sctl-config.yaml", + mock.MatchedBy(func(data []byte) bool { + return strings.Contains(string(data), "default_pull_policy: Never") + }), + os.FileMode(0644), + ).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("installs an existing k0sctl config without generating one", func() { + c.Opts.K0sctlConfig = "/etc/codesphere/k0sctl-config.yaml" + + setupWorkdirMocks() + mockFileWriter.EXPECT().Exists("/etc/codesphere/k0sctl-config.yaml").Return(true) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) + mockK0sctl.EXPECT().Apply("/etc/codesphere/k0sctl-config.yaml", "/tmp/k0sctl", false).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("fails when the existing k0sctl config does not exist", func() { + c.Opts.K0sctlConfig = "/nonexistent/k0sctl-config.yaml" + + setupWorkdirMocks() + mockFileWriter.EXPECT().Exists("/nonexistent/k0sctl-config.yaml").Return(false) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(MatchError(ContainSubstring("k0sctl config '/nonexistent/k0sctl-config.yaml' does not exist"))) + }) + + It("fails when an existing k0sctl config is combined with generation flags", func() { + c.Opts.K0sctlConfig = "/etc/codesphere/k0sctl-config.yaml" + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + + setupWorkdirMocks() + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).To(MatchError(ContainSubstring("cannot be combined with --install-config"))) + }) + + It("takes the k0s binary from the package but still ensures the airgap bundle", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + c.Opts.Package = "test-package.tar.gz" + + setupWorkdirMocks() + mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false, false).Return(nil) + mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") + mockK0s.EXPECT().EnsureAirgapBundle("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/cache/bundle", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("resolves the latest k0s version when airgapped without a version", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Airgap = true + c.Opts.AirgapBundlePath = "/cache/bundle" + + setupWorkdirMocks() + mockK0s.EXPECT().GetLatestVersion().Return("v1.30.0+k0s.0", nil) + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{}).Return("/downloaded/k0s", nil) + mockFileWriter.EXPECT().Exists("/cache/bundle").Return(true) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + + It("forces a fresh airgap bundle download with --force", func() { + c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) + c.Opts.Version = "v1.30.0+k0s.0" + c.Opts.Airgap = true + c.Opts.Force = true + + setupWorkdirMocks() + mockK0s.EXPECT().Download("v1.30.0+k0s.0", installer.DownloadOptions{Force: true}).Return("/downloaded/k0s", nil) + mockK0s.EXPECT().EnsureAirgapBundle("v1.30.0+k0s.0", installer.DownloadOptions{Force: true}).Return("/cache/bundle", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{Force: true}).Return("/tmp/k0sctl", nil) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", true).Return(nil) + + err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) + Expect(err).NotTo(HaveOccurred()) + }) + It("fails when k0sctl download fails", func() { c.Opts.InstallConfig = writeTestConfig(createTestConfig(true)) c.Opts.Package = "test-package.tar.gz" c.Opts.Version = "v1.30.0+k0s.0" - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false, false).Return(nil) mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") - mockK0sctl.EXPECT().Download("", false, false).Return("", os.ErrPermission) + mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("", os.ErrPermission) err := c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) @@ -228,11 +433,10 @@ var _ = Describe("InstallK0sCmd", func() { c.Opts.Package = "test-package.tar.gz" c.Opts.Version = "v1.30.0+k0s.0" - mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) - mockFileWriter.EXPECT().MkdirAll(tempDir, os.FileMode(0755)).Return(nil) + setupWorkdirMocks() mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false, false).Return(nil) mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") - mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) mockFileWriter.EXPECT().WriteFile(mock.Anything, mock.Anything, mock.Anything).Return(nil) mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(os.ErrPermission) @@ -245,7 +449,7 @@ var _ = Describe("InstallK0sCmd", func() { mockEnv.EXPECT().GetOmsWorkdir().Return(tempDir) mockPM.EXPECT().ExtractDependency("kubernetes/files/k0s", false, false).Return(nil) mockPM.EXPECT().GetDependencyPath("kubernetes/files/k0s").Return("/test/path/k0s") - mockK0sctl.EXPECT().Download("", false, false).Return("/tmp/k0sctl", nil) + mockK0sctl.EXPECT().Download("", installer.DownloadOptions{}).Return("/tmp/k0sctl", nil) mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) } @@ -254,6 +458,7 @@ var _ = Describe("InstallK0sCmd", func() { if !testutil.SopsAndAgeAvailable() { Skip("sops and age-keygen not available") } + c.FileWriter = intutil.NewFilesystemWriter() }) @@ -278,6 +483,7 @@ var _ = Describe("InstallK0sCmd", func() { Expect(err).NotTo(HaveOccurred()) loaded, err := backend.Load() Expect(err).NotTo(HaveOccurred()) + secret := loaded.GetSecret(files.SecretKubeConfig) Expect(secret).NotTo(BeNil()) Expect(secret.File.Content).To(Equal("apiVersion: v1\nkind: Config")) @@ -313,6 +519,7 @@ var _ = Describe("InstallK0sCmd", func() { vaultYAML, err := existingVault.Marshal() Expect(err).NotTo(HaveOccurred()) + plainPath := c.Opts.Vault + ".plain" err = os.WriteFile(plainPath, vaultYAML, 0600) Expect(err).NotTo(HaveOccurred()) @@ -365,6 +572,7 @@ var _ = Describe("InstallK0sCmd", func() { } vaultYAML, err := existingVault.Marshal() Expect(err).NotTo(HaveOccurred()) + plainPath := c.Opts.Vault + ".plain" err = os.WriteFile(plainPath, vaultYAML, 0600) Expect(err).NotTo(HaveOccurred()) @@ -382,6 +590,7 @@ var _ = Describe("InstallK0sCmd", func() { Expect(err).NotTo(HaveOccurred()) loaded, err := backend.Load() Expect(err).NotTo(HaveOccurred()) + secret := loaded.GetSecret(files.SecretKubeConfig) Expect(secret).NotTo(BeNil()) Expect(secret.File.Content).To(Equal("apiVersion: v1\nkind: Config\nnew: true")) @@ -411,6 +620,7 @@ var _ = Describe("InstallK0sCmd", func() { Expect(err).NotTo(HaveOccurred()) loaded, err := backend.Load() Expect(err).NotTo(HaveOccurred()) + secret := loaded.GetSecret(files.SecretKubeConfig) Expect(secret).NotTo(BeNil()) Expect(secret.File.Content).To(Equal("apiVersion: v1\nkind: Config")) @@ -459,6 +669,7 @@ var _ = Describe("InstallK0sCmd", func() { } vaultYAML, err := existingVault.Marshal() Expect(err).NotTo(HaveOccurred()) + plainPath := vaultPath + ".plain" err = os.WriteFile(plainPath, vaultYAML, 0600) Expect(err).NotTo(HaveOccurred()) @@ -513,6 +724,7 @@ var _ = Describe("InstallK0sCmd", func() { } vaultYAML, err := existingVault.Marshal() Expect(err).NotTo(HaveOccurred()) + plainPath := vaultPath + ".plain" err = os.WriteFile(plainPath, vaultYAML, 0600) Expect(err).NotTo(HaveOccurred()) @@ -577,6 +789,7 @@ var _ = Describe("InstallK0sCmd", func() { } vaultYAML, err := existingVault.Marshal() Expect(err).NotTo(HaveOccurred()) + plainPath := vaultPath + ".plain" err = os.WriteFile(plainPath, vaultYAML, 0600) Expect(err).NotTo(HaveOccurred()) @@ -595,7 +808,8 @@ var _ = Describe("InstallK0sCmd", func() { c.Opts.Vault = vaultPath c.Opts.VaultPrivKey = ageKeyPath - // Use mockFileWriter to simulate an I/O error during encryption. + // The vault itself is written through the real filesystem, so the + // failure is provoked on the filesystem instead of via the mock. c.FileWriter = mockFileWriter setupCommonMocks() @@ -607,28 +821,21 @@ var _ = Describe("InstallK0sCmd", func() { }), mock.Anything, mock.Anything).Return(nil) mockK0sctl.EXPECT().Apply(mock.Anything, "/tmp/k0sctl", false).Return(nil) mockK0sctl.EXPECT().GetKubeconfig(mock.Anything, "/tmp/k0sctl").Return("apiVersion: v1\nkind: Config\n", nil) - mockFileWriter.EXPECT().MkdirAll(mock.Anything, os.FileMode(0755)).Return(nil) - - // Vault exists and is encrypted. - mockFileWriter.EXPECT().Exists(vaultPath).Return(true) - // Simulate encryption failure: writing the temporary vault file fails. - mockFileWriter.EXPECT().WriteFile(vaultPath+".tmp", mock.Anything, os.FileMode(0600)). - Return(os.ErrPermission) + // Make the vault directory unwritable so the SOPS backend cannot create + // its temporary files while the existing vault file stays readable. + Expect(os.Chmod(tempDir, 0500)).To(Succeed()) + DeferCleanup(func() { _ = os.Chmod(tempDir, 0755) }) err = c.InstallK0s(mockPM, mockK0s, mockK0sctl) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to write temporary vault file")) + Expect(err.Error()).To(ContainSubstring("failed to create temporary plaintext vault")) // Verify the vault file is unchanged. currentData, err := os.ReadFile(vaultPath) Expect(err).NotTo(HaveOccurred()) Expect(string(currentData)).To(Equal(string(origData)), "vault should be untouched when encryption fails") - - // Verify no tmp file is left behind. - tmpPath := vaultPath + ".tmp" - Expect(tmpPath).NotTo(BeAnExistingFile()) }) }) }) diff --git a/cli/cmd/k0s/k0s_suite_test.go b/cli/cmd/k0s/k0s_suite_test.go new file mode 100644 index 000000000..1e5bd8f0c --- /dev/null +++ b/cli/cmd/k0s/k0s_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package k0s_test + +import ( + "testing" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" +) + +func TestK0s(t *testing.T) { + gomega.RegisterFailHandler(ginkgo.Fail) + ginkgo.RunSpecs(t, "K0s Command Suite") +} diff --git a/docs/oms_beta_bootstrap-gcp.md b/docs/oms_beta_bootstrap-gcp.md index 072855440..9783b4184 100644 --- a/docs/oms_beta_bootstrap-gcp.md +++ b/docs/oms_beta_bootstrap-gcp.md @@ -18,6 +18,7 @@ oms beta bootstrap-gcp [flags] ``` --acme-staging Use the Let's Encrypt staging ACME endpoint (certificates are not browser-trusted) + --airgapped Install k0s from the k0s airgap image bundle, so the cluster nodes pull no k0s images from the internet (default: false) --azure-devops-app-client-id string Azure DevOps App Client ID (optional) --azure-devops-app-client-secret string Azure DevOps App Client Secret (optional) --base-domain string Base domain for Codesphere (required) diff --git a/docs/oms_download_k0s.md b/docs/oms_download_k0s.md index 64e381303..0d2748bdf 100644 --- a/docs/oms_download_k0s.md +++ b/docs/oms_download_k0s.md @@ -23,11 +23,15 @@ $ oms download k0s --version 1.22.0 # Force download even if k0s binary exists $ oms download k0s --force +# Also download the airgap image bundle for that version +$ oms download k0s --airgapped + ``` ### Options ``` + -a, --airgapped Downloads the airgapped bundle for that version -f, --force Force download even if k0s binary exists -h, --help help for k0s -v, --version string Version of k0s to download diff --git a/docs/oms_install_k0s.md b/docs/oms_install_k0s.md index c3fc12167..d2aadd963 100644 --- a/docs/oms_install_k0s.md +++ b/docs/oms_install_k0s.md @@ -40,14 +40,30 @@ $ oms install k0s --force # Skip downloading k0s binary (expects it to be on remote nodes) $ oms install k0s --no-download +# Install k0s without internet access using an airgap image bundle +$ oms install k0s --airgapped + +# Install k0s airgapped from a local airgap image bundle +$ oms install k0s --airgapped --airgap-bundle + +# Only generate the k0sctl config without installing k0s +$ oms install k0s --install-config --config-only --k0sctl-config + +# Install k0s from a previously generated k0sctl config +$ oms install k0s --k0sctl-config + ``` ### Options ``` + --airgap-bundle string Path to the k0s airgap image bundle to install from (requires --airgapped) + --airgapped Install k0s without internet access by uploading the airgap image bundle to the workers + --config-only Only generate the k0sctl config, without installing k0s -f, --force Force new download and installation -h, --help help for k0s - --install-config string Path to Codesphere install-config file (required) + --install-config string Path to Codesphere install-config file (required unless --k0sctl-config is installed) + --k0sctl-config string With --config-only, where to write the generated k0sctl config; otherwise an existing k0sctl config to install k0s from instead of generating one --k0sctl-version string Version of k0sctl to use (default "v0.33.1") --no-download Skip downloading k0s binary -p, --package string Package file (e.g. codesphere-v1.2.3-installer-lite.tar.gz) to load k0s from diff --git a/internal/bootstrap/datacenter/datacenter.go b/internal/bootstrap/datacenter/datacenter.go index 1cb152e77..ab876e550 100644 --- a/internal/bootstrap/datacenter/datacenter.go +++ b/internal/bootstrap/datacenter/datacenter.go @@ -94,6 +94,12 @@ func (dc *DataCenter) K0sConfigScriptPath() string { return fmt.Sprintf("configure-k0s%s.sh", dc.Suffix) } +// RemoteK0sctlConfigPath returns the path on the jumpbox where an airgapped bootstrap generates +// this data center's k0sctl config, next to its install config. +func (dc *DataCenter) RemoteK0sctlConfigPath() string { + return SuffixedPath(filepath.Join(filepath.Dir(dc.RemoteConfigPath), "k0sctl-config.yaml"), dc.Suffix) +} + // StepName qualifies a bootstrap step name with the data center it applies to. Single-DC // bootstraps keep their unqualified step names. func (dc *DataCenter) StepName(name string) string { diff --git a/internal/bootstrap/datacenter/datacenter_test.go b/internal/bootstrap/datacenter/datacenter_test.go index edaeaec17..56a761117 100644 --- a/internal/bootstrap/datacenter/datacenter_test.go +++ b/internal/bootstrap/datacenter/datacenter_test.go @@ -26,3 +26,14 @@ var _ = Describe("SuffixedPath", func() { Entry("absolute path", "/etc/codesphere/config.yaml", "-dc2", "/etc/codesphere/config-dc2.yaml"), ) }) + +var _ = Describe("RemoteK0sctlConfigPath", func() { + DescribeTable("places the k0sctl config next to the install config", + func(remoteConfigPath, suffix, expected string) { + dc := &datacenter.DataCenter{RemoteConfigPath: remoteConfigPath, Suffix: suffix} + Expect(dc.RemoteK0sctlConfigPath()).To(Equal(expected)) + }, + Entry("primary", "/etc/codesphere/config.yaml", "", "/etc/codesphere/k0sctl-config.yaml"), + Entry("secondary", "/etc/codesphere/config-dc2.yaml", "-dc2", "/etc/codesphere/k0sctl-config-dc2.yaml"), + ) +}) diff --git a/internal/bootstrap/gcp/gcp.go b/internal/bootstrap/gcp/gcp.go index 5e8b12e40..b8bc8740b 100644 --- a/internal/bootstrap/gcp/gcp.go +++ b/internal/bootstrap/gcp/gcp.go @@ -153,6 +153,7 @@ type CodesphereEnvironment struct { InstallSkipSteps []string `json:"install_skip_steps"` Preemptible bool `json:"preemptible"` SpotVMs bool `json:"spot_vms"` + Airgapped bool `json:"airgapped"` WriteConfig bool `json:"-"` RecoverConfig bool `json:"-"` GatewayIP string `json:"gateway_ip"` diff --git a/internal/bootstrap/gcp/gcp_test.go b/internal/bootstrap/gcp/gcp_test.go index 2adde6ef4..fb74b58e2 100644 --- a/internal/bootstrap/gcp/gcp_test.go +++ b/internal/bootstrap/gcp/gcp_test.go @@ -1831,6 +1831,48 @@ var _ = Describe("GCP Bootstrapper", func() { err := bs.InstallK0s() Expect(err).To(MatchError(ContainSubstring("failed to install k0s from jumpbox"))) }) + + It("installs the generated airgap k0sctl config when airgapped", func() { + csEnv.Airgapped = true + + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install k0s --k0sctl-config /etc/codesphere/k0sctl-config.yaml --vault /etc/codesphere/secrets/prod.vault.yaml --vault-priv-key /etc/codesphere/secrets/age_key.txt").Return(nil) + + err := bs.InstallK0s() + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Describe("DownloadK0sAirgapBundle", func() { + It("caches the k0s binary and airgap bundle on the jumpbox", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms download k0s --version v1.31.14+k0s.0 --airgapped").Return(nil) + + Expect(bs.DownloadK0sAirgapBundle()).To(Succeed()) + }) + + It("reports a download failure", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", mock.Anything).Return(fmt.Errorf("offline")) + + err := bs.DownloadK0sAirgapBundle() + Expect(err).To(MatchError(ContainSubstring("failed to download k0s airgap bundle on jumpbox"))) + }) + }) + + Describe("GenerateK0sAirgapConfig", func() { + It("generates the airgap k0sctl config on the jumpbox without installing it", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", + "oms install k0s --version v1.31.14+k0s.0 --install-config /etc/codesphere/config.yaml --airgapped --config-only --k0sctl-config /etc/codesphere/k0sctl-config.yaml").Return(nil) + + Expect(bs.GenerateK0sAirgapConfig()).To(Succeed()) + }) + + It("reports a generation failure", func() { + nodeClient.EXPECT().RunCommand(mock.MatchedBy(jumpboxMatcher), "root", mock.Anything).Return(fmt.Errorf("no bundle")) + + err := bs.GenerateK0sAirgapConfig() + Expect(err).To(MatchError(ContainSubstring("failed to generate k0s airgap config on jumpbox"))) + }) }) Describe("WaitForK0sNodes", func() { diff --git a/internal/bootstrap/gcp/k0s.go b/internal/bootstrap/gcp/k0s.go index 293dbc4b4..b3cd057b2 100644 --- a/internal/bootstrap/gcp/k0s.go +++ b/internal/bootstrap/gcp/k0s.go @@ -13,6 +13,8 @@ import ( // EnsureK0s executed all steps to ensure a k0s cluster in gcp for every data center. // Only executing the config script needs to be done after installing codesphere, as crucial parts are still in the ts-installer. +// Airgapped bootstraps download the airgap bundle and generate an airgap k0sctl config first, +// which InstallK0s then installs. // Returns an error if k0s could not be ensured. func (b *GCPBootstrapper) EnsureK0s() error { err := b.GenerateK0sConfigScript() @@ -20,6 +22,18 @@ func (b *GCPBootstrapper) EnsureK0s() error { return fmt.Errorf("failed to generate k0s config script: %w", err) } + if b.Env.Airgapped { + err = b.stlog.Step("Download k0s airgap bundle", b.DownloadK0sAirgapBundle) + if err != nil { + return fmt.Errorf("failed to download k0s airgap bundle: %w", err) + } + + err = b.GenerateK0sAirgapConfig() + if err != nil { + return fmt.Errorf("failed to generate k0s airgap config: %w", err) + } + } + err = b.InstallK0s() if err != nil { return fmt.Errorf("failed to install k0s: %w", err) @@ -196,9 +210,50 @@ func (b *GCPBootstrapper) runK0sConfigScript(dc *datacenter.DataCenter) error { return nil } +// DownloadK0sAirgapBundle caches the k0s binary and its airgap image bundle on the jumpbox. All +// data centers install the same k0s version, so they share the cached bundle. +func (b *GCPBootstrapper) DownloadK0sAirgapBundle() error { + downloadCmd := fmt.Sprintf("oms download k0s --version %s --airgapped", installer.DefaultK0sVersion) + if err := b.Env.Jumpbox.RunSSHCommand("root", downloadCmd); err != nil { + return fmt.Errorf("failed to download k0s airgap bundle on jumpbox: %w", err) + } + + return nil +} + +// GenerateK0sAirgapConfig generates every data center's airgap k0sctl config on the jumpbox from +// the cached airgap bundle, without installing it yet. +func (b *GCPBootstrapper) GenerateK0sAirgapConfig() error { + if err := b.ensureDataCenters(); err != nil { + return err + } + + for _, dc := range b.Env.DataCenters { + err := b.stlog.Step(dc.StepName("Generate k0s airgap config"), func() error { + return b.generateK0sAirgapConfig(dc) + }) + if err != nil { + return fmt.Errorf("failed to generate k0s airgap config (data center %d): %w", dc.ID, err) + } + } + + return nil +} + +func (b *GCPBootstrapper) generateK0sAirgapConfig(dc *datacenter.DataCenter) error { + generateCmd := fmt.Sprintf("oms install k0s --version %s --install-config %s --airgapped --config-only --k0sctl-config %s", + installer.DefaultK0sVersion, dc.RemoteConfigPath, dc.RemoteK0sctlConfigPath()) + if err := b.Env.Jumpbox.RunSSHCommand("root", generateCmd); err != nil { + return fmt.Errorf("failed to generate k0s airgap config on jumpbox (data center %d): %w", dc.ID, err) + } + + return nil +} + // InstallK0s deploys k0s into every data center with the native OMS installer. Each data center // gets its own cluster, so every run stores its kubeconfig in that data center's encrypted -// install vault for the remaining installer steps. +// install vault for the remaining installer steps. Airgapped bootstraps install the k0sctl config +// that GenerateK0sAirgapConfig generated. func (b *GCPBootstrapper) InstallK0s() error { if err := b.ensureDataCenters(); err != nil { return err @@ -219,8 +274,13 @@ func (b *GCPBootstrapper) InstallK0s() error { func (b *GCPBootstrapper) installK0s(dc *datacenter.DataCenter) error { // Reuse matching cached binaries and let k0sctl reconcile normally. Without // --force, an unchanged cluster remains untouched on bootstrap retries. - installCmd := fmt.Sprintf("oms install k0s --version %s --install-config %s --vault %s --vault-priv-key %s", - installer.DefaultK0sVersion, dc.RemoteConfigPath, dc.RemoteVaultPath(), dc.RemoteAgeKeyPath()) + configFlags := fmt.Sprintf("--version %s --install-config %s", installer.DefaultK0sVersion, dc.RemoteConfigPath) + if b.Env.Airgapped { + configFlags = "--k0sctl-config " + dc.RemoteK0sctlConfigPath() + } + + installCmd := fmt.Sprintf("oms install k0s %s --vault %s --vault-priv-key %s", + configFlags, dc.RemoteVaultPath(), dc.RemoteAgeKeyPath()) if err := b.Env.Jumpbox.RunSSHCommand("root", installCmd); err != nil { return fmt.Errorf("failed to install k0s from jumpbox (data center %d): %w", dc.ID, err) } diff --git a/internal/installer/airgap.go b/internal/installer/airgap.go new file mode 100644 index 000000000..4ab632405 --- /dev/null +++ b/internal/installer/airgap.go @@ -0,0 +1,109 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package installer + +import ( + "fmt" + "os" + "path/filepath" + "slices" + "strings" + + "github.com/codesphere-cloud/cs-go/pkg/io" +) + +const ( + // AirgapBundleName is the name of target airgap-bundle for oms to download to + AirgapBundleName = "k0s-airgap-bundle" + + // AirgapImagesDir is the k0s data dir from which worker nodes import the + // images of an airgap bundle. Controllers that do not run a worker ignore it. + AirgapImagesDir = "/var/lib/k0s/images" +) + +// EnsureAirgapBundle makes sure the airgap image bundle of the given version is +// available in the OMS cache dir and returns its path. +func (k *K0s) EnsureAirgapBundle(version string, opts DownloadOptions) (string, error) { + cacheDir, err := ensureCacheDir(k.FileWriter, k.Env) + if err != nil { + return "", err + } + + cachePath, found := k.cachedAirgapBundle(cacheDir, version) + if !found { + assetName, err := k.resolveAirgapBundleAssetName(version) + if err != nil { + return "", err + } + + cachePath = filepath.Join(cacheDir, assetName) + } + + if k.FileWriter.Exists(cachePath) && !opts.Force { + io.Verbosef(!opts.Quiet, "Using cached airgap bundle %s", cachePath) + + return cachePath, nil + } + + downloadURL := releaseAssetURL(k0sReleaseURL, version, filepath.Base(cachePath)) + io.Verbosef(!opts.Quiet, "Downloading k0s airgap bundle from %s", downloadURL) + + if err := downloadToPath(k.FileWriter, k.Http, cachePath, downloadURL, opts.Quiet); err != nil { + return "", err + } + + return cachePath, nil +} + +// cachedAirgapBundle returns the path of the cached airgap bundle of the given +// version, if there is one. An unreadable cache is not an error: the release +// metadata then decides which bundle to look for. +func (k *K0s) cachedAirgapBundle(cacheDir, version string) (string, bool) { + entries, err := k.FileWriter.ReadDir(cacheDir) + if err != nil { + return "", false + } + + i := slices.IndexFunc(entries, func(entry os.DirEntry) bool { return k.isAirgapBundleFor(entry.Name(), version) }) + if i < 0 { + return "", false + } + + return filepath.Join(cacheDir, entries[i].Name()), true +} + +// resolveAirgapBundleAssetName returns the release asset name of the airgap image +// bundle of the given version and architecture. +// +// k0s changed the asset naming scheme in v1.36, from +// "k0s-airgap-bundle--" to +// "k0s-airgap-bundle---.tar", so the name is resolved from the +// release metadata instead of being constructed. +func (k *K0s) resolveAirgapBundleAssetName(version string) (string, error) { + release, err := getGitHubRelease(k.Http, k0sReleaseAPIURL+"/"+version, "k0s release "+version) + if err != nil { + return "", err + } + + i := slices.IndexFunc(release.Assets, func(asset githubReleaseAsset) bool { return k.isAirgapBundleFor(asset.Name, version) }) + if i < 0 { + return "", fmt.Errorf("no airgap bundle for %s/%s found in k0s release %s", k.Goos, k.Goarch, version) + } + + return release.Assets[i].Name, nil +} + +// isAirgapBundleFor reports whether name is the airgap image bundle asset of the +// given version that matches the configured OS and architecture. It accepts both the +// legacy and the current upstream naming scheme. +func (k *K0s) isAirgapBundleFor(name, version string) bool { + platform, ok := strings.CutPrefix(name, fmt.Sprintf("%s-%s-", AirgapBundleName, version)) + if !ok { + return false + } + + platform = strings.TrimSuffix(platform, ".tar") + + return platform == k.Goarch || platform == k.Goos+"-"+k.Goarch +} diff --git a/internal/installer/download.go b/internal/installer/download.go index bb46b3d92..6bb995d5d 100644 --- a/internal/installer/download.go +++ b/internal/installer/download.go @@ -4,29 +4,118 @@ package installer import ( + "encoding/json" "fmt" "strings" + "github.com/codesphere-cloud/cs-go/pkg/io" + "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) -func downloadBinaryToPath(fw util.FileIO, http portal.Http, binaryPath, binaryName, downloadURL string, quiet bool) (string, error) { - dstFile, err := fw.Create(binaryPath) +// DownloadOptions configures how oms fetches an artifact from a remote release +// into the OMS cache. +type DownloadOptions struct { + // Force downloads the artifact even when a matching cached copy exists. + Force bool + // Quiet suppresses progress output. + Quiet bool +} + +// githubReleaseAsset is a single downloadable asset of a GitHub release. +type githubReleaseAsset struct { + Name string `json:"name"` +} + +// githubRelease is the subset of the GitHub release API response that oms uses. +type githubRelease struct { + TagName string `json:"tag_name"` + Assets []githubReleaseAsset `json:"assets"` +} + +// getGitHubRelease fetches the GitHub release at url and decodes it. subject names +// the release in error messages. +func getGitHubRelease(h portal.Http, url, subject string) (*githubRelease, error) { + responseBody, err := h.Get(url) + if err != nil { + return nil, fmt.Errorf("failed to fetch %s: %w", subject, err) + } + + var release githubRelease + if err := json.Unmarshal(responseBody, &release); err != nil { + return nil, fmt.Errorf("failed to parse %s: %w", subject, err) + } + + return &release, nil +} + +func ensureCacheDir(fw util.FileIO, environment env.Env) (string, error) { + cacheDir, err := environment.GetOmsCacheDir() if err != nil { - return "", fmt.Errorf("failed to create %s binary file: %w", binaryName, err) + return "", fmt.Errorf("failed to determine cache directory: %w", err) + } + + if err := fw.MkdirAll(cacheDir, 0755); err != nil { + return "", fmt.Errorf("failed to create workdir: %w", err) + } + + return cacheDir, nil +} + +// reuseCachedBinary returns the cached binary at cachePath and true when it exists, +// matches requestedVersion and opts do not force a fresh download. +func reuseCachedBinary(fw util.FileIO, cachePath, requestedVersion, name string, opts DownloadOptions) (string, bool) { + if !fw.Exists(cachePath) || opts.Force { + return "", false + } + + cachedVersion, versionErr := localBinaryVersion(cachePath) + if versionErr == nil && cachedVersion == requestedVersion { + io.Verbosef(!opts.Quiet, "Using cached %s %s at %s", name, requestedVersion, cachePath) + + return cachePath, true + } + + if versionErr != nil { + io.Verbosef(!opts.Quiet, "Replacing cached %s: version could not be determined: %v", name, versionErr) + } else { + io.Verbosef(!opts.Quiet, "Replacing cached %s %s: requested version %s", name, cachedVersion, requestedVersion) + } + + return "", false +} + +func releaseAssetURL(releaseURL, version, assetName string) string { + return fmt.Sprintf("%s/%s/%s", releaseURL, version, assetName) +} + +func downloadToPath(fw util.FileIO, http portal.Http, path, downloadURL string, quiet bool) error { + dstFile, err := fw.Create(path) + if err != nil { + return fmt.Errorf("failed to create file: %s: %w", path, err) } defer util.CloseFileIgnoreError(dstFile) if err := http.Download(downloadURL, dstFile, quiet); err != nil { - return "", fmt.Errorf("failed to download %s binary: %w", binaryName, err) + _ = fw.Remove(path) + + return fmt.Errorf("failed to download %s: %w", path, err) + } + + return nil +} + +func downloadBinaryToPath(fw util.FileIO, http portal.Http, binaryPath, binaryName, downloadURL string, quiet bool) error { + if err := downloadToPath(fw, http, binaryPath, downloadURL, quiet); err != nil { + return fmt.Errorf("failed to download: %w", err) } if err := fw.Chmod(binaryPath, 0755); err != nil { - return "", fmt.Errorf("failed to make %s binary executable: %w", binaryName, err) + return fmt.Errorf("failed to make %s binary executable: %w", binaryName, err) } - return binaryPath, nil + return nil } func localBinaryVersion(binaryPath string) (string, error) { @@ -37,13 +126,15 @@ func localBinaryVersion(binaryPath string) (string, error) { for _, line := range strings.Split(output, "\n") { line = strings.TrimSpace(line) + if line == "" { + continue + } + if version, found := strings.CutPrefix(line, "version:"); found { return strings.TrimSpace(version), nil } - if line != "" { - return line, nil - } + return line, nil } return "", fmt.Errorf("version output is empty") diff --git a/internal/installer/k0s.go b/internal/installer/k0s.go index ed052267b..f2ef813a1 100644 --- a/internal/installer/k0s.go +++ b/internal/installer/k0s.go @@ -10,20 +10,24 @@ import ( "runtime" "strings" - "github.com/codesphere-cloud/cs-go/pkg/io" "github.com/codesphere-cloud/oms/internal/env" "github.com/codesphere-cloud/oms/internal/portal" "github.com/codesphere-cloud/oms/internal/util" ) -// DefaultK0sVersion is the currently verified k0s version -// Use of newer versions should work in most cases but can't be guaranteed -const DefaultK0sVersion = "v1.31.14+k0s.0" +const ( + // DefaultK0sVersion is the k0s version installed unless another one is requested. + DefaultK0sVersion = "v1.31.14+k0s.0" + k0sReleaseURL = "https://github.com/k0sproject/k0s/releases/download" + k0sReleaseAPIURL = "https://api.github.com/repos/k0sproject/k0s/releases/tags" + k0sBinaryName = "k0s" +) //mockery:generate: true type K0sManager interface { GetLatestVersion() (string, error) - Download(version string, force bool, quiet bool) (string, error) + Download(version string, opts DownloadOptions) (string, error) + EnsureAirgapBundle(version string, opts DownloadOptions) (string, error) } type K0s struct { @@ -59,46 +63,31 @@ func (k *K0s) GetLatestVersion() (string, error) { } // Download downloads the k0s binary for the specified version and saves it to the OMS cache dir. -func (k *K0s) Download(version string, force bool, quiet bool) (string, error) { +func (k *K0s) Download(version string, opts DownloadOptions) (string, error) { if k.Goos != "linux" || k.Goarch != "amd64" { return "", fmt.Errorf("codesphere installation is only supported on Linux amd64. Current platform: %s/%s", k.Goos, k.Goarch) } log.Printf("Downloading k0s version %s", version) - cacheDir, err := k.Env.GetOmsCacheDir() + cacheDir, err := ensureCacheDir(k.FileWriter, k.Env) if err != nil { - return "", fmt.Errorf("failed to determine cache directory: %w", err) - } - - if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { - return "", fmt.Errorf("failed to create workdir: %w", err) + return "", err } - cachePath := filepath.Join(cacheDir, "k0s") - if k.FileWriter.Exists(cachePath) && !force { - cachedVersion, versionErr := localBinaryVersion(cachePath) - if versionErr == nil && cachedVersion == version { - io.Verbosef(!quiet, "Using cached k0s %s at %s", version, cachePath) - return cachePath, nil - } - - replaceReason := fmt.Sprintf("Cached k0s version %s does not match requested version %s; replacing it", cachedVersion, version) - if versionErr != nil { - replaceReason = "Cached k0s version could not be determined: " + versionErr.Error() - } - - io.Verbosef(!quiet, "Replacing existing k0s binary: %s", replaceReason) + cachePath := filepath.Join(cacheDir, k0sBinaryName) + if cachedPath, cached := reuseCachedBinary(k.FileWriter, cachePath, version, k0sBinaryName, opts); cached { + return cachedPath, nil } - downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0s/releases/download/%s/k0s-%s-%s", version, version, k.Goarch) + assetName := fmt.Sprintf("%s-%s-%s", k0sBinaryName, version, k.Goarch) + downloadURL := releaseAssetURL(k0sReleaseURL, version, assetName) - path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0s", downloadURL, quiet) - if err != nil { - return "", err + if err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, k0sBinaryName, downloadURL, opts.Quiet); err != nil { + return "", fmt.Errorf("failed to download k0s binary: %w", err) } - log.Printf("k0s binary downloaded and made executable at '%s'", path) + log.Printf("k0s binary downloaded and made executable at '%s'", cachePath) - return path, nil + return cachePath, nil } diff --git a/internal/installer/k0s_config.go b/internal/installer/k0s_config.go index 99d6cc592..a0cc82911 100644 --- a/internal/installer/k0s_config.go +++ b/internal/installer/k0s_config.go @@ -4,6 +4,7 @@ package installer import ( + "cmp" "fmt" "github.com/codesphere-cloud/oms/internal/installer/files" @@ -66,6 +67,7 @@ type K0sKonnectivity struct { AgentPort int `yaml:"agentPort,omitempty"` } +// GenerateK0sConfig generates a K0sConfig based on the provided RootConfig. func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { if installConfig == nil { return nil, fmt.Errorf("installConfig cannot be nil") @@ -88,6 +90,7 @@ func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { for _, cp := range installConfig.Kubernetes.ControlPlanes { sans = append(sans, cp.IPAddress) } + if installConfig.Kubernetes.APIServerHost != "" { sans = append(sans, installConfig.Kubernetes.APIServerHost) } @@ -109,8 +112,8 @@ func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { k0sConfig.Spec.Network = &K0sNetwork{ Provider: "calico", - PodCIDR: defaultIfEmpty(installConfig.Kubernetes.PodCIDR, "100.96.0.0/11"), - ServiceCIDR: defaultIfEmpty(installConfig.Kubernetes.ServiceCIDR, "100.64.0.0/13"), + PodCIDR: cmp.Or(installConfig.Kubernetes.PodCIDR, "100.96.0.0/11"), + ServiceCIDR: cmp.Or(installConfig.Kubernetes.ServiceCIDR, "100.64.0.0/13"), ClusterDomain: "cluster.local", } @@ -131,13 +134,6 @@ func GenerateK0sConfig(installConfig *files.RootConfig) (*K0sConfig, error) { return k0sConfig, nil } -func defaultIfEmpty(value, defaultValue string) string { - if value != "" { - return value - } - return defaultValue -} - func (c *K0sConfig) Marshal() ([]byte, error) { return yaml.Marshal(c) } diff --git a/internal/installer/k0s_test.go b/internal/installer/k0s_test.go index f59be7419..6caecf69a 100644 --- a/internal/installer/k0s_test.go +++ b/internal/installer/k0s_test.go @@ -4,6 +4,7 @@ package installer_test import ( + "encoding/json" "errors" "os" "path/filepath" @@ -57,7 +58,7 @@ var _ = Describe("K0s", func() { }) It("implements K0sManager interface", func() { - var manager = installer.NewK0s(mockHttp, mockEnv, mockFileWriter) + manager := installer.NewK0s(mockHttp, mockEnv, mockFileWriter) Expect(manager).ToNot(BeNil()) }) }) @@ -113,7 +114,7 @@ var _ = Describe("K0s", func() { k0sImpl.Goos = "windows" k0sImpl.Goarch = "amd64" - _, err := k0s.Download("v1.29.1+k0s.0", false, false) + _, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("codesphere installation is only supported on Linux amd64")) Expect(err.Error()).To(ContainSubstring("windows/amd64")) @@ -123,7 +124,7 @@ var _ = Describe("K0s", func() { k0sImpl.Goos = "linux" k0sImpl.Goarch = "arm64" - _, err := k0s.Download("v1.29.1+k0s.0", false, false) + _, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("codesphere installation is only supported on Linux amd64")) Expect(err.Error()).To(ContainSubstring("linux/arm64")) @@ -148,13 +149,14 @@ var _ = Describe("K0s", func() { // Create a real file for the test realFile, err := os.Create(k0sPath) Expect(err).ToNot(HaveOccurred()) + defer util.CloseFileIgnoreError(realFile) mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) - path, err := k0s.Download("v1.29.1+k0s.0", false, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) @@ -164,6 +166,7 @@ var _ = Describe("K0s", func() { BeforeEach(func() { k0sImpl.Goos = "linux" k0sImpl.Goarch = "amd64" + mockEnv.EXPECT().GetOmsCacheDir().Return(workDir, nil) mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(nil) }) @@ -175,7 +178,7 @@ var _ = Describe("K0s", func() { Expect(err).ToNot(HaveOccurred()) mockFileWriter.EXPECT().Exists(k0sPath).Return(true) - path, err := k0s.Download("v1.29.1+k0s.0", false, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) @@ -196,7 +199,7 @@ var _ = Describe("K0s", func() { mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) - path, err := k0s.Download("v1.29.1+k0s.0", false, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) @@ -211,13 +214,14 @@ var _ = Describe("K0s", func() { // Create a real file for the test realFile, err := os.Create(k0sPath) Expect(err).ToNot(HaveOccurred()) + defer util.CloseFileIgnoreError(realFile) mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) - path, err := k0s.Download("v1.29.1+k0s.0", true, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{Force: true}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) @@ -227,6 +231,7 @@ var _ = Describe("K0s", func() { BeforeEach(func() { k0sImpl.Goos = "linux" k0sImpl.Goarch = "amd64" + mockEnv.EXPECT().GetOmsCacheDir().Return(workDir, nil) mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(nil) mockFileWriter.EXPECT().Exists(k0sPath).Return(false) @@ -235,9 +240,9 @@ var _ = Describe("K0s", func() { It("should fail when file creation fails", func() { mockFileWriter.EXPECT().Create(k0sPath).Return(nil, errors.New("permission denied")) - _, err := k0s.Download("v1.29.1+k0s.0", false, false) + _, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("failed to create k0s binary file")) + Expect(err.Error()).To(ContainSubstring("failed to download k0s binary")) Expect(err.Error()).To(ContainSubstring("permission denied")) }) @@ -245,6 +250,7 @@ var _ = Describe("K0s", func() { // Create a mock file for the test mockFile, err := os.CreateTemp("", "k0s-test") Expect(err).ToNot(HaveOccurred()) + defer func() { _ = os.Remove(mockFile.Name()) }() @@ -252,8 +258,10 @@ var _ = Describe("K0s", func() { mockFileWriter.EXPECT().Create(k0sPath).Return(mockFile, nil) mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", mockFile, false).Return(errors.New("download failed")) + // The truncated destination must not stay behind as a reusable cache entry. + mockFileWriter.EXPECT().Remove(k0sPath).Return(nil) - _, err = k0s.Download("v1.29.1+k0s.0", false, false) + _, err = k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("failed to download k0s binary")) Expect(err.Error()).To(ContainSubstring("download failed")) @@ -266,13 +274,14 @@ var _ = Describe("K0s", func() { realFile, err := os.Create(k0sPath) Expect(err).ToNot(HaveOccurred()) + defer util.CloseFileIgnoreError(realFile) mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) - path, err := k0s.Download("v1.29.1+k0s.0", false, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) @@ -281,6 +290,7 @@ var _ = Describe("K0s", func() { Context("URL construction", func() { BeforeEach(func() { k0sImpl.Goos = "linux" + mockEnv.EXPECT().GetOmsCacheDir().Return(workDir, nil) mockFileWriter.EXPECT().Exists(k0sPath).Return(false) }) @@ -297,16 +307,263 @@ var _ = Describe("K0s", func() { // Create a real file for the test realFile, err := os.Create(k0sPath) Expect(err).ToNot(HaveOccurred()) + defer util.CloseFileIgnoreError(realFile) mockFileWriter.EXPECT().Create(k0sPath).Return(realFile, nil) mockHttp.EXPECT().Download("https://github.com/k0sproject/k0s/releases/download/v1.29.1+k0s.0/k0s-v1.29.1+k0s.0-amd64", realFile, false).Return(nil) mockFileWriter.EXPECT().Chmod(k0sPath, os.FileMode(0755)).Return(nil) - path, err := k0s.Download("v1.29.1+k0s.0", false, false) + path, err := k0s.Download("v1.29.1+k0s.0", installer.DownloadOptions{}) Expect(err).ToNot(HaveOccurred()) Expect(path).To(Equal(k0sPath)) }) }) }) + + Describe("EnsureAirgapBundle", func() { + const ( + legacyVersion = "v1.31.14+k0s.0" + modernVersion = "v1.36.3+k0s.2" + ) + + var ( + legacyAsset = "k0s-airgap-bundle-" + legacyVersion + "-amd64" + modernAsset = "k0s-airgap-bundle-" + modernVersion + "-linux-amd64.tar" + ) + + // expectAirgapDownload sets up the file and http mocks for a fresh download of + // assetName and returns the expected bundle path. otherAssets are listed by the + // release before assetName. + expectAirgapDownload := func(version, assetName string, otherAssets ...string) string { + bundlePath := filepath.Join(workDir, assetName) + + mockFileWriter.EXPECT().ReadDir(workDir).Return(nil, nil) + mockHttp.EXPECT().Get("https://api.github.com/repos/k0sproject/k0s/releases/tags/"+version). + Return(releaseJSON(append(otherAssets, assetName)...), nil) + + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + realFile, err := os.Create(bundlePath) + Expect(err).ToNot(HaveOccurred()) + + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Exists(bundlePath).Return(false) + mockFileWriter.EXPECT().Create(bundlePath).Return(realFile, nil) + mockHttp.EXPECT().Download( + "https://github.com/k0sproject/k0s/releases/download/"+version+"/"+assetName, realFile, false, + ).Return(nil) + + return bundlePath + } + + BeforeEach(func() { + k0sImpl.Goos = "linux" + k0sImpl.Goarch = "amd64" + + mockEnv.EXPECT().GetOmsCacheDir().Return(workDir, nil) + mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(nil) + }) + + It("resolves the legacy asset name used up to k0s v1.35", func() { + bundlePath := expectAirgapDownload(legacyVersion, legacyAsset) + + path, err := k0s.EnsureAirgapBundle(legacyVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("resolves the current asset name with OS and extension used since k0s v1.36", func() { + bundlePath := expectAirgapDownload(modernVersion, modernAsset) + + path, err := k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("accepts the legacy asset name with a tar extension", func() { + bundlePath := expectAirgapDownload(legacyVersion, legacyAsset+".tar") + + path, err := k0s.EnsureAirgapBundle(legacyVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("ignores assets of other platforms", func() { + bundlePath := expectAirgapDownload( + modernVersion, + modernAsset, + "k0s-airgap-bundle-"+modernVersion+"-linux-arm64.tar", + "k0s-airgap-bundle-"+modernVersion+"-windows2022-amd64.tar", + ) + + path, err := k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("reuses a cached bundle without contacting the release API", func() { + bundlePath := filepath.Join(workDir, modernAsset) + + mockFileWriter.EXPECT().ReadDir(workDir).Return([]os.DirEntry{fakeDirEntry{name: modernAsset}}, nil) + mockFileWriter.EXPECT().Exists(bundlePath).Return(true) + + path, err := k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("re-downloads a cached bundle when force is set", func() { + bundlePath := filepath.Join(workDir, modernAsset) + + mockFileWriter.EXPECT().ReadDir(workDir).Return([]os.DirEntry{fakeDirEntry{name: modernAsset}}, nil) + + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + realFile, err := os.Create(bundlePath) + Expect(err).ToNot(HaveOccurred()) + + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Exists(bundlePath).Return(true) + mockFileWriter.EXPECT().Create(bundlePath).Return(realFile, nil) + mockHttp.EXPECT().Download( + "https://github.com/k0sproject/k0s/releases/download/"+modernVersion+"/"+modernAsset, realFile, false, + ).Return(nil) + + path, err := k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{Force: true}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("falls back to the release API when the cache cannot be read", func() { + bundlePath := filepath.Join(workDir, modernAsset) + + mockFileWriter.EXPECT().ReadDir(workDir).Return(nil, errors.New("permission denied")) + mockHttp.EXPECT().Get("https://api.github.com/repos/k0sproject/k0s/releases/tags/"+modernVersion). + Return(releaseJSON(modernAsset), nil) + + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + realFile, err := os.Create(bundlePath) + Expect(err).ToNot(HaveOccurred()) + + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Exists(bundlePath).Return(false) + mockFileWriter.EXPECT().Create(bundlePath).Return(realFile, nil) + mockHttp.EXPECT().Download( + "https://github.com/k0sproject/k0s/releases/download/"+modernVersion+"/"+modernAsset, realFile, false, + ).Return(nil) + + path, err := k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{}) + Expect(err).ToNot(HaveOccurred()) + Expect(path).To(Equal(bundlePath)) + }) + + It("removes a partial bundle when the transfer fails", func() { + bundlePath := filepath.Join(workDir, modernAsset) + + mockFileWriter.EXPECT().ReadDir(workDir).Return(nil, nil) + mockHttp.EXPECT().Get("https://api.github.com/repos/k0sproject/k0s/releases/tags/"+modernVersion). + Return(releaseJSON(modernAsset), nil) + + err := os.MkdirAll(workDir, 0755) + Expect(err).ToNot(HaveOccurred()) + + realFile, err := os.Create(bundlePath) + Expect(err).ToNot(HaveOccurred()) + + defer util.CloseFileIgnoreError(realFile) + + mockFileWriter.EXPECT().Exists(bundlePath).Return(false) + mockFileWriter.EXPECT().Create(bundlePath).Return(realFile, nil) + mockHttp.EXPECT().Download( + "https://github.com/k0sproject/k0s/releases/download/"+modernVersion+"/"+modernAsset, realFile, false, + ).Return(errors.New("connection reset")) + // The cache only checks for existence, so the partial bundle must go. + mockFileWriter.EXPECT().Remove(bundlePath).Return(nil) + + _, err = k0s.EnsureAirgapBundle(modernVersion, installer.DownloadOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to download")) + }) + + It("fails when the release metadata cannot be fetched", func() { + mockFileWriter.EXPECT().ReadDir(workDir).Return(nil, nil) + mockHttp.EXPECT().Get("https://api.github.com/repos/k0sproject/k0s/releases/tags/"+legacyVersion). + Return(nil, errors.New("network error")) + + _, err := k0s.EnsureAirgapBundle(legacyVersion, installer.DownloadOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to fetch k0s release")) + }) + + It("fails when the release has no airgap bundle for the platform", func() { + mockFileWriter.EXPECT().ReadDir(workDir).Return(nil, nil) + mockHttp.EXPECT().Get("https://api.github.com/repos/k0sproject/k0s/releases/tags/"+legacyVersion). + Return(releaseJSON("k0s-v1.31.14+k0s.0-amd64"), nil) + + _, err := k0s.EnsureAirgapBundle(legacyVersion, installer.DownloadOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("no airgap bundle for linux/amd64")) + }) + }) + + Describe("cache failures", func() { + BeforeEach(func() { + k0sImpl.Goos = "linux" + k0sImpl.Goarch = "amd64" + }) + + It("fails when the cache directory cannot be determined", func() { + mockEnv.EXPECT().GetOmsCacheDir().Return("", errors.New("no cache dir")) + + _, err := k0s.EnsureAirgapBundle("v1.31.14+k0s.0", installer.DownloadOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to determine cache directory")) + }) + + It("fails when the cache directory cannot be created", func() { + mockEnv.EXPECT().GetOmsCacheDir().Return(workDir, nil) + mockFileWriter.EXPECT().MkdirAll(workDir, os.FileMode(0755)).Return(errors.New("permission denied")) + + _, err := k0s.EnsureAirgapBundle("v1.31.14+k0s.0", installer.DownloadOptions{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("failed to create workdir")) + }) + }) }) + +// fakeDirEntry is a minimal os.DirEntry for cache lookups in tests. +type fakeDirEntry struct { + name string +} + +func (f fakeDirEntry) Name() string { return f.name } +func (f fakeDirEntry) IsDir() bool { return false } +func (f fakeDirEntry) Type() os.FileMode { return 0 } +func (f fakeDirEntry) Info() (os.FileInfo, error) { return nil, nil } + +// releaseJSON renders a GitHub release API response listing the given assets. +func releaseJSON(names ...string) []byte { + type asset struct { + Name string `json:"name"` + } + + assets := make([]asset, 0, len(names)) + for _, name := range names { + assets = append(assets, asset{Name: name}) + } + + data, err := json.Marshal(struct { + Assets []asset `json:"assets"` + }{Assets: assets}) + Expect(err).ToNot(HaveOccurred()) + + return data +} diff --git a/internal/installer/k0sctl.go b/internal/installer/k0sctl.go index af0ba27ac..a8339ee4e 100644 --- a/internal/installer/k0sctl.go +++ b/internal/installer/k0sctl.go @@ -4,7 +4,6 @@ package installer import ( - "encoding/json" "fmt" "log" "path/filepath" @@ -17,9 +16,14 @@ import ( "github.com/codesphere-cloud/oms/internal/util" ) -// DefaultK0sctlVersion is the currently verified k0sctl version. It mirrors -// DefaultK0sVersion in k0s.go: the pair is the version combination we test -// against, while users can override k0sctl via --k0sctl-version. +const ( + k0sctlBinaryName = "k0sctl" + k0sctlReleaseURL = "https://github.com/k0sproject/k0sctl/releases/download" + k0sctlReleaseAPIURL = "https://api.github.com/repos/k0sproject/k0sctl/releases/latest" +) + +// DefaultK0sctlVersion is the currently verified k0sctl version, mirroring +// DefaultK0sVersion in k0s.go. Users can override it via --k0sctl-version. // // renovate: datasource=github-releases depName=k0sproject/k0sctl const DefaultK0sctlVersion = "v0.33.1" @@ -27,7 +31,7 @@ const DefaultK0sctlVersion = "v0.33.1" //mockery:generate: true type K0sctlManager interface { GetLatestVersion() (string, error) - Download(version string, force bool, quiet bool) (string, error) + Download(version string, opts DownloadOptions) (string, error) Apply(configPath string, k0sctlPath string, force bool) error Reset(configPath string, k0sctlPath string) error GetKubeconfig(configPath string, k0sctlPath string) (string, error) @@ -51,20 +55,10 @@ func NewK0sctl(hw portal.Http, env env.Env, fw util.FileIO) *K0sctl { } } -type githubRelease struct { - TagName string `json:"tag_name"` -} - func (k *K0sctl) GetLatestVersion() (string, error) { - releaseURL := "https://api.github.com/repos/k0sproject/k0sctl/releases/latest" - responseBody, err := k.Http.Get(releaseURL) + release, err := getGitHubRelease(k.Http, k0sctlReleaseAPIURL, "latest k0sctl release") if err != nil { - return "", fmt.Errorf("failed to fetch latest k0sctl release: %w", err) - } - - var release githubRelease - if err := json.Unmarshal(responseBody, &release); err != nil { - return "", fmt.Errorf("failed to parse GitHub API response: %w", err) + return "", err } if release.TagName == "" { @@ -74,69 +68,55 @@ func (k *K0sctl) GetLatestVersion() (string, error) { return release.TagName, nil } -func (k *K0sctl) Download(version string, force bool, quiet bool) (string, error) { - cacheDir, err := k.Env.GetOmsCacheDir() +// Download stores the k0sctl binary of the requested version in the OMS cache dir and +// returns its path. An empty version resolves to the latest release. +func (k *K0sctl) Download(version string, opts DownloadOptions) (string, error) { + cacheDir, err := ensureCacheDir(k.FileWriter, k.Env) if err != nil { - return "", fmt.Errorf("failed to determine cache directory: %w", err) - } - - if err := k.FileWriter.MkdirAll(cacheDir, 0755); err != nil { - return "", fmt.Errorf("failed to create workdir: %w", err) + return "", err } if version == "" { - var err error version, err = k.GetLatestVersion() if err != nil { return "", fmt.Errorf("failed to get latest version: %w", err) } - io.Verbosef(!quiet, "Using latest k0sctl version: %s", version) + + io.Verbosef(!opts.Quiet, "Using latest k0sctl version: %s", version) } if !strings.HasPrefix(version, "v") { version = "v" + version } - cachePath := filepath.Join(cacheDir, "k0sctl") - if k.FileWriter.Exists(cachePath) && !force { - cachedVersion, versionErr := localBinaryVersion(cachePath) - if versionErr == nil && cachedVersion == version { - io.Verbosef(!quiet, "Using cached k0sctl %s at %s", version, cachePath) - - return cachePath, nil - } - - if versionErr != nil { - io.Verbosef(!quiet, "Cached k0sctl version could not be determined; replacing it: %v", versionErr) - } else { - io.Verbosef(!quiet, "Cached k0sctl version %s does not match requested version %s; replacing it", cachedVersion, version) - } + cachePath := filepath.Join(cacheDir, k0sctlBinaryName) + if cachedPath, cached := reuseCachedBinary(k.FileWriter, cachePath, version, k0sctlBinaryName, opts); cached { + return cachedPath, nil } - binaryName := fmt.Sprintf("k0sctl-%s-%s", k.Goos, k.Goarch) - downloadURL := fmt.Sprintf("https://github.com/k0sproject/k0sctl/releases/download/%s/%s", version, binaryName) + assetName := fmt.Sprintf("%s-%s-%s", k0sctlBinaryName, k.Goos, k.Goarch) + downloadURL := releaseAssetURL(k0sctlReleaseURL, version, assetName) - io.Verbosef(!quiet, "Downloading k0sctl %s from %s", version, downloadURL) + io.Verbosef(!opts.Quiet, "Downloading k0sctl %s from %s", version, downloadURL) - path, err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, "k0sctl", downloadURL, quiet) - if err != nil { + if err := downloadBinaryToPath(k.FileWriter, k.Http, cachePath, k0sctlBinaryName, downloadURL, opts.Quiet); err != nil { return "", err } - io.Verbosef(!quiet, "k0sctl downloaded successfully to %s", path) + io.Verbosef(!opts.Quiet, "k0sctl downloaded successfully to %s", cachePath) - return path, nil + return cachePath, nil } -// requireBinaryAndConfig checks that both the k0sctl binary and config exist, -// returning an error if either is missing. func (k *K0sctl) requireBinaryAndConfig(configPath, k0sctlPath string) error { if !k.FileWriter.Exists(k0sctlPath) { return fmt.Errorf("k0sctl binary does not exist at '%s', please download first", k0sctlPath) } + if !k.FileWriter.Exists(configPath) { return fmt.Errorf("k0sctl config does not exist at '%s'", configPath) } + return nil } @@ -151,7 +131,6 @@ func (k *K0sctl) Apply(configPath string, k0sctlPath string, force bool) error { args = append(args, "--force") } - // Add debug flag for more verbose output args = append(args, "--debug") log.Printf("Running k0sctl apply with config: %s", configPath) @@ -162,6 +141,7 @@ func (k *K0sctl) Apply(configPath string, k0sctlPath string, force bool) error { } log.Println("k0sctl apply completed successfully") + return nil } @@ -169,6 +149,7 @@ func (k *K0sctl) Reset(configPath string, k0sctlPath string) error { if !k.FileWriter.Exists(k0sctlPath) { return nil } + if err := k.requireBinaryAndConfig(configPath, k0sctlPath); err != nil { return err } @@ -183,6 +164,7 @@ func (k *K0sctl) Reset(configPath string, k0sctlPath string) error { } log.Println("k0sctl reset completed successfully") + return nil } @@ -194,6 +176,7 @@ func (k *K0sctl) GetKubeconfig(configPath string, k0sctlPath string) (string, er args := []string{"kubeconfig", "--config", configPath} log.Println("Retrieving kubeconfig from k0sctl...") + output, err := util.RunCommandWithOutput(k0sctlPath, args, "") if err != nil { return "", fmt.Errorf("k0sctl kubeconfig failed: %w", err) diff --git a/internal/installer/k0sctl_config.go b/internal/installer/k0sctl_config.go index 30791b0d6..a6bb3ef4f 100644 --- a/internal/installer/k0sctl_config.go +++ b/internal/installer/k0sctl_config.go @@ -37,9 +37,31 @@ type K0sctlHost struct { Environment map[string]string `yaml:"environment,omitempty"` UploadBinary bool `yaml:"uploadBinary,omitempty"` K0sBinaryPath string `yaml:"k0sBinaryPath,omitempty"` + Files []K0sctlFile `yaml:"files,omitempty"` Hooks *K0sctlHooks `yaml:"hooks,omitempty"` } +// K0sctlFile is a file that k0sctl uploads to a node before installing k0s, for +// example the airgap image bundle that worker nodes import. +type K0sctlFile struct { + Src string `yaml:"src"` + DstDir string `yaml:"dstDir"` + Perm string `yaml:"perm,omitempty"` +} + +// K0sctlOptions configures the k0sctl cluster configuration that oms generates from +// an install-config. +type K0sctlOptions struct { + // K0sVersion is the version of k0s that k0sctl installs on the nodes. + K0sVersion string + // SSHKeyPath is the private key k0sctl uses to connect to the nodes. + SSHKeyPath string + // K0sBinaryPath is the local k0s binary that k0sctl uploads to the nodes. + K0sBinaryPath string + // AirgapBundlePath is the image bundle that k0sctl uploads to the worker nodes. + AirgapBundlePath string +} + type K0sctlSSH struct { Address string `yaml:"address"` User string `yaml:"user"` @@ -69,19 +91,23 @@ type K0sctlApplyHooks struct { After []string `yaml:"after,omitempty"` } -func (k *K0sctlSpec) addUniqueK0sctlHost(node files.K8sNode, role string, installFlags []string, sshKeyPath string, k0sBinaryPath string) { +// addUniqueK0sctlHost appends a host to the cluster config unless its address is +// already present. runsWorker marks hosts that run the k0s worker role, either +// dedicated workers or control planes installed with --enable-worker. +func (k *K0sctlSpec) addUniqueK0sctlHost(node files.K8sNode, role string, installFlags []string, runsWorker bool, options K0sctlOptions) { for _, host := range k.Hosts { if host.PrivateAddress == node.IPAddress { return } } + host := K0sctlHost{ Role: role, SSH: K0sctlSSH{ Address: node.IPAddress, User: "root", Port: 22, - KeyPath: sshKeyPath, + KeyPath: options.SSHKeyPath, }, InstallFlags: installFlags, PrivateAddress: node.IPAddress, @@ -90,16 +116,20 @@ func (k *K0sctlSpec) addUniqueK0sctlHost(node files.K8sNode, role string, instal }, } - if k0sBinaryPath != "" { + if options.K0sBinaryPath != "" { host.UploadBinary = true - host.K0sBinaryPath = k0sBinaryPath + host.K0sBinaryPath = options.K0sBinaryPath + } + + if runsWorker && options.AirgapBundlePath != "" { + host.Files = []K0sctlFile{{Src: options.AirgapBundlePath, DstDir: AirgapImagesDir, Perm: "0644"}} } k.Hosts = append(k.Hosts, host) } // GenerateK0sctlConfig generates a k0sctl configuration from a Codesphere install-config -func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, sshKeyPath string, k0sBinaryPath string) (*K0sctlConfig, error) { +func GenerateK0sctlConfig(installConfig *files.RootConfig, options K0sctlOptions) (*K0sctlConfig, error) { if installConfig == nil { return nil, fmt.Errorf("installConfig cannot be nil") } @@ -108,12 +138,15 @@ func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, ss return nil, fmt.Errorf("k0sctl is only supported for Codesphere-managed Kubernetes") } - // Generate k0s config that will be embedded in k0sctl config k0sConfig, err := GenerateK0sConfig(installConfig) if err != nil { return nil, fmt.Errorf("failed to generate k0s config: %w", err) } + if options.AirgapBundlePath != "" { + k0sConfig.Spec.Images.DefaultPullPolicy = "Never" + } + k0sctlConfig := &K0sctlConfig{ APIVersion: "k0sctl.k0sproject.io/v1beta1", Kind: "Cluster", @@ -123,7 +156,7 @@ func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, ss Spec: K0sctlSpec{ Hosts: []K0sctlHost{}, K0s: K0sctlK0s{ - Version: k0sVersion, + Version: options.K0sVersion, Config: k0sConfig, }, }, @@ -133,16 +166,16 @@ func GenerateK0sctlConfig(installConfig *files.RootConfig, k0sVersion string, ss for _, cp := range installConfig.Kubernetes.ControlPlanes { var installFlags []string // A node may intentionally be listed as both a control plane and a worker. - if slices.Contains(installConfig.Kubernetes.Workers, cp) { + runsWorker := slices.Contains(installConfig.Kubernetes.Workers, cp) + if runsWorker { installFlags = []string{"--enable-worker", "--no-taints=true"} } - k0sctlConfig.Spec.addUniqueK0sctlHost(cp, "controller", installFlags, sshKeyPath, k0sBinaryPath) + k0sctlConfig.Spec.addUniqueK0sctlHost(cp, "controller", installFlags, runsWorker, options) } - // Add dedicated worker nodes if present for _, worker := range installConfig.Kubernetes.Workers { - k0sctlConfig.Spec.addUniqueK0sctlHost(worker, "worker", nil, sshKeyPath, k0sBinaryPath) + k0sctlConfig.Spec.addUniqueK0sctlHost(worker, "worker", nil, true, options) } return k0sctlConfig, nil diff --git a/internal/installer/k0sctl_config_test.go b/internal/installer/k0sctl_config_test.go index 55ca856a3..7f720d911 100644 --- a/internal/installer/k0sctl_config_test.go +++ b/internal/installer/k0sctl_config_test.go @@ -21,7 +21,11 @@ var _ = Describe("K0sctlConfig", func() { installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "/path/to/k0s") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, installer.K0sctlOptions{ + K0sVersion: "v1.30.0+k0s.0", + SSHKeyPath: "/path/to/key", + K0sBinaryPath: "/path/to/k0s", + }) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig).ToNot(BeNil()) @@ -40,7 +44,7 @@ var _ = Describe("K0sctlConfig", func() { It("should assign controller role to control plane nodes", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) @@ -55,7 +59,7 @@ var _ = Describe("K0sctlConfig", func() { {IPAddress: "10.0.2.11"}, } - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(3)) @@ -76,7 +80,7 @@ var _ = Describe("K0sctlConfig", func() { {IPAddress: "10.0.2.10"}, // Unique } - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) // The overlapping node is emitted once, but retains both roles in @@ -92,7 +96,7 @@ var _ = Describe("K0sctlConfig", func() { It("should enable UploadBinary when k0sBinaryPath is provided", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "/path/to/k0s") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "/path/to/k0s")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].UploadBinary).To(BeTrue()) @@ -102,7 +106,7 @@ var _ = Describe("K0sctlConfig", func() { It("should not enable UploadBinary when k0sBinaryPath is empty", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].UploadBinary).To(BeFalse()) @@ -112,7 +116,7 @@ var _ = Describe("K0sctlConfig", func() { It("should set SSH key path correctly", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/home/user/.ssh/id_rsa", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/home/user/.ssh/id_rsa", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].SSH.KeyPath).To(Equal("/home/user/.ssh/id_rsa")) @@ -121,7 +125,7 @@ var _ = Describe("K0sctlConfig", func() { It("should set SSH user to root", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].SSH.User).To(Equal("root")) @@ -130,18 +134,105 @@ var _ = Describe("K0sctlConfig", func() { It("should set KUBELET_EXTRA_ARGS environment variable", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].Environment).To(HaveKeyWithValue("KUBELET_EXTRA_ARGS", "--node-ip=10.0.1.10")) }) + It("should upload the airgap bundle to worker nodes when airgapped", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + installConfig.Kubernetes.Workers = []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlAirgapOptions("v1.30.0+k0s.0", "/cache/k0s-airgap-bundle-amd64")) + Expect(err).ToNot(HaveOccurred()) + + // Controllers without the worker role do not import image bundles. + Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("controller")) + Expect(k0sctlConfig.Spec.Hosts[0].Files).To(BeEmpty()) + + Expect(k0sctlConfig.Spec.Hosts[1].Role).To(Equal("worker")) + Expect(k0sctlConfig.Spec.Hosts[1].Files).To(Equal([]installer.K0sctlFile{{ + Src: "/cache/k0s-airgap-bundle-amd64", + DstDir: "/var/lib/k0s/images", + Perm: "0644", + }})) + }) + + It("should upload the airgap bundle to control planes that also run a worker", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + installConfig.Kubernetes.Workers = []files.K8sNode{ + {IPAddress: "10.0.1.10"}, // Same node, so it also runs a worker + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlAirgapOptions("v1.30.0+k0s.0", "/cache/bundle")) + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) + Expect(k0sctlConfig.Spec.Hosts[0].InstallFlags).To(Equal([]string{"--enable-worker", "--no-taints=true"})) + Expect(k0sctlConfig.Spec.Hosts[0].Files).To(HaveLen(1)) + }) + + It("should stop pulling images when airgapped", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlAirgapOptions("v1.30.0+k0s.0", "/cache/bundle")) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sctlConfig.Spec.K0s.Config.Spec.Images.DefaultPullPolicy).To(Equal("Never")) + + yamlData, err := k0sctlConfig.Marshal() + Expect(err).ToNot(HaveOccurred()) + Expect(string(yamlData)).To(ContainSubstring("default_pull_policy: Never")) + }) + + It("should keep the default pull policy for installations with internet access", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) + Expect(err).ToNot(HaveOccurred()) + Expect(k0sctlConfig.Spec.K0s.Config.Spec.Images.DefaultPullPolicy).To(Equal("IfNotPresent")) + }) + + It("should not upload files for installations with internet access", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + installConfig.Kubernetes.Workers = []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts[0].Files).To(BeEmpty()) + Expect(k0sctlConfig.Spec.Hosts[1].Files).To(BeEmpty()) + }) + + It("should marshal the airgap upload with k0sctl field names", func() { + installConfig := newTestConfig("test-dc", true, "10.0.1.10") + installConfig.Kubernetes.Workers = []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlAirgapOptions("v1.30.0+k0s.0", "/cache/bundle")) + Expect(err).ToNot(HaveOccurred()) + + yamlData, err := k0sctlConfig.Marshal() + Expect(err).ToNot(HaveOccurred()) + + yamlString := string(yamlData) + Expect(yamlString).To(ContainSubstring("files:")) + Expect(yamlString).To(ContainSubstring("src: /cache/bundle")) + Expect(yamlString).To(ContainSubstring("dstDir: /var/lib/k0s/images")) + Expect(yamlString).To(ContainSubstring("perm:")) + }) + It("should generate valid YAML", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") installConfig.Kubernetes.PodCIDR = "10.244.0.0/16" installConfig.Kubernetes.ServiceCIDR = "10.96.0.0/12" - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) yamlData, err := k0sctlConfig.Marshal() @@ -150,6 +241,7 @@ var _ = Describe("K0sctlConfig", func() { // Verify it can be unmarshalled back var parsedConfig installer.K0sctlConfig + err = yaml.Unmarshal(yamlData, &parsedConfig) Expect(err).ToNot(HaveOccurred()) Expect(parsedConfig.Metadata.Name).To(Equal("codesphere-test-dc")) @@ -158,7 +250,7 @@ var _ = Describe("K0sctlConfig", func() { Context("with invalid input", func() { It("should return error for nil install-config", func() { - k0sctlConfig, err := installer.GenerateK0sctlConfig(nil, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(nil, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("installConfig cannot be nil")) Expect(k0sctlConfig).To(BeNil()) @@ -167,7 +259,7 @@ var _ = Describe("K0sctlConfig", func() { It("should return error for non-managed Kubernetes", func() { installConfig := newTestConfig("test-dc", false) - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("k0sctl is only supported for Codesphere-managed Kubernetes")) Expect(k0sctlConfig).To(BeNil()) @@ -179,7 +271,7 @@ var _ = Describe("K0sctlConfig", func() { installConfig := newTestConfig("test-dc", true) installConfig.Kubernetes.ControlPlanes = []files.K8sNode{} - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts).To(BeEmpty()) }) @@ -187,7 +279,7 @@ var _ = Describe("K0sctlConfig", func() { It("should handle nil control plane list", func() { installConfig := newTestConfig("test-dc", true) - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts).To(BeEmpty()) }) @@ -199,17 +291,36 @@ var _ = Describe("K0sctlConfig", func() { {IPAddress: "10.0.2.10"}, } - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) // Workers should still be added even without control planes Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) Expect(k0sctlConfig.Spec.Hosts[0].Role).To(Equal("worker")) }) + It("should upload the airgap bundle to worker-only clusters", func() { + installConfig := newTestConfig("test-dc", true) + installConfig.Kubernetes.ControlPlanes = []files.K8sNode{} + installConfig.Kubernetes.Workers = []files.K8sNode{ + {IPAddress: "10.0.2.10"}, + } + + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, + k0sctlAirgapOptions("v1.30.0+k0s.0", "/cache/bundle")) + Expect(err).ToNot(HaveOccurred()) + + Expect(k0sctlConfig.Spec.Hosts).To(HaveLen(1)) + Expect(k0sctlConfig.Spec.Hosts[0].Files).To(Equal([]installer.K0sctlFile{{ + Src: "/cache/bundle", + DstDir: "/var/lib/k0s/images", + Perm: "0644", + }})) + }) + It("should handle empty SSH key path", func() { installConfig := newTestConfig("test-dc", true, "10.0.1.10") - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "", "")) Expect(err).ToNot(HaveOccurred()) Expect(k0sctlConfig.Spec.Hosts[0].SSH.KeyPath).To(BeEmpty()) @@ -221,7 +332,7 @@ var _ = Describe("K0sctlConfig", func() { {IPAddress: "10.0.2.10"}, } - k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, "v1.30.0+k0s.0", "/path/to/key", "") + k0sctlConfig, err := installer.GenerateK0sctlConfig(installConfig, k0sctlOptions("v1.30.0+k0s.0", "/path/to/key", "")) Expect(err).ToNot(HaveOccurred()) // Both hosts should have PrivateAddress set to the internal IP @@ -231,3 +342,21 @@ var _ = Describe("K0sctlConfig", func() { }) }) }) + +// k0sctlOptions builds k0sctl options for installations with internet access. +func k0sctlOptions(k0sVersion, sshKeyPath, k0sBinaryPath string) installer.K0sctlOptions { + return installer.K0sctlOptions{ + K0sVersion: k0sVersion, + SSHKeyPath: sshKeyPath, + K0sBinaryPath: k0sBinaryPath, + } +} + +// k0sctlAirgapOptions builds k0sctl options for airgapped installations. +func k0sctlAirgapOptions(k0sVersion, bundlePath string) installer.K0sctlOptions { + return installer.K0sctlOptions{ + K0sVersion: k0sVersion, + SSHKeyPath: "/path/to/key", + AirgapBundlePath: bundlePath, + } +} diff --git a/internal/installer/k0sctl_test.go b/internal/installer/k0sctl_test.go index 8eb02d649..5e70d94a7 100644 --- a/internal/installer/k0sctl_test.go +++ b/internal/installer/k0sctl_test.go @@ -58,7 +58,7 @@ var _ = Describe("K0sctl", func() { writeCachedVersion("v0.32.1") mockFileWriter.EXPECT().Exists(cachedPath).Return(true) - path, err := k0sctl.Download("v0.32.1", false, false) + path, err := k0sctl.Download("v0.32.1", installer.DownloadOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(path).To(Equal(cachedPath)) @@ -69,7 +69,7 @@ var _ = Describe("K0sctl", func() { mockFileWriter.EXPECT().Exists(cachedPath).Return(true) expectDownload("v0.32.1") - path, err := k0sctl.Download("v0.32.1", false, false) + path, err := k0sctl.Download("v0.32.1", installer.DownloadOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(path).To(Equal(cachedPath)) @@ -81,7 +81,7 @@ var _ = Describe("K0sctl", func() { Return([]byte(`{"tag_name":"v0.32.1"}`), nil) mockFileWriter.EXPECT().Exists(cachedPath).Return(true) - path, err := k0sctl.Download("", false, false) + path, err := k0sctl.Download("", installer.DownloadOptions{}) Expect(err).NotTo(HaveOccurred()) Expect(path).To(Equal(cachedPath)) diff --git a/internal/installer/mocks.go b/internal/installer/mocks.go index bf6256390..4cc97f91d 100644 --- a/internal/installer/mocks.go +++ b/internal/installer/mocks.go @@ -1208,8 +1208,8 @@ func (_m *MockK0sManager) EXPECT() *MockK0sManager_Expecter { } // Download provides a mock function for the type MockK0sManager -func (_mock *MockK0sManager) Download(version string, force bool, quiet bool) (string, error) { - ret := _mock.Called(version, force, quiet) +func (_mock *MockK0sManager) Download(version string, opts DownloadOptions) (string, error) { + ret := _mock.Called(version, opts) if len(ret) == 0 { panic("no return value specified for Download") @@ -1217,16 +1217,16 @@ func (_mock *MockK0sManager) Download(version string, force bool, quiet bool) (s var r0 string var r1 error - if returnFunc, ok := ret.Get(0).(func(string, bool, bool) (string, error)); ok { - return returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) (string, error)); ok { + return returnFunc(version, opts) } - if returnFunc, ok := ret.Get(0).(func(string, bool, bool) string); ok { - r0 = returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) string); ok { + r0 = returnFunc(version, opts) } else { r0 = ret.Get(0).(string) } - if returnFunc, ok := ret.Get(1).(func(string, bool, bool) error); ok { - r1 = returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(1).(func(string, DownloadOptions) error); ok { + r1 = returnFunc(version, opts) } else { r1 = ret.Error(1) } @@ -1240,30 +1240,24 @@ type MockK0sManager_Download_Call struct { // Download is a helper method to define mock.On call // - version string -// - force bool -// - quiet bool -func (_e *MockK0sManager_Expecter) Download(version any, force any, quiet any) *MockK0sManager_Download_Call { - return &MockK0sManager_Download_Call{Call: _e.mock.On("Download", version, force, quiet)} +// - opts DownloadOptions +func (_e *MockK0sManager_Expecter) Download(version any, opts any) *MockK0sManager_Download_Call { + return &MockK0sManager_Download_Call{Call: _e.mock.On("Download", version, opts)} } -func (_c *MockK0sManager_Download_Call) Run(run func(version string, force bool, quiet bool)) *MockK0sManager_Download_Call { +func (_c *MockK0sManager_Download_Call) Run(run func(version string, opts DownloadOptions)) *MockK0sManager_Download_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { arg0 = args[0].(string) } - var arg1 bool + var arg1 DownloadOptions if args[1] != nil { - arg1 = args[1].(bool) - } - var arg2 bool - if args[2] != nil { - arg2 = args[2].(bool) + arg1 = args[1].(DownloadOptions) } run( arg0, arg1, - arg2, ) }) return _c @@ -1274,7 +1268,73 @@ func (_c *MockK0sManager_Download_Call) Return(s string, err error) *MockK0sMana return _c } -func (_c *MockK0sManager_Download_Call) RunAndReturn(run func(version string, force bool, quiet bool) (string, error)) *MockK0sManager_Download_Call { +func (_c *MockK0sManager_Download_Call) RunAndReturn(run func(version string, opts DownloadOptions) (string, error)) *MockK0sManager_Download_Call { + _c.Call.Return(run) + return _c +} + +// EnsureAirgapBundle provides a mock function for the type MockK0sManager +func (_mock *MockK0sManager) EnsureAirgapBundle(version string, opts DownloadOptions) (string, error) { + ret := _mock.Called(version, opts) + + if len(ret) == 0 { + panic("no return value specified for EnsureAirgapBundle") + } + + var r0 string + var r1 error + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) (string, error)); ok { + return returnFunc(version, opts) + } + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) string); ok { + r0 = returnFunc(version, opts) + } else { + r0 = ret.Get(0).(string) + } + if returnFunc, ok := ret.Get(1).(func(string, DownloadOptions) error); ok { + r1 = returnFunc(version, opts) + } else { + r1 = ret.Error(1) + } + return r0, r1 +} + +// MockK0sManager_EnsureAirgapBundle_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnsureAirgapBundle' +type MockK0sManager_EnsureAirgapBundle_Call struct { + *mock.Call +} + +// EnsureAirgapBundle is a helper method to define mock.On call +// - version string +// - opts DownloadOptions +func (_e *MockK0sManager_Expecter) EnsureAirgapBundle(version any, opts any) *MockK0sManager_EnsureAirgapBundle_Call { + return &MockK0sManager_EnsureAirgapBundle_Call{Call: _e.mock.On("EnsureAirgapBundle", version, opts)} +} + +func (_c *MockK0sManager_EnsureAirgapBundle_Call) Run(run func(version string, opts DownloadOptions)) *MockK0sManager_EnsureAirgapBundle_Call { + _c.Call.Run(func(args mock.Arguments) { + var arg0 string + if args[0] != nil { + arg0 = args[0].(string) + } + var arg1 DownloadOptions + if args[1] != nil { + arg1 = args[1].(DownloadOptions) + } + run( + arg0, + arg1, + ) + }) + return _c +} + +func (_c *MockK0sManager_EnsureAirgapBundle_Call) Return(s string, err error) *MockK0sManager_EnsureAirgapBundle_Call { + _c.Call.Return(s, err) + return _c +} + +func (_c *MockK0sManager_EnsureAirgapBundle_Call) RunAndReturn(run func(version string, opts DownloadOptions) (string, error)) *MockK0sManager_EnsureAirgapBundle_Call { _c.Call.Return(run) return _c } @@ -1423,8 +1483,8 @@ func (_c *MockK0sctlManager_Apply_Call) RunAndReturn(run func(configPath string, } // Download provides a mock function for the type MockK0sctlManager -func (_mock *MockK0sctlManager) Download(version string, force bool, quiet bool) (string, error) { - ret := _mock.Called(version, force, quiet) +func (_mock *MockK0sctlManager) Download(version string, opts DownloadOptions) (string, error) { + ret := _mock.Called(version, opts) if len(ret) == 0 { panic("no return value specified for Download") @@ -1432,16 +1492,16 @@ func (_mock *MockK0sctlManager) Download(version string, force bool, quiet bool) var r0 string var r1 error - if returnFunc, ok := ret.Get(0).(func(string, bool, bool) (string, error)); ok { - return returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) (string, error)); ok { + return returnFunc(version, opts) } - if returnFunc, ok := ret.Get(0).(func(string, bool, bool) string); ok { - r0 = returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(0).(func(string, DownloadOptions) string); ok { + r0 = returnFunc(version, opts) } else { r0 = ret.Get(0).(string) } - if returnFunc, ok := ret.Get(1).(func(string, bool, bool) error); ok { - r1 = returnFunc(version, force, quiet) + if returnFunc, ok := ret.Get(1).(func(string, DownloadOptions) error); ok { + r1 = returnFunc(version, opts) } else { r1 = ret.Error(1) } @@ -1455,30 +1515,24 @@ type MockK0sctlManager_Download_Call struct { // Download is a helper method to define mock.On call // - version string -// - force bool -// - quiet bool -func (_e *MockK0sctlManager_Expecter) Download(version any, force any, quiet any) *MockK0sctlManager_Download_Call { - return &MockK0sctlManager_Download_Call{Call: _e.mock.On("Download", version, force, quiet)} +// - opts DownloadOptions +func (_e *MockK0sctlManager_Expecter) Download(version any, opts any) *MockK0sctlManager_Download_Call { + return &MockK0sctlManager_Download_Call{Call: _e.mock.On("Download", version, opts)} } -func (_c *MockK0sctlManager_Download_Call) Run(run func(version string, force bool, quiet bool)) *MockK0sctlManager_Download_Call { +func (_c *MockK0sctlManager_Download_Call) Run(run func(version string, opts DownloadOptions)) *MockK0sctlManager_Download_Call { _c.Call.Run(func(args mock.Arguments) { var arg0 string if args[0] != nil { arg0 = args[0].(string) } - var arg1 bool + var arg1 DownloadOptions if args[1] != nil { - arg1 = args[1].(bool) - } - var arg2 bool - if args[2] != nil { - arg2 = args[2].(bool) + arg1 = args[1].(DownloadOptions) } run( arg0, arg1, - arg2, ) }) return _c @@ -1489,7 +1543,7 @@ func (_c *MockK0sctlManager_Download_Call) Return(s string, err error) *MockK0sc return _c } -func (_c *MockK0sctlManager_Download_Call) RunAndReturn(run func(version string, force bool, quiet bool) (string, error)) *MockK0sctlManager_Download_Call { +func (_c *MockK0sctlManager_Download_Call) RunAndReturn(run func(version string, opts DownloadOptions) (string, error)) *MockK0sctlManager_Download_Call { _c.Call.Return(run) return _c } diff --git a/internal/installer/node/agent_forwarding_test.go b/internal/installer/node/agent_forwarding_test.go new file mode 100644 index 000000000..d894f7277 --- /dev/null +++ b/internal/installer/node/agent_forwarding_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package node + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/pem" + "os" + "path/filepath" + + "github.com/codesphere-cloud/oms/internal/util" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" +) + +// writeKeyPair writes a new ed25519 key pair to dir and returns the private key path and the key. +func writeKeyPair(dir string, name string) (string, ed25519.PrivateKey) { + _, priv, err := ed25519.GenerateKey(rand.Reader) + Expect(err).NotTo(HaveOccurred()) + + block, err := ssh.MarshalPrivateKey(priv, "") + Expect(err).NotTo(HaveOccurred()) + signer, err := ssh.NewSignerFromKey(priv) + Expect(err).NotTo(HaveOccurred()) + + keyPath := filepath.Join(dir, name) + Expect(os.WriteFile(keyPath, pem.EncodeToMemory(block), 0600)).To(Succeed()) + Expect(os.WriteFile(keyPath+".pub", ssh.MarshalAuthorizedKey(signer.PublicKey()), 0644)).To(Succeed()) + + return keyPath, priv +} + +func listedKeys(a agent.Agent) []string { + keys, err := a.List() + Expect(err).NotTo(HaveOccurred()) + + blobs := make([]string, 0, len(keys)) + for _, k := range keys { + blobs = append(blobs, string(k.Marshal())) + } + + return blobs +} + +func publicBlob(priv ed25519.PrivateKey) string { + signer, err := ssh.NewSignerFromKey(priv) + Expect(err).NotTo(HaveOccurred()) + + return string(signer.PublicKey().Marshal()) +} + +var _ = Describe("forwardedAgent", func() { + var ( + dir string + keyPath string + deploy ed25519.PrivateKey + n *Node + ) + + BeforeEach(func() { + dir = GinkgoT().TempDir() + keyPath, deploy = writeKeyPair(dir, "deploy") + n = &Node{FileIO: util.NewFilesystemWriter(), KeyPath: keyPath} + }) + + It("forwards nothing without a local agent or key path", func() { + n.KeyPath = "" + + forwarded, err := n.forwardedAgent(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(forwarded).To(BeNil()) + }) + + It("forwards the local agent as-is when it holds the deploy key", func() { + local := agent.NewKeyring().(agent.ExtendedAgent) + Expect(local.Add(agent.AddedKey{PrivateKey: deploy})).To(Succeed()) + + forwarded, err := n.forwardedAgent(local) + Expect(err).NotTo(HaveOccurred()) + Expect(forwarded).To(BeIdenticalTo(local)) + }) + + It("forwards only the deploy key without a local agent", func() { + forwarded, err := n.forwardedAgent(nil) + Expect(err).NotTo(HaveOccurred()) + Expect(listedKeys(forwarded)).To(ConsistOf(publicBlob(deploy))) + }) + + It("adds the deploy key to a local agent that lacks it and signs with both keys", func() { + _, other := writeKeyPair(dir, "other") + local := agent.NewKeyring().(agent.ExtendedAgent) + Expect(local.Add(agent.AddedKey{PrivateKey: other})).To(Succeed()) + + forwarded, err := n.forwardedAgent(local) + Expect(err).NotTo(HaveOccurred()) + Expect(listedKeys(forwarded)).To(ConsistOf(publicBlob(deploy), publicBlob(other))) + + signers, err := forwarded.Signers() + Expect(err).NotTo(HaveOccurred()) + Expect(signers).To(HaveLen(2)) + + data := []byte("challenge") + for _, s := range signers { + sig, err := forwarded.Sign(s.PublicKey(), data) + Expect(err).NotTo(HaveOccurred()) + Expect(s.PublicKey().Verify(data, sig)).To(Succeed()) + } + }) + + It("falls back to the local agent when the deploy key cannot be loaded", func() { + n.KeyPath = filepath.Join(dir, "missing") + local := agent.NewKeyring().(agent.ExtendedAgent) + + forwarded, err := n.forwardedAgent(local) + Expect(err).NotTo(HaveOccurred()) + Expect(forwarded).To(BeIdenticalTo(local)) + }) + + It("fails without a local agent when the deploy key cannot be loaded", func() { + n.KeyPath = filepath.Join(dir, "missing") + + _, err := n.forwardedAgent(nil) + Expect(err).To(HaveOccurred()) + }) +}) diff --git a/internal/installer/node/node.go b/internal/installer/node/node.go index 8d30e7074..2339f3c38 100644 --- a/internal/installer/node/node.go +++ b/internal/installer/node/node.go @@ -1,6 +1,7 @@ // Copyright (c) Codesphere Inc. // SPDX-License-Identifier: Apache-2.0 +// Package node manages the remote hosts of an installation over SSH. package node import ( @@ -31,6 +32,7 @@ type Node struct { Name string `json:"name"` ExternalIP string `json:"external_ip"` InternalIP string `json:"internal_ip"` + cachedKey any `json:"-"` cachedSigner ssh.Signer `json:"-"` sshQuiet bool `json:"-"` @@ -623,8 +625,7 @@ func (n *Node) getAuthMethods() ([]ssh.AuthMethod, error) { // Else load from file with passphrase prompt if needed if shouldLoad { - if signer, err := n.loadPrivateKey(); err == nil { - n.cachedSigner = signer + if _, signer, err := n.loadKey(); err == nil { signers = append(signers, signer) } else { log.Printf("Warning: failed to load private key: %v\n", err) @@ -639,16 +640,39 @@ func (n *Node) getAuthMethods() ([]ssh.AuthMethod, error) { return []ssh.AuthMethod{ssh.PublicKeys(signers...)}, nil } -// loadPrivateKey reads and parses the private key, prompting for passphrase if needed. -func (n *Node) loadPrivateKey() (ssh.Signer, error) { +// loadKey returns the private key at KeyPath and its signer. The key is read only once, so an +// encrypted key prompts for its passphrase at most once. +func (n *Node) loadKey() (any, ssh.Signer, error) { + if n.cachedSigner != nil && n.cachedKey != nil { + return n.cachedKey, n.cachedSigner, nil + } + + key, err := n.loadPrivateKey() + if err != nil { + return nil, nil, err + } + + signer, err := ssh.NewSignerFromKey(key) + if err != nil { + return nil, nil, fmt.Errorf("failed to create signer from private key: %v", err) + } + + n.cachedKey = key + n.cachedSigner = signer + + return key, signer, nil +} + +// loadPrivateKey reads and parses the raw private key, prompting for passphrase if needed. +func (n *Node) loadPrivateKey() (any, error) { key, err := n.FileIO.ReadFile(n.KeyPath) if err != nil { return nil, fmt.Errorf("failed to read private key file %s: %v", n.KeyPath, err) } - signer, err := ssh.ParsePrivateKey(key) + rawKey, err := ssh.ParseRawPrivateKey(key) if err == nil { - return signer, nil + return rawKey, nil } if _, ok := err.(*ssh.PassphraseMissingError); !ok { @@ -663,7 +687,7 @@ func (n *Node) loadPrivateKey() (ssh.Signer, error) { return nil, fmt.Errorf("failed to read passphrase: %v", err) } - signer, err = ssh.ParsePrivateKeyWithPassphrase(key, passphrase) + rawKey, err = ssh.ParseRawPrivateKeyWithPassphrase(key, passphrase) // Clear passphrase from memory for i := range passphrase { passphrase[i] = 0 @@ -672,20 +696,148 @@ func (n *Node) loadPrivateKey() (ssh.Signer, error) { return nil, fmt.Errorf("failed to parse private key with passphrase: %v", err) } - return signer, nil + return rawKey, nil } -// setupAgentForwarding sets up SSH agent forwarding on the client (best effort) +// setupAgentForwarding sets up SSH agent forwarding on the client (best effort). Tools on the +// remote host, such as k0sctl on the jumpbox, authenticate to further hosts through the +// forwarded agent, so it always offers the key at KeyPath, even when the local agent lacks it. func (n *Node) setupAgentForwarding(client *ssh.Client) error { - authSocket := os.Getenv("SSH_AUTH_SOCK") - if authSocket == "" { + var localAgent agent.ExtendedAgent + + if authSocket := os.Getenv("SSH_AUTH_SOCK"); authSocket != "" { + conn, err := net.Dial("unix", authSocket) + if err != nil { + log.Printf("Warning: failed to connect to SSH agent: %v", err) + } else { + localAgent = agent.NewClient(conn) + } + } + + forwarded, err := n.forwardedAgent(localAgent) + if err != nil { + return err + } + + if forwarded == nil { return nil } - conn, err := net.Dial("unix", authSocket) + if err := agent.ForwardToAgent(client, forwarded); err != nil { + return fmt.Errorf("failed to forward SSH agent: %w", err) + } + + return nil +} + +// forwardedAgent returns the agent to forward to remote hosts: the local agent when it already +// holds the key at KeyPath, otherwise an agent that adds that key to the local agent's keys. +func (n *Node) forwardedAgent(localAgent agent.ExtendedAgent) (agent.Agent, error) { + if n.KeyPath == "" || (localAgent != nil && agentHoldsKey(localAgent, n.KeyPath, n.FileIO)) { + return localAgent, nil + } + + key, signer, err := n.loadKey() + if err != nil { + if localAgent == nil { + return nil, fmt.Errorf("no SSH agent to forward and failed to load private key: %w", err) + } + + log.Printf("Warning: forwarding SSH agent without the key at %s: %v", n.KeyPath, err) + + return localAgent, nil + } + + keyring := agent.NewKeyring() + if err := keyring.Add(agent.AddedKey{PrivateKey: key, Comment: n.KeyPath}); err != nil { + return nil, fmt.Errorf("failed to add private key to forwarded agent: %w", err) + } + + if localAgent == nil { + return keyring, nil + } + + return &keyAddingAgent{ + ExtendedAgent: localAgent, + keyring: keyring.(agent.ExtendedAgent), + keyBlob: signer.PublicKey().Marshal(), + }, nil +} + +// agentHoldsKey reports whether the agent holds the private key whose public key is stored next +// to keyPath. +func agentHoldsKey(a agent.Agent, keyPath string, fileIO util.FileIO) bool { + pubBytes, err := fileIO.ReadFile(keyPath + ".pub") + if err != nil { + return false + } + + pub, _, _, _, err := ssh.ParseAuthorizedKey(pubBytes) + if err != nil { + return false + } + + keys, err := a.List() + if err != nil { + return false + } + + target := string(pub.Marshal()) + for _, k := range keys { + if string(k.Marshal()) == target { + return true + } + } + + return false +} + +// keyAddingAgent serves the local agent with one extra key from a keyring. Signing requests for +// that key go to the keyring, everything else to the local agent. +type keyAddingAgent struct { + agent.ExtendedAgent + keyring agent.ExtendedAgent + keyBlob []byte +} + +func (a *keyAddingAgent) List() ([]*agent.Key, error) { + return withExtra(a.keyring.List, a.ExtendedAgent.List) +} + +func (a *keyAddingAgent) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return a.SignWithFlags(key, data, 0) +} + +func (a *keyAddingAgent) SignWithFlags(key ssh.PublicKey, data []byte, flags agent.SignatureFlags) (*ssh.Signature, error) { + signer := a.ExtendedAgent + if bytes.Equal(key.Marshal(), a.keyBlob) { + signer = a.keyring + } + + signature, err := signer.SignWithFlags(key, data, flags) + if err != nil { + return nil, fmt.Errorf("failed to sign with SSH agent: %w", err) + } + + return signature, nil +} + +func (a *keyAddingAgent) Signers() ([]ssh.Signer, error) { + return withExtra(a.keyring.Signers, a.ExtendedAgent.Signers) +} + +// withExtra prepends the keyring's entries to the local agent's. A failing local agent is +// ignored, as it may be locked or broken while the extra key still works. +func withExtra[T any](extra, local func() ([]T, error)) ([]T, error) { + extraItems, err := extra() + if err != nil { + return nil, err + } + + localItems, err := local() if err != nil { - return fmt.Errorf("failed to connect to SSH agent: %v", err) + return extraItems, nil } - return agent.ForwardToAgent(client, agent.NewClient(conn)) + return append(extraItems, localItems...), nil } diff --git a/internal/installer/node/node_suite_test.go b/internal/installer/node/node_suite_test.go new file mode 100644 index 000000000..35b32ade3 --- /dev/null +++ b/internal/installer/node/node_suite_test.go @@ -0,0 +1,16 @@ +// Copyright (c) Codesphere Inc. +// SPDX-License-Identifier: Apache-2.0 + +package node + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestNode(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Node Suite") +} diff --git a/internal/tmpl/NOTICE b/internal/tmpl/NOTICE index 377368eb1..8e9dfb6fd 100644 --- a/internal/tmpl/NOTICE +++ b/internal/tmpl/NOTICE @@ -199,7 +199,7 @@ License URL: https://github.com/beorn7/perks/blob/v1.0.1/LICENSE Module: github.com/blang/semver/v4 Version: v4.0.0 License: MIT -License URL: https://github.com/blang/semver/blob/v4.0.0/v4/LICENSE +License URL: https://github.com/blang/semver/blob/v4.0.0/LICENSE ---------- Module: github.com/bmatcuk/doublestar/v4 @@ -397,7 +397,7 @@ License URL: https://github.com/emirpasic/gods/blob/v1.18.1/LICENSE Module: github.com/evanphx/json-patch/v5 Version: v5.9.11 License: BSD-3-Clause -License URL: https://github.com/evanphx/json-patch/blob/v5.9.11/v5/LICENSE +License URL: https://github.com/evanphx/json-patch/blob/v5.9.11/LICENSE ---------- Module: github.com/exponent-io/jsonpath @@ -721,7 +721,7 @@ License URL: https://github.com/googleapis/enterprise-certificate-proxy/blob/v0. Module: github.com/googleapis/gax-go/v2 Version: v2.26.2 License: BSD-3-Clause -License URL: https://github.com/googleapis/gax-go/blob/v2.26.2/v2/LICENSE +License URL: https://github.com/googleapis/gax-go/blob/v2.26.2/LICENSE ---------- Module: github.com/gorilla/websocket @@ -1507,7 +1507,7 @@ License URL: https://cs.opensource.google/go/x/time/+/v0.16.0:LICENSE Module: gomodules.xyz/jsonpatch/v2 Version: v2.5.0 License: Apache-2.0 -License URL: https://github.com/gomodules/jsonpatch/blob/v2.5.0/v2/LICENSE +License URL: https://github.com/gomodules/jsonpatch/blob/v2.5.0/LICENSE ---------- Module: google.golang.org/api