diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 7bef48cc7a7..a6b926aa186 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -15,6 +15,7 @@ builds: - -X go.jetify.com/devbox/internal/build.CommitDate={{.CommitDate}} - -X go.jetify.com/devbox/internal/build.SentryDSN={{ .Env.SENTRY_DSN }} - -X go.jetify.com/devbox/internal/build.TelemetryKey={{ .Env.TELEMETRY_KEY }} + - -X go.jetify.com/devbox/internal/build.Launcher=true env: - CGO_ENABLED=0 - GO111MODULE=on diff --git a/internal/boxcli/root.go b/internal/boxcli/root.go index be6f041ad54..c8bda151779 100644 --- a/internal/boxcli/root.go +++ b/internal/boxcli/root.go @@ -16,6 +16,7 @@ import ( "go.jetify.com/devbox/internal/boxcli/midcobra" "go.jetify.com/devbox/internal/cmdutil" "go.jetify.com/devbox/internal/debug" + "go.jetify.com/devbox/internal/launcher" "go.jetify.com/devbox/internal/telemetry" "go.jetify.com/devbox/internal/vercheck" ) @@ -112,6 +113,10 @@ func Execute(ctx context.Context, args []string) int { } func Main() { + // Runs first because it might run another devbox version instead of this + // one, which makes any work done before it wasted. + launcher.Main() + timer := debug.Timer(strings.Join(os.Args, " ")) setSystemBinaryPaths() ctx := context.Background() diff --git a/internal/boxcli/version.go b/internal/boxcli/version.go index 15b47a1cf44..e8d1d6987de 100644 --- a/internal/boxcli/version.go +++ b/internal/boxcli/version.go @@ -39,14 +39,22 @@ func versionCmd() *cobra.Command { } func selfUpdateCmd() *cobra.Command { + updateLauncher := false command := &cobra.Command{ Use: "update", Short: "Update devbox launcher and binary", - Args: cobra.ExactArgs(0), + Long: "Update devbox to the latest version.\n\n" + + "This also replaces the launcher, the devbox in your PATH, when it's out of date. " + + "Use --launcher to replace it with the latest devbox even when it's up to date, " + + "so that commands run it directly instead of switching to the latest version on every run. " + + "Replacing the launcher may ask for sudo.", + Args: cobra.ExactArgs(0), RunE: func(cmd *cobra.Command, args []string) error { - return vercheck.SelfUpdate(cmd.OutOrStdout(), cmd.ErrOrStderr()) + return vercheck.SelfUpdate(cmd.ErrOrStderr(), updateLauncher) }, } + command.Flags().BoolVar(&updateLauncher, "launcher", false, + "also replace the launcher with the latest devbox, even when it's up to date (may ask for sudo)") return command } diff --git a/internal/build/build.go b/internal/build/build.go index 970422c281d..215d66d00f8 100644 --- a/internal/build/build.go +++ b/internal/build/build.go @@ -32,6 +32,12 @@ var ( // https://segment.com/docs/connections/sources/catalog/libraries/server/go/quickstart/ // It is disabled by default. TelemetryKey = "" + + // Launcher is "true" in official release builds, which makes devbox its + // own launcher (see the launcher package). Builds from package managers + // such as nixpkgs leave it unset, so the package manager stays in charge + // of the installed devbox version. + Launcher = "" ) // User-presentable names of operating systems supported by Devbox. diff --git a/internal/launcher/files.go b/internal/launcher/files.go new file mode 100644 index 00000000000..f4f196c08a9 --- /dev/null +++ b/internal/launcher/files.go @@ -0,0 +1,161 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package launcher + +import ( + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "go.jetify.com/devbox/internal/xdg" +) + +// The files below match the bash launcher's, because older devbox versions +// read and write them too. + +// currentVersionFile holds the devbox version to run when DEVBOX_USE_VERSION +// isn't set. Its modification time is when the launcher last checked for a +// new version. +func currentVersionFile() string { return xdg.CacheSubpath("devbox/current-version") } + +// latestVersionFile holds the latest devbox version from the last check. +func latestVersionFile() string { return xdg.CacheSubpath("devbox/latest-version") } + +// channelFile, if it exists, holds the URL to get the latest version from +// instead of the stable channel. +func channelFile() string { return xdg.ConfigSubpath("devbox/channel") } + +// autoUpdateFile turns on switching to new versions automatically, unless +// it's empty or holds "0" or "false". +func autoUpdateFile() string { return xdg.ConfigSubpath("devbox/auto_update") } + +// installDir is where the launcher downloads devbox versions to. Unlike the +// files above, it's always in $HOME/.cache, even when XDG_CACHE_HOME is set. +func installDir() string { + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, ".cache", "devbox", "bin") +} + +// BinaryPath returns where the launcher installs devbox version. +func BinaryPath(version string) string { + return filepath.Join(installDir(), normalizeVersion(version)+"_"+platform(), "devbox") +} + +// InstalledBinary returns the path to the devbox binary for version if it's +// installed, or an empty string otherwise. +func InstalledBinary(version string) string { + if installDir() == "" || !validVersion(version) { + return "" + } + bin := BinaryPath(version) + if !isExecutable(bin) { + return "" + } + return bin +} + +// platform returns the platform part of release archive names, which is also +// the suffix of each version's directory in the install dir. +func platform() string { + arch := runtime.GOARCH + if arch == "arm" { + // Releases only build 32-bit ARM for ARMv7, and name it like + // `uname -m` does. + arch = "armv7l" + } + return runtime.GOOS + "_" + arch +} + +func isExecutable(path string) bool { + fi, err := os.Stat(path) + return err == nil && fi.Mode().IsRegular() && fi.Mode().Perm()&0o111 != 0 +} + +// versionRegexp matches devbox release versions such as 0.18.4, v0.18.4, and +// 0.0.0-edge.2026-10-25. It never matches path separators, so a version is +// safe to use in file paths and URLs. +var versionRegexp = regexp.MustCompile(`^v?[0-9]+\.[0-9]+\.[0-9]+(-?[0-9A-Za-z][0-9A-Za-z.-]*)?$`) + +func validVersion(version string) bool { + return versionRegexp.MatchString(version) +} + +// normalizeVersion removes the "v" prefix, because release tags and archive +// names don't have one. +func normalizeVersion(version string) string { + return strings.TrimPrefix(version, "v") +} + +func sameVersion(a, b string) bool { + return normalizeVersion(a) == normalizeVersion(b) +} + +// readVersion returns the version in path, or an empty string if the file +// doesn't exist or doesn't hold a valid version. +func readVersion(path string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + version := strings.TrimSpace(string(data)) + if !validVersion(version) { + return "" + } + return normalizeVersion(version) +} + +// SetCurrentVersion makes version the devbox version that runs when +// DEVBOX_USE_VERSION isn't set. +func SetCurrentVersion(version string) error { + return writeVersion(currentVersionFile(), version) +} + +// SetLatestVersion saves version as the latest devbox version. +func SetLatestVersion(version string) error { + return writeVersion(latestVersionFile(), version) +} + +// writeVersion writes version to path atomically, so that a launcher that +// runs at the same time never reads part of it. +func writeVersion(path, version string) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(path), "."+filepath.Base(path)+"-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if _, err := tmp.WriteString(normalizeVersion(version) + "\n"); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + if err := os.Chmod(tmp.Name(), 0o644); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +// readConfig returns the trimmed contents of path, or an empty string if it +// doesn't exist. +func readConfig(path string) string { + data, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(data)) +} + +func autoUpdate() bool { + value := strings.ToLower(readConfig(autoUpdateFile())) + return value != "" && value != "0" && value != "false" +} diff --git a/internal/launcher/install.go b/internal/launcher/install.go new file mode 100644 index 00000000000..cc4304ab4fd --- /dev/null +++ b/internal/launcher/install.go @@ -0,0 +1,200 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package launcher + +import ( + "bufio" + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "go.jetify.com/devbox/internal/fileutil" +) + +// These URLs are variables so that tests can serve them locally. +var ( + // releasesURL serves the latest version of each channel. + releasesURL = "https://releases.jetify.com/devbox" + // downloadURL serves the release archives and checksums. + downloadURL = "https://github.com/jetify-com/devbox/releases/download" +) + +// httpClient's timeout is a backstop: callers set their own deadlines. +var httpClient = &http.Client{Timeout: 10 * time.Minute} + +// FetchLatestVersion returns the latest devbox version from the release +// channel, which is the stable channel unless the channel config file names +// another URL. +func FetchLatestVersion(ctx context.Context) (string, error) { + url := readConfig(channelFile()) + if url == "" { + url = releasesURL + "/stable/version" + } + resp, err := get(ctx, url) + if err != nil { + return "", err + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, 1024)) + if err != nil { + return "", fmt.Errorf("read %s: %w", url, err) + } + version := strings.TrimSpace(string(data)) + if !validVersion(version) { + return "", fmt.Errorf("%s returned an invalid version %q", url, version) + } + return normalizeVersion(version), nil +} + +// Install downloads devbox version unless it's already installed, and returns +// the path to its binary. It prints progress to w. +func Install(ctx context.Context, version string, w io.Writer) (string, error) { + if !validVersion(version) { + return "", fmt.Errorf("invalid devbox version %q", version) + } + if installDir() == "" { + return "", errors.New("can't install devbox: the home directory is unknown") + } + version = normalizeVersion(version) + bin := BinaryPath(version) + if isExecutable(bin) { + return bin, nil + } + + fmt.Fprintf(w, "Downloading devbox %s...\n", version) + if err := download(ctx, version, filepath.Dir(bin)); err != nil { + return "", fmt.Errorf("download devbox %s: %w", version, err) + } + fmt.Fprintf(w, "Downloaded devbox %s.\n", version) + return bin, nil +} + +// download installs the release archive for version into dir. It unpacks the +// archive into a temporary directory and then renames that to dir, so that +// dir only ever holds a complete binary. +func download(ctx context.Context, version, dir string) error { + if err := os.MkdirAll(filepath.Dir(dir), 0o755); err != nil { + return err + } + tmp, err := os.MkdirTemp(filepath.Dir(dir), ".download-") + if err != nil { + return err + } + defer os.RemoveAll(tmp) + + archiveName := fmt.Sprintf("devbox_%s_%s.tar.gz", version, platform()) + wantSum, err := fetchChecksum(ctx, version, archiveName) + if err != nil { + return err + } + + archivePath := filepath.Join(tmp, archiveName) + gotSum, err := fetchFile(ctx, fmt.Sprintf("%s/%s/%s", downloadURL, version, archiveName), archivePath) + if err != nil { + return err + } + if gotSum != wantSum { + return fmt.Errorf("checksum mismatch for %s: expected %s, got %s", archiveName, wantSum, gotSum) + } + + unpacked := filepath.Join(tmp, "unpacked") + if err := os.Mkdir(unpacked, 0o755); err != nil { + return err + } + archive, err := os.Open(archivePath) + if err != nil { + return err + } + err = fileutil.Untar(archive, unpacked) + archive.Close() + if err != nil { + return fmt.Errorf("unpack %s: %w", archiveName, err) + } + if !isExecutable(filepath.Join(unpacked, "devbox")) { + return fmt.Errorf("%s doesn't contain a devbox binary", archiveName) + } + + // Remove whatever is left of an earlier failed install. + if err := os.RemoveAll(dir); err != nil { + return err + } + if err := os.Rename(unpacked, dir); err != nil { + // Another launcher may have installed the same version first. + if isExecutable(filepath.Join(dir, "devbox")) { + return nil + } + return err + } + return nil +} + +// fetchChecksum returns the SHA-256 checksum of archiveName from the +// release's checksums.txt. +func fetchChecksum(ctx context.Context, version, archiveName string) (string, error) { + url := fmt.Sprintf("%s/%s/checksums.txt", downloadURL, version) + resp, err := get(ctx, url) + if err != nil { + return "", err + } + defer resp.Body.Close() + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + fields := strings.Fields(scanner.Text()) + if len(fields) == 2 && fields[1] == archiveName { + return fields[0], nil + } + } + if err := scanner.Err(); err != nil { + return "", fmt.Errorf("read %s: %w", url, err) + } + return "", fmt.Errorf("%s has no checksum for %s", url, archiveName) +} + +// fetchFile downloads url to path and returns the file's SHA-256 checksum. +func fetchFile(ctx context.Context, url, path string) (string, error) { + resp, err := get(ctx, url) + if err != nil { + return "", err + } + defer resp.Body.Close() + f, err := os.Create(path) + if err != nil { + return "", err + } + hash := sha256.New() + _, err = io.Copy(io.MultiWriter(f, hash), resp.Body) + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return "", fmt.Errorf("download %s: %w", url, err) + } + return hex.EncodeToString(hash.Sum(nil)), nil +} + +// get returns the response to a GET request for url, which the caller must +// close. It returns an error unless the status is 200 OK. +func get(ctx context.Context, url string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, fmt.Errorf("GET %s: %s", url, resp.Status) + } + return resp, nil +} diff --git a/internal/launcher/launcher.go b/internal/launcher/launcher.go new file mode 100644 index 00000000000..4fef7a740ae --- /dev/null +++ b/internal/launcher/launcher.go @@ -0,0 +1,234 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +// Package launcher makes release builds of devbox their own launcher. The +// devbox that users run, such as /usr/local/bin/devbox, picks the devbox +// version to use, downloads it if needed, and runs it. +// +// It replaces the bash launcher that the installer used to put in +// /usr/local/bin. Older devbox versions rely on how that script behaves, so +// this package keeps its environment variables, version files, and download +// directory. See specs/native-launcher.md. +package launcher + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + "time" + + "go.jetify.com/devbox/internal/build" + "go.jetify.com/devbox/internal/envir" +) + +// Version is the launcher version, which devbox passes to the version it runs +// as LAUNCHER_VERSION. Bump it only when installed launchers need to change, +// because a newer version makes devbox ask users to replace their launcher, +// which can need sudo. The bash launcher's versions are all below 1.0.0. +const Version = "1.0.0" + +// refreshArg is a hidden command that checks for a new devbox version. The +// launcher runs it in a detached child process so that commands don't wait +// for the network. +const refreshArg = "__launcher-refresh" + +// versionCheckInterval is how often the launcher checks for a new devbox +// version. It matches the bash launcher's default VERSION_CACHE_TTL. +const versionCheckInterval = 24 * time.Hour + +const ( + // firstRunTimeout limits how long the first run waits to learn the latest + // version. On a timeout, it runs the installed version. + firstRunTimeout = 5 * time.Second + // refreshTimeout limits how long the background version check runs, + // including downloading the new version when auto_update is on. + refreshTimeout = 5 * time.Minute +) + +// These variables let tests replace process-level behavior. +var ( + execFunc = syscall.Exec + spawnFunc = spawnDetached +) + +// Main runs the launcher in release builds. It returns when this process +// should run the command itself. Otherwise it replaces this process with the +// devbox version to use, and exits if it can't. +func Main() { + if build.Launcher != "true" { + return + } + if len(os.Args) > 1 && os.Args[1] == refreshArg { + ctx, cancel := context.WithTimeout(context.Background(), refreshTimeout) + refresh(ctx) + cancel() + os.Exit(0) + } + exe, err := os.Executable() + if err != nil { + slog.Debug("launcher: can't find this binary, so running it as is", "err", err) + return + } + if err := run(exe, build.Version, os.Args, os.Stderr); err != nil { + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + os.Exit(1) + } +} + +// run starts the devbox version to use for args, or returns nil if it's +// version, the version of the binary at exe. +func run(exe, version string, args []string, stderr io.Writer) error { + // A launcher already picked this version: launchers only run binaries + // from the install dir, and this binary is one of them. + if inInstallDir(exe) { + return nil + } + + target, err := selectVersion(exe, version) + if err != nil { + return err + } + env := map[string]string{ + envir.LauncherVersion: Version, + envir.LauncherPath: exe, + envir.DevboxLatestVersion: readVersion(latestVersionFile()), + } + if sameVersion(target, version) { + for name, value := range env { + if err := os.Setenv(name, value); err != nil { + return err + } + } + return nil + } + + bin, err := Install(context.Background(), target, stderr) + if err != nil { + return err + } + argv := append([]string{bin}, args[1:]...) + if err := execFunc(bin, argv, withEnv(os.Environ(), env)); err != nil { + return fmt.Errorf("run devbox %s from %s: %w", target, bin, err) + } + return nil +} + +// selectVersion returns the devbox version to run: DEVBOX_USE_VERSION if it's +// set, or the version in the current-version file. It starts a background +// check for a new version when the last check was too long ago. +func selectVersion(exe, version string) (string, error) { + if useVersion := os.Getenv(envir.DevboxUseVersion); useVersion != "" { + if !validVersion(useVersion) { + return "", fmt.Errorf("invalid %s %q", envir.DevboxUseVersion, useVersion) + } + return normalizeVersion(useVersion), nil + } + + currentFile := currentVersionFile() + if fi, err := os.Stat(currentFile); err == nil { + if current := readVersion(currentFile); current != "" { + if time.Since(fi.ModTime()) >= versionCheckInterval { + startRefresh(exe, currentFile) + } + return current, nil + } + } + + // There's no current version. Either this is the first run after an + // install that didn't write one, or an older devbox's `devbox version + // update` deleted it to make the launcher pick the latest version. Like the + // bash launcher, wait for the latest version and keep using it. + ctx, cancel := context.WithTimeout(context.Background(), firstRunTimeout) + defer cancel() + latest, err := FetchLatestVersion(ctx) + if err != nil { + // Run this version without saving it, so the next run tries again. + slog.Debug("launcher: can't get the latest devbox version", "err", err) + return version, nil + } + if err := SetLatestVersion(latest); err != nil { + slog.Debug("launcher: can't save the latest devbox version", "err", err) + } + if err := SetCurrentVersion(latest); err != nil { + slog.Debug("launcher: can't save the current devbox version", "err", err) + } + return latest, nil +} + +// startRefresh checks for a new devbox version in a detached child process. +func startRefresh(exe, currentFile string) { + // Reset the check interval first, so that commands that start before the + // check finishes don't start another one. + now := time.Now() + if err := os.Chtimes(currentFile, now, now); err != nil { + slog.Debug("launcher: can't reset the version check interval", "err", err) + return + } + if err := spawnFunc(exe, refreshArg); err != nil { + slog.Debug("launcher: can't start the version check", "err", err) + } +} + +func spawnDetached(name string, args ...string) error { + cmd := exec.Command(name, args...) + cmd.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + if err := cmd.Start(); err != nil { + return err + } + return cmd.Process.Release() +} + +// refresh saves the latest devbox version. When auto_update is on, it also +// downloads that version and makes it the current version. +func refresh(ctx context.Context) { + latest, err := FetchLatestVersion(ctx) + if err != nil { + return + } + if err := SetLatestVersion(latest); err != nil || !autoUpdate() { + return + } + // Download before switching, so that the next command doesn't wait for it. + if _, err := Install(ctx, latest, io.Discard); err != nil { + return + } + _ = SetCurrentVersion(latest) +} + +// inInstallDir reports whether exe is a binary in the install dir. +func inInstallDir(exe string) bool { + dir := installDir() + if dir == "" { + return false + } + if resolved, err := filepath.EvalSymlinks(dir); err == nil { + dir = resolved + } + if resolved, err := filepath.EvalSymlinks(exe); err == nil { + exe = resolved + } + rel, err := filepath.Rel(dir, exe) + return err == nil && filepath.IsLocal(rel) +} + +// withEnv returns environ with the variables in vars set, replacing any +// existing values. +func withEnv(environ []string, vars map[string]string) []string { + result := make([]string, 0, len(environ)+len(vars)) + for _, kv := range environ { + name, _, _ := strings.Cut(kv, "=") + if _, ok := vars[name]; !ok { + result = append(result, kv) + } + } + for name, value := range vars { + result = append(result, name+"="+value) + } + return result +} diff --git a/internal/launcher/launcher_test.go b/internal/launcher/launcher_test.go new file mode 100644 index 00000000000..cd9396dc72b --- /dev/null +++ b/internal/launcher/launcher_test.go @@ -0,0 +1,501 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package launcher + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "go.jetify.com/devbox/internal/envir" +) + +const ( + testVersion = "0.18.4" // the version of the binary under test + latestVersion = "0.19.0" +) + +// fakeReleases serves the stable channel and release archives that hold a +// devbox binary whose contents name the version. +type fakeReleases struct { + t *testing.T + latest string + versions map[string][]byte // archive by version + badSum bool + requests []string +} + +func (f *fakeReleases) ServeHTTP(w http.ResponseWriter, r *http.Request) { + f.requests = append(f.requests, r.URL.Path) + if r.URL.Path == "/releases/stable/version" { + if f.latest == "" { + http.NotFound(w, r) + return + } + fmt.Fprintln(w, f.latest) + return + } + var version, file string + if _, err := fmt.Sscanf(strings.ReplaceAll(strings.TrimPrefix(r.URL.Path, "/download/"), "/", " "), "%s %s", &version, &file); err != nil { + http.NotFound(w, r) + return + } + archive, ok := f.versions[version] + if !ok { + http.NotFound(w, r) + return + } + archiveName := fmt.Sprintf("devbox_%s_%s.tar.gz", version, platform()) + switch file { + case "checksums.txt": + sum := sha256.Sum256(archive) + if f.badSum { + sum = sha256.Sum256([]byte("something else")) + } + fmt.Fprintf(w, "%s devbox_%s_other_os.tar.gz\n", strings.Repeat("0", 64), version) + fmt.Fprintf(w, "%s %s\n", hex.EncodeToString(sum[:]), archiveName) + case archiveName: + _, _ = w.Write(archive) + default: + http.NotFound(w, r) + } +} + +func (f *fakeReleases) addVersion(version string) { + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + content := []byte("devbox " + version) + if err := tw.WriteHeader(&tar.Header{Name: "devbox", Mode: 0o755, Size: int64(len(content))}); err != nil { + f.t.Fatal(err) + } + if _, err := tw.Write(content); err != nil { + f.t.Fatal(err) + } + if err := tw.Close(); err != nil { + f.t.Fatal(err) + } + if err := gzw.Close(); err != nil { + f.t.Fatal(err) + } + f.versions[version] = buf.Bytes() +} + +type execCall struct { + argv0 string + argv []string + env map[string]string +} + +type testEnv struct { + home string + exe string + releases *fakeReleases + execs []execCall + spawns [][]string +} + +// setup isolates the launcher's files, network, and process calls. +func setup(t *testing.T) *testEnv { + t.Helper() + env := &testEnv{home: t.TempDir()} + t.Setenv("HOME", env.home) + t.Setenv(envir.XDGCacheHome, filepath.Join(env.home, "xdg-cache")) + t.Setenv(envir.XDGConfigHome, filepath.Join(env.home, "xdg-config")) + for _, name := range []string{envir.DevboxUseVersion, envir.LauncherPath, envir.LauncherVersion, envir.DevboxLatestVersion} { + t.Setenv(name, "") + } + + env.exe = filepath.Join(env.home, "usr-local-bin", "devbox") + writeExecutable(t, env.exe, "devbox "+testVersion) + + env.releases = &fakeReleases{t: t, latest: latestVersion, versions: map[string][]byte{}} + env.releases.addVersion(latestVersion) + server := httptest.NewServer(env.releases) + t.Cleanup(server.Close) + setVar(t, &releasesURL, server.URL+"/releases") + setVar(t, &downloadURL, server.URL+"/download") + + setVar(t, &execFunc, func(argv0 string, argv, envv []string) error { + call := execCall{argv0: argv0, argv: argv, env: map[string]string{}} + for _, kv := range envv { + name, value, _ := strings.Cut(kv, "=") + call.env[name] = value + } + env.execs = append(env.execs, call) + return nil + }) + setVar(t, &spawnFunc, func(name string, args ...string) error { + env.spawns = append(env.spawns, append([]string{name}, args...)) + return nil + }) + return env +} + +func setVar[T any](t *testing.T, v *T, value T) { + t.Helper() + old := *v + t.Cleanup(func() { *v = old }) + *v = value +} + +func writeExecutable(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o755); err != nil { + t.Fatal(err) + } +} + +func writeVersionFile(t *testing.T, path, version string, modTime time.Time) { + t.Helper() + if err := writeVersion(path, version); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(path, modTime, modTime); err != nil { + t.Fatal(err) + } +} + +func (env *testEnv) run(t *testing.T) error { + t.Helper() + return run(env.exe, testVersion, []string{env.exe, "shell", "--pure"}, io.Discard) +} + +func readFile(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + +func TestRunsItselfWhenItIsTheCurrentVersion(t *testing.T) { + env := setup(t) + writeVersionFile(t, currentVersionFile(), testVersion, time.Now()) + writeVersionFile(t, latestVersionFile(), latestVersion, time.Now()) + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if len(env.execs) != 0 { + t.Errorf("got exec %v, want none", env.execs) + } + if len(env.spawns) != 0 { + t.Errorf("got version check %v, want none", env.spawns) + } + for name, want := range map[string]string{ + envir.LauncherPath: env.exe, + envir.LauncherVersion: Version, + envir.DevboxLatestVersion: latestVersion, + } { + if got := os.Getenv(name); got != want { + t.Errorf("got %s=%q, want %q", name, got, want) + } + } + if len(env.releases.requests) != 0 { + t.Errorf("got requests %v, want none", env.releases.requests) + } +} + +func TestRunsInstalledCurrentVersion(t *testing.T) { + env := setup(t) + writeVersionFile(t, currentVersionFile(), "0.17.0", time.Now()) + bin := BinaryPath("0.17.0") + writeExecutable(t, bin, "devbox 0.17.0") + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if len(env.execs) != 1 { + t.Fatalf("got %d execs, want 1", len(env.execs)) + } + call := env.execs[0] + if call.argv0 != bin { + t.Errorf("got exec of %s, want %s", call.argv0, bin) + } + if want := []string{bin, "shell", "--pure"}; !slices.Equal(call.argv, want) { + t.Errorf("got argv %q, want %q", call.argv, want) + } + if call.env[envir.LauncherPath] != env.exe || call.env[envir.LauncherVersion] != Version { + t.Errorf("got launcher env %q and %q, want %q and %q", + call.env[envir.LauncherPath], call.env[envir.LauncherVersion], env.exe, Version) + } + if len(env.releases.requests) != 0 { + t.Errorf("got requests %v, want none", env.releases.requests) + } +} + +func TestDownloadsMissingVersion(t *testing.T) { + env := setup(t) + env.releases.addVersion("0.17.0") + writeVersionFile(t, currentVersionFile(), "0.17.0", time.Now()) + + if err := env.run(t); err != nil { + t.Fatal(err) + } + bin := BinaryPath("0.17.0") + if got := readFile(t, bin); got != "devbox 0.17.0" { + t.Errorf("got installed binary %q, want %q", got, "devbox 0.17.0") + } + if len(env.execs) != 1 || env.execs[0].argv0 != bin { + t.Errorf("got execs %v, want one of %s", env.execs, bin) + } + leftovers, _ := filepath.Glob(filepath.Join(installDir(), ".download-*")) + if len(leftovers) != 0 { + t.Errorf("got leftover download dirs %v", leftovers) + } +} + +func TestRejectsArchiveWithWrongChecksum(t *testing.T) { + env := setup(t) + env.releases.addVersion("0.17.0") + env.releases.badSum = true + writeVersionFile(t, currentVersionFile(), "0.17.0", time.Now()) + + err := env.run(t) + if err == nil || !strings.Contains(err.Error(), "checksum mismatch") { + t.Fatalf("got error %v, want a checksum mismatch", err) + } + if InstalledBinary("0.17.0") != "" { + t.Error("installed a binary with the wrong checksum") + } + if len(env.execs) != 0 { + t.Errorf("got execs %v, want none", env.execs) + } +} + +func TestUseVersionOverridesCurrentVersion(t *testing.T) { + env := setup(t) + writeVersionFile(t, currentVersionFile(), testVersion, time.Now()) + bin := BinaryPath("0.17.0") + writeExecutable(t, bin, "devbox 0.17.0") + t.Setenv(envir.DevboxUseVersion, "v0.17.0") + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if len(env.execs) != 1 || env.execs[0].argv0 != bin { + t.Errorf("got execs %v, want one of %s", env.execs, bin) + } +} + +func TestRejectsInvalidUseVersion(t *testing.T) { + env := setup(t) + t.Setenv(envir.DevboxUseVersion, "../../evil") + + if err := env.run(t); err == nil { + t.Fatal("got no error for an invalid DEVBOX_USE_VERSION") + } + if len(env.execs) != 0 { + t.Errorf("got execs %v, want none", env.execs) + } +} + +func TestDoesNothingForInstalledVersions(t *testing.T) { + env := setup(t) + // The bash launcher (or another devbox) sets DEVBOX_USE_VERSION and runs a + // binary from the install dir. That binary must run itself. + t.Setenv(envir.DevboxUseVersion, "0.17.0") + bin := BinaryPath(testVersion) + writeExecutable(t, bin, "devbox "+testVersion) + + if err := run(bin, testVersion, []string{bin, "version"}, io.Discard); err != nil { + t.Fatal(err) + } + if len(env.execs) != 0 { + t.Errorf("got execs %v, want none", env.execs) + } + if got := os.Getenv(envir.LauncherPath); got != "" { + t.Errorf("got %s=%q, want it left alone", envir.LauncherPath, got) + } +} + +func TestFirstRunPinsLatestVersion(t *testing.T) { + env := setup(t) + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if got := readVersion(currentVersionFile()); got != latestVersion { + t.Errorf("got current version %q, want %q", got, latestVersion) + } + if got := readVersion(latestVersionFile()); got != latestVersion { + t.Errorf("got latest version %q, want %q", got, latestVersion) + } + if len(env.execs) != 1 || env.execs[0].argv0 != BinaryPath(latestVersion) { + t.Errorf("got execs %v, want one of %s", env.execs, BinaryPath(latestVersion)) + } +} + +func TestFirstRunOfflineRunsItself(t *testing.T) { + env := setup(t) + env.releases.latest = "" + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if len(env.execs) != 0 { + t.Errorf("got execs %v, want none", env.execs) + } + if _, err := os.Stat(currentVersionFile()); err == nil { + t.Error("saved a current version, want the next run to try again") + } +} + +func TestFirstRunUsesChannel(t *testing.T) { + env := setup(t) + env.releases.addVersion("0.0.0-edge.2026-10-25") + edge := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "0.0.0-edge.2026-10-25") + })) + t.Cleanup(edge.Close) + if err := os.MkdirAll(filepath.Dir(channelFile()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(channelFile(), []byte(edge.URL+"\n"), 0o644); err != nil { + t.Fatal(err) + } + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if got := readVersion(currentVersionFile()); got != "0.0.0-edge.2026-10-25" { + t.Errorf("got current version %q, want the edge version", got) + } +} + +func TestStaleVersionStartsBackgroundCheck(t *testing.T) { + env := setup(t) + writeVersionFile(t, currentVersionFile(), testVersion, time.Now().Add(-25*time.Hour)) + + if err := env.run(t); err != nil { + t.Fatal(err) + } + if want := [][]string{{env.exe, refreshArg}}; !slices.EqualFunc(env.spawns, want, slices.Equal) { + t.Errorf("got spawns %v, want %v", env.spawns, want) + } + fi, err := os.Stat(currentVersionFile()) + if err != nil { + t.Fatal(err) + } + if time.Since(fi.ModTime()) > time.Minute { + t.Errorf("didn't reset the check interval: modified %v", fi.ModTime()) + } + // The command must not wait for the network. + if len(env.releases.requests) != 0 { + t.Errorf("got requests %v, want none", env.releases.requests) + } + if len(env.execs) != 0 { + t.Errorf("got execs %v, want none", env.execs) + } +} + +func TestRefresh(t *testing.T) { + t.Run("saves latest version", func(t *testing.T) { + setup(t) + writeVersionFile(t, currentVersionFile(), testVersion, time.Now()) + + refresh(context.Background()) + if got := readVersion(latestVersionFile()); got != latestVersion { + t.Errorf("got latest version %q, want %q", got, latestVersion) + } + if got := readVersion(currentVersionFile()); got != testVersion { + t.Errorf("got current version %q, want it unchanged at %q", got, testVersion) + } + if InstalledBinary(latestVersion) != "" { + t.Error("downloaded the latest version without auto_update") + } + }) + + t.Run("auto_update switches to latest version", func(t *testing.T) { + setup(t) + writeVersionFile(t, currentVersionFile(), testVersion, time.Now()) + if err := os.MkdirAll(filepath.Dir(autoUpdateFile()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(autoUpdateFile(), []byte("true\n"), 0o644); err != nil { + t.Fatal(err) + } + + refresh(context.Background()) + if got := readVersion(currentVersionFile()); got != latestVersion { + t.Errorf("got current version %q, want %q", got, latestVersion) + } + if InstalledBinary(latestVersion) == "" { + t.Error("didn't download the latest version before switching to it") + } + }) +} + +func TestAutoUpdate(t *testing.T) { + for content, want := range map[string]bool{"": false, "0": false, "FALSE\n": false, "1": true, "yes": true} { + setup(t) + if err := os.MkdirAll(filepath.Dir(autoUpdateFile()), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(autoUpdateFile(), []byte(content), 0o644); err != nil { + t.Fatal(err) + } + if got := autoUpdate(); got != want { + t.Errorf("auto_update %q: got %v, want %v", content, got, want) + } + } +} + +func TestReplaceLauncher(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "new-devbox") + writeExecutable(t, src, "devbox binary") + real := filepath.Join(dir, "launch.sh") + if err := os.WriteFile(real, []byte("#!/bin/bash\n"), 0o755); err != nil { + t.Fatal(err) + } + link := filepath.Join(dir, "devbox") + if err := os.Symlink(real, link); err != nil { + t.Fatal(err) + } + + if err := ReplaceLauncher(src, link, io.Discard); err != nil { + t.Fatal(err) + } + if got := readFile(t, real); got != "devbox binary" { + t.Errorf("got launcher %q, want the new binary", got) + } + if fi, err := os.Lstat(link); err != nil || fi.Mode()&os.ModeSymlink == 0 { + t.Error("replaced the symlink instead of the file it points to") + } + if !isExecutable(real) { + t.Error("new launcher isn't executable") + } +} + +func TestValidVersion(t *testing.T) { + for _, v := range []string{"0.18.4", "v0.18.4", "0.0.0-edge.2026-10-25", "0.8.3-beta"} { + if !validVersion(v) { + t.Errorf("validVersion(%q) = false, want true", v) + } + } + for _, v := range []string{"", "latest", "0.18", "0.18.4/../x", "0.18.4 ", "1.2.3-"} { + if validVersion(v) { + t.Errorf("validVersion(%q) = true, want false", v) + } + } +} diff --git a/internal/launcher/replace.go b/internal/launcher/replace.go new file mode 100644 index 00000000000..028a3ef3b3d --- /dev/null +++ b/internal/launcher/replace.go @@ -0,0 +1,77 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package launcher + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "os/exec" + "path/filepath" +) + +// sudoCommand is a variable so that tests can replace it. +var sudoCommand = "sudo" + +// ReplaceLauncher replaces the launcher at dst, which is either the bash +// launcher or a devbox binary, with the devbox binary at src. It runs sudo +// when it can't write to dst's directory, which is common for /usr/local/bin. +func ReplaceLauncher(src, dst string, stderr io.Writer) error { + // Replace the file that a symlink points to, not the symlink. + if resolved, err := filepath.EvalSymlinks(dst); err == nil { + dst = resolved + } + + err := replaceFile(src, dst) + if err == nil || !errors.Is(err, fs.ErrPermission) { + return err + } + + sudo, lookErr := exec.LookPath(sudoCommand) + if lookErr != nil { + return fmt.Errorf("%w (and sudo isn't available)", err) + } + fmt.Fprintf(stderr, "Replacing the devbox launcher at %s requires sudo.\n", dst) + cmd := exec.Command( + sudo, "sh", "-c", `cp "$1" "$2.tmp" && chmod 755 "$2.tmp" && mv -f "$2.tmp" "$2"`, + "sh", src, dst, + ) + cmd.Stdin = os.Stdin + cmd.Stdout = stderr + cmd.Stderr = stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("replace %s with sudo: %w", dst, err) + } + return nil +} + +// replaceFile copies src to a temporary file next to dst and renames it to +// dst, so that commands that start at the same time never see a partial +// file. +func replaceFile(src, dst string) error { + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + + tmp, err := os.CreateTemp(filepath.Dir(dst), "."+filepath.Base(dst)+"-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + _, err = io.Copy(tmp, srcFile) + if closeErr := tmp.Close(); err == nil { + err = closeErr + } + if err != nil { + return err + } + if err := os.Chmod(tmp.Name(), 0o755); err != nil { + return err + } + return os.Rename(tmp.Name(), dst) +} diff --git a/internal/vercheck/project.go b/internal/vercheck/project.go index 576929da618..81e1f9327aa 100644 --- a/internal/vercheck/project.go +++ b/internal/vercheck/project.go @@ -8,8 +8,6 @@ import ( "io" "log/slog" "os" - "path/filepath" - "runtime" "strings" "syscall" @@ -18,6 +16,7 @@ import ( "go.jetify.com/devbox/internal/boxcli/usererr" "go.jetify.com/devbox/internal/devconfig/configfile" "go.jetify.com/devbox/internal/envir" + "go.jetify.com/devbox/internal/launcher" "go.jetify.com/devbox/internal/ux" ) @@ -161,8 +160,8 @@ func switchVersion(w io.Writer, configPath string, required *configfile.DevboxVe ) } - launcher := os.Getenv(envir.LauncherPath) - if launcher == "" { + launcherPath := os.Getenv(envir.LauncherPath) + if launcherPath == "" { return usererr.New( "%s\nDevbox can't switch versions automatically because it wasn't started by the devbox launcher. "+ "Install devbox with `curl -fsSL https://get.jetify.com/devbox | bash` to enable automatic switching.", @@ -173,9 +172,9 @@ func switchVersion(w io.Writer, configPath string, required *configfile.DevboxVe // Running the downloaded binary directly skips the launcher's startup // cost. It still gets the launcher's environment variables because this // process inherited them. - bin := installedBinary(target) + bin := launcher.InstalledBinary(target) if bin == "" { - bin = launcher + bin = launcherPath } slog.Debug("switching devbox version for devbox_version", "from", currentDevboxVersion, "to", target, "bin", bin) env := withEnv(os.Environ(), map[string]string{ @@ -189,25 +188,6 @@ func switchVersion(w io.Writer, configPath string, required *configfile.DevboxVe return nil } -// installedBinary returns the path to the devbox binary for version if the -// launcher already downloaded it, or an empty string otherwise. -// -// Note: keep this in sync with INSTALL_DIR and install_flow in launch.sh, -// which use $HOME/.cache even when XDG_CACHE_HOME is set. -func installedBinary(version string) string { - home, err := os.UserHomeDir() - if err != nil { - return "" - } - platform := runtime.GOOS + "_" + runtime.GOARCH - bin := filepath.Join(home, ".cache", "devbox", "bin", version+"_"+platform, "devbox") - fi, err := os.Stat(bin) - if err != nil || !fi.Mode().IsRegular() || fi.Mode().Perm()&0o111 == 0 { - return "" - } - return bin -} - // warnOnce prints msg as a warning unless it was already printed for the // project at configPath by this process or a parent devbox process. func warnOnce(w io.Writer, configPath, msg string) error { diff --git a/internal/vercheck/vercheck.go b/internal/vercheck/vercheck.go index 777927c1591..2c9029eccd8 100644 --- a/internal/vercheck/vercheck.go +++ b/internal/vercheck/vercheck.go @@ -4,14 +4,13 @@ package vercheck import ( - "bytes" + "context" "fmt" "io" - "io/fs" "os" - "os/exec" "path/filepath" "strings" + "time" "github.com/fatih/color" "github.com/pkg/errors" @@ -20,15 +19,21 @@ import ( "go.jetify.com/devbox/internal/boxcli/usererr" "go.jetify.com/devbox/internal/build" - "go.jetify.com/devbox/internal/cmdutil" "go.jetify.com/devbox/internal/envir" + "go.jetify.com/devbox/internal/launcher" "go.jetify.com/devbox/internal/ux" "go.jetify.com/devbox/internal/xdg" ) -// Keep this in-sync with latest version in launch.sh. -// If this version is newer than the version in launch.sh, we'll print a notice. -const expectedLauncherVersion = "v0.2.2" +// expectedLauncherVersion is the launcher version that this devbox expects. +// When the launcher that ran devbox is older, devbox prints a notice asking +// users to replace it with `devbox version update`. +const expectedLauncherVersion = "v" + launcher.Version + +// launcherNoticeInterval limits how often devbox prints the new launcher +// notice. Replacing the launcher can need sudo, so it shouldn't nag on every +// command. +const launcherNoticeInterval = 24 * time.Hour // envName determines whether the version check has already occurred. // We set this env-var so that this devbox command invoking other devbox commands @@ -65,6 +70,11 @@ func CheckVersion(w io.Writer, commandPath string) { return } + // Don't use up the once-a-day launcher notice on a --quiet command. + if w == io.Discard { + return + } + if envir.IsDevboxCloud() { return } @@ -94,112 +104,59 @@ func CheckVersion(w io.Writer, commandPath string) { os.Setenv(envName, "1") } -// SelfUpdate updates the devbox launcher and devbox CLI binary. -// It ignores and deletes the version cache. -// -// The launcher is a wrapper bash script introduced to manage the auto-update process -// for devbox. The production devbox application is actually this launcher script -// that acts as "devbox" and delegates commands to the devbox CLI binary. -func SelfUpdate(stdOut, stdErr io.Writer) error { - if isNewLauncherAvailable() { - return selfUpdateLauncher(stdOut, stdErr) +// SelfUpdate updates devbox to the latest version. It also replaces the +// launcher, the devbox that users run, when updateLauncher is true or when the +// launcher is older than this devbox expects. Replacing the launcher can need +// sudo, which is why it doesn't happen on every update. +func SelfUpdate(stdErr io.Writer, updateLauncher bool) error { + launcherPath := os.Getenv(envir.LauncherPath) + if launcherPath == "" { + return usererr.New( + "Devbox can't update itself because it wasn't installed with the devbox installer. " + + "Update it with the package manager you installed it with, or reinstall it with " + + "`curl -fsSL https://get.jetify.com/devbox | bash`.", + ) } - return selfUpdateDevbox(stdErr) -} - -func selfUpdateLauncher(stdOut, stdErr io.Writer) error { - installScript := "" - if cmdutil.Exists("curl") { - installScript = "curl -fsSL https://get.jetpack.io/devbox | bash" - } else if cmdutil.Exists("wget") { - installScript = "wget -qO- https://get.jetpack.io/devbox | bash" - } else { - return usererr.New("curl or wget is required to update devbox. Please install either and try again.") + ctx := context.Background() + latest, err := launcher.FetchLatestVersion(ctx) + if err != nil { + return usererr.WithUserMessage(err, "Failed to get the latest devbox version.") } - - // Delete current version file. This will trigger an update when invoking any devbox command; - // in this case, inside triggerUpdate function. - if err := removeCurrentVersionFile(); err != nil { - return err + bin, err := launcher.Install(ctx, latest, stdErr) + if err != nil { + return usererr.WithUserMessage(err, "Failed to download devbox %s.", latest) } - - // Fetch the new launcher. And installs the new devbox CLI binary. - cmd := exec.Command("sh", "-c", installScript) - cmd.Stdout = stdOut - cmd.Stderr = stdErr - if err := cmd.Run(); err != nil { + if err := launcher.SetLatestVersion(latest); err != nil { return errors.WithStack(err) } - - // Previously, we have already updated the binary. So, we call triggerUpdate - // just to get the new version information. - updated, err := triggerUpdate(stdErr) - if err != nil { + if err := launcher.SetCurrentVersion(latest); err != nil { return errors.WithStack(err) } - - printSuccessMessage(stdErr, "Launcher", currentLauncherVersion(), updated.launcherVersion) - printSuccessMessage(stdErr, "Devbox", currentDevboxVersion, updated.devboxVersion) - - return nil -} - -// selfUpdateDevbox will update the devbox CLI binary to the latest version. -func selfUpdateDevbox(stdErr io.Writer) error { - // Delete current version file. This will trigger an update when the next devbox command is run; - // in this case, inside triggerUpdate function. - if err := removeCurrentVersionFile(); err != nil { - return err + printSuccessMessage(stdErr, "Devbox", currentDevboxVersion, latest) + if useVersion := userChosenVersion(); useVersion != "" { + ux.Fwarningf(stdErr, "%s=%s is set, so devbox will keep running version %s until you unset it.\n", + envir.DevboxUseVersion, useVersion, useVersion) } - updated, err := triggerUpdate(stdErr) - if err != nil { - return errors.WithStack(err) + if !updateLauncher && !isNewLauncherAvailable() { + return nil } - - printSuccessMessage(stdErr, "Devbox", currentDevboxVersion, updated.devboxVersion) - - return nil -} - -type updatedVersions struct { - devboxVersion string - launcherVersion string -} - -// triggerUpdate runs `devbox version -v` and triggers an update since a new -// version is available. It parses the output to get the new launcher and -// devbox versions. -func triggerUpdate(stdErr io.Writer) (*updatedVersions, error) { - exePath := os.Getenv(envir.LauncherPath) - if exePath == "" { - ux.Fwarningf(stdErr, "expected LAUNCHER_PATH to be set. Defaulting to \"devbox\".") - exePath = "devbox" - } - - // TODO savil. Add a --json flag to devbox version and parse the output as JSON - cmd := exec.Command(exePath, "version", "-v") - - buf := new(bytes.Buffer) - cmd.Stdout = io.MultiWriter(stdErr, buf) - cmd.Stderr = stdErr - if err := cmd.Run(); err != nil { - return nil, errors.WithStack(err) - } - - // Parse the output to ascertain the new devbox and launcher versions - updated := &updatedVersions{} - for _, line := range strings.Split(buf.String(), "\n") { - if strings.HasPrefix(line, "Version:") { - updated.devboxVersion = strings.TrimSpace(strings.TrimPrefix(line, "Version:")) - } - - if strings.HasPrefix(line, "Launcher:") { - updated.launcherVersion = strings.TrimSpace(strings.TrimPrefix(line, "Launcher:")) + // The new launcher must be a devbox with a built-in launcher. This devbox + // has one, and so does every later version. + src, srcVersion := bin, latest + if SemverCompare(latest, currentDevboxVersion) < 0 { + if src, err = os.Executable(); err != nil { + return errors.WithStack(err) } + srcVersion = currentDevboxVersion } - return updated, nil + if err := launcher.ReplaceLauncher(src, launcherPath, stdErr); err != nil { + return usererr.WithUserMessage(err, "Failed to replace the devbox launcher at %s.", launcherPath) + } + fmt.Fprintf(stdErr, "%sreplaced the launcher at %s with devbox %s\n", + color.New(color.FgGreen).Sprint("Success: "), launcherPath, strings.TrimPrefix(srcVersion, "v")) + return nil } func printSuccessMessage(w io.Writer, toolName, oldVersion, newVersion string) { @@ -220,8 +177,17 @@ func launcherVersionNotice() string { return "" } + // Print the notice at most once per interval. + noticeFile := xdg.CacheSubpath("devbox/launcher-notice") + if fi, err := os.Stat(noticeFile); err == nil && time.Since(fi.ModTime()) < launcherNoticeInterval { + return "" + } + if err := os.MkdirAll(filepath.Dir(noticeFile), 0o755); err == nil { + _ = os.WriteFile(noticeFile, nil, 0o644) + } + return fmt.Sprintf( - "New launcher available: %s -> %s. Please run `devbox version update`.\n", + "New launcher available: %s -> %s. Please run `devbox version update` (it may ask for sudo).\n", currentLauncherVersion(), expectedLauncherVersion, ) @@ -267,24 +233,6 @@ func currentLauncherVersion() string { return "v" + launcherVersion } -func removeCurrentVersionFile() error { - // currentVersionFilePath is the path to the file that contains the cached - // version. The launcher checks this file to see if a new version is available. - // If the version is newer, then the launcher updates. - // - // Note: keep this in sync with launch.sh code - currentVersionFilePath := filepath.Join(xdg.CacheSubpath("devbox"), "current-version") - - if err := os.Remove(currentVersionFilePath); err != nil && !errors.Is(err, fs.ErrNotExist) { - return usererr.WithLoggedUserMessage( - err, - "Failed to delete version-cache at %s. Please manually delete it and try again.", - currentVersionFilePath, - ) - } - return nil -} - func SemverCompare(ver1, ver2 string) int { if !strings.HasPrefix(ver1, "v") { ver1 = "v" + ver1 diff --git a/internal/vercheck/vercheck_test.go b/internal/vercheck/vercheck_test.go index ca4ee8f457c..3c817f3c1f8 100644 --- a/internal/vercheck/vercheck_test.go +++ b/internal/vercheck/vercheck_test.go @@ -14,6 +14,7 @@ import ( func TestCheckVersion(t *testing.T) { isDevBuild = false + t.Setenv(envir.XDGCacheHome, t.TempDir()) t.Run("skip_if_devbox_cloud", func(t *testing.T) { defer os.Unsetenv(envName) @@ -51,6 +52,21 @@ func TestCheckVersion(t *testing.T) { } }) + t.Run("print_launcher_notice_once_per_interval", func(t *testing.T) { + t.Setenv(envir.XDGCacheHome, t.TempDir()) + t.Setenv(envir.LauncherVersion, "0.2.2") + + for i, want := range []bool{true, false} { + os.Unsetenv(envName) + buf := new(bytes.Buffer) + CheckVersion(buf, "devbox shell") + if got := strings.Contains(buf.String(), "New launcher available"); got != want { + t.Errorf("run %d: got notice %v, want %v (output %q)", i, got, want, buf.String()) + } + } + os.Unsetenv(envName) + }) + t.Run("print_if_binary_version_outdated", func(t *testing.T) { defer os.Unsetenv(envName) // set the launcher version so that it is not outdated diff --git a/scripts/install.sh b/scripts/install.sh new file mode 100755 index 00000000000..d17130cefa6 --- /dev/null +++ b/scripts/install.sh @@ -0,0 +1,296 @@ +#!/bin/bash +# +# Install script +# +# Downloads the latest devbox release and installs its binary. Release builds +# of devbox are their own launcher: the installed binary picks the devbox +# version to run, downloads it if needed, and checks for updates. + +set -euo pipefail + +# ======================== +# Customize install script +# ======================== + +# This script is published at get.jetify.com/devbox so users can install via: +# curl -fsSL https://get.jetify.com/devbox | bash + +readonly INSTALL_DIR="/usr/local/bin" +readonly BIN="devbox" +readonly RELEASES_URL="https://releases.jetify.com/devbox" +readonly DOWNLOAD_URL="https://github.com/jetify-com/devbox/releases/download" +# Must match the version files that devbox reads (internal/launcher/files.go). +readonly CACHE_DIR="${XDG_CACHE_HOME:-${HOME}/.cache}/${BIN}" + +readonly TITLE="Devbox 📦 by Jetify" +readonly DESCRIPTION=$( + cat <&2 +} + +success() { + local -r text="$*" + printf "%s\n" "${GREEN}✓${NO_COLOR} ${text}" +} + +start_task() { + local -r text="$*" + printf "%s\n" "${BOLD}${GREY}→${NO_COLOR} ${text}..." +} + +end_task() { + local -r text="$*" + printf "${CLEAR_LAST_MSG}%s\n" "${GREEN}✓${NO_COLOR} ${text}... [DONE]" +} + +fail_task() { + local -r text="$*" + printf "${CLEAR_LAST_MSG}%s\n" "${RED}✘ ${text}... [FAILED]${NO_COLOR}" >&2 +} + +confirm() { + if [ ${FORCE-} -ne 1 ]; then + printf "%s " "${MAGENTA}?${NO_COLOR} $* ${BOLD}[Y/n]${NO_COLOR}" + set +e + read -r yn /dev/null 2>&1 +} + +download() { + local -r url="$1" + local -r file="$2" + local cmd="" + + if has curl; then + cmd="curl --fail --silent --location --output $file $url" + elif has wget; then + cmd="wget --quiet --output-document=$file $url" + elif has fetch; then + cmd="fetch --quiet --output=$file $url" + else + error "No program to download files found. Please install one of: curl, wget, fetch" + exit 1 + fi + + $cmd && return 0 || rc=$? + error "Command failed (exit code $rc): ${BLUE}${cmd}${NO_COLOR}" + exit $rc +} + +sha256() { + if has sha256sum; then + sha256sum "$1" | cut -d ' ' -f 1 + elif has shasum; then + shasum -a 256 "$1" | cut -d ' ' -f 1 + else + error "No program to verify checksums found. Please install one of: sha256sum, shasum" + exit 1 + fi +} + +# platform prints the platform part of release archive names, such as +# darwin_arm64 or linux_amd64. +platform() { + local os arch + os="$(uname -s | tr '[:upper:]' '[:lower:]')" + arch="$(uname -m | tr '[:upper:]' '[:lower:]')" + case "${arch}" in + x86_64 | amd64) arch="amd64" ;; + aarch64 | arm64) arch="arm64" ;; + armv7*) arch="armv7l" ;; + i386 | i686) arch="386" ;; + esac + case "${os}" in + linux | darwin) ;; + *) + error "Unsupported operating system: ${os}" + exit 1 + ;; + esac + printf '%s_%s' "${os}" "${arch}" +} + +# ============== +# Implementation +# ============== +intro_msg() { + title "${TITLE}" + plain "${DESCRIPTION}" + printf "\n" + header "Confirm Installation Details" + plain " Location: ${GREEN}${INSTALL_DIR}/${BIN}${NO_COLOR}" + plain " Download URL: ${UNDERLINE}${BLUE}${DOWNLOAD_URL}${NO_COLOR}" + printf "\n" +} + +install_flow() { + confirm "Install ${GREEN}${BIN}${NO_COLOR} to ${GREEN}${INSTALL_DIR}${NO_COLOR} (requires sudo)?" + printf "\n" + header "Downloading and Installing" + + TMP_DIR="$(mktemp -d)" + trap 'rm -rf "${TMP_DIR}"' EXIT + local -r tmp_dir="${TMP_DIR}" + + start_task "Finding the latest version" + download "${RELEASES_URL}/stable/version" "${tmp_dir}/version" + local version + version="$(tr -d '[:space:]' <"${tmp_dir}/version")" + if [[ ! ${version} =~ ^v?[0-9]+\.[0-9]+\.[0-9]+ ]]; then + fail_task "Finding the latest version" + error "Unexpected version: ${version}" + exit 1 + fi + version="${version#v}" + end_task "Finding the latest version (${version})" + + local -r archive="${BIN}_${version}_$(platform).tar.gz" + start_task "Downloading ${BIN} ${version}" + download "${DOWNLOAD_URL}/${version}/checksums.txt" "${tmp_dir}/checksums.txt" + download "${DOWNLOAD_URL}/${version}/${archive}" "${tmp_dir}/${archive}" + end_task "Downloading ${BIN} ${version}" + + start_task "Verifying checksum" + local -r expected="$(awk -v name="${archive}" '$2 == name { print $1 }' "${tmp_dir}/checksums.txt")" + local -r actual="$(sha256 "${tmp_dir}/${archive}")" + if [[ -z ${expected} || ${actual} != "${expected}" ]]; then + fail_task "Verifying checksum" + plain " Expected checksum (sha256): ${expected:-none}" + plain " Actual checksum (sha256): ${actual}" + exit 1 + fi + end_task "Verifying checksum" + + mkdir "${tmp_dir}/unpacked" + tar -xzof "${tmp_dir}/${archive}" -C "${tmp_dir}/unpacked" + chmod 755 "${tmp_dir}/unpacked/${BIN}" + + start_task "Installing in ${INSTALL_DIR}/${BIN} (requires sudo)" + # Only use sudo when the install dir (or its closest existing parent, if + # it doesn't exist yet) isn't writable. + local writable_dir="${INSTALL_DIR}" + while [[ ! -e ${writable_dir} ]]; do + writable_dir="$(dirname "${writable_dir}")" + done + local sudo="" + if [[ ! -w ${writable_dir} ]]; then + sudo="$(command -v sudo || true)" + fi + # Copy next to the destination and rename, so that a devbox command that + # starts during the install never runs a partial binary. + ${sudo} bash -c 'mkdir -p "$1" && cp "$2" "$1/.$3.tmp" && mv -f "$1/.$3.tmp" "$1/$3"' \ + bash "${INSTALL_DIR}" "${tmp_dir}/unpacked/${BIN}" "${BIN}" + end_task "Installing in ${INSTALL_DIR}/${BIN}" + + # Run the installed version from now on, and don't wait to check for a + # newer one on the first run. + if mkdir -p "${CACHE_DIR}" 2>/dev/null && + printf '%s\n' "${version}" >"${CACHE_DIR}/current-version" && + printf '%s\n' "${version}" >"${CACHE_DIR}/latest-version"; then + : + else + warn "Couldn't write ${CACHE_DIR}/current-version. ${BIN} will check for the latest version on its first run." + fi + + success "${BOLD}Successfully installed ${GREEN}${BIN} ${version}${NO_COLOR}${BOLD}${NO_COLOR} 🚀" + printf "\n" +} + +next_steps_msg() { + header "Next Steps" + plain " 1. ${BOLD}Learn how to use ${BIN}${NO_COLOR}" + plain " ${GREY}Run ${CYAN}${BIN} help${GREY} or read the docs at ${UNDERLINE}${BLUE}${DOCS_URL}${NO_COLOR}" + plain " 2. ${BOLD}Get help and give feedback${NO_COLOR}" + plain " ${GREY}Join our community at ${UNDERLINE}${BLUE}${COMMUNITY_URL}${NO_COLOR}" +} + +main() { + parse_flags "$@" + intro_msg + install_flow + next_steps_msg +} + +main "$@" diff --git a/specs/native-launcher.md b/specs/native-launcher.md new file mode 100644 index 00000000000..d453cfa8c87 --- /dev/null +++ b/specs/native-launcher.md @@ -0,0 +1,152 @@ +# Devbox as its own launcher + +Replace the bash launcher (`releases.jetify.com/devbox`, installed at +`/usr/local/bin/devbox`) with the devbox binary itself. The binary that's +installed in `/usr/local/bin` (the "front binary") picks the devbox version to +run, downloads it if needed, and either runs the command itself or execs that +version. All of this then lives in this repo, in `internal/launcher`. + +**Status:** implemented, except publishing the new installer +(`scripts/install.sh`) to `get.jetify.com/devbox` and retiring the launcher in +the axiom repo. See [Rollout](#rollout). + +## Why + +- The bash launcher adds about 70ms to every command, mostly from subshells + and chained `exec env`. When its version cache is more than 24h old, it also + does a blocking `curl` with no timeout. +- The launcher lives in another repo (`axiom/installers/devbox`), so fixing it + means changing two repos that have to stay in sync. `vercheck` already + mirrored launcher internals, such as the `current-version` file and + `INSTALL_DIR`. + +Measured with `devbox version` (median of 30 runs, macOS arm64): + +| Path | Time | +|---|---| +| binary run directly | 9.0ms | +| bash launcher, then the binary | 78.2ms | +| native launcher, front binary is the current version (no exec) | 8.4ms | +| native launcher, then another version | 17.3ms | + +## Terms + +- **Front binary:** the devbox binary at `/usr/local/bin/devbox`. +- **Install dir:** `$HOME/.cache/devbox/bin`, where each version lives in + `__/devbox`. It's always under `$HOME`, even when + `XDG_CACHE_HOME` is set, to match the bash launcher. +- **Launcher version:** `launcher.Version`, separate from the devbox version. + The front binary passes it to the version it runs as `LAUNCHER_VERSION`. The + bash launcher's versions are below 1.0.0, and the native launcher starts at + 1.0.0. Bump it only when installed launchers need to change (see + [Updates](#updates)). +- **Selected version:** the version that should run: `DEVBOX_USE_VERSION` if + set, otherwise `current-version`. + +## How the launcher works + +`launcher.Main` runs first in `boxcli.Main`, before cobra, telemetry, or +anything that opens a project. + +1. **Only in release builds.** It does nothing unless the + `build.Launcher=true` ldflag is set, which only `.goreleaser.yaml` sets. + Dev builds and package-manager builds (nixpkgs, Homebrew) stay plain + binaries, so the package manager stays in charge of their version. +2. **Not for installed versions.** It does nothing when the binary is in the + install dir. Launchers (bash or native) only run binaries from there, so + such a binary is already the version a launcher picked. This rule, rather + than an env var guard, prevents exec loops, and env vars that leak into a + `devbox shell` can't confuse it. +3. **Fast path.** If the selected version is the front binary's own version, + it sets the launcher env vars and runs the command in-process. +4. **Otherwise** it execs the selected version from the install dir. If that + version isn't installed, it downloads the release archive and its + `checksums.txt` from GitHub first, checks the archive's SHA-256, unpacks + it into a temp dir, and renames that into place. +5. **Version checks don't block.** If `current-version` was last modified + more than 24h ago, it resets the file's modification time and starts a + detached `devbox __launcher-refresh` process. That process saves the + latest version to `latest-version`. With `auto_update` on, it also + downloads that version and then makes it the current version. A goroutine + wouldn't work, because a short command exits before it finishes. +6. **No current version.** This happens on the first run after an install + that didn't write `current-version`, or after an older devbox's `devbox + version update` deleted the file. Like the bash launcher, it waits (up to + 5s) for the latest version and saves it as the current version. If that + fails, it runs itself without saving anything, so the next run tries again. + +### Compatibility with older versions + +The front binary runs older versions (anything a project or user pins), and +older binaries may run under the bash launcher, so both sides keep the bash +launcher's contract: + +- `DEVBOX_USE_VERSION` picks the version, including for the `"auto"` policy's + re-exec through `LAUNCHER_PATH` (`vercheck/project.go`). +- `current-version` and `latest-version` in `$XDG_CACHE_HOME/devbox`, and the + `auto_update` and `channel` files in `$XDG_CONFIG_HOME/devbox`, keep their + meaning. +- The launcher sets `LAUNCHER_PATH` (its own path), `LAUNCHER_VERSION`, and + `DEVBOX_LATEST_VERSION`. Older binaries compare `LAUNCHER_VERSION` to their + expected 0.2.2, so 1.0.0 doesn't make them ask for a new launcher. +- The install dir layout is unchanged, so versions that the bash launcher + already downloaded are reused, and `vercheck` execs them directly as before + (#3023). 32-bit ARM uses `armv7l` like `uname -m` and the release archives. + +## Updates + +Most updates never touch the front binary, so they never need sudo. + +- **`devbox version update`** gets the latest version, downloads it into the + install dir, and saves it as the current and latest version. The front + binary then execs it. It no longer reruns the installer or the launcher. +- **The front binary is replaced only when the launcher version is bumped.** + When `LAUNCHER_VERSION` is older than `expectedLauncherVersion` (now + `launcher.Version`), devbox prints "New launcher available" and `devbox + version update` also replaces the front binary. +- **`devbox version update --launcher`** replaces the front binary even when + the launcher is up to date. It's for people who want the fast path back, + meaning no exec on every command, and don't mind sudo. +- **How it's replaced:** with the latest version's binary, or with the running + binary if the latest is older (for example on a channel), because the new + front binary must have the native launcher. It copies to a temp file next to + the destination and renames it, resolving symlinks first. If the directory + isn't writable, it runs the same copy and rename with `sudo`. +- **Not installed with the installer:** without `LAUNCHER_PATH`, `devbox + version update` returns an error that says to update devbox with the + package manager it came from. + +## Migration and installer + +- **Existing bash-launcher users** see "New launcher available: v0.2.2 -> + v1.0.0" once this release is the version they run. `devbox version update` + then replaces `/usr/local/bin/devbox` (the script) with the binary itself, + without the installer. So the migration doesn't depend on publishing the new + installer. Users who never update keep the bash launcher, which still works. +- **The notice is throttled** to once a day using a `launcher-notice` file in + the cache. It isn't printed (and doesn't use up the day's notice) when + `--quiet` discards stderr. It's still skipped for `shellenv` and the other + commands in `commandSkipList`, and it never prompts. +- **The new installer**, `scripts/install.sh`, gets the latest version, + downloads the archive and `checksums.txt` from GitHub, checks the archive, + and installs the binary to `/usr/local/bin/devbox`. It uses sudo only when + that directory isn't writable. It writes `current-version` and + `latest-version`, so the first run doesn't wait for the network. It keeps + the old installer's `-f` flag and its curl/wget/fetch support. It always + installs the latest version, because the front binary honors + `DEVBOX_USE_VERSION` (which the generated Dockerfiles set) by downloading + that version on the first run. + +## Rollout + +1. Ship this in a release. New and existing installs keep working as before + until users run `devbox version update`. +2. After that release is the stable version, publish `scripts/install.sh` to + `get.jetify.com/devbox` (it's currently published from + `axiom/installers/devbox/install.sh`). It needs a release with the native + launcher, because it installs the latest release as the front binary. + Older devbox versions' `devbox version update` runs this installer too, + through `get.jetpack.io/devbox`, which redirects to the same script. +3. Keep `releases.jetify.com/devbox` (the bash launcher) published for + anything that downloads it directly. Later, retire it and + `axiom/installers/devbox`. diff --git a/specs/perf-improvements.md b/specs/perf-improvements.md index b3d1eb39c5f..b55a796d56c 100644 --- a/specs/perf-improvements.md +++ b/specs/perf-improvements.md @@ -27,7 +27,7 @@ direnv calls devbox on every `cd`. | 3 | Cache macOS machine ID | ~20ms every command (macOS) | ~30–50 | | 4 | Throttle `upload-telemetry` child process | 1 fork + HTTPS per command | ~40–60 | | 5 | Lightweight `generate direnv --print-envrc` | every direnv reload | ~15–30 | -| 6 | Faster launcher script | 100–200ms every command | ~50–80 (launcher repo) | +| 6 | Make devbox its own launcher (instead of a faster launcher script) | ~70ms every command | ~1,500 incl. tests and installer | | 7 | Early version switch for `devbox_version: "auto"` | one full startup | ~100–150 | | 8 | Split state hash: nix vs non-nix inputs | ~1s+ per config edit | ~150–250 | | 9 | Persist narinfo lookups on disk | network round-trips per recompute | ~60–100 | @@ -116,16 +116,22 @@ inherits `PersistentPreRunE: ensureNixInstalled` from `generateCmd` `PersistentPreRunE` on the direnv subcommand (or check the flag) to skip the nix check, and skip telemetry for it. -### 6. Speed up the launcher script (launcher repo) +### 6. Make devbox its own launcher (done in #3024) -**Win:** 100–200ms on every command through `/usr/local/bin/devbox`. -**Effort:** ~50–80 LOC changed, in the launcher repo, not this one. +**Win:** about 70ms on every command through `/usr/local/bin/devbox`. With +`devbox version`, the bash launcher took 78ms. The native launcher takes 8ms +when the installed binary is the current version, about the same as running +the binary directly, and 17ms when it switches to another version. +**Effort:** ~1,500 LOC including tests and the new installer, all in this repo. -The launcher forks about 10 subshells (`tr`, `uname`, `date -r`, `cat`) and -chains three `exec env` calls. When its version cache is more than 24h old it -also does a blocking `curl` with no timeout. Fix: use bash built-ins and -parameter expansion, collapse to a single `exec`, and refresh the version -check in the background (with a timeout). +The bash launcher forks about 10 subshells (`tr`, `uname`, `date -r`, `cat`) +and chains three `exec env` calls. When its version cache is more than 24h old +it also does a blocking `curl` with no timeout. Rather than speed up the +script in the launcher repo, release builds of devbox now do the launcher's +job themselves (`internal/launcher`), and version checks run in a detached +child process. See `specs/native-launcher.md` for the design, compatibility +with older versions, and the rollout. The rollout's remaining step is +publishing the new installer. ### 7. Switch versions early for `devbox_version: "auto"` (done in #3023) @@ -287,4 +293,5 @@ nixpkgs resolution is already cached. 1. Items 1–5 (~200–330 LOC): biggest win for terminal start and direnv `cd`. 2. Item 8 (~150–250 LOC): biggest win for editing `devbox.json`. 3. Item 12 (~30–50 LOC): easiest win for `devbox update`. -4. The rest as time allows; item 6 needs a launcher-repo PR. +4. The rest as time allows. Item 6 is done except for publishing the new + installer (see `specs/native-launcher.md`).