fix(agents): reload the launchd agents after an upgrade; tests stop writing the real log

.github/workflows/release.yml

@@ -171,7 +171,15 @@ jobs:
 
       - name: Build CloudMachine.app (universal)
         working-directory: mac-app
-        run: swift run cloudmachine-agent build-app --universal
+        run: |
+          # Commit count as the build number, computed here so a failing git
+          # call inside build-app cannot quietly turn it into a date (v1.3.1).
+          CM_BUILD_NUMBER="$(git rev-list --count HEAD)"
+          if [ -z "$CM_BUILD_NUMBER" ]; then
+            echo "::error::git rev-list --count HEAD returned nothing."; exit 1
+          fi
+          export CM_BUILD_NUMBER
+          swift run cloudmachine-agent build-app --universal
 
       - name: Verify architectures and signature
         working-directory: mac-app

README.md

@@ -100,8 +100,11 @@ keep `mac-studio`, which is where their backup already is.
 
 ### Upgrading and uninstalling
 
-`brew upgrade` replaces the app without restarting the Google Drive mount; the
-agents pick up the new version on their next run. Neither `brew uninstall` nor
+`brew upgrade` replaces the app without restarting the Google Drive mount.
+When Homebrew reopens the app, it reloads the background agents so they run
+the new version; `cloudmachine-agent drive-status` shows `Agents: OK`. If a
+new version does not show up, run `brew update` first - Homebrew refreshes the
+tap only now and then. Neither `brew uninstall` nor
 `--zap` touches the launchd agents or the upload buffer in `~/.cloudmachine`,
 which may hold backups that have not reached Google Drive yet. Run
 `cloudmachine-agent prepare-shutdown` before uninstalling.

mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift

@@ -207,13 +207,23 @@ struct BuildApp: AsyncParsableCommand {
   }
 
   private func resolveBuildNumber(projectRoot: URL) async -> String {
-    if let result = try? await ProcessRunner.run(
-      "/usr/bin/git", ["-C", projectRoot.path, "rev-list", "--count", "HEAD"]),
-      result.succeeded
+    // The release workflow computes it itself: v1.3.1 came out with a date
+    // here instead of the commit count, and nothing said why git failed.
+    if let given = ProcessInfo.processInfo.environment["CM_BUILD_NUMBER"]?
+      .trimmingCharacters(in: .whitespacesAndNewlines), !given.isEmpty
     {
+      return given
+    }
+    let result = try? await ProcessRunner.run(
+      "/usr/bin/git", ["-C", projectRoot.path, "rev-list", "--count", "HEAD"])
+    if let result, result.succeeded {
       let count = result.stdout.trimmingCharacters(in: .whitespacesAndNewlines)
       if !count.isEmpty { return count }
     }
+    print(
+      L10n.tr(
+        "==> WARNING: git rev-list failed (%@) - using the date as the build number.",
+        result?.stderr.trimmingCharacters(in: .whitespacesAndNewlines) ?? "no answer"))
     let formatter = DateFormatter()
     formatter.dateFormat = "yyyyMMddHHmm"
     return formatter.string(from: Date())

mac-app/Sources/CloudMachineAgent/DriveCommands.swift

@@ -262,6 +262,15 @@ struct BackupHealthCommand: AsyncParsableCommand {
     // `WatchdogHeartbeat`).
     WatchdogHeartbeat.record()
 
+    // The watchdog runs every 30 minutes whether or not anyone opened the app,
+    // so it is also what brings back an agent launchd stopped being able to
+    // start (see `AgentRepair`) - most importantly the mount.
+    let repaired = await AgentRepair.repairBroken()
+    if !repaired.isEmpty {
+      print(
+        L10n.tr("Reloaded agents that could not start: %@", repaired.joined(separator: ", ")))
+    }
+
     if let lastSuccess = report.lastSuccess {
       print(L10n.tr("Last successful backup: %@", BackupHealth.stamp(lastSuccess)))
     } else {
@@ -442,6 +451,15 @@ struct DriveStatus: AsyncParsableCommand {
     // Who watches the watchdog. Without this line "no alarm" meant both
     // "the backup works" and "nobody checked" at once - see `WatchdogHeartbeat`.
     print(L10n.tr("Backup watchdog:  %@", StatusLines.watchdogRun(WatchdogHeartbeat.current())))
+    let broken = await AgentRepair.brokenAgents()
+    print(
+      L10n.tr(
+        "Agents:           %@",
+        broken.isEmpty
+          ? "OK"
+          : L10n.tr(
+            "CANNOT START: %@ - open CloudMachine or run backup-health to reload them",
+            broken.joined(separator: ", "))))
 
     // At the very end and not aligned to the column - this is not another
     // status line but something meant to interrupt the reading. Since recently

mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift

@@ -32,8 +32,17 @@ final class CloudMachineController: ObservableObject {
   func startAutoRefresh(interval: TimeInterval = 10) {
     refreshTask?.cancel()
     refreshTask = Task { [weak self] in
+      // Homebrew quits the app for an upgrade and reopens it afterwards, so
+      // this is the first code that runs after the bundle was replaced - see
+      // `AgentRepair` for what breaks if nobody reloads the agents.
+      await AgentRepair.afterLaunch()
+      var cycles = 0
       while !Task.isCancelled {
         await self?.refreshAll()
+        // An agent can stop being startable later too (gdrive-buffer only
+        // notices when its rclone exits), so check every few minutes.
+        cycles += 1
+        if cycles % 30 == 0 { await AgentRepair.repairBroken() }
         try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000)
       }
     }

mac-app/Sources/CloudMachineCore/AgentRepair.swift

@@ -0,0 +1,133 @@
+import Foundation
+
+/// Keeps the launchd agents startable after the app bundle is replaced.
+///
+/// Replacing `/Applications/CloudMachine.app` - by `brew upgrade`, or by hand -
+/// leaves the loaded jobs pointing at the old code signature of that path.
+/// From then on launchd refuses to start them: `job state = spawn failed`,
+/// `last exit reason = OS_REASON_CODESIGNING`, and nothing in any log. Seen on
+/// 6 Oct 2026 after the 1.3.0 -> 1.3.1 upgrade: the backup watchdog and the
+/// image attach stopped running, which looks exactly like a quiet, healthy day.
+/// Only `bootout` + `bootstrap` clears it.
+///
+/// Two moments:
+/// - after a version change, the agents that are safe to restart are reloaded
+///   at once, so they run the new code;
+/// - at any time, an agent that launchd failed to spawn is reloaded. This is
+///   what covers `gdrive-buffer`: restarting it while it runs would drop the
+///   Google Drive mount, so it is left alone until it has actually died - and
+///   then reloading costs nothing, because the mount is already gone.
+public enum AgentRepair {
+  static let prefix = "com.renacode.cloudmachine."
+
+  /// Restarting these does not touch the mount (measured 26 Sep 2026; the
+  /// mount lives in the rclone process of `gdrive-buffer`).
+  public static let safeToReload = ["backup-health", "gdrive-attach", "buffer-guard"]
+  /// Every agent that runs our binary.
+  public static let all = ["gdrive-buffer"] + safeToReload
+
+  // MARK: - Reading launchd
+
+  /// Whether `launchctl print` output describes a job launchd cannot start.
+  public static func cannotStart(printOutput: String) -> Bool {
+    let lines = printOutput.split(separator: "\n").map {
+      $0.trimmingCharacters(in: .whitespaces)
+    }
+    if lines.contains("job state = spawn failed") { return true }
+    let running = lines.contains("state = running")
+    return !running && lines.contains("last exit reason = OS_REASON_CODESIGNING")
+  }
+
+  static func printOutput(label: String) async -> String? {
+    guard
+      let result = try? await ProcessRunner.run(
+        "/bin/launchctl", ["print", "gui/\(getuid())/\(prefix)\(label)"], timeout: 15),
+      result.succeeded
+    else { return nil }
+    return result.stdout
+  }
+
+  /// Agents that are loaded but cannot start. Not loaded = not listed:
+  /// installing agents is a setup step, not a repair.
+  public static func brokenAgents() async -> [String] {
+    var broken: [String] = []
+    for name in all {
+      if let output = await printOutput(label: name), cannotStart(printOutput: output) {
+        broken.append(name)
+      }
+    }
+    return broken
+  }
+
+  // MARK: - Reloading
+
+  static func plist(_ name: String) -> URL {
+    FileManager.default.homeDirectoryForCurrentUser
+      .appendingPathComponent("Library/LaunchAgents/\(prefix)\(name).plist")
+  }
+
+  /// `bootout` + `bootstrap`. `bootout` finishes asynchronously and a
+  /// `bootstrap` right after it fails with error 5, so it is retried.
+  @discardableResult
+  static func reload(_ name: String) async -> Bool {
+    let plist = plist(name)
+    guard FileManager.default.fileExists(atPath: plist.path) else { return false }
+    let domain = "gui/\(getuid())"
+    _ = try? await ProcessRunner.run(
+      "/bin/launchctl", ["bootout", "\(domain)/\(prefix)\(name)"], timeout: 30)
+    for _ in 0..<10 {
+      try? await Task.sleep(nanoseconds: 1_000_000_000)
+      if let result = try? await ProcessRunner.run(
+        "/bin/launchctl", ["bootstrap", domain, plist.path], timeout: 30),
+        result.succeeded
+      {
+        CMLogger.log("Reloaded launchd agent \(name)")
+        return true
+      }
+    }
+    CMLogger.log("Could not reload launchd agent \(name) - bootstrap kept failing")
+    return false
+  }
+
+  /// Reloads every agent launchd cannot start. Returns the ones reloaded.
+  @discardableResult
+  public static func repairBroken() async -> [String] {
+    var repaired: [String] = []
+    for name in await brokenAgents() {
+      CMLogger.log("launchd agent \(name) cannot start (spawn failed) - reloading")
+      if await reload(name) { repaired.append(name) }
+    }
+    return repaired
+  }
+
+  // MARK: - After an upgrade
+
+  static var stampFile: URL {
+    CMPaths.appSupportDir.appendingPathComponent("agents-version")
+  }
+
+  /// Decides whether the safe agents need a reload: the running app is a
+  /// different build than the one that last checked. A missing stamp counts
+  /// as different - the first launch of a version with this code is exactly
+  /// the launch after an upgrade from one without it.
+  public static func versionChanged(current: String, stamp: String?) -> Bool {
+    current != stamp?.trimmingCharacters(in: .whitespacesAndNewlines)
+  }
+
+  /// Called when the app starts: after an upgrade, reload the agents that are
+  /// safe to restart, then repair any that cannot start. Does nothing for a
+  /// build run outside a bundle (`swift run`), which has no version to compare.
+  public static func afterLaunch() async {
+    guard let version = AppVersionReader.current() else { return }
+    let current = version.summary
+    let stamp = try? String(contentsOf: stampFile, encoding: .utf8)
+    if versionChanged(current: current, stamp: stamp) {
+      CMLogger.log("App version is now \(current) - reloading the launchd agents")
+      for name in safeToReload where FileManager.default.fileExists(atPath: plist(name).path) {
+        await reload(name)
+      }
+      try? current.write(to: stampFile, atomically: true, encoding: .utf8)
+    }
+    await repairBroken()
+  }
+}

mac-app/Sources/CloudMachineCore/CMPaths.swift

@@ -77,9 +77,18 @@ public enum CMPaths {
 
   public static var configPath: URL { appSupportDir.appendingPathComponent("machines.json") }
 
+  /// Under XCTest, a temporary directory of this test process. `swift test`
+  /// used to append to the real `cloudmachine.log`: lock messages from image
+  /// tests and a "BACKUP FAILURE: ... test canary" line from HealthAlertTests,
+  /// which in the production log reads exactly like an alarm.
   public static var logDir: URL {
-    let base = FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask)[0]
-    let dir = base.appendingPathComponent("Logs/CloudMachine")
+    let base =
+      NSClassFromString("XCTestCase") != nil
+      ? FileManager.default.temporaryDirectory
+        .appendingPathComponent("CloudMachineTestLogs-\(ProcessInfo.processInfo.processIdentifier)")
+      : FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask)[0]
+        .appendingPathComponent("Logs")
+    let dir = base.appendingPathComponent("CloudMachine")
     try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
     return dir
   }

mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift

@@ -161,6 +161,14 @@ extension L10nPolish {
       "brakuje: %@",
     "Drive mount:      %@":
       "Montowanie Drive: %@",
+    "==> WARNING: git rev-list failed (%@) - using the date as the build number.":
+      "==> UWAGA: git rev-list nie powiódł się (%@) - jako numer budowy idzie data.",
+    "Agents:           %@":
+      "Agenci:           %@",
+    "CANNOT START: %@ - open CloudMachine or run backup-health to reload them":
+      "NIE STARTUJĄ: %@ - otwórz CloudMachine albo uruchom backup-health, żeby je przeładować",
+    "Reloaded agents that could not start: %@":
+      "Przeładowano agentów, którzy nie mogli wystartować: %@",
     "Drive folder:     %@":
       "Folder na Drive:  %@",
     "Name of this Mac's folder on Google Drive (default: derived from the computer name). Set once; it cannot be changed later.":

mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift

@@ -0,0 +1,62 @@
+import XCTest
+
+@testable import CloudMachineCore
+
+/// The states below are copied from `launchctl print` on 6 Oct 2026, after
+/// `brew upgrade` replaced the app: the watchdog had stopped starting and
+/// nothing said so.
+final class AgentRepairTests: XCTestCase {
+
+  func testSpawnFailedAfterUpgradeIsBroken() {
+    let output = """
+      \tstate = not running
+      \truns = 27
+      \tlast exit reason = OS_REASON_CODESIGNING
+      \tspawn type = daemon (3)
+      \tjob state = spawn failed
+      """
+    XCTAssertTrue(AgentRepair.cannotStart(printOutput: output))
+  }
+
+  func testCodesigningExitWithoutSpawnFailedLineIsStillBroken() {
+    let output = """
+      \tstate = not running
+      \tlast exit reason = OS_REASON_CODESIGNING
+      """
+    XCTAssertTrue(AgentRepair.cannotStart(printOutput: output))
+  }
+
+  func testRunningMountIsLeftAlone() {
+    // gdrive-buffer keeps running the old process after an upgrade; reloading
+    // it would drop the Google Drive mount.
+    let output = """
+      \tstate = running
+      \truns = 1
+      \tlast exit code = (never exited)
+      \tjob state = running
+      """
+    XCTAssertFalse(AgentRepair.cannotStart(printOutput: output))
+  }
+
+  func testIdlePeriodicAgentIsHealthy() {
+    let output = """
+      \tstate = not running
+      \truns = 2
+      \tlast exit code = 0
+      """
+    XCTAssertFalse(AgentRepair.cannotStart(printOutput: output))
+  }
+
+  func testVersionChangeTriggersReload() {
+    XCTAssertTrue(
+      AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: "1.3.1 (125) def5678"))
+    XCTAssertTrue(AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: nil))
+    XCTAssertFalse(
+      AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: "1.3.2 (130) abc1234\n"))
+  }
+
+  func testMountAgentIsNeverReloadedPreemptively() {
+    XCTAssertFalse(AgentRepair.safeToReload.contains("gdrive-buffer"))
+    XCTAssertTrue(AgentRepair.all.contains("gdrive-buffer"))
+  }
+}

mac-app/Tests/CloudMachineAppTests/HealthAlertTests.swift

@@ -88,8 +88,10 @@ final class HealthAlertTests: XCTestCase {
       "the test must not append a single line to \(CMPaths.combinedLogFile.path)")
   }
 
-  /// The other side of the same fix - and the only test in this class that
-  /// DELIBERATELY writes to the production log (one line, marked as a canary).
+  /// The other side of the same fix: a report with the default `log:` must
+  /// reach the file `CMLogger` writes. During tests that file is in a
+  /// temporary directory (see `CMPaths.logDir`), so the canary no longer
+  /// lands in the real `cloudmachine.log`, where it read like an alarm.
   ///
   /// Without this test the fix could have silenced REAL alarms and nobody
   /// would have noticed: a backup failure does not get in the way of daily

packaging/README.md

@@ -98,10 +98,12 @@ prints the cask, so a missing tap update cannot go unnoticed.
 - **Does not stop launchd agents** on uninstall. Homebrew runs `uninstall`
   directives on `brew upgrade` too, and stopping `gdrive-buffer` kills the
   rclone process that holds the mount.
-- **Does not reload agents** after an upgrade. Cask install steps run in a
-  sandbox without access to `~/Library/LaunchAgents`, and an upgrade replaces
-  the bundle with new files, after which the agents start normally.
-  `drive-status` reports a watchdog that stopped running.
+- **Does not reload agents itself** - but they must be reloaded: after the
+  bundle is replaced, launchd refuses to start them (`spawn failed`,
+  `OS_REASON_CODESIGNING`). Cask steps run in a sandbox without
+  `~/Library/LaunchAgents`, so the app does it when Homebrew reopens it after
+  the upgrade, and the backup watchdog repairs any agent that cannot start.
+  `drive-status` shows the agents' state.
 - **`zap` leaves `~/.cloudmachine` alone.** It holds the upload buffer, which
   may contain backups that have not reached Google Drive yet.