diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a0bbcd..fc34dbf 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -171,7 +171,15 @@ jobs: - name: Build CloudMachine.app (universal) working-directory: mac-app - run: swift run cloudmachine-agent build-app --universal + run: | + # Commit count as the build number, computed here so a failing git + # call inside build-app cannot quietly turn it into a date (v1.3.1). + CM_BUILD_NUMBER="$(git rev-list --count HEAD)" + if [ -z "$CM_BUILD_NUMBER" ]; then + echo "::error::git rev-list --count HEAD returned nothing."; exit 1 + fi + export CM_BUILD_NUMBER + swift run cloudmachine-agent build-app --universal - name: Verify architectures and signature working-directory: mac-app diff --git a/README.md b/README.md index 4cd651c..1729a62 100644 --- a/README.md +++ b/README.md @@ -100,8 +100,11 @@ keep `mac-studio`, which is where their backup already is. ### Upgrading and uninstalling -`brew upgrade` replaces the app without restarting the Google Drive mount; the -agents pick up the new version on their next run. Neither `brew uninstall` nor +`brew upgrade` replaces the app without restarting the Google Drive mount. +When Homebrew reopens the app, it reloads the background agents so they run +the new version; `cloudmachine-agent drive-status` shows `Agents: OK`. If a +new version does not show up, run `brew update` first - Homebrew refreshes the +tap only now and then. Neither `brew uninstall` nor `--zap` touches the launchd agents or the upload buffer in `~/.cloudmachine`, which may hold backups that have not reached Google Drive yet. Run `cloudmachine-agent prepare-shutdown` before uninstalling. diff --git a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift index fe03404..a0bb6fa 100644 --- a/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift +++ b/mac-app/Sources/CloudMachineAgent/BuildAppCommand.swift @@ -207,13 +207,23 @@ struct BuildApp: AsyncParsableCommand { } private func resolveBuildNumber(projectRoot: URL) async -> String { - if let result = try? await ProcessRunner.run( - "/usr/bin/git", ["-C", projectRoot.path, "rev-list", "--count", "HEAD"]), - result.succeeded + // The release workflow computes it itself: v1.3.1 came out with a date + // here instead of the commit count, and nothing said why git failed. + if let given = ProcessInfo.processInfo.environment["CM_BUILD_NUMBER"]? + .trimmingCharacters(in: .whitespacesAndNewlines), !given.isEmpty { + return given + } + let result = try? await ProcessRunner.run( + "/usr/bin/git", ["-C", projectRoot.path, "rev-list", "--count", "HEAD"]) + if let result, result.succeeded { let count = result.stdout.trimmingCharacters(in: .whitespacesAndNewlines) if !count.isEmpty { return count } } + print( + L10n.tr( + "==> WARNING: git rev-list failed (%@) - using the date as the build number.", + result?.stderr.trimmingCharacters(in: .whitespacesAndNewlines) ?? "no answer")) let formatter = DateFormatter() formatter.dateFormat = "yyyyMMddHHmm" return formatter.string(from: Date()) diff --git a/mac-app/Sources/CloudMachineAgent/DriveCommands.swift b/mac-app/Sources/CloudMachineAgent/DriveCommands.swift index 0713471..7e276bf 100644 --- a/mac-app/Sources/CloudMachineAgent/DriveCommands.swift +++ b/mac-app/Sources/CloudMachineAgent/DriveCommands.swift @@ -262,6 +262,15 @@ struct BackupHealthCommand: AsyncParsableCommand { // `WatchdogHeartbeat`). WatchdogHeartbeat.record() + // The watchdog runs every 30 minutes whether or not anyone opened the app, + // so it is also what brings back an agent launchd stopped being able to + // start (see `AgentRepair`) - most importantly the mount. + let repaired = await AgentRepair.repairBroken() + if !repaired.isEmpty { + print( + L10n.tr("Reloaded agents that could not start: %@", repaired.joined(separator: ", "))) + } + if let lastSuccess = report.lastSuccess { print(L10n.tr("Last successful backup: %@", BackupHealth.stamp(lastSuccess))) } else { @@ -442,6 +451,15 @@ struct DriveStatus: AsyncParsableCommand { // Who watches the watchdog. Without this line "no alarm" meant both // "the backup works" and "nobody checked" at once - see `WatchdogHeartbeat`. print(L10n.tr("Backup watchdog: %@", StatusLines.watchdogRun(WatchdogHeartbeat.current()))) + let broken = await AgentRepair.brokenAgents() + print( + L10n.tr( + "Agents: %@", + broken.isEmpty + ? "OK" + : L10n.tr( + "CANNOT START: %@ - open CloudMachine or run backup-health to reload them", + broken.joined(separator: ", ")))) // At the very end and not aligned to the column - this is not another // status line but something meant to interrupt the reading. Since recently diff --git a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift index bb49d30..47a9486 100644 --- a/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift +++ b/mac-app/Sources/CloudMachineApp/Services/CloudMachineController.swift @@ -32,8 +32,17 @@ final class CloudMachineController: ObservableObject { func startAutoRefresh(interval: TimeInterval = 10) { refreshTask?.cancel() refreshTask = Task { [weak self] in + // Homebrew quits the app for an upgrade and reopens it afterwards, so + // this is the first code that runs after the bundle was replaced - see + // `AgentRepair` for what breaks if nobody reloads the agents. + await AgentRepair.afterLaunch() + var cycles = 0 while !Task.isCancelled { await self?.refreshAll() + // An agent can stop being startable later too (gdrive-buffer only + // notices when its rclone exits), so check every few minutes. + cycles += 1 + if cycles % 30 == 0 { await AgentRepair.repairBroken() } try? await Task.sleep(nanoseconds: UInt64(interval) * 1_000_000_000) } } diff --git a/mac-app/Sources/CloudMachineCore/AgentRepair.swift b/mac-app/Sources/CloudMachineCore/AgentRepair.swift new file mode 100644 index 0000000..293ade3 --- /dev/null +++ b/mac-app/Sources/CloudMachineCore/AgentRepair.swift @@ -0,0 +1,133 @@ +import Foundation + +/// Keeps the launchd agents startable after the app bundle is replaced. +/// +/// Replacing `/Applications/CloudMachine.app` - by `brew upgrade`, or by hand - +/// leaves the loaded jobs pointing at the old code signature of that path. +/// From then on launchd refuses to start them: `job state = spawn failed`, +/// `last exit reason = OS_REASON_CODESIGNING`, and nothing in any log. Seen on +/// 6 Oct 2026 after the 1.3.0 -> 1.3.1 upgrade: the backup watchdog and the +/// image attach stopped running, which looks exactly like a quiet, healthy day. +/// Only `bootout` + `bootstrap` clears it. +/// +/// Two moments: +/// - after a version change, the agents that are safe to restart are reloaded +/// at once, so they run the new code; +/// - at any time, an agent that launchd failed to spawn is reloaded. This is +/// what covers `gdrive-buffer`: restarting it while it runs would drop the +/// Google Drive mount, so it is left alone until it has actually died - and +/// then reloading costs nothing, because the mount is already gone. +public enum AgentRepair { + static let prefix = "com.renacode.cloudmachine." + + /// Restarting these does not touch the mount (measured 26 Sep 2026; the + /// mount lives in the rclone process of `gdrive-buffer`). + public static let safeToReload = ["backup-health", "gdrive-attach", "buffer-guard"] + /// Every agent that runs our binary. + public static let all = ["gdrive-buffer"] + safeToReload + + // MARK: - Reading launchd + + /// Whether `launchctl print` output describes a job launchd cannot start. + public static func cannotStart(printOutput: String) -> Bool { + let lines = printOutput.split(separator: "\n").map { + $0.trimmingCharacters(in: .whitespaces) + } + if lines.contains("job state = spawn failed") { return true } + let running = lines.contains("state = running") + return !running && lines.contains("last exit reason = OS_REASON_CODESIGNING") + } + + static func printOutput(label: String) async -> String? { + guard + let result = try? await ProcessRunner.run( + "/bin/launchctl", ["print", "gui/\(getuid())/\(prefix)\(label)"], timeout: 15), + result.succeeded + else { return nil } + return result.stdout + } + + /// Agents that are loaded but cannot start. Not loaded = not listed: + /// installing agents is a setup step, not a repair. + public static func brokenAgents() async -> [String] { + var broken: [String] = [] + for name in all { + if let output = await printOutput(label: name), cannotStart(printOutput: output) { + broken.append(name) + } + } + return broken + } + + // MARK: - Reloading + + static func plist(_ name: String) -> URL { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/LaunchAgents/\(prefix)\(name).plist") + } + + /// `bootout` + `bootstrap`. `bootout` finishes asynchronously and a + /// `bootstrap` right after it fails with error 5, so it is retried. + @discardableResult + static func reload(_ name: String) async -> Bool { + let plist = plist(name) + guard FileManager.default.fileExists(atPath: plist.path) else { return false } + let domain = "gui/\(getuid())" + _ = try? await ProcessRunner.run( + "/bin/launchctl", ["bootout", "\(domain)/\(prefix)\(name)"], timeout: 30) + for _ in 0..<10 { + try? await Task.sleep(nanoseconds: 1_000_000_000) + if let result = try? await ProcessRunner.run( + "/bin/launchctl", ["bootstrap", domain, plist.path], timeout: 30), + result.succeeded + { + CMLogger.log("Reloaded launchd agent \(name)") + return true + } + } + CMLogger.log("Could not reload launchd agent \(name) - bootstrap kept failing") + return false + } + + /// Reloads every agent launchd cannot start. Returns the ones reloaded. + @discardableResult + public static func repairBroken() async -> [String] { + var repaired: [String] = [] + for name in await brokenAgents() { + CMLogger.log("launchd agent \(name) cannot start (spawn failed) - reloading") + if await reload(name) { repaired.append(name) } + } + return repaired + } + + // MARK: - After an upgrade + + static var stampFile: URL { + CMPaths.appSupportDir.appendingPathComponent("agents-version") + } + + /// Decides whether the safe agents need a reload: the running app is a + /// different build than the one that last checked. A missing stamp counts + /// as different - the first launch of a version with this code is exactly + /// the launch after an upgrade from one without it. + public static func versionChanged(current: String, stamp: String?) -> Bool { + current != stamp?.trimmingCharacters(in: .whitespacesAndNewlines) + } + + /// Called when the app starts: after an upgrade, reload the agents that are + /// safe to restart, then repair any that cannot start. Does nothing for a + /// build run outside a bundle (`swift run`), which has no version to compare. + public static func afterLaunch() async { + guard let version = AppVersionReader.current() else { return } + let current = version.summary + let stamp = try? String(contentsOf: stampFile, encoding: .utf8) + if versionChanged(current: current, stamp: stamp) { + CMLogger.log("App version is now \(current) - reloading the launchd agents") + for name in safeToReload where FileManager.default.fileExists(atPath: plist(name).path) { + await reload(name) + } + try? current.write(to: stampFile, atomically: true, encoding: .utf8) + } + await repairBroken() + } +} diff --git a/mac-app/Sources/CloudMachineCore/CMPaths.swift b/mac-app/Sources/CloudMachineCore/CMPaths.swift index 2b727e4..a22cbdb 100644 --- a/mac-app/Sources/CloudMachineCore/CMPaths.swift +++ b/mac-app/Sources/CloudMachineCore/CMPaths.swift @@ -77,9 +77,18 @@ public enum CMPaths { public static var configPath: URL { appSupportDir.appendingPathComponent("machines.json") } + /// Under XCTest, a temporary directory of this test process. `swift test` + /// used to append to the real `cloudmachine.log`: lock messages from image + /// tests and a "BACKUP FAILURE: ... test canary" line from HealthAlertTests, + /// which in the production log reads exactly like an alarm. public static var logDir: URL { - let base = FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask)[0] - let dir = base.appendingPathComponent("Logs/CloudMachine") + let base = + NSClassFromString("XCTestCase") != nil + ? FileManager.default.temporaryDirectory + .appendingPathComponent("CloudMachineTestLogs-\(ProcessInfo.processInfo.processIdentifier)") + : FileManager.default.urls(for: .libraryDirectory, in: .userDomainMask)[0] + .appendingPathComponent("Logs") + let dir = base.appendingPathComponent("CloudMachine") try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) return dir } diff --git a/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift b/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift index dd8ea40..04d57ea 100644 --- a/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift +++ b/mac-app/Sources/CloudMachineCore/L10nPolish+Agent.swift @@ -161,6 +161,14 @@ extension L10nPolish { "brakuje: %@", "Drive mount: %@": "Montowanie Drive: %@", + "==> WARNING: git rev-list failed (%@) - using the date as the build number.": + "==> UWAGA: git rev-list nie powiódł się (%@) - jako numer budowy idzie data.", + "Agents: %@": + "Agenci: %@", + "CANNOT START: %@ - open CloudMachine or run backup-health to reload them": + "NIE STARTUJĄ: %@ - otwórz CloudMachine albo uruchom backup-health, żeby je przeładować", + "Reloaded agents that could not start: %@": + "Przeładowano agentów, którzy nie mogli wystartować: %@", "Drive folder: %@": "Folder na Drive: %@", "Name of this Mac's folder on Google Drive (default: derived from the computer name). Set once; it cannot be changed later.": diff --git a/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift new file mode 100644 index 0000000..dc2f00d --- /dev/null +++ b/mac-app/Tests/CloudMachineAppTests/AgentRepairTests.swift @@ -0,0 +1,62 @@ +import XCTest + +@testable import CloudMachineCore + +/// The states below are copied from `launchctl print` on 6 Oct 2026, after +/// `brew upgrade` replaced the app: the watchdog had stopped starting and +/// nothing said so. +final class AgentRepairTests: XCTestCase { + + func testSpawnFailedAfterUpgradeIsBroken() { + let output = """ + \tstate = not running + \truns = 27 + \tlast exit reason = OS_REASON_CODESIGNING + \tspawn type = daemon (3) + \tjob state = spawn failed + """ + XCTAssertTrue(AgentRepair.cannotStart(printOutput: output)) + } + + func testCodesigningExitWithoutSpawnFailedLineIsStillBroken() { + let output = """ + \tstate = not running + \tlast exit reason = OS_REASON_CODESIGNING + """ + XCTAssertTrue(AgentRepair.cannotStart(printOutput: output)) + } + + func testRunningMountIsLeftAlone() { + // gdrive-buffer keeps running the old process after an upgrade; reloading + // it would drop the Google Drive mount. + let output = """ + \tstate = running + \truns = 1 + \tlast exit code = (never exited) + \tjob state = running + """ + XCTAssertFalse(AgentRepair.cannotStart(printOutput: output)) + } + + func testIdlePeriodicAgentIsHealthy() { + let output = """ + \tstate = not running + \truns = 2 + \tlast exit code = 0 + """ + XCTAssertFalse(AgentRepair.cannotStart(printOutput: output)) + } + + func testVersionChangeTriggersReload() { + XCTAssertTrue( + AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: "1.3.1 (125) def5678")) + XCTAssertTrue(AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: nil)) + XCTAssertFalse( + AgentRepair.versionChanged(current: "1.3.2 (130) abc1234", stamp: "1.3.2 (130) abc1234\n")) + } + + func testMountAgentIsNeverReloadedPreemptively() { + XCTAssertFalse(AgentRepair.safeToReload.contains("gdrive-buffer")) + XCTAssertTrue(AgentRepair.all.contains("gdrive-buffer")) + } +} diff --git a/mac-app/Tests/CloudMachineAppTests/HealthAlertTests.swift b/mac-app/Tests/CloudMachineAppTests/HealthAlertTests.swift index ef510d0..c3ee202 100644 --- a/mac-app/Tests/CloudMachineAppTests/HealthAlertTests.swift +++ b/mac-app/Tests/CloudMachineAppTests/HealthAlertTests.swift @@ -88,8 +88,10 @@ final class HealthAlertTests: XCTestCase { "the test must not append a single line to \(CMPaths.combinedLogFile.path)") } - /// The other side of the same fix - and the only test in this class that - /// DELIBERATELY writes to the production log (one line, marked as a canary). + /// The other side of the same fix: a report with the default `log:` must + /// reach the file `CMLogger` writes. During tests that file is in a + /// temporary directory (see `CMPaths.logDir`), so the canary no longer + /// lands in the real `cloudmachine.log`, where it read like an alarm. /// /// Without this test the fix could have silenced REAL alarms and nobody /// would have noticed: a backup failure does not get in the way of daily diff --git a/packaging/README.md b/packaging/README.md index 8aca787..36ce741 100644 --- a/packaging/README.md +++ b/packaging/README.md @@ -98,10 +98,12 @@ prints the cask, so a missing tap update cannot go unnoticed. - **Does not stop launchd agents** on uninstall. Homebrew runs `uninstall` directives on `brew upgrade` too, and stopping `gdrive-buffer` kills the rclone process that holds the mount. -- **Does not reload agents** after an upgrade. Cask install steps run in a - sandbox without access to `~/Library/LaunchAgents`, and an upgrade replaces - the bundle with new files, after which the agents start normally. - `drive-status` reports a watchdog that stopped running. +- **Does not reload agents itself** - but they must be reloaded: after the + bundle is replaced, launchd refuses to start them (`spawn failed`, + `OS_REASON_CODESIGNING`). Cask steps run in a sandbox without + `~/Library/LaunchAgents`, so the app does it when Homebrew reopens it after + the upgrade, and the backup watchdog repairs any agent that cannot start. + `drive-status` shows the agents' state. - **`zap` leaves `~/.cloudmachine` alone.** It holds the upload buffer, which may contain backups that have not reached Google Drive yet. diff --git a/packaging/homebrew/cloudmachine.rb.in b/packaging/homebrew/cloudmachine.rb.in index 0889349..baae869 100644 --- a/packaging/homebrew/cloudmachine.rb.in +++ b/packaging/homebrew/cloudmachine.rb.in @@ -24,12 +24,12 @@ cask "cloudmachine" do # would block the downloaded copy. The cask comes from the author's tap, and # the file matches the sha256 above. # - # The cask does NOT reload the launchd agents: the steps run in the Homebrew - # sandbox without access to ~/Library/LaunchAgents. It is not needed - - # the upgrade replaces the bundle with new files (new inodes), and that is - # exactly the procedure after which the agents start correctly. Should the - # watchdog stop anyway, `drive-status` and the app window will show a - # "THE WATCHDOG MAY NOT BE RUNNING" warning. + # The agents DO need a reload after an upgrade: launchd refuses to start + # them from the replaced bundle (spawn failed, OS_REASON_CODESIGNING) - seen + # on the 1.3.0 -> 1.3.1 upgrade. The cask cannot do it (its steps run in the + # Homebrew sandbox, without ~/Library/LaunchAgents), so the app does: Homebrew + # quits it for the upgrade and reopens it, and on launch it reloads the + # agents (`AgentRepair`). The backup watchdog repairs them too, every 30 min. postflight_steps do run "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "{{appdir}}/CloudMachine.app"] end @@ -55,8 +55,9 @@ cask "cloudmachine" do remaining setup steps for this Mac, with a button for each. See https://github.com/RenaCode/CloudMachine#getting-started - `brew upgrade` keeps the Google Drive mount running. Afterwards, check - that the backup watchdog still runs: `cloudmachine-agent drive-status`. + `brew upgrade` keeps the Google Drive mount running; the app reloads the + background agents when it reopens. `cloudmachine-agent drive-status` + shows "Agents: OK" once they run the new version. Uninstalling does NOT stop the launchd agents or delete the upload buffer in ~/.cloudmachine (it may hold backups not yet sent to Google Drive).