diff --git a/loopx/control_plane/coordination/authority_source_capture.py b/loopx/control_plane/coordination/authority_source_capture.py index 45fbbaaec2..777acc0b24 100644 --- a/loopx/control_plane/coordination/authority_source_capture.py +++ b/loopx/control_plane/coordination/authority_source_capture.py @@ -193,11 +193,18 @@ def _source_lease_bytes(path: Path) -> bytes: data = stream.read() after = os.fstat(stream.fileno()) current = path.lstat() + # Tolerate Windows ctime-only skew only with a volume and file index + # in every observation; POSIX keeps ctime in the lease fence. + stability_fields: tuple[str, ...] = ("st_mtime_ns",) + if os.name != "nt" or not all( + observed.st_dev and observed.st_ino for observed in (before, after, current) + ): + stability_fields += ("st_ctime_ns",) if ((current.st_dev, current.st_ino) != (opened.st_dev, opened.st_ino) or not stat.S_ISREG(current.st_mode) or before.st_size != len(data) or after.st_size != len(data) or current.st_size != len(data) - or before.st_mtime_ns != after.st_mtime_ns or before.st_ctime_ns != after.st_ctime_ns - or after.st_mtime_ns != current.st_mtime_ns or after.st_ctime_ns != current.st_ctime_ns): + or any(getattr(before, field) != getattr(after, field) for field in stability_fields) + or any(getattr(after, field) != getattr(current, field) for field in stability_fields)): raise ShadowManagementError("source_changed_retry") return data diff --git a/loopx/control_plane/coordination/runtime_shadow.ts b/loopx/control_plane/coordination/runtime_shadow.ts index 7c2d2780ca..1a33c7f189 100644 --- a/loopx/control_plane/coordination/runtime_shadow.ts +++ b/loopx/control_plane/coordination/runtime_shadow.ts @@ -148,10 +148,14 @@ async function sourceLeaseBytes(path: string): Promise { const bytes = await handle.readFile(); const after = await handle.stat(); const current = await lstat(path); + // Tolerate Windows ctime-only skew only with a volume and file index in + // every observation; POSIX keeps ctime in the lease fence. + const fenceCtime = process.platform !== "win32" || + !(before.dev && before.ino && after.dev && after.ino && current.dev && current.ino); if (current.isSymbolicLink() || current.dev !== opened.dev || current.ino !== opened.ino || before.size !== bytes.length || after.size !== bytes.length || current.size !== bytes.length || - before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || - after.mtimeMs !== current.mtimeMs || after.ctimeMs !== current.ctimeMs) { + before.mtimeMs !== after.mtimeMs || after.mtimeMs !== current.mtimeMs || + (fenceCtime && (before.ctimeMs !== after.ctimeMs || after.ctimeMs !== current.ctimeMs))) { throw new ShadowManagementError("source_changed_retry"); } return bytes; diff --git a/tests/control_plane/test_shadow_registry_source.py b/tests/control_plane/test_shadow_registry_source.py index d5c8a5a0ea..fed2844154 100644 --- a/tests/control_plane/test_shadow_registry_source.py +++ b/tests/control_plane/test_shadow_registry_source.py @@ -4,7 +4,9 @@ import hashlib import json +import os from pathlib import Path +from types import SimpleNamespace import pytest @@ -400,3 +402,76 @@ def sync(): action() assert (registry.read_bytes(), global_path.read_bytes()) == before assert action()["ok"] + + +@pytest.mark.parametrize("unavailable_identity", [None, "st_dev", "st_ino"]) +@pytest.mark.parametrize("platform", ["nt", "posix"]) +@pytest.mark.parametrize("field", ["st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns"]) +def test_lease_read_identity_change_policy(tmp_path, monkeypatch, platform, field, unavailable_identity): + from loopx.control_plane.coordination import authority_source_capture + + state, runtime, registry, goal, _ = workspace(tmp_path) + directory = runtime / "goals" / goal["id"] / "task-leases" + directory.mkdir(parents=True) + lease = directory / "todo_policy.json" + original = json.dumps({"schema_version": "task_lease_v0", "goal_id": goal["id"], + "todo_id": "todo_policy", "owner": "agent-a", "status": "released", "version": 1}).encode() + lease.write_bytes(original) + lease_stat = lease.stat() + assert lease_stat.st_dev and lease_stat.st_ino + original_fstat = os.fstat + original_lstat = Path.lstat + descriptor_observations = 0 + path_observations = 0 + + def partial_identity(observed): + if unavailable_identity is None or (observed.st_dev, observed.st_ino) != ( + lease_stat.st_dev, lease_stat.st_ino + ): + return observed + # Unavailable volume/file indexes are injected only at the stat boundary. + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields[unavailable_identity] = 0 + return SimpleNamespace(**fields) + + def path_stat(path, *args, **kwargs): + nonlocal path_observations + raw = original_lstat(path, *args, **kwargs) + is_lease = Path(path) == lease + if is_lease: + path_observations += 1 + observed = partial_identity(raw) + if not is_lease or path_observations != 2 or field not in ("st_dev", "st_ino"): + return observed + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields[field] += 1 + return SimpleNamespace(**fields) + + def descriptor_stat(fd): + nonlocal descriptor_observations + observed = original_fstat(fd) + if (observed.st_dev, observed.st_ino) != (lease_stat.st_dev, lease_stat.st_ino): + return observed + descriptor_observations += 1 + observed = partial_identity(observed) + if descriptor_observations != 2 or field not in ("st_size", "st_mtime_ns", "st_ctime_ns"): + return observed + fields = {name: getattr(observed, name) for name in dir(observed) if name.startswith("st_")} + fields[field] += 1 + return SimpleNamespace(**fields) + + monkeypatch.setattr(Path, "lstat", path_stat) + monkeypatch.setattr(authority_source_capture, "os", SimpleNamespace(**{ + **vars(os), "name": platform, "fstat": descriptor_stat, + })) + if platform == "nt" and field == "st_ctime_ns" and unavailable_identity is None: + projection, snapshot = capture(state, runtime, registry, goal) + assert snapshot["lease_inventory"] == [{ + "name": lease.name, + "bytes_sha256": "sha256:" + hashlib.sha256(original).hexdigest(), + }] + else: + with pytest.raises(ShadowManagementError, match="source_changed_retry"): + capture(state, runtime, registry, goal) + assert descriptor_observations == 2 + assert path_observations == 2 diff --git a/tests/control_plane_ts/coordination_runtime_shadow.test.ts b/tests/control_plane_ts/coordination_runtime_shadow.test.ts index fc6b3941be..c59a4bd26c 100644 --- a/tests/control_plane_ts/coordination_runtime_shadow.test.ts +++ b/tests/control_plane_ts/coordination_runtime_shadow.test.ts @@ -1,5 +1,7 @@ import assert from "node:assert/strict"; -import { mkdir, readFile, readdir, rm, writeFile, unlink, symlink } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, readdir, rm, writeFile, unlink, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; import type { JsonObject } from "../../loopx/control_plane/effect_program.ts"; @@ -287,3 +289,98 @@ test("archiving a Todo drops its retained lease from the candidate head", async assert.deepEqual(leased.leases, []); assert.deepEqual(leased.todos, [todo("todo_one")]); }); + + +// An applied bootstrap rechecks each lease twice (admission plus the final +// pre-write recheck), so both passes must tolerate the injected skew. The +// ctime-kept fence branch (POSIX policy) is exercised through an incomplete +// volume identity, which is the same code path on every real platform; +// reverse platform spoofing on Windows leaks EPERM from unrelated owners. +for (const scenario of [ + {name: "Windows ctime-only skew with a full identity bootstraps", platform: "win32", skew: "ctime", zeroIdentity: false, expected: "applied"}, + {name: "an incomplete volume identity keeps ctime and fails closed", platform: "win32", skew: "ctime", zeroIdentity: true, expected: "source_changed_retry"}, + {name: "Windows mtime skew still rejects", platform: "win32", skew: "mtime", zeroIdentity: false, expected: "source_changed_retry"}, + {name: "Windows without skew is the applied control", platform: "win32", skew: "none", zeroIdentity: false, expected: "applied"}, +]) { + test(`lease recheck metadata policy: ${scenario.name}`, async t => { + const root = await mkdtemp(join(tmpdir(), "loopx-lease-ctime-")); + t.after(() => rm(root, { recursive: true, force: true })); + const statePath = join(root, "ACTIVE_GOAL_STATE.md"); + await writeFile(statePath, "---\ngoal_id: goal-a\nhandoff_mode: hard_lease\n---\n\n## Agent Todo\n\n"); + const leaseRecord = { schema_version: "task_lease_v0", goal_id: "goal-a", todo_id: "todo_one", + owner: "agent-a", status: "released", version: 1 }; + const leasePath = join(root, "goals", "goal-a", "task-leases", "todo_one.json"); + await mkdir(join(root, "goals", "goal-a", "task-leases"), { recursive: true }); + await writeFile(leasePath, JSON.stringify(leaseRecord)); + const head = projection([todo("todo_one", "done")], [leaseRecord]); + const request = await sourceRequest({ root, statePath, baseline: head } as unknown as ShadowFixture, head); + // Only the lease's stat observations and the platform predicate are + // injected in an isolated child; every byte, identity, lock and digest + // uses the production filesystem owner through a real bootstrap. + const script = ` + import * as fs from "node:fs/promises"; + import {join} from "node:path"; + import {mock} from "node:test"; + const [shadowUrl, schemasUrl, request, scenario] = JSON.parse(process.argv[1]); + const leasePath = join(request.runtime_root, "goals", request.goal_id, "task-leases", "todo_one.json"); + let lstatCalls = 0, statCalls = 0; + function copy(observed) { + return Object.assign(Object.create(Object.getPrototypeOf(observed)), observed); + } + const {default: ignored, ...namedExports} = fs; + mock.module("node:fs/promises", {namedExports: {...namedExports, + lstat: async (file, ...args) => { + const observed = await fs.lstat(file, ...args); + if (file !== leasePath) return observed; + lstatCalls++; + if (!scenario.zeroIdentity) return observed; + const skewed = copy(observed); + skewed.dev = 0; + return skewed; + }, + open: async (file, flags, ...args) => { + const handle = await fs.open(file, flags, ...args); + if (file !== leasePath) return handle; + const realStat = handle.stat.bind(handle); + handle.stat = async (...statArgs) => { + const observed = await realStat(...statArgs); + statCalls++; + const skewed = copy(observed); + if (scenario.zeroIdentity) skewed.dev = 0; + if (scenario.skew === "ctime") skewed.ctimeMs += 1; + if (scenario.skew === "mtime") skewed.mtimeMs += 1; + return skewed; + }; + return handle; + }, + }}); + const shadow = await import(shadowUrl); + const schemas = await import(schemasUrl); + Object.defineProperty(process, "platform", {value: scenario.platform}); + let status = null, reason_code = null; + try { + const result = await shadow.bootstrapCoordinationRuntimeShadow({...request, + schema_version: schemas.COORDINATION_RUNTIME_SHADOW_BOOTSTRAP_REQUEST_SCHEMA, + operation_id: "bootstrap:test:ctime-policy", source_version: "source:initial"}); + status = result.status; + reason_code = result.reason_code ?? null; + } catch (error) { status = "threw"; reason_code = String(error); } + console.log(JSON.stringify({status, reason_code, lstatCalls, statCalls})); + `; + const result = spawnSync(process.execPath, [ + "--no-warnings", "--experimental-test-module-mocks", "--experimental-strip-types", + "--experimental-sqlite", "--input-type=module", "-e", script, + JSON.stringify([ + new URL("../../loopx/control_plane/coordination/runtime_shadow.ts", import.meta.url).href, + new URL("../../loopx/control_plane/coordination/coordination_state_contract.generated.ts", import.meta.url).href, + request, scenario, + ]), + ], {encoding: "utf8", timeout: 60_000}); + assert.equal(result.status, 0, result.stderr); + const observed = JSON.parse(result.stdout.trim().split("\n").at(-1)); + assert.equal(observed.status === "failed" ? observed.reason_code : observed.status, scenario.expected); + const passes = scenario.expected === "applied" ? 2 : 1; + assert.equal(observed.lstatCalls, 2 * passes); + assert.equal(observed.statCalls, 2 * passes); + }); +}