diff --git a/apps/desktop/src/main/__tests__/archive-retention-candidates.test.ts b/apps/desktop/src/main/__tests__/archive-retention-candidates.test.ts new file mode 100644 index 0000000000..a9234bff9b --- /dev/null +++ b/apps/desktop/src/main/__tests__/archive-retention-candidates.test.ts @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { strict as assert } from 'node:assert'; +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { test } from 'node:test'; +import { + type SessionHeader, + type SessionSummary, + sessionRevisionFamilyId, +} from '@maka/core/session'; +import { createSqliteSessionMetadataStore } from '@maka/storage/sqlite-session-metadata-store'; +import { archivedTaskRows } from '../../renderer/features/session-navigation/testing.js'; + +/** + * Retention may delete only what Settings › Archived tasks shows. Both read + * one catalog: the page through the rail's projection of catalog rows, the + * Host through its candidate query. Agent Graph operators, which retire only + * with their root, are the one deliberate difference, and none is an + * archived-task row here because each has a live root. + */ +test('the retention candidates are exactly the families Archived tasks lists', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-retention-rows-')); + const path = join(root, 'state.sqlite'); + const store = createSqliteSessionMetadataStore(path, { now: () => 10 }); + try { + const create = (overrides: Partial) => store.create(header(overrides)); + await create({ id: 'root', isArchived: true }); + await create({ ...revisionOf('root'), id: 'root-revision', isArchived: true }); + await create({ id: 'active' }); + await create({ id: 'archived-parent', isArchived: true }); + await create({ id: 'child-of-archived', isArchived: true, subagentParent: parent('archived-parent') }); + await create({ id: 'child-of-active', isArchived: true, subagentParent: parent('active') }); + await create({ id: 'orphan', isArchived: true, subagentParent: parent('gone') }); + await create({ + id: 'operator', + isArchived: true, + subagentParent: { ...parent('root'), graph: { graphId: 'g', workId: 'w', operatorId: 'o' } }, + }); + await create({ id: 'preparing', isArchived: true }); + await create({ id: 'unprojected', isArchived: true }); + // A subtask whose only parent is a row the catalog does not list is an orphan on the page. + await create({ id: 'hidden-parent', isArchived: true }); + await create({ id: 'child-of-hidden', isArchived: true, subagentParent: parent('hidden-parent') }); + const database = new DatabaseSync(path); + try { + database + .prepare( + `UPDATE session_metadata + SET payload_json = json_set(payload_json, '$.conversationCopy', json(?)) + WHERE session_id IN ('preparing', 'hidden-parent')`, + ) + .run( + JSON.stringify({ + kind: 'branch', + sourceSessionId: 'active', + sourceTurnId: 'turn-1', + requestFingerprint: `sha256:${'a'.repeat(64)}`, + state: 'preparing', + }), + ); + database.prepare('DELETE FROM session_catalog_projection WHERE session_id = ?').run('unprojected'); + } finally { + database.close(); + } + + const catalog = (await store.listCatalogPage({}, undefined, 128)).records.map( + (record) => record.header as unknown as SessionSummary, + ); + const pageFamilies = new Set(archivedTaskRows(catalog).map(sessionRevisionFamilyId)); + const candidateFamilies = new Set( + (await store.listArchiveRetentionCandidates({ limit: 256 })).map((row) => { + assert.ok(!('undecodable' in row)); + return sessionRevisionFamilyId(row.header); + }), + ); + assert.deepEqual([...candidateFamilies].sort(), [...pageFamilies].sort()); + assert.deepEqual([...pageFamilies].sort(), [ + 'archived-parent', + 'child-of-hidden', + 'orphan', + 'root', + ]); + } finally { + store.close(); + await rm(root, { recursive: true, force: true }); + } +}); + +function parent(parentSessionId: string): NonNullable { + return { + kind: 'subagent', + parentSessionId, + spawnedBy: { parentRunId: 'run', parentTurnId: 'turn', toolCallId: 'call' }, + lifecycle: 'foreground', + }; +} + +function revisionOf(rootId: string): Partial { + return { + revisionRootSessionId: rootId, + revisionParentSessionId: rootId, + revisionOfTurnId: 'turn-1', + revisionIndex: 2, + revisionState: 'committed', + }; +} + +function header(overrides: Partial): SessionHeader { + return { + id: 'session', + workspaceRoot: '/workspace', + cwd: '/workspace/repo', + createdAt: 1, + lastMessageAt: 3, + name: 'Session', + titleIsManual: true, + isFlagged: false, + labels: [], + isArchived: false, + status: 'active', + statusUpdatedAt: 4, + hasUnread: false, + backend: 'ai-sdk', + llmConnectionSlug: 'openai', + connectionLocked: true, + model: 'gpt-5', + toolProfile: 'headless-coding-v1', + thinkingLevel: 'high', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'swarm', + schemaVersion: 1, + ...overrides, + }; +} diff --git a/apps/desktop/src/main/__tests__/archive-retention-section.test.ts b/apps/desktop/src/main/__tests__/archive-retention-section.test.ts new file mode 100644 index 0000000000..eba1e3800f --- /dev/null +++ b/apps/desktop/src/main/__tests__/archive-retention-section.test.ts @@ -0,0 +1,295 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { afterEach, test } from 'node:test'; +import { parseHTML } from 'linkedom'; +import { act, createElement } from 'react'; +import { createRoot } from 'react-dom/client'; +import { formatAbsoluteTimestamp } from '@maka/core/relative-time'; +import type { + StorageRetentionQueryResult, + StorageRetentionSetInput, +} from '@maka/runtime-host/protocol'; +import { AstryxLocaleProvider, LocaleProvider, ToastProvider } from '@maka/ui'; +import { + ArchiveRetentionSection, + StorageUsageServicesProvider, + type StorageUsageServices, +} from '../../renderer/features/storage-usage/index.js'; +import { + applyArchiveRetentionChange, + archiveRetentionConfirm, + getArchiveRetentionCopy, +} from '../../renderer/features/storage-usage/testing.js'; +import { RuntimeHostSettingsTarget } from '../../renderer/settings/runtime-host-settings-target.js'; + +const HOST = { profileId: 'profile-1', hostId: 'host-1' }; +const DAY = 24 * 60 * 60 * 1000; +const copy = getArchiveRetentionCopy('en'); + +const originalGlobals = { + document: globalThis.document, + window: globalThis.window, + HTMLElement: globalThis.HTMLElement, + getComputedStyle: globalThis.getComputedStyle, + IS_REACT_ACT_ENVIRONMENT: (globalThis as { IS_REACT_ACT_ENVIRONMENT?: boolean }) + .IS_REACT_ACT_ENVIRONMENT, +}; + +afterEach(() => { + Object.assign(globalThis, originalGlobals); +}); + +function services(retention: StorageRetentionQueryResult) { + const reads = { count: 0 }; + const writes: StorageRetentionSetInput[] = []; + const value: StorageUsageServices = { + loadUsage: async () => assert.fail('usage is not read here'), + loadSessionUsage: async () => ({}), + loadRetention: async (host) => { + assert.deepEqual(host, HOST); + reads.count += 1; + return retention; + }, + setRetention: async (host, input) => { + assert.deepEqual(host, HOST); + writes.push(input); + return { + kind: 'committed', + setting: { + revision: input.expectedRevision + 1, + enabled: input.enabled, + days: input.days, + ...(input.enabled ? { enabledAt: 1 } : {}), + }, + }; + }, + }; + return { value, reads, writes }; +} + +test('a change that starts the clock is confirmed with the Host count first', async () => { + const off = { revision: 2, enabled: false, days: 30, preview: { count: 4 } } as const; + const fake = services(off); + const asked: Array<{ count: number; days: number }> = []; + const answer = { value: false }; + const change = ( + current: StorageRetentionQueryResult, + enabled: boolean, + days: 30 | 60 | 90, + ) => + applyArchiveRetentionChange({ + services: fake.value, + host: HOST, + current, + next: { enabled, days }, + confirm: async (count, applied) => { + asked.push({ count, days: applied.days }); + return answer.value; + }, + }); + + // Declined: the count was read fresh from the Host, and nothing changed. + assert.deepEqual(await change(off, true, 60), { kind: 'cancelled' }); + assert.equal(fake.reads.count, 1); + assert.deepEqual(asked, [{ count: 4, days: 60 }]); + assert.deepEqual(fake.writes, []); + + answer.value = true; + const enabled = await change(off, true, 60); + assert.equal(enabled.kind, 'committed'); + assert.deepEqual(fake.writes, [{ expectedRevision: 2, enabled: true, days: 60 }]); + + // Turning it off deletes nothing, so it is neither previewed nor confirmed. + const on = { + revision: 3, + enabled: true, + days: 60, + enabledAt: 1, + preview: { count: 0 }, + } as const; + assert.equal((await change(on, false, 60)).kind, 'committed'); + assert.equal(asked.length, 2); + assert.deepEqual(fake.writes.at(-1), { expectedRevision: 3, enabled: false, days: 60 }); + + // The confirm states the count as a floor and an approximate earliest date: + // this Client's now plus the new days. + const confirm = archiveRetentionConfirm({ + copy, + locale: 'en', + current: off, + change: { enabled: true, days: 60 }, + count: 4, + now: 10 * DAY, + }); + assert.equal(confirm.title, copy.confirmEnableTitle); + assert.match(confirm.description ?? '', /4 archived tasks are subject to automatic cleanup now/); + assert.ok( + confirm.description?.includes(`before about ${formatAbsoluteTimestamp(70 * DAY, 'en')}`), + ); + assert.match(confirm.description ?? '', /Pinned tasks are kept\. Deleted tasks cannot be restored\./); + assert.equal( + archiveRetentionConfirm({ + copy, + locale: 'en', + current: on, + change: { enabled: true, days: 30 }, + count: 0, + now: 0, + }).title, + copy.confirmChangeTitle, + ); +}); + +test('a stale revision reports a conflict instead of committing', async () => { + const result = await applyArchiveRetentionChange({ + services: { + loadRetention: async () => assert.fail('disabling needs no preview'), + setRetention: async () => ({ kind: 'revision_conflict', expectedRevision: 1, actualRevision: 2 }), + }, + host: HOST, + current: { revision: 1, enabled: true, days: 30, enabledAt: 1 }, + next: { enabled: false, days: 30 }, + confirm: async () => assert.fail('disabling is not confirmed'), + }); + assert.deepEqual(result, { kind: 'conflict' }); +}); + +async function render(retention: StorageRetentionQueryResult) { + const { document, window } = parseHTML('
'); + // The days selector reads the pointer and motion media queries. + Object.assign(window, { + matchMedia: () => ({ + matches: false, + addEventListener() {}, + removeEventListener() {}, + addListener() {}, + removeListener() {}, + }), + }); + Object.assign(globalThis, { + document, + window, + HTMLElement: window.HTMLElement, + getComputedStyle: (element: Element) => + ({ color: (element as HTMLElement).style?.color || 'currentColor' }) as CSSStyleDeclaration, + IS_REACT_ACT_ENVIRONMENT: true, + }); + const fake = services(retention); + const container = document.getElementById('root') as unknown as HTMLElement; + const root = createRoot(container); + await act(async () => { + root.render( + createElement(LocaleProvider, { + locale: 'en', + children: createElement(AstryxLocaleProvider, { + children: createElement(ToastProvider, { + children: createElement(StorageUsageServicesProvider, { + services: fake.value, + children: createElement(RuntimeHostSettingsTarget, { + host: HOST, + label: 'Build box', + children: createElement(ArchiveRetentionSection), + }), + }), + }), + }), + }), + ); + }); + const text = container.textContent ?? ''; + await act(async () => root.unmount()); + return { text, fake }; +} + +test('the section states the rules, the preview, the last cleanup and what needs review', async () => { + const sweptAt = 40 * DAY; + const { text, fake } = await render({ + revision: 1, + enabled: true, + days: 30, + enabledAt: 1, + preview: { count: 5, eligibleAt: 31 * DAY }, + lastSweep: { at: sweptAt, deleted: 3, skippedBusy: 1, needsReview: 2, failed: 0 }, + lastDeletion: { at: sweptAt, count: 3, bytes: 3 * 1024 }, + }); + assert.equal(fake.reads.count, 1); + // The switch names its Host; the list below spans every Host. + assert.match(text, /Applies to the Runtime Host “Build box” only/); + assert.match(text, /applies to every archived task/); + assert.match(text, /The clock starts when you turn it on/); + assert.match(text, /Pinned tasks are kept\. Deletion is permanent\./); + assert.ok( + text.includes( + `5 archived tasks are subject to automatic cleanup; the first can be deleted after ${formatAbsoluteTimestamp(31 * DAY, 'en')}.`, + ), + ); + assert.ok( + text.includes( + `Last automatic cleanup: deleted 3 tasks (about 3.0 KB) on ${formatAbsoluteTimestamp(sweptAt, 'en')}`, + ), + ); + assert.match(text, /2 tasks need review/); + assert.doesNotMatch(text, /paused/); +}); + +test('the section shows a paused sweep and hides what it has not done', async () => { + const { text } = await render({ + revision: 1, + enabled: true, + days: 30, + enabledAt: 1, + preview: { count: 0 }, + lastSweep: { at: 5, deleted: 0, skippedBusy: 0, needsReview: 0, failed: 0, paused: true }, + }); + assert.match(text, /Automatic cleanup is paused/); + assert.doesNotMatch(text, /resumes after/); + assert.match(text, /No archived tasks would be deleted yet\./); + assert.doesNotMatch(text, /Last automatic cleanup/); + assert.doesNotMatch(text, /need review/); +}); + +test('the legacy Archived tasks page renders the section and makes no bridge calls of its own', () => { + const source = readFileSync( + new URL('../../../src/renderer/settings/tasks-settings-page.tsx', import.meta.url), + 'utf8', + ); + assert.match(source, //); + assert.doesNotMatch(source, /window\.maka|\bmaka\.storage\b|useState|useEffect/); +}); + +test('the section says when a held cleanup resumes and how far the clock moved', async () => { + const until = 50 * DAY; + const { text } = await render({ + revision: 1, + enabled: true, + days: 30, + enabledAt: 1, + preview: { count: 2, eligibleAt: 31 * DAY }, + hold: { since: 10 * DAY, detectedAt: 49 * DAY, until }, + }); + assert.ok( + text.includes( + `Automatic cleanup resumes after ${formatAbsoluteTimestamp(until, 'en')} because the system clock moved ahead by about 39 days since this Host last ran.`, + ), + ); + assert.match(text, /if it is wrong, turn cleanup off/); +}); diff --git a/apps/desktop/src/main/__tests__/storage-usage-section.test.ts b/apps/desktop/src/main/__tests__/storage-usage-section.test.ts index a62532fd98..5f13570c4d 100644 --- a/apps/desktop/src/main/__tests__/storage-usage-section.test.ts +++ b/apps/desktop/src/main/__tests__/storage-usage-section.test.ts @@ -46,6 +46,11 @@ afterEach(() => { Object.assign(globalThis, originalGlobals); }); +const UNUSED_RETENTION = { + loadRetention: async () => assert.fail('retention is not read here'), + setRetention: async () => assert.fail('retention is not changed here'), +}; + test('the Storage section reads the selected Host once and states its caveats', async () => { const { document, window } = parseHTML('
'); Object.assign(globalThis, { @@ -83,6 +88,7 @@ test('the Storage section reads the selected Host once and states its caveats', return usage; }, loadSessionUsage: async () => ({}), + ...UNUSED_RETENTION, }, children: createElement(RuntimeHostSettingsTarget, { host, @@ -142,6 +148,7 @@ test('a task row is measured only after it scrolls into view', async () => { }); const requested: string[][] = []; const services = { + ...UNUSED_RETENTION, loadUsage: async (): Promise => { throw new Error('not used'); }, diff --git a/apps/desktop/src/main/runtime-host-renderer-ipc-main.ts b/apps/desktop/src/main/runtime-host-renderer-ipc-main.ts index 54c7f104fe..8f741feed3 100644 --- a/apps/desktop/src/main/runtime-host-renderer-ipc-main.ts +++ b/apps/desktop/src/main/runtime-host-renderer-ipc-main.ts @@ -100,6 +100,10 @@ function request( return client.request(operation, HOST_OPERATION_SPECS[operation].decodeInput(value)); case 'scheduled-task.query': return client.request(operation, HOST_OPERATION_SPECS[operation].decodeInput(value)); + case 'storage.retention.query': + return client.request(operation, HOST_OPERATION_SPECS[operation].decodeInput(value)); + case 'storage.retention.set': + return client.request(operation, HOST_OPERATION_SPECS[operation].decodeInput(value)); case 'storage.usage.query': return client.request(operation, HOST_OPERATION_SPECS[operation].decodeInput(value)); case 'storage.usage.sessions.query': diff --git a/apps/desktop/src/preload/bridge-contract.d.ts b/apps/desktop/src/preload/bridge-contract.d.ts index 52c50ef981..74cd4d4488 100644 --- a/apps/desktop/src/preload/bridge-contract.d.ts +++ b/apps/desktop/src/preload/bridge-contract.d.ts @@ -158,6 +158,9 @@ import type { ContextDiagnosticsResult, SessionRemovePreviewResult, SessionStorageUsage, + StorageRetentionQueryResult, + StorageRetentionSetInput, + StorageRetentionSetResult, StorageUsageQueryResult, } from '@maka/runtime-host/protocol'; import type { TestProxyInput } from '@maka/core/settings/network-settings'; @@ -1846,6 +1849,13 @@ export interface MakaBridge { * Runtime Host is unavailable or fails, or when that Host no longer holds it. */ sessionUsage(sessionIds: readonly string[]): Promise>; + /** One Runtime Host's archived-task retention setting, its preview and its latest results. */ + retention(host?: DesktopRuntimeHostRef): Promise; + /** Changes that setting, fenced by the revision the caller read. The Host stamps the time. */ + setRetention( + input: StorageRetentionSetInput, + host?: DesktopRuntimeHostRef, + ): Promise; }; dailyReview: { day(offsetDays: number, daySpan?: number, host?: DesktopRuntimeHostRef): Promise>; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index eb4c9e8885..5b13b61377 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -259,6 +259,9 @@ import { type SessionTurnAccessRequest, type SessionRemovePreviewResult, type SessionStorageUsage, + type StorageRetentionQueryResult, + type StorageRetentionSetInput, + type StorageRetentionSetResult, type StorageUsageQueryResult, } from '@maka/runtime-host/protocol'; import type { PlanControlIpcResult } from '../shared/plan-mode-ipc.js'; @@ -3746,6 +3749,15 @@ const makaBridge = { sessionUsage(sessionIds: readonly string[]): Promise> { return loadDesktopSessionStorageUsage(sessionIds); }, + async retention(host?: DesktopRuntimeHostRef): Promise { + return scopedRuntimeHost(await selectedRuntimeHostScope(host)).query('storage.retention.query', {}); + }, + async setRetention( + input: StorageRetentionSetInput, + host?: DesktopRuntimeHostRef, + ): Promise { + return scopedRuntimeHost(await selectedRuntimeHostScope(host)).command('storage.retention.set', input); + }, }, dailyReview: { day(offsetDays: number, daySpan?: number, host?: DesktopRuntimeHostRef): Promise> { diff --git a/apps/desktop/src/preload/runtime-host-renderer-operations.ts b/apps/desktop/src/preload/runtime-host-renderer-operations.ts index d327cb205d..7d1a23ce4a 100644 --- a/apps/desktop/src/preload/runtime-host-renderer-operations.ts +++ b/apps/desktop/src/preload/runtime-host-renderer-operations.ts @@ -30,11 +30,16 @@ export const RENDERER_RUNTIME_HOST_QUERY_OPERATIONS = [ // Read-only State Root and per-task size measurement; it reclaims nothing. 'storage.usage.query', 'storage.usage.sessions.query', + // The archived-task retention setting, its preview and latest results. + 'storage.retention.query', ] as const satisfies readonly (keyof OperationSpecMap)[]; export const RENDERER_RUNTIME_HOST_COMMAND_OPERATIONS = [ 'daily-review.mutate', 'scheduled-task.mutate', + // The only writer of the retention setting. The Host stamps its time and + // fences it by revision; agent settings and config import never reach it. + 'storage.retention.set', 'web-search.execute', ] as const satisfies readonly (keyof OperationSpecMap)[]; diff --git a/apps/desktop/src/renderer/application/contracts/settings-presentation/runtime-host-settings-target.tsx b/apps/desktop/src/renderer/application/contracts/settings-presentation/runtime-host-settings-target.tsx index 78ad53e018..bf1633e87e 100644 --- a/apps/desktop/src/renderer/application/contracts/settings-presentation/runtime-host-settings-target.tsx +++ b/apps/desktop/src/renderer/application/contracts/settings-presentation/runtime-host-settings-target.tsx @@ -36,6 +36,8 @@ interface RuntimeHostSettingsTargetValue { * key to retire their own async work without remounting the Settings page. */ readonly generationKey: string; + /** The Host's profile name, for copy that must say which Host it means. */ + readonly label?: string; } const RuntimeHostSettingsTargetContext = @@ -44,6 +46,7 @@ const RuntimeHostSettingsTargetContext = export function RuntimeHostSettingsTarget(props: { readonly host?: SettingsHostTarget; readonly generation?: string; + readonly label?: string; readonly children: ReactNode; }) { const value = useMemo(() => { @@ -52,8 +55,9 @@ export function RuntimeHostSettingsTarget(props: { host: props.host, generationKey: `${props.host.profileId}:${props.host.hostId}@${props.generation ?? "unversioned"}`, + ...(props.label === undefined ? {} : { label: props.label }), }; - }, [props.generation, props.host]); + }, [props.generation, props.host, props.label]); return ( {props.children} @@ -71,6 +75,10 @@ export function useOptionalRuntimeHostSettingsTarget(): SettingsHostTarget | und return useContext(RuntimeHostSettingsTargetContext)?.host; } +export function useOptionalRuntimeHostSettingsLabel(): string | undefined { + return useContext(RuntimeHostSettingsTargetContext)?.label; +} + export function useOptionalRuntimeHostSettingsGenerationKey(): string | undefined { return useContext(RuntimeHostSettingsTargetContext)?.generationKey; } diff --git a/apps/desktop/src/renderer/features/storage-usage/index.ts b/apps/desktop/src/renderer/features/storage-usage/index.ts index f64555059a..757562520b 100644 --- a/apps/desktop/src/renderer/features/storage-usage/index.ts +++ b/apps/desktop/src/renderer/features/storage-usage/index.ts @@ -20,6 +20,7 @@ // Public API of the storage usage feature. Legacy Settings pages import only // from this barrel and render these surfaces; they hold no storage state. +export { ArchiveRetentionSection } from './ui/archive-retention-section.js'; export { StorageUsageSection } from './ui/storage-usage-section.js'; export { TaskStorageSize } from './ui/task-storage-size.js'; export { StorageUsageServicesProvider } from './services-context.js'; diff --git a/apps/desktop/src/renderer/features/storage-usage/locales/archive-retention-copy.ts b/apps/desktop/src/renderer/features/storage-usage/locales/archive-retention-copy.ts new file mode 100644 index 0000000000..fdd5dba2fa --- /dev/null +++ b/apps/desktop/src/renderer/features/storage-usage/locales/archive-retention-copy.ts @@ -0,0 +1,170 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import type { UiCatalog, UiLocale } from '@maka/core/ui-locale'; + +export interface ArchiveRetentionCopy { + readonly title: string; + /** What the setting covers, when its clock starts, what it keeps, and that it is final. */ + readonly help: string; + readonly enable: string; + /** Names the one Runtime Host the switch applies to; the list below spans every Host. */ + hostScope(hostName: string | undefined): string; + readonly days: string; + readonly daysHelp: string; + dayOption(days: number): string; + readonly loadFailed: string; + readonly saveFailed: string; + readonly conflict: string; + /** Nothing archived would be deleted. */ + readonly previewNone: string; + /** + * Neither bound: the count includes families a sweep keeps (busy, for + * review) and misses subtasks a deletion orphans into the archive later. + */ + preview(count: number, date: string): string; + lastCleanup(count: number, size: string | undefined, date: string): string; + needsReview(count: number): string; + readonly paused: string; + /** Deletions held after the clock moved ahead by about `days` days. */ + held(until: string, days: number): string; + readonly confirmEnableTitle: string; + readonly confirmChangeTitle: string; + /** `date` is the Client's own now plus `days`, so it is approximate. */ + confirmDescription(days: number, count: number, date: string): string; + readonly confirmEnable: string; + readonly confirmChange: string; + readonly cancel: string; +} + +const COPY_BY_LOCALE = { + 'zh-CN': { + title: '自动清理', + help: '开启后,此运行时主机会删除归档超过所选天数的任务。适用于所有已归档任务,无论是手动还是自动归档的。计时从开启或修改天数时开始,因此不会立即删除积压的任务。已置顶的任务会保留。删除后无法恢复。', + enable: '自动删除已归档任务', + hostScope: (hostName: string | undefined) => + hostName + ? `只作用于运行时主机「${hostName}」;下方列表包含所有主机的任务。` + : '只作用于当前运行时主机;下方列表包含所有主机的任务。', + days: '归档后保留', + daysHelp: '修改天数会重新开始计时。', + dayOption: (days: number) => `${days} 天`, + loadFailed: '无法读取自动清理设置', + saveFailed: '无法更改自动清理设置', + conflict: '设置已在别处更改,已重新读取。', + previewNone: '目前没有会被删除的已归档任务。', + preview: (count: number, date: string) => + `${count} 个已归档任务适用自动清理,最早在 ${date} 之后删除。`, + lastCleanup: (count: number, size: string | undefined, date: string) => + `上次自动清理:${date} 删除了 ${count} 个任务${size ? `(约 ${size})` : ''}`, + needsReview: (count: number) => + `${count} 个任务需要你处理:它们使用子代理工作树,或仍有进行中的子任务,请手动删除。`, + paused: '自动清理已暂停:本机时钟早于已记录的时间。时钟追上后会自动恢复。', + held: (until: string, days: number) => + `自动清理将在 ${until} 之后恢复:自此主机上次运行以来,系统时钟向前跳了约 ${days} 天。请检查系统时间;如果时间有误,请关闭自动清理。`, + confirmEnableTitle: '开启自动清理?', + confirmChangeTitle: '修改保留天数?', + confirmDescription: (days: number, count: number, date: string) => + `已归档任务将在归档 ${days} 天后删除,计时最早从现在开始。` + + (count > 0 + ? `目前有 ${count} 个已归档任务适用自动清理,最早约在 ${date} 之后删除。` + : '目前没有涵盖的任务。') + + '已置顶的任务会保留。删除后无法恢复。', + confirmEnable: '开启', + confirmChange: '修改', + cancel: '取消', + }, + 'zh-TW': { + title: '自動清理', + help: '開啟後,此執行階段主機會刪除歸檔超過所選天數的任務。適用於所有已歸檔任務,無論是手動或自動歸檔的。計時從開啟或修改天數時開始,因此不會立即刪除積壓的任務。已置頂的任務會保留。刪除後無法復原。', + enable: '自動刪除已歸檔任務', + hostScope: (hostName: string | undefined) => + hostName + ? `只作用於執行階段主機「${hostName}」;下方清單包含所有主機的任務。` + : '只作用於目前的執行階段主機;下方清單包含所有主機的任務。', + days: '歸檔後保留', + daysHelp: '修改天數會重新開始計時。', + dayOption: (days: number) => `${days} 天`, + loadFailed: '無法讀取自動清理設定', + saveFailed: '無法變更自動清理設定', + conflict: '設定已在別處變更,已重新讀取。', + previewNone: '目前沒有會被刪除的已歸檔任務。', + preview: (count: number, date: string) => + `${count} 個已歸檔任務適用自動清理,最早在 ${date} 之後刪除。`, + lastCleanup: (count: number, size: string | undefined, date: string) => + `上次自動清理:${date} 刪除了 ${count} 個任務${size ? `(約 ${size})` : ''}`, + needsReview: (count: number) => + `${count} 個任務需要你處理:它們使用子代理工作樹,或仍有進行中的子任務,請手動刪除。`, + paused: '自動清理已暫停:本機時鐘早於已記錄的時間。時鐘追上後會自動恢復。', + held: (until: string, days: number) => + `自動清理將在 ${until} 之後恢復:自此主機上次執行以來,系統時鐘向前跳了約 ${days} 天。請檢查系統時間;如果時間有誤,請關閉自動清理。`, + confirmEnableTitle: '開啟自動清理?', + confirmChangeTitle: '修改保留天數?', + confirmDescription: (days: number, count: number, date: string) => + `已歸檔任務將在歸檔 ${days} 天後刪除,計時最早從現在開始。` + + (count > 0 + ? `目前有 ${count} 個已歸檔任務適用自動清理,最早約在 ${date} 之後刪除。` + : '目前沒有涵蓋的任務。') + + '已置頂的任務會保留。刪除後無法復原。', + confirmEnable: '開啟', + confirmChange: '修改', + cancel: '取消', + }, + en: { + title: 'Automatic cleanup', + help: 'When on, this Runtime Host deletes tasks that have been archived for longer than the period you choose. It applies to every archived task, whether you archived it or it was archived automatically. The clock starts when you turn it on or change the period, so no backlog is deleted at once. Pinned tasks are kept. Deletion is permanent.', + enable: 'Delete archived tasks automatically', + hostScope: (hostName: string | undefined) => + hostName + ? `Applies to the Runtime Host “${hostName}” only. The list below shows tasks from every Host.` + : 'Applies to this Runtime Host only. The list below shows tasks from every Host.', + days: 'Keep archived tasks for', + daysHelp: 'Changing the period restarts the clock.', + dayOption: (days: number) => `${days} days`, + loadFailed: 'Could not load automatic cleanup', + saveFailed: 'Could not change automatic cleanup', + conflict: 'The setting changed elsewhere and has been reloaded.', + previewNone: 'No archived tasks would be deleted yet.', + preview: (count: number, date: string) => + `${count === 1 ? '1 archived task is' : `${count} archived tasks are`} subject to automatic cleanup; the first can be deleted after ${date}.`, + lastCleanup: (count: number, size: string | undefined, date: string) => + `Last automatic cleanup: deleted ${count === 1 ? '1 task' : `${count} tasks`}${size ? ` (about ${size})` : ''} on ${date}`, + needsReview: (count: number) => + `${count === 1 ? '1 task needs' : `${count} tasks need`} review: they use a subagent worktree or have active subtasks, so delete them by hand.`, + paused: + 'Automatic cleanup is paused: this computer’s clock reads earlier than a time already recorded. It resumes once the clock catches up.', + held: (until: string, days: number) => + `Automatic cleanup resumes after ${until} because the system clock moved ahead by about ${days === 1 ? '1 day' : `${days} days`} since this Host last ran. Check your system time; if it is wrong, turn cleanup off.`, + confirmEnableTitle: 'Turn on automatic cleanup?', + confirmChangeTitle: 'Change the period?', + confirmDescription: (days: number, count: number, date: string) => + `Archived tasks will be deleted ${days} days after they were archived, counting from now at the earliest. ` + + (count > 0 + ? `${count === 1 ? '1 archived task is' : `${count} archived tasks are`} subject to automatic cleanup now; none is deleted before about ${date}. ` + : 'No tasks are covered yet. ') + + 'Pinned tasks are kept. Deleted tasks cannot be restored.', + confirmEnable: 'Turn on', + confirmChange: 'Change', + cancel: 'Cancel', + }, +} satisfies UiCatalog; + +export function getArchiveRetentionCopy(locale: UiLocale): ArchiveRetentionCopy { + return COPY_BY_LOCALE[locale]; +} diff --git a/apps/desktop/src/renderer/features/storage-usage/model/archive-retention.ts b/apps/desktop/src/renderer/features/storage-usage/model/archive-retention.ts new file mode 100644 index 0000000000..96f3ce6103 --- /dev/null +++ b/apps/desktop/src/renderer/features/storage-usage/model/archive-retention.ts @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { ARCHIVE_RETENTION_DAY_MS, type ArchiveRetentionDays } from '@maka/core/archive-retention'; +import { formatAbsoluteTimestamp } from '@maka/core/relative-time'; +import type { UiLocale } from '@maka/core/ui-locale'; +import type { StorageRetentionSetting } from '@maka/runtime-host/protocol'; +import type { ConfirmInput } from '@maka/ui'; +import type { ArchiveRetentionCopy } from '../locales/archive-retention-copy.js'; +import type { StorageUsageHostTarget, StorageUsageServices } from '../ports.js'; + +export interface ArchiveRetentionChange { + readonly enabled: boolean; + readonly days: ArchiveRetentionDays; +} + +export type ArchiveRetentionChangeResult = + | { readonly kind: 'cancelled' } + | { readonly kind: 'committed'; readonly setting: StorageRetentionSetting } + /** The setting moved on the Host since it was read; read it again. */ + | { readonly kind: 'conflict' }; + +/** + * The confirm a change that starts the clock asks. The Host counts what it + * covers; the date is this Client's now plus the new days, since any change + * restarts every clock, so it is stated as approximate. + */ +export function archiveRetentionConfirm(input: { + readonly copy: ArchiveRetentionCopy; + readonly locale: UiLocale; + readonly current: StorageRetentionSetting; + readonly change: ArchiveRetentionChange; + readonly count: number; + readonly now: number; +}): ConfirmInput { + const { copy, change } = input; + return { + title: input.current.enabled ? copy.confirmChangeTitle : copy.confirmEnableTitle, + description: copy.confirmDescription( + change.days, + input.count, + formatAbsoluteTimestamp(input.now + change.days * ARCHIVE_RETENTION_DAY_MS, input.locale), + ), + confirmLabel: input.current.enabled ? copy.confirmChange : copy.confirmEnable, + cancelLabel: copy.cancel, + destructive: true, + }; +} + +/** + * Applies one change to a Host's retention setting. A change that starts the + * clock — enabling, or new days while enabled — is confirmed first with the + * Host's current count of what it would cover; turning the setting off, or + * picking days while it is off, deletes nothing and is applied directly. + */ +export async function applyArchiveRetentionChange(input: { + readonly services: Pick; + readonly host: StorageUsageHostTarget; + readonly current: StorageRetentionSetting; + readonly next: ArchiveRetentionChange; + readonly confirm: (count: number, change: ArchiveRetentionChange) => Promise; +}): Promise { + const { current, next } = input; + if (next.enabled) { + const { preview } = await input.services.loadRetention(input.host); + if (!(await input.confirm(preview.count, next))) return { kind: 'cancelled' }; + } + const result = await input.services.setRetention(input.host, { + expectedRevision: current.revision, + enabled: next.enabled, + days: next.days, + }); + return result.kind === 'committed' + ? { kind: 'committed', setting: result.setting } + : { kind: 'conflict' }; +} diff --git a/apps/desktop/src/renderer/features/storage-usage/ports.ts b/apps/desktop/src/renderer/features/storage-usage/ports.ts index dbc4c7235c..2b69724914 100644 --- a/apps/desktop/src/renderer/features/storage-usage/ports.ts +++ b/apps/desktop/src/renderer/features/storage-usage/ports.ts @@ -17,7 +17,13 @@ * under the License. */ -import type { SessionStorageUsage, StorageUsageQueryResult } from '@maka/runtime-host/protocol'; +import type { + SessionStorageUsage, + StorageRetentionQueryResult, + StorageRetentionSetInput, + StorageRetentionSetResult, + StorageUsageQueryResult, +} from '@maka/runtime-host/protocol'; /** The Runtime Host a Settings page is pointed at, stated structurally. */ export interface StorageUsageHostTarget { @@ -27,7 +33,8 @@ export interface StorageUsageHostTarget { /** * What the storage usage feature needs from the Desktop: two read-only - * measurements. Nothing here reclaims or deletes data. + * measurements, and one Host's archived-task retention setting. Nothing here + * deletes data itself; a Host with retention enabled does that on its own. */ export interface StorageUsageServices { /** One Runtime Host's State Root footprint. */ @@ -39,4 +46,11 @@ export interface StorageUsageServices { loadSessionUsage( sessionIds: readonly string[], ): Promise>>; + /** One Host's retention setting, its preview and its latest results. */ + loadRetention(host: StorageUsageHostTarget): Promise; + /** The Desktop's only way to change that setting. */ + setRetention( + host: StorageUsageHostTarget, + input: StorageRetentionSetInput, + ): Promise; } diff --git a/apps/desktop/src/renderer/features/storage-usage/testing.ts b/apps/desktop/src/renderer/features/storage-usage/testing.ts index 36f85638e0..684f176ba7 100644 --- a/apps/desktop/src/renderer/features/storage-usage/testing.ts +++ b/apps/desktop/src/renderer/features/storage-usage/testing.ts @@ -27,3 +27,5 @@ export { SESSION_STORAGE_FAILURE_COOLDOWN_MS, SESSION_STORAGE_RESULT_TTL_MS, } from './model/session-storage-loader.js'; +export { applyArchiveRetentionChange, archiveRetentionConfirm } from './model/archive-retention.js'; +export { getArchiveRetentionCopy } from './locales/archive-retention-copy.js'; diff --git a/apps/desktop/src/renderer/features/storage-usage/ui/archive-retention-section.tsx b/apps/desktop/src/renderer/features/storage-usage/ui/archive-retention-section.tsx new file mode 100644 index 0000000000..b392735fc7 --- /dev/null +++ b/apps/desktop/src/renderer/features/storage-usage/ui/archive-retention-section.tsx @@ -0,0 +1,235 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { useEffect, useState } from 'react'; +import { Banner } from '@astryxdesign/core/Banner'; +import { Selector } from '@astryxdesign/core/Selector'; +import { Text } from '@astryxdesign/core/Text'; +import { ARCHIVE_RETENTION_DAY_MS, ARCHIVE_RETENTION_DAYS } from '@maka/core/archive-retention'; +import { formatAbsoluteTimestamp } from '@maka/core/relative-time'; +import type { UiLocale } from '@maka/core/ui-locale'; +import type { StorageRetentionQueryResult } from '@maka/runtime-host/protocol'; +import { formatBytes, Switch, useMountedRef, useToast, useUiLocale } from '@maka/ui'; +import { + SettingsRow, + SettingsSection, + useOptionalRuntimeHostSettingsGenerationKey, + useOptionalRuntimeHostSettingsLabel, + useOptionalRuntimeHostSettingsTarget, +} from '../../../application/contracts/settings-presentation/index.js'; +import { + type ArchiveRetentionCopy, + getArchiveRetentionCopy, +} from '../locales/archive-retention-copy.js'; +import { + type ArchiveRetentionChange, + applyArchiveRetentionChange, + archiveRetentionConfirm, +} from '../model/archive-retention.js'; +import { useOptionalStorageUsageServices } from '../services-context.js'; + +type RetentionState = + | { readonly status: 'loading' } + | { readonly status: 'ready'; readonly retention: StorageRetentionQueryResult } + | { readonly status: 'failed' }; + +/** A reading belongs to one Host generation; another Host's setting is never shown. */ +interface ScopedRetention { + readonly hostKey: string | undefined; + readonly state: RetentionState; +} + +/** + * Settings › Archived tasks › Automatic cleanup: the selected Runtime Host's + * opt-in retention for archived tasks. The Host decides and deletes; this + * section shows the setting, the Host's preview and what the last sweep did, + * and changes the setting only after a confirm that states the preview. + */ +export function ArchiveRetentionSection() { + const host = useOptionalRuntimeHostSettingsTarget(); + const hostKey = useOptionalRuntimeHostSettingsGenerationKey(); + const hostName = useOptionalRuntimeHostSettingsLabel(); + const services = useOptionalStorageUsageServices(); + const toast = useToast(); + const locale = useUiLocale(); + const copy = getArchiveRetentionCopy(locale); + const mountedRef = useMountedRef(); + const [scoped, setScoped] = useState({ hostKey, state: { status: 'loading' } }); + const [attempt, setAttempt] = useState(0); + const [saving, setSaving] = useState(false); + + useEffect(() => { + if (!host || !services) return; + let current = true; + services.loadRetention(host).then( + (retention) => { + if (current) setScoped({ hostKey, state: { status: 'ready', retention } }); + }, + () => { + if (current) setScoped({ hostKey, state: { status: 'failed' } }); + }, + ); + return () => { + current = false; + }; + }, [attempt, host, hostKey, services]); + + if (!host || !services) return null; + const state: RetentionState = scoped.hostKey === hostKey ? scoped.state : { status: 'loading' }; + const retention = state.status === 'ready' ? state.retention : undefined; + + async function change(next: ArchiveRetentionChange) { + if (!host || !services || !retention) return; + setSaving(true); + try { + const result = await applyArchiveRetentionChange({ + services, + host, + current: retention, + next, + confirm: (count, applied) => + toast.confirm( + archiveRetentionConfirm({ + copy, + locale, + current: retention, + change: applied, + count, + now: Date.now(), + }), + ), + }); + if (result.kind === 'conflict') toast.warning(copy.conflict); + if (result.kind !== 'cancelled' && mountedRef.current) setAttempt((value) => value + 1); + } catch { + toast.error(copy.saveFailed); + } finally { + if (mountedRef.current) setSaving(false); + } + } + + return ( + + {state.status === 'failed' ? ( + + ) : null} + {retention?.lastSweep?.paused ? ( + + ) : null} + {retention?.hold ? ( + + ) : null} + + {copy.hostScope(hostName)} + {retention?.enabled ? ` ${retentionSummary(retention, copy, locale)}` : null} + + )} + align="start" + end={( + + retention ? change({ enabled, days: retention.days }) : undefined + } + /> + )} + /> + ({ + value: String(days), + label: copy.dayOption(days), + }))} + onChange={(value) => { + const days = ARCHIVE_RETENTION_DAYS.find((option) => String(option) === value); + if (retention && days !== undefined && days !== retention.days) { + void change({ enabled: retention.enabled, days }); + } + }} + /> + )} + /> + {retention ? : null} + + ); +} + +function retentionSummary( + retention: StorageRetentionQueryResult, + copy: ArchiveRetentionCopy, + locale: UiLocale, +): string { + const preview = retention.preview; + return preview.count === 0 || preview.eligibleAt === undefined + ? copy.previewNone + : copy.preview(preview.count, formatAbsoluteTimestamp(preview.eligibleAt, locale)); +} + +function RetentionResults(props: { + readonly retention: StorageRetentionQueryResult; + readonly copy: ArchiveRetentionCopy; + readonly locale: UiLocale; +}) { + const { lastDeletion, lastSweep } = props.retention; + const needsReview = lastSweep?.paused ? 0 : (lastSweep?.needsReview ?? 0); + if (!lastDeletion && needsReview === 0) return null; + return ( + <> + {lastDeletion ? ( + + {props.copy.lastCleanup( + lastDeletion.count, + lastDeletion.bytes === undefined ? undefined : formatBytes(lastDeletion.bytes, props.locale), + formatAbsoluteTimestamp(lastDeletion.at, props.locale), + )} + + ) : null} + {needsReview > 0 ? ( + + {props.copy.needsReview(needsReview)} + + ) : null} + + ); +} diff --git a/apps/desktop/src/renderer/platform/desktop/create-storage-usage-services.ts b/apps/desktop/src/renderer/platform/desktop/create-storage-usage-services.ts index d38bfff68b..356632da4d 100644 --- a/apps/desktop/src/renderer/platform/desktop/create-storage-usage-services.ts +++ b/apps/desktop/src/renderer/platform/desktop/create-storage-usage-services.ts @@ -31,5 +31,7 @@ export function createDesktopStorageUsageServices( // No host argument: each task is measured by the Host that holds it, and // the bridge routes by the projected id for exactly that reason. loadSessionUsage: (sessionIds) => bridge.storage.sessionUsage(sessionIds), + loadRetention: (host) => bridge.storage.retention(host), + setRetention: (host, input) => bridge.storage.setRetention(input, host), }; } diff --git a/apps/desktop/src/renderer/settings/settings-nav.ts b/apps/desktop/src/renderer/settings/settings-nav.ts index 47b830b333..cdca681fb1 100644 --- a/apps/desktop/src/renderer/settings/settings-nav.ts +++ b/apps/desktop/src/renderer/settings/settings-nav.ts @@ -113,7 +113,8 @@ const SETTINGS_SECTION_SCOPES: Record< 'bot-chat': 'client', search: 'runtime-host', usage: 'runtime-host', - 'archived-tasks': 'client', + // The list spans every Host; automatic cleanup is the selected Host's. + 'archived-tasks': 'mixed', 'import-tasks': 'runtime-host', 'daily-review': 'runtime-host', data: 'mixed', diff --git a/apps/desktop/src/renderer/settings/settings-surface.tsx b/apps/desktop/src/renderer/settings/settings-surface.tsx index 3071bb5210..cca509e9ff 100644 --- a/apps/desktop/src/renderer/settings/settings-surface.tsx +++ b/apps/desktop/src/renderer/settings/settings-surface.tsx @@ -1043,6 +1043,7 @@ function SettingsSurfaceContent( ? `${selectedRuntimeHost.profileId}:${selectedRuntimeHost.hostId}` : 'client'} host={selectedRuntimeHost} + label={selectedRuntimeHostEntry?.profile.name} generation={selectedProfileId ? runtimeHostLifecycleByProfile.get(selectedProfileId)?.epoch : undefined} diff --git a/apps/desktop/src/renderer/settings/tasks-settings-page.tsx b/apps/desktop/src/renderer/settings/tasks-settings-page.tsx index a2c868b53a..0b5cf43581 100644 --- a/apps/desktop/src/renderer/settings/tasks-settings-page.tsx +++ b/apps/desktop/src/renderer/settings/tasks-settings-page.tsx @@ -31,7 +31,7 @@ import type { DesktopSessionSummary } from '../../preload/bridge-contract.js'; import type { SessionCatalogController } from '../application/contracts/session-catalog/session-catalog-state.js'; import { getSettingsTasksCopy } from '../locales/settings-tasks-copy.js'; import { getStorageUsageCopy } from '../locales/storage-usage-copy.js'; -import { TaskStorageSize } from '../features/storage-usage/index.js'; +import { ArchiveRetentionSection, TaskStorageSize } from '../features/storage-usage/index.js'; import { SettingsPage, SettingsSection } from './settings-section'; import { isOrphanedSubagentTask } from './task-catalog-rows'; @@ -89,6 +89,7 @@ export function TasksSettingsPage( if (props.scope.rows.length === 0) { return ( + ); @@ -96,6 +97,8 @@ export function TasksSettingsPage( return ( + {/* The selected Host's automatic cleanup; that Host decides and deletes. */} + {props.scope.controls} {visible.length === 0 ? ( diff --git a/docs/astryx-surface-file-inventory.md b/docs/astryx-surface-file-inventory.md index 7a699d4889..2241d8c3e3 100644 --- a/docs/astryx-surface-file-inventory.md +++ b/docs/astryx-surface-file-inventory.md @@ -6,7 +6,7 @@ Generated against `@astryxdesign/core@0.6.3` (195 component exports). Wiki bar: Design Conventions · API Use-the-System · Theming · Container Padding. -**Totals:** 328 files — blocker 0, reimplementation 0, polish 4, aligned 324. +**Totals:** 329 files — blocker 0, reimplementation 0, polish 4, aligned 325. ## Exclusions (explicit) @@ -124,6 +124,7 @@ Wiki bar: Design Conventions · API Use-the-System · Theming · Container Paddi | `apps/desktop/src/renderer/features/session-settings/services-context.tsx` | shell-chrome-or-panel | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `apps/desktop/src/renderer/features/session-settings/ui/session-settings-provider.tsx` | shell-chrome-or-panel | none | aligned — no raw controls; no Astryx JSX usage | aligned | | `apps/desktop/src/renderer/features/storage-usage/services-context.tsx` | other | none | aligned — no raw controls; no Astryx JSX usage | aligned | +| `apps/desktop/src/renderer/features/storage-usage/ui/archive-retention-section.tsx` | other | Banner, Selector, Switch, Text | aligned — uses Astryx (Banner, Selector, Switch, Text) | aligned | | `apps/desktop/src/renderer/features/storage-usage/ui/storage-usage-section.tsx` | other | Banner, Button | aligned — uses Astryx (Banner, Button) | aligned | | `apps/desktop/src/renderer/features/storage-usage/ui/task-storage-size.tsx` | other | Text | aligned — uses Astryx (Text) | aligned | | `apps/desktop/src/renderer/features/task-entry/services-context.tsx` | other | none | aligned — no raw controls; no Astryx JSX usage | aligned | diff --git a/docs/astryx-surface-file-inventory.paths b/docs/astryx-surface-file-inventory.paths index 7baaba7b86..73ff2f73de 100644 --- a/docs/astryx-surface-file-inventory.paths +++ b/docs/astryx-surface-file-inventory.paths @@ -94,6 +94,7 @@ apps/desktop/src/renderer/features/session-navigation/ui/session-navigation-prov apps/desktop/src/renderer/features/session-settings/services-context.tsx apps/desktop/src/renderer/features/session-settings/ui/session-settings-provider.tsx apps/desktop/src/renderer/features/storage-usage/services-context.tsx +apps/desktop/src/renderer/features/storage-usage/ui/archive-retention-section.tsx apps/desktop/src/renderer/features/storage-usage/ui/storage-usage-section.tsx apps/desktop/src/renderer/features/storage-usage/ui/task-storage-size.tsx apps/desktop/src/renderer/features/task-entry/services-context.tsx diff --git a/packages/core/package.json b/packages/core/package.json index ab6f92bdba..33ba1939d9 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -144,6 +144,7 @@ "./project": "./dist/project.js", "./pty-output-view": "./dist/pty-output-view.js", "./relative-time": "./dist/relative-time.js", + "./archive-retention": "./dist/archive-retention.js", "./run-composition": "./dist/run-composition.js", "./runtime-event-store": "./dist/runtime-event-store.js", "./side-conversation": "./dist/side-conversation.js", diff --git a/packages/core/src/archive-retention.ts b/packages/core/src/archive-retention.ts new file mode 100644 index 0000000000..fa4738225b --- /dev/null +++ b/packages/core/src/archive-retention.ts @@ -0,0 +1,184 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** + * Opt-in retention for archived tasks (#5899): when a Runtime Host enables it, + * the Host deletes archived tasks once they have been archived for longer than + * the chosen number of days. These are the rules both the Host and its Clients + * read; the Host alone decides and deletes. + */ + +/** The retention periods a Host offers. */ +export const ARCHIVE_RETENTION_DAYS = [30, 60, 90] as const; + +export type ArchiveRetentionDays = (typeof ARCHIVE_RETENTION_DAYS)[number]; + +export const ARCHIVE_RETENTION_DAY_MS = 24 * 60 * 60 * 1000; + +export function isArchiveRetentionDays(value: unknown): value is ArchiveRetentionDays { + return ARCHIVE_RETENTION_DAYS.some((days) => days === value); +} + +/** + * When a task's retention clock starts: when it was archived, but never before + * the policy was enabled. Enabling therefore never deletes a backlog at once, + * and a task archived before the Host recorded the time counts from enablement. + */ +export function archiveRetentionClockStart( + archivedAt: number | undefined, + enabledAt: number, +): number { + return Math.max(archivedAt ?? enabledAt, enabledAt); +} + +/** A task whose clock started at `start` is eligible once the clock passes this instant. */ +export function archiveRetentionDeadline(start: number, days: ArchiveRetentionDays): number { + return start + days * ARCHIVE_RETENTION_DAY_MS; +} + +/** + * A forward gap in the wall clock larger than this, between two times the Host + * observed, holds deletions: the smaller of the retention window and 7 days. + * A wrong clock set far ahead would otherwise make a whole backlog eligible at + * once, and so would a long time offline, which a hold costs only a day. + */ +export function archiveRetentionGapThreshold(days: ArchiveRetentionDays): number { + return Math.min(days, 7) * ARCHIVE_RETENTION_DAY_MS; +} + +/** How long a forward gap holds deletions after it is observed. */ +export const ARCHIVE_RETENTION_HOLD_MS = 24 * 60 * 60 * 1000; + +/** Deletions held after the wall clock moved ahead further than a sweep expects. */ +export interface ArchiveRetentionHold { + /** The last Host time observed before the gap. */ + readonly since: number; + /** When the gap was observed. */ + readonly detectedAt: number; + /** Deletions resume once the Host clock reaches this. */ + readonly until: number; +} + +/** The latest automatic sweep, as the Host last recorded it. */ +export interface ArchiveRetentionSweep { + /** Host clock, epoch milliseconds. */ + readonly at: number; + /** Tasks (revision families) deleted. */ + readonly deleted: number; + /** Tasks kept because something was still running in them. */ + readonly skippedBusy: number; + /** + * Tasks kept because deleting them would also reclaim a subagent worktree or + * archive a still-active subtask; they stay for manual cleanup. + */ + readonly needsReview: number; + /** Tasks whose deletion failed. */ + readonly failed: number; + /** Present when the sweep paused because the clock moved backwards. */ + readonly paused?: true; +} + +/** The latest sweep that deleted anything. */ +export interface ArchiveRetentionDeletion { + readonly at: number; + readonly count: number; + /** Measured before deletion; an estimate, and absent when it could not be measured. */ + readonly bytes?: number; +} + +/** + * The one decoder of the results the Host records and reports, shared by the + * State Root document and the protocol. `fail` builds the caller's own error. + */ +export function decodeArchiveRetentionSweep( + value: unknown, + fail: (message: string) => Error, +): ArchiveRetentionSweep { + const sweep = exactRecord( + value, + 'retention sweep', + ['at', 'deleted', 'skippedBusy', 'needsReview', 'failed'], + ['paused'], + fail, + ); + if (sweep.paused !== undefined && sweep.paused !== true) { + throw fail('Invalid retention sweep paused'); + } + return { + at: count(sweep.at, 'retention sweep at', fail), + deleted: count(sweep.deleted, 'retention sweep deleted', fail), + skippedBusy: count(sweep.skippedBusy, 'retention sweep skippedBusy', fail), + needsReview: count(sweep.needsReview, 'retention sweep needsReview', fail), + failed: count(sweep.failed, 'retention sweep failed', fail), + ...(sweep.paused === true ? { paused: true as const } : {}), + }; +} + +export function decodeArchiveRetentionDeletion( + value: unknown, + fail: (message: string) => Error, +): ArchiveRetentionDeletion { + const deletion = exactRecord(value, 'retention deletion', ['at', 'count'], ['bytes'], fail); + return { + at: count(deletion.at, 'retention deletion at', fail), + count: count(deletion.count, 'retention deletion count', fail), + ...(deletion.bytes === undefined + ? {} + : { bytes: count(deletion.bytes, 'retention deletion bytes', fail) }), + }; +} + +export function decodeArchiveRetentionHold( + value: unknown, + fail: (message: string) => Error, +): ArchiveRetentionHold { + const hold = exactRecord(value, 'retention hold', ['since', 'detectedAt', 'until'], [], fail); + const since = count(hold.since, 'retention hold since', fail); + const detectedAt = count(hold.detectedAt, 'retention hold detectedAt', fail); + const until = count(hold.until, 'retention hold until', fail); + if (since > detectedAt || detectedAt > until) throw fail('Invalid retention hold order'); + return { since, detectedAt, until }; +} + +function exactRecord( + value: unknown, + label: string, + required: readonly string[], + optional: readonly string[], + fail: (message: string) => Error, +): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw fail(`Invalid ${label}`); + } + const record = value as Record; + if (Object.keys(record).some((key) => !required.includes(key) && !optional.includes(key))) { + throw fail(`Unknown ${label} field`); + } + if (required.some((key) => !Object.hasOwn(record, key))) { + throw fail(`Invalid ${label} fields`); + } + return record; +} + +function count(value: unknown, label: string, fail: (message: string) => Error): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw fail(`Invalid ${label}`); + } + return value; +} diff --git a/packages/runtime-host/src/__tests__/archive-retention-coordinator.test.ts b/packages/runtime-host/src/__tests__/archive-retention-coordinator.test.ts new file mode 100644 index 0000000000..aa65d82d96 --- /dev/null +++ b/packages/runtime-host/src/__tests__/archive-retention-coordinator.test.ts @@ -0,0 +1,733 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import type { SessionHeader } from '@maka/core/session'; +import type { + ArchiveRetentionDocument, + ArchiveRetentionDocumentRead, +} from '@maka/storage/archive-retention-store'; +import type { + ArchiveRetentionCandidateRow, + SessionCatalogRecord, +} from '@maka/storage/execution-stores'; +import type { ConnectionContext } from '../server/operation-dispatcher.js'; +import { + ARCHIVE_RETENTION_FAMILIES_PER_TICK, + HostArchiveRetentionCoordinator, +} from '../server/archive-retention-coordinator.js'; +import type { + RetentionHoldReason, + RetentionRemovalOutcome, + RetentionRemovalPlan, +} from '../server/session-retirement-coordinator.js'; + +const DAY = 24 * 60 * 60 * 1000; +const CONTEXT: ConnectionContext = { + hostEpoch: 'retention-test', + connectionId: 'retention-test-connection', + principal: 'local_os_user', + acquireResidency: () => ({ release() {} }), +}; + +interface Task { + readonly id: string; + family?: string; + archivedAt?: number; + isFlagged?: boolean; + /** False once restored. */ + isArchived?: boolean; + /** What the removal path answers once the guard admits the task. */ + outcome?: RetentionRemovalOutcome; + /** Metadata that no longer decodes. */ + undecodable?: boolean; +} + +function rig(options: { read?: ArchiveRetentionDocumentRead; tasks?: Task[] } = {}) { + let clock = 1_000 * DAY; + let read: ArchiveRetentionDocumentRead = options.read ?? { kind: 'absent' }; + const tasks = new Map((options.tasks ?? []).map((task) => [task.id, task])); + const calls = { list: 0, latest: 0 }; + const writes: ArchiveRetentionDocument[] = []; + const removed: string[] = []; + const held: RetentionHoldReason[] = []; + let newest: number | undefined; + /** Runs inside a removal admission, before the guard: a mid-sweep change. */ + let beforeGuard: (() => Promise) | undefined; + /** The last guard the removal path ran, with the plan it ran on. */ + let lastGuard: { run: () => Promise } | undefined; + const header = (task: Task): SessionHeader => + ({ + id: task.id, + isArchived: task.isArchived ?? true, + isFlagged: task.isFlagged ?? false, + ...(task.family ? { revisionRootSessionId: task.family } : {}), + }) as SessionHeader; + const order = (task: { archivedAt?: number }) => task.archivedAt ?? -1; + const create = () => + new HostArchiveRetentionCoordinator({ + document: { + read: async () => read, + write: async (document) => { + writes.push(document); + read = { kind: 'valid', document }; + }, + }, + catalog: { + listArchiveRetentionCandidates: async (query) => { + calls.list += 1; + return [...tasks.values()] + .filter((task) => !task.isFlagged) + .filter( + (task) => + query.archivedBefore === undefined || + task.archivedAt === undefined || + task.archivedAt < query.archivedBefore, + ) + .sort((a, b) => order(a) - order(b) || a.id.localeCompare(b.id)) + .filter( + (task) => + !query.after || + order(task) > order(query.after) || + (order(task) === order(query.after) && task.id > query.after.sessionId), + ) + .slice(0, query.limit) + .map((task): ArchiveRetentionCandidateRow => { + const position = task.archivedAt === undefined ? {} : { archivedAt: task.archivedAt }; + return task.undecodable + ? { undecodable: true, sessionId: task.id, ...position } + : { header: header(task), revision: 1, committedAt: 0, ...position }; + }); + }, + countArchiveRetentionCandidates: async (enabledAt) => { + const starts = new Map(); + for (const task of tasks.values()) { + if (task.isFlagged) continue; + const start = Math.max(task.archivedAt ?? enabledAt, enabledAt); + const family = task.family ?? task.id; + starts.set(family, Math.max(starts.get(family) ?? start, start)); + } + return starts.size === 0 + ? { families: 0 } + : { families: starts.size, firstStart: Math.min(...starts.values()) }; + }, + readCatalogRecord: async (sessionId) => + ({ + summary: { archivedAt: tasks.get(sessionId)?.archivedAt }, + }) as unknown as SessionCatalogRecord, + readLatestSessionMetadataTime: async () => { + calls.latest += 1; + return newest; + }, + }, + retirement: { + removeForRetention: async (target, guard) => { + const task = tasks.get(target.sessionId)!; + await beforeGuard?.(); + const plan: RetentionRemovalPlan = { + remove: [{ header: header(task), revision: 1, committedAt: 0 }], + archiveSessionIds: [], + worktreeCount: 0, + }; + lastGuard = { run: () => guard(plan) }; + const reason = await guard(plan); + if (reason) { + held.push(reason); + return { kind: 'held', reason }; + } + const outcome = task.outcome ?? { kind: 'removed', bytes: 10 }; + if (outcome.kind === 'removed') { + removed.push(task.id); + tasks.delete(task.id); + } + return outcome; + }, + }, + now: () => clock, + log: () => undefined, + }); + let retention = create(); + const query = async () => { + const result = await retention.handlers['storage.retention.query']({}, CONTEXT); + assert.ok(result.ok); + return result.result; + }; + const set = async (enabled: boolean, days: 30 | 60 | 90, expectedRevision?: number) => { + const result = await retention.handlers['storage.retention.set']( + { expectedRevision: expectedRevision ?? (await query()).revision, enabled, days }, + CONTEXT, + ); + assert.ok(result.ok); + return result.result; + }; + /** + * Moves the clock as a Host that keeps running sees it: a sweep at least + * every six days on the way, so no step reads as a forward clock jump. + */ + const advanceTo = async (to: number) => { + while (to - clock > 6 * DAY) { + clock += 6 * DAY; + await retention.sweep(); + } + clock = to; + }; + return { + get retention() { + return retention; + }, + /** A new Host process over the same State Root and catalog. */ + restart() { + retention = create(); + }, + advanceTo, + advance: (ms: number) => advanceTo(clock + ms), + calls, + writes, + removed, + held, + tasks, + query, + set, + get now() { + return clock; + }, + set now(value: number) { + clock = value; + }, + set newest(value: number | undefined) { + newest = value; + }, + set beforeGuard(value: (() => Promise) | undefined) { + beforeGuard = value; + }, + get lastGuard() { + return lastGuard; + }, + }; +} + +test('enabling or changing the days restamps enabledAt on the Host clock; disabling clears it', async () => { + const r = rig(); + assert.deepEqual(await r.query(), { + revision: 0, + enabled: false, + days: 30, + preview: { count: 0 }, + }); + + const enabled = await r.set(true, 30); + assert.deepEqual(enabled, { + kind: 'committed', + setting: { revision: 1, enabled: true, days: 30, enabledAt: r.now }, + }); + + const enabledAt = r.now; + await r.advance(10 * DAY); + const changed = await r.set(true, 60); + assert.deepEqual(changed, { + kind: 'committed', + setting: { revision: 2, enabled: true, days: 60, enabledAt: enabledAt + 10 * DAY }, + }); + + // Setting what is already set is no change and keeps the clock. + r.now += DAY; + const writesAfterChange = r.writes.length; + assert.deepEqual(await r.set(true, 60), changed); + assert.equal(r.writes.length, writesAfterChange); + + const disabled = await r.set(false, 60); + assert.deepEqual(disabled, { + kind: 'committed', + setting: { revision: 3, enabled: false, days: 60 }, + }); + assert.equal(r.writes.at(-1)?.enabledAt, undefined); +}); + +test('a stale revision is rejected without writing', async () => { + const r = rig(); + await r.set(true, 30); + const stale = await r.set(false, 30, 0); + assert.deepEqual(stale, { kind: 'revision_conflict', expectedRevision: 0, actualRevision: 1 }); + assert.equal(r.writes.length, 1); + assert.equal((await r.query()).enabled, true); +}); + +test('no candidate is read and nothing is eligible until the policy is older than its days', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + const enabledAt = r.now; + + const before = { ...r.calls }; + await r.advanceTo(enabledAt + 30 * DAY); + assert.equal(await r.retention.sweep(), false); + assert.equal(r.calls.list, before.list); + + r.now = enabledAt + 30 * DAY + 1; + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.removed, ['legacy']); +}); + +test('changing the days restarts the clock, so nothing becomes eligible at once', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 60); + await r.advance(29 * DAY); + // Shortening to 30 days is not immediate: the clock restarts at the change, + // so two days later nothing is 30 days old. + await r.set(true, 30); + const restartedAt = r.now; + r.now = restartedAt + 2 * DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + await r.advanceTo(restartedAt + 30 * DAY + 1); + await r.retention.sweep(); + assert.deepEqual(r.removed, ['legacy']); +}); + +test('a sweep deletes at most eight families a tick and reports when work remains', async () => { + const tasks: Task[] = []; + for (let index = 0; index < 20; index += 1) { + const id = `task-${String(index).padStart(2, '0')}`; + tasks.push({ id, archivedAt: index }); + // A revision of the first task: one family, tried once. + if (index === 0) tasks.push({ id: 'task-00-revision', family: id, archivedAt: index }); + } + const r = rig({ tasks }); + await r.set(true, 30); + await r.advance(31 * DAY); + + assert.equal(await r.retention.sweep(), true); + assert.equal(r.removed.length, ARCHIVE_RETENTION_FAMILIES_PER_TICK); + assert.equal(await r.retention.sweep(), true); + assert.equal(r.removed.length, 16); + assert.equal(await r.retention.sweep(), false); + // The revision shared its family with task-00 and was not tried on its own. + assert.deepEqual(r.removed.length, 20); + assert.ok(!r.removed.includes('task-00-revision')); +}); + +test('a setting change pending at the admission keeps the task and waits for the tick', async () => { + const r = rig({ tasks: [{ id: 'legacy' }, { id: 'later' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + let changed: Promise | undefined; + r.beforeGuard = async () => { + r.beforeGuard = undefined; + // Started, not awaited: it waits for this tick, which must not delete. + changed = r.retention.handlers['storage.retention.set']( + { expectedRevision: 1, enabled: true, days: 60 }, + CONTEXT, + ); + }; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + assert.deepEqual(r.held, ['ineligible']); + // The tick stopped before the next family. + assert.equal(r.tasks.has('later'), true); + assert.deepEqual(await changed, { + ok: true, + result: { + kind: 'committed', + setting: { revision: 2, enabled: true, days: 60, enabledAt: r.now }, + }, + }); +}); + +test('a guard run after the setting moved on holds the task by its revision alone', async () => { + const r = rig({ tasks: [{ id: 'legacy', outcome: { kind: 'skipped' } }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + await r.retention.sweep(); + const guard = r.lastGuard!; + // Same days, so only the revision tells the two settings apart. + await r.set(false, 30); + await r.set(true, 30); + await r.advance(31 * DAY); + assert.equal(await guard.run(), 'ineligible'); +}); + +test('a clock that goes back between the candidate read and the admission keeps the task', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + r.beforeGuard = async () => { + // Still past every deadline, but behind a time this sweep already saw. + r.now -= 1; + }; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + assert.deepEqual(r.held, ['ineligible']); +}); + +test('draining stops a sweep before the next family', async () => { + const r = rig({ tasks: [{ id: 'a' }, { id: 'b' }, { id: 'c' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + r.beforeGuard = async () => { + r.beforeGuard = undefined; + r.retention.beginDrain(); + }; + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.removed, ['a']); + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.removed, ['a']); +}); + +test('a row that no longer decodes is counted as failed and passed over', async () => { + const r = rig({ + tasks: [ + { id: 'a-bad', archivedAt: 1, undecodable: true }, + { id: 'b-good', archivedAt: 2 }, + ], + }); + await r.set(true, 30); + await r.advance(31 * DAY); + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.removed, ['b-good']); + assert.equal((await r.query()).lastSweep?.failed, 1); +}); + +test('a setting change clears a recorded pause and never backdates enabledAt', async () => { + const r = rig({ tasks: [] }); + await r.set(true, 30); + await r.advance(40 * DAY); + await r.retention.sweep(); + const observed = r.now; + r.now = observed - 10; + await r.retention.sweep(); + assert.equal((await r.query()).lastSweep?.paused, true); + + // Behind what the Host saw: the new clock starts where the Host already was. + const changed = await r.set(true, 60); + assert.equal(changed.kind === 'committed' && changed.setting.enabledAt, observed); + assert.equal((await r.query()).lastSweep?.paused, undefined); + + // Behind the newest metadata time: the clock starts there. + r.newest = observed + 500; + const enabled = await r.set(true, 90); + assert.equal(enabled.kind === 'committed' && enabled.setting.enabledAt, observed + 500); +}); + +test('a restore, re-archive or pin between the candidate read and the admission keeps the task', async () => { + const r = rig({ tasks: [{ id: 'pinned' }, { id: 'rearchived' }, { id: 'restored' }] }); + await r.set(true, 30); + const enabledAt = r.now; + await r.advance(31 * DAY); + r.beforeGuard = async () => { + r.tasks.get('pinned')!.isFlagged = true; + // Archived again exactly the period ago: not longer than it. + r.tasks.get('rearchived')!.archivedAt = enabledAt + DAY; + r.tasks.get('restored')!.isArchived = false; + }; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + assert.deepEqual(r.held, ['ineligible', 'ineligible', 'ineligible']); +}); + +test('a wall clock behind the high-water mark pauses the sweep once, with no deletions', async () => { + const r = rig({ tasks: [] }); + await r.set(true, 30); + await r.advance(40 * DAY); + await r.retention.sweep(); + const observed = r.now; + r.tasks.set('legacy', { id: 'legacy' }); + + r.now = observed - 1; + const lists = r.calls.list; + assert.equal(await r.retention.sweep(), false); + assert.equal(r.calls.list, lists); + assert.deepEqual(r.removed, []); + assert.deepEqual((await r.query()).lastSweep, { + at: observed - 1, + deleted: 0, + skippedBusy: 0, + needsReview: 0, + failed: 0, + paused: true, + }); + const writes = r.writes.length; + await r.retention.sweep(); + assert.equal(r.writes.length, writes, 'a clock that stays behind writes nothing further'); + + // Once the clock catches up, the sweep resumes and clears the pause. + r.now = observed + 1; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['legacy']); + assert.equal((await r.query()).lastSweep?.paused, undefined); +}); + +test('a wall clock behind the newest recorded metadata time pauses the sweep', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + r.newest = r.now + 1; + assert.equal(await r.retention.sweep(), false); + assert.equal(r.calls.list, 0); + assert.deepEqual(r.removed, []); + assert.equal((await r.query()).lastSweep?.paused, true); +}); + +test('a document that cannot be validated is treated as disabled until it is set again', async () => { + const r = rig({ + read: { kind: 'invalid', reason: 'unsupported version' }, + tasks: [{ id: 'legacy' }], + }); + assert.deepEqual(await r.query(), { + revision: 0, + enabled: false, + days: 30, + preview: { count: 1 }, + }); + await r.advance(365 * DAY); + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.calls, { list: 0, latest: 0 }); + assert.deepEqual(r.removed, []); + assert.equal((await r.set(true, 30)).kind, 'committed'); +}); + +test('the document records a daily heartbeat and sweep results without duplicate writes', async () => { + const r = rig({ tasks: [] }); + await r.set(true, 30); + await r.advance(31 * DAY); + const writes = r.writes.length; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 1, 'an empty sweep records the heartbeat'); + assert.equal(r.writes.at(-1)?.latest?.observedAt, r.now); + assert.equal((await r.query()).lastSweep, undefined); + assert.equal((await r.query()).lastDeletion, undefined); + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 1, 'an immediate repeat writes nothing'); + + r.tasks.set('busy', { id: 'busy', outcome: { kind: 'busy' } }); + r.now += DAY; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 2); + const lastSweep = (await r.query()).lastSweep; + assert.deepEqual((await r.query()).lastSweep, { + at: r.now, + deleted: 0, + skippedBusy: 1, + needsReview: 0, + failed: 0, + }); + r.now += DAY; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 3, 'the next heartbeat is written once'); + assert.equal(r.writes.at(-1)?.latest?.observedAt, r.now); + assert.deepEqual( + (await r.query()).lastSweep, + lastSweep, + 'unchanged results keep their timestamp', + ); + + r.tasks.set('legacy', { id: 'legacy', outcome: { kind: 'removed', bytes: 40 } }); + r.tasks.set('unmeasured', { id: 'unmeasured', outcome: { kind: 'removed' } }); + r.now += DAY; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 4); + const result = await r.query(); + assert.equal(result.lastSweep?.deleted, 2); + // One deletion could not be measured, so no estimate is claimed. + assert.deepEqual(result.lastDeletion, { at: r.now, count: 2 }); +}); + +test('the preview counts current candidates by family and says when the first is eligible', async () => { + const r = rig({ + tasks: [ + { id: 'legacy' }, + { id: 'revision', family: 'legacy' }, + { id: 'early', archivedAt: 1_000 * DAY - 100 * DAY }, + { id: 'late', archivedAt: 1_000 * DAY + 5 * DAY }, + ], + }); + const now = r.now; + // Disabled: a count, never a date. + assert.deepEqual((await r.query()).preview, { count: 3 }); + + await r.set(true, 30); + assert.deepEqual((await r.query()).preview, { count: 3, eligibleAt: now + 30 * DAY }); + r.tasks.clear(); + assert.deepEqual((await r.query()).preview, { count: 0 }); +}); + +test('a forward clock jump holds deletions for a day, then they resume', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(29 * DAY); + await r.retention.sweep(); + const before = r.now; + // The clock leaps ahead past every deadline at once. + r.now += 10 * DAY; + assert.equal(await r.retention.sweep(), false); + assert.deepEqual(r.removed, []); + assert.deepEqual((await r.query()).hold, { + since: before, + detectedAt: r.now, + until: r.now + DAY, + }); + + const until = r.now + DAY; + r.now = until - 1; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + r.now = until; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['legacy']); + assert.equal((await r.query()).hold, undefined); +}); + +test('a further jump while held arms the hold again', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(29 * DAY); + r.now += 10 * DAY; + await r.retention.sweep(); + const first = r.now; + r.now += 8 * DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, []); + assert.deepEqual((await r.query()).hold, { + since: first, + detectedAt: r.now, + until: r.now + DAY, + }); +}); + +test('an advance within the threshold never holds', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(25 * DAY); + await r.retention.sweep(); + // Exactly the threshold is not more than it. + r.now += 7 * DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['legacy']); + assert.equal((await r.query()).hold, undefined); +}); + +test('a Host back after twenty days offline holds for a day and then cleans up', async () => { + const r = rig({ tasks: [{ id: 'first' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + await r.retention.sweep(); + assert.deepEqual(r.removed, ['first']); + const lastRan = r.now; + r.tasks.set('second', { id: 'second' }); + + r.restart(); + r.now = lastRan + 20 * DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['first']); + assert.equal((await r.query()).hold?.since, lastRan); + r.now += DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['first', 'second']); +}); + +test('after a restart, recent metadata writes say the Host was running', async () => { + const r = rig({ tasks: [{ id: 'first' }] }); + await r.set(true, 30); + await r.advance(31 * DAY); + await r.retention.sweep(); + const lastRan = r.now; + r.tasks.set('second', { id: 'second' }); + + r.restart(); + r.now = lastRan + 20 * DAY; + r.newest = r.now - DAY; + await r.retention.sweep(); + assert.deepEqual(r.removed, ['first', 'second']); + assert.equal((await r.query()).hold, undefined); +}); + +test('a running Host records a heartbeat so an idle restart does not hold', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(6 * DAY); + await r.retention.sweep(); + const heartbeat = r.writes.at(-1)?.latest?.observedAt; + assert.equal(heartbeat, r.now); + + r.restart(); + r.now += 2 * DAY; + await r.retention.sweep(); + assert.equal((await r.query()).hold, undefined); +}); + +test('a setting change clears a hold', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(29 * DAY); + r.now += 10 * DAY; + await r.retention.sweep(); + assert.ok((await r.query()).hold); + await r.set(true, 60); + assert.equal((await r.query()).hold, undefined); +}); + +test('a restart before the deadline, with recent metadata, does not hold', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(9 * DAY); + await r.retention.sweep(); + r.restart(); + r.now += 60 * 60 * 1000; + r.newest = r.now - 60 * 1000; + await r.retention.sweep(); + assert.equal((await r.query()).hold, undefined); +}); + +test('a fresh process still holds after a genuine forward jump', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + const enabledAt = r.now; + await r.advance(2 * DAY); + await r.retention.sweep(); + r.restart(); + r.newest = enabledAt + 2 * DAY; + r.now = enabledAt + 12 * DAY; + await r.retention.sweep(); + assert.deepEqual((await r.query()).hold, { + since: enabledAt + 2 * DAY, + detectedAt: r.now, + until: r.now + DAY, + }); +}); + +test('a hold expires before the deadline: cleared once and never reported after its day', async () => { + const r = rig({ tasks: [{ id: 'legacy' }] }); + await r.set(true, 30); + await r.advance(DAY); + await r.retention.sweep(); + r.now += 8 * DAY; + await r.retention.sweep(); + const hold = (await r.query()).hold; + assert.ok(hold); + r.now = hold.until; + // Reported as over even before a sweep writes it away. + assert.equal((await r.query()).hold, undefined); + const writes = r.writes.length; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 1); + assert.equal(r.writes.at(-1)?.latest?.hold, undefined); + r.now += 1; + await r.retention.sweep(); + assert.equal(r.writes.length, writes + 1, 'cleared once'); + assert.deepEqual(r.removed, [], 'still before the deadline'); +}); diff --git a/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts index ac9272b7ce..98153a796e 100644 --- a/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts @@ -46,6 +46,9 @@ import type { ConnectionContext } from '../server/operation-dispatcher.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; import { MemoryExtractionSessionLane } from '../server/memory-extraction-session-lane.js'; import { HostSessionRetirementCoordinator } from '../server/session-retirement-coordinator.js'; +import { HostArchiveRetentionCoordinator } from '../server/archive-retention-coordinator.js'; +import type { StorageRetentionQueryResult } from '../protocol/index.js'; +import type { ArchiveRetentionDocument } from '@maka/storage/archive-retention-store'; import { purgeSessionSidecars } from '../server/session-sidecar-purge.js'; import { waitFor as pollFor } from '@maka/core/test-only/async-primitives'; @@ -1386,6 +1389,372 @@ describe('Host Session retirement coordinator', () => { }); }); +describe('archive retention through the removal path', () => { + test("a task's clock starts no earlier than enablement and must run longer than the period", async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + const legacy = await rig.archivedTask('Legacy'); + const early = await rig.archivedTask('Archived before enablement'); + const late = await rig.archivedTask('Archived after enablement'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([legacy], null); + rig.setArchivedAt([early], enabledAt - 10 * DAY); + rig.setArchivedAt([late], enabledAt + 5 * DAY); + + // Exactly the period after enablement is not "longer than" it. + await rig.advanceTo(enabledAt + 30 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([legacy, early, late]), [legacy, early, late]); + + // An unknown or earlier archive time counts from enablement. + await rig.advanceTo(enabledAt + 30 * DAY + 1); + const first = await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([legacy, early, late]), [late]); + assert.equal(first.lastSweep?.deleted, 2); + assert.deepEqual(first.lastDeletion, { + at: enabledAt + 30 * DAY + 1, + count: 2, + bytes: 2 * MEASURED_SESSION_TOTAL, + }); + + // A later archive time counts from itself, with the same strict boundary. + await rig.advanceTo(enabledAt + 35 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([late]), [late]); + await rig.advanceTo(enabledAt + 35 * DAY + 1); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([late]), []); + }); + }); + }); + + test('restoring a task cancels its deadline and archiving it again starts a fresh one', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + const task = await rig.archivedTask('Restored'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([task], enabledAt); + + await rig.advanceTo(enabledAt + 31 * DAY); + await rig.setLifecycle(task, 'active'); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), [task]); + + await rig.setLifecycle(task, 'archived'); + rig.setArchivedAt([task], enabledAt + 31 * DAY); + await rig.advanceTo(enabledAt + 61 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), [task]); + await rig.advanceTo(enabledAt + 61 * DAY + 1); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), []); + }); + }); + }); + + test('a pinned revision keeps its whole family', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + await rig.setLifecycle(harness.rootId, 'archived'); + await harness.store.setFlagged(harness.revisionId, true); + const enabledAt = await rig.enable(30); + rig.setArchivedAt(harness.familyIds, null); + + await rig.advanceTo(enabledAt + 31 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present(harness.familyIds), [...harness.familyIds]); + + await harness.store.setFlagged(harness.revisionId, false); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present(harness.familyIds), []); + }); + }); + }); + + test('a family that would archive an active subtask or reclaim a worktree needs review', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + // An active subtask the deletion would move to the archive. + const parent = await rig.archivedTask('Parent of an active subtask'); + const active = await createClosedSubagent(harness, parent, 1); + // An orphaned archived subtask whose deletion would reclaim its worktree. + const owner = (await harness.store.create(sessionInput('Worktree owner'))).id; + const orphan = await createClosedSubagent(harness, owner, 2, worktreeBinding('f')); + await rig.setLifecycle(orphan, 'archived'); + const removed = await harness.coordinator.handlers['session.remove']( + { + sessionId: owner, + expectedRevision: (await harness.store.readHeaderRecordSnapshot(owner)).revision, + }, + CONNECTION_CONTEXT, + ); + assert.equal(removed.ok, true); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([parent, orphan], null); + + await rig.advanceTo(enabledAt + 31 * DAY); + const result = await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([parent, active, orphan]), [parent, active, orphan]); + assert.equal(result.lastSweep?.needsReview, 2); + assert.equal(result.lastSweep?.deleted, 0); + assert.equal(result.lastDeletion, undefined); + assert.deepEqual(harness.actions.retiredWorktrees, []); + }); + }); + }); + + test('a family is as young as its most recently archived member', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + await rig.setLifecycle(harness.rootId, 'archived'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([harness.rootId], null); + rig.setArchivedAt([harness.revisionId], enabledAt + 5 * DAY); + + // The root alone is due; its revision is not, so neither is deleted. + await rig.advanceTo(enabledAt + 31 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present(harness.familyIds), [...harness.familyIds]); + await rig.advanceTo(enabledAt + 35 * DAY + 1); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present(harness.familyIds), []); + }); + }); + }); + + test('a revision pinned after the candidate read keeps its whole family at the admission', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + await rig.setLifecycle(harness.rootId, 'archived'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt(harness.familyIds, null); + rig.afterCandidates(() => harness.store.setFlagged(harness.revisionId, true)); + + await rig.advanceTo(enabledAt + 31 * DAY); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present(harness.familyIds), [...harness.familyIds]); + }); + }); + }); + + test('a task already removed by hand is not counted as this sweep deletion', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + const task = await rig.archivedTask('Removed by hand'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([task], null); + rig.afterCandidates(async () => { + const removed = await harness.coordinator.handlers['session.remove']( + { + sessionId: task, + expectedRevision: (await harness.store.readHeaderRecordSnapshot(task)).revision, + }, + CONNECTION_CONTEXT, + ); + assert.equal(removed.ok, true); + }); + + await rig.advanceTo(enabledAt + 31 * DAY); + const result = await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), []); + assert.equal(result.lastDeletion, undefined); + assert.equal(result.lastSweep, undefined); + }); + }); + }); + + test('turning retention off waits for the family in flight, which is recorded', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + const first = await rig.archivedTask('In flight'); + const second = await rig.archivedTask('Not yet admitted'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([first], enabledAt - 2 * DAY); + rig.setArchivedAt([second], enabledAt - DAY); + let settled = false; + let atAnswer: { present: string[]; deleted?: number } | undefined; + let disabling: Promise | undefined; + // After the guard admitted `first`, before its tombstone is written. + harness.disposeBackend = async (sessionId) => { + harness.disposeBackend = undefined; + disabling = rig.disable().then(async (answer) => { + settled = true; + atAnswer = { + present: await rig.present([first]), + deleted: (await rig.query()).lastDeletion?.count, + }; + return answer; + }); + for (let turn = 0; turn < 20; turn += 1) await Promise.resolve(); + assert.equal(settled, false, 'the change waits for the family in flight'); + harness.actions.disposed.push(sessionId); + }; + + await rig.advanceTo(enabledAt + 31 * DAY); + await rig.sweepUntilIdle(); + await disabling; + // When the change answered, the admitted family was gone and recorded; + // nothing was admitted after it. + assert.deepEqual(atAnswer, { present: [], deleted: 1 }); + assert.deepEqual(await rig.present([first, second]), [second]); + const result = await rig.query(); + assert.equal(result.enabled, false); + assert.equal(result.lastDeletion?.count, 1); + }); + }); + }); + + test('a busy task is skipped as busy and deleted by a later sweep', async () => { + await withHarness(async (harness) => { + await withRetention(harness, async (rig) => { + const task = await rig.archivedTask('Busy'); + const enabledAt = await rig.enable(30); + rig.setArchivedAt([task], null); + harness.blockers.message.add(task); + + await rig.advanceTo(enabledAt + 31 * DAY); + const busy = await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), [task]); + assert.equal(busy.lastSweep?.skippedBusy, 1); + + harness.blockers.message.delete(task); + await rig.sweepUntilIdle(); + assert.deepEqual(await rig.present([task]), []); + }); + }); + }); +}); + +const DAY = 24 * 60 * 60 * 1000; + +interface RetentionRig { + archivedTask(name: string): Promise; + setLifecycle(sessionId: string, state: 'active' | 'archived'): Promise; + /** Overrides the recorded archive time; null is a task archived before it was recorded. */ + setArchivedAt(sessionIds: readonly string[], archivedAt: number | null): void; + /** Enables the policy at the harness clock and returns `enabledAt`. */ + enable(days: 30 | 60 | 90): Promise; + /** Disables the policy, not waiting for anything first. */ + disable(): Promise; + query(): Promise; + /** Runs once, right after the next candidate read and before any admission. */ + afterCandidates(hook: () => Promise): void; + sweepUntilIdle(): Promise; + /** + * Moves the Host clock as a running Host sees it, sweeping at least every + * six days on the way so no step reads as a forward clock jump. + */ + advanceTo(time: number): Promise; + /** The given Sessions that still exist, in order. */ + present(sessionIds: readonly string[]): Promise; +} + +async function withRetention( + harness: RetirementHarness, + operation: (rig: RetentionRig) => Promise, +): Promise { + // Far enough ahead of the real clock that metadata writes stamped by it never + // read as a clock that went back. + harness.now = Date.now() + 1_000 * DAY; + let document: ArchiveRetentionDocument | undefined; + let afterCandidates: (() => Promise) | undefined; + const retention = new HostArchiveRetentionCoordinator({ + document: { + read: async () => (document ? { kind: 'valid', document } : { kind: 'absent' }), + write: async (next) => { + document = next; + }, + }, + catalog: { + listArchiveRetentionCandidates: async (query) => { + const rows = await harness.store.listArchiveRetentionCandidates(query); + const hook = afterCandidates; + afterCandidates = undefined; + await hook?.(); + return rows; + }, + countArchiveRetentionCandidates: (enabledAt) => + harness.store.countArchiveRetentionCandidates(enabledAt), + readLatestSessionMetadataTime: () => harness.store.readLatestSessionMetadataTime(), + readCatalogRecord: (sessionId) => harness.store.readCatalogRecord(sessionId), + }, + retirement: harness.coordinator, + now: () => harness.now ?? Date.now(), + log: () => undefined, + }); + const query = async () => { + const result = await retention.handlers['storage.retention.query']({}, CONNECTION_CONTEXT); + assert.ok(result.ok); + return result.result; + }; + const rig: RetentionRig = { + archivedTask: async (name) => { + const { id } = await harness.store.create(sessionInput(name)); + await rig.setLifecycle(id, 'archived'); + return id; + }, + setLifecycle: async (sessionId, state) => { + const result = await harness.coordinator.handlers['session.lifecycle.set']( + { sessionId, state }, + CONNECTION_CONTEXT, + ); + assert.equal(result.ok, true); + }, + setArchivedAt: (sessionIds, archivedAt) => { + const database = new DatabaseSync( + join(harness.workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), + ); + try { + const update = database.prepare( + 'UPDATE session_metadata SET archived_at = ? WHERE session_id = ?', + ); + for (const sessionId of sessionIds) update.run(archivedAt, sessionId); + } finally { + database.close(); + } + }, + enable: async (days) => { + const set = await retention.handlers['storage.retention.set']( + { expectedRevision: (await query()).revision, enabled: true, days }, + CONNECTION_CONTEXT, + ); + assert.ok(set.ok && set.result.kind === 'committed'); + assert.equal(set.result.setting.enabledAt, harness.now); + return set.result.setting.enabledAt!; + }, + disable: async () => + retention.handlers['storage.retention.set']( + { expectedRevision: document?.revision ?? 0, enabled: false, days: document?.days ?? 30 }, + CONNECTION_CONTEXT, + ), + query, + afterCandidates: (hook) => { + afterCandidates = hook; + }, + advanceTo: async (time) => { + while (time - (harness.now ?? Date.now()) > 6 * DAY) { + harness.now = (harness.now ?? Date.now()) + 6 * DAY; + await retention.sweep(); + } + harness.now = time; + }, + sweepUntilIdle: async () => { + for (let tick = 0; await retention.sweep(); tick += 1) assert.ok(tick < 100); + return query(); + }, + present: async (sessionIds) => { + const present: string[] = []; + for (const sessionId of sessionIds) { + if ((await harness.store.probeSessionRemoval(sessionId)).kind === 'present') { + present.push(sessionId); + } + } + return present; + }, + }; + await operation(rig); +} + /** What the fake footprint reports for any Session: 124 bytes in all. */ const MEASURED_SESSION_BYTES = { transcript: 100, runtime: 20, artifacts: 3, context: 1 }; const MEASURED_SESSION_TOTAL = 124; diff --git a/packages/runtime-host/src/__tests__/storage-maintenance.test.ts b/packages/runtime-host/src/__tests__/storage-maintenance.test.ts index 24a3a901de..a9f6054525 100644 --- a/packages/runtime-host/src/__tests__/storage-maintenance.test.ts +++ b/packages/runtime-host/src/__tests__/storage-maintenance.test.ts @@ -185,3 +185,42 @@ test('failed paths do not pin the pagination cursor, and another sweep retries t assert.deepEqual(cursors, [undefined, 'failed-path', undefined]); await maintenance.close(); }); + +test('the retention lane sweeps a second apart while work remains and every quarter hour otherwise', async (t) => { + t.mock.timers.enable({ apis: ['setTimeout'] }); + const remaining = [true, true, false]; + let sweeps = 0; + const maintenance = new HostStorageMaintenance({ + artifacts: { + reclaimUpgradeResidue: async () => ({ nextAfter: null, processedPaths: 0, failedPaths: 0 }), + }, + retention: { + sweep: async () => { + sweeps += 1; + return remaining.shift() ?? false; + }, + }, + onError: assert.fail, + }); + maintenance.start(); + t.mock.timers.tick(999); + await settle(); + assert.equal(sweeps, 0, 'the first sweep waits a second after Ready'); + t.mock.timers.tick(1); + await settle(); + assert.equal(sweeps, 1); + t.mock.timers.tick(1_000); + await settle(); + assert.equal(sweeps, 2); + t.mock.timers.tick(1_000); + await settle(); + assert.equal(sweeps, 3); + // Nothing left: the next sweep is fifteen minutes away, not the other lanes' minute. + t.mock.timers.tick(15 * 60_000 - 1); + await settle(); + assert.equal(sweeps, 3); + t.mock.timers.tick(1); + await settle(); + assert.equal(sweeps, 4); + await maintenance.close(); +}); diff --git a/packages/runtime-host/src/__tests__/storage-retention-protocol.test.ts b/packages/runtime-host/src/__tests__/storage-retention-protocol.test.ts new file mode 100644 index 0000000000..28b0a7291e --- /dev/null +++ b/packages/runtime-host/src/__tests__/storage-retention-protocol.test.ts @@ -0,0 +1,182 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import { RuntimeHostProtocolError } from '../protocol/errors.js'; +import { + decodeHostFrame, + HOST_OPERATION_SPECS, + REMOTE_OWNER_OPERATION_GRANTS, + type StorageRetentionQueryResult, + type StorageRetentionSetResult, +} from '../protocol/index.js'; + +const querySpec = HOST_OPERATION_SPECS['storage.retention.query']; +const setSpec = HOST_OPERATION_SPECS['storage.retention.set']; + +const queried: StorageRetentionQueryResult = { + revision: 3, + enabled: true, + days: 60, + enabledAt: 1_700_000_000_000, + preview: { count: 4, eligibleAt: 1_705_184_000_000 }, + lastSweep: { at: 1_706_000_000_000, deleted: 2, skippedBusy: 1, needsReview: 1, failed: 0 }, + lastDeletion: { at: 1_706_000_000_000, count: 2, bytes: 4096 }, + hold: { since: 1_705_000_000_000, detectedAt: 1_706_000_000_000, until: 1_706_086_400_000 }, +}; + +function minimal() { + return { revision: 0, enabled: false, days: 30 } as const; +} + +function rejects(decode: () => unknown): void { + assert.throws(decode, RuntimeHostProtocolError); +} + +describe('storage retention protocol', () => { + test('round-trips the query and set shapes', () => { + assert.deepEqual(querySpec.decodeInput({}), {}); + assert.deepEqual(querySpec.decodeOutput(JSON.parse(JSON.stringify(queried))), queried); + // Disabled: a count, never a date. + const minimal = { revision: 0, enabled: false, days: 30, preview: { count: 3 } }; + assert.deepEqual(querySpec.decodeOutput(minimal), minimal); + const paused = { + ...minimal, + lastSweep: { at: 1, deleted: 0, skippedBusy: 0, needsReview: 0, failed: 0, paused: true }, + preview: { count: 0 }, + }; + const enabledEmpty = { ...queried, preview: { count: 0 } }; + assert.deepEqual(querySpec.decodeOutput(enabledEmpty), enabledEmpty); + assert.deepEqual(querySpec.decodeOutput(paused), paused); + + const input = { expectedRevision: 3, enabled: true, days: 30 }; + assert.deepEqual(setSpec.decodeInput(input), input); + const committed: StorageRetentionSetResult = { + kind: 'committed', + setting: { revision: 4, enabled: true, days: 30, enabledAt: 5 }, + }; + assert.deepEqual(setSpec.decodeOutput(committed), committed); + const conflict: StorageRetentionSetResult = { + kind: 'revision_conflict', + expectedRevision: 3, + actualRevision: 4, + }; + assert.deepEqual(setSpec.decodeOutput(conflict), conflict); + assert.deepEqual( + decodeHostFrame({ + requestId: 'request-set', + operation: 'storage.retention.set', + ok: true, + result: committed, + }), + { requestId: 'request-set', operation: 'storage.retention.set', ok: true, result: committed }, + ); + }); + + test('rejects malformed inputs and results', () => { + rejects(() => querySpec.decodeInput({ previewDays: 30 })); + rejects(() => querySpec.decodeInput({ days: 30 })); + rejects(() => setSpec.decodeInput({ expectedRevision: 0, enabled: true })); + rejects(() => setSpec.decodeInput({ expectedRevision: -1, enabled: true, days: 30 })); + rejects(() => setSpec.decodeInput({ expectedRevision: 0, enabled: 'yes', days: 30 })); + // The client never stamps the time: the Host does. + rejects(() => + setSpec.decodeInput({ expectedRevision: 0, enabled: true, days: 30, enabledAt: 1 }), + ); + + // enabledAt is present exactly while enabled. + rejects(() => + querySpec.decodeOutput({ revision: 1, enabled: true, days: 30, preview: { count: 0 } }), + ); + rejects(() => + querySpec.decodeOutput({ + revision: 1, + enabled: false, + days: 30, + enabledAt: 1, + preview: { count: 0 }, + }), + ); + // The preview is always there. + const { preview: _preview, ...withoutPreview } = queried; + rejects(() => querySpec.decodeOutput(withoutPreview)); + rejects(() => querySpec.decodeOutput({ ...queried, days: 7 })); + rejects(() => querySpec.decodeOutput({ ...queried, unknown: true })); + // An enabled preview with tasks says when; one without tasks, or a disabled one, does not. + rejects(() => querySpec.decodeOutput({ ...queried, preview: { count: 2 } })); + rejects(() => querySpec.decodeOutput({ ...queried, preview: { count: 0, eligibleAt: 1 } })); + rejects(() => querySpec.decodeOutput({ ...minimal(), preview: { count: 2, eligibleAt: 1 } })); + rejects(() => + querySpec.decodeOutput({ ...queried, lastSweep: { ...queried.lastSweep, paused: false } }), + ); + rejects(() => + querySpec.decodeOutput({ ...queried, lastDeletion: { at: 1, count: 1, bytes: -1 } }), + ); + // A hold resumes after it was detected, which is after the time it measures from. + rejects(() => + querySpec.decodeOutput({ ...queried, hold: { since: 3, detectedAt: 2, until: 4 } }), + ); + rejects(() => querySpec.decodeOutput({ ...queried, hold: { since: 1, until: 4 } })); + rejects(() => setSpec.decodeOutput({ kind: 'committed' })); + rejects(() => setSpec.decodeOutput({ kind: 'stale' })); + rejects(() => + setSpec.decodeOutput({ + kind: 'revision_conflict', + expectedRevision: 1, + actualRevision: 2, + setting: {}, + }), + ); + }); + + test('a set result must answer the request it was given', () => { + const input = { expectedRevision: 3, enabled: true, days: 60 } as const; + assert.throws( + () => + setSpec.assertOutputForInput!(input, { + kind: 'committed', + setting: { revision: 4, enabled: true, days: 30, enabledAt: 1 }, + }), + RuntimeHostProtocolError, + ); + assert.throws( + () => + setSpec.assertOutputForInput!(input, { + kind: 'revision_conflict', + expectedRevision: 2, + actualRevision: 4, + }), + RuntimeHostProtocolError, + ); + setSpec.assertOutputForInput!(input, { + kind: 'committed', + setting: { revision: 4, enabled: true, days: 60, enabledAt: 1 }, + }); + }); + + test('is a Ready query and command a remote owner may use', () => { + assert.equal(querySpec.mode, 'query'); + assert.equal(setSpec.mode, 'command'); + assert.equal(querySpec.availability, 'ready'); + assert.equal(setSpec.availability, 'ready'); + assert.ok(REMOTE_OWNER_OPERATION_GRANTS.includes('storage.retention.query')); + assert.ok(REMOTE_OWNER_OPERATION_GRANTS.includes('storage.retention.set')); + }); +}); diff --git a/packages/runtime-host/src/protocol/index.ts b/packages/runtime-host/src/protocol/index.ts index 712f0c82c0..405047936b 100644 --- a/packages/runtime-host/src/protocol/index.ts +++ b/packages/runtime-host/src/protocol/index.ts @@ -104,7 +104,11 @@ export const RUNTIME_HOST_REGISTRATION_SCHEMA_VERSION = 1 as const; export const RUNTIME_HOST_PROTOCOL_VERSION = 0 as const; // Increment when the same protocol version no longer guarantees safe Client-Host // interoperability. Mismatches are rejected before domain commands are admitted. -export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 204 as const; +export const RUNTIME_HOST_COMPATIBILITY_EPOCH = 205 as const; +// 205: `storage.retention.query` and `storage.retention.set` read and change the +// opt-in retention for archived tasks. An epoch-204 Client could not show or +// turn off a Host that deletes archived tasks on its own, and an epoch-204 Host +// rejects the unknown operations, so the pair must fail admission. // 204: Executor catalogs and Session configuration carry opaque mode IDs; // catalog queries may request a provider refresh. Older peers reject these fields. // 202: `session.remove.preview` takes a bounded list of Sessions and reports the diff --git a/packages/runtime-host/src/protocol/operations.ts b/packages/runtime-host/src/protocol/operations.ts index 308ed53848..fba841c427 100644 --- a/packages/runtime-host/src/protocol/operations.ts +++ b/packages/runtime-host/src/protocol/operations.ts @@ -64,6 +64,7 @@ import { SESSION_RETIREMENT_OPERATION_SPECS } from './session-retirement.js'; import { PROMPT_SUGGESTION_OPERATION_SPECS } from './prompt-suggestions.js'; import { SESSION_EFFECT_OPERATION_SPECS } from './session-effects.js'; import { SKILL_CATALOG_OPERATION_SPECS } from './skill-catalog.js'; +import { STORAGE_RETENTION_OPERATION_SPECS } from './storage-retention.js'; import { STORAGE_USAGE_OPERATION_SPECS } from './storage-usage.js'; import { TURN_OPERATION_SPECS } from './turn.js'; import { USAGE_PRICING_OPERATION_SPECS } from './usage-pricing.js'; @@ -192,6 +193,7 @@ export * from './session-todo.js'; export * from './session-effects.js'; export * from './prompt-suggestions.js'; export * from './skill-catalog.js'; +export * from './storage-retention.js'; export * from './storage-usage.js'; export * from './usage-pricing.js'; export * from './web-search.js'; @@ -234,6 +236,7 @@ export const HOST_OPERATION_SPECS = composeOperationSpecMaps( SKILL_CATALOG_OPERATION_SPECS, USAGE_PRICING_OPERATION_SPECS, STORAGE_USAGE_OPERATION_SPECS, + STORAGE_RETENTION_OPERATION_SPECS, MEMORY_OPERATION_SPECS, OAUTH_OPERATION_SPECS, EXTERNAL_AGENT_SETUP_OPERATION_SPECS, @@ -359,6 +362,8 @@ export const REMOTE_OWNER_OPERATION_GRANTS = Object.freeze([ 'skill.catalog.mutate', 'skill.catalog.preview-update', 'skill.catalog.query', + 'storage.retention.query', + 'storage.retention.set', 'storage.usage.query', 'storage.usage.sessions.query', 'subscription.close', diff --git a/packages/runtime-host/src/protocol/storage-retention.ts b/packages/runtime-host/src/protocol/storage-retention.ts new file mode 100644 index 0000000000..5afff36e86 --- /dev/null +++ b/packages/runtime-host/src/protocol/storage-retention.ts @@ -0,0 +1,260 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + type ArchiveRetentionDays, + type ArchiveRetentionDeletion, + type ArchiveRetentionHold, + type ArchiveRetentionSweep, + decodeArchiveRetentionDeletion, + decodeArchiveRetentionHold, + decodeArchiveRetentionSweep, + isArchiveRetentionDays, +} from '@maka/core/archive-retention'; +import { requireCount, requireShapedRecord } from './codec.js'; +import { invalidProtocolFrame } from './errors.js'; +import { defineOperation } from './operation-spec.js'; + +/** + * One Host's archived-task retention setting. Off by default; while enabled, + * the Host deletes archived tasks whose clock has run longer than `days`. + */ +export interface StorageRetentionSetting { + /** Moves with every setting change; `storage.retention.set` must name it. */ + readonly revision: number; + readonly enabled: boolean; + readonly days: ArchiveRetentionDays; + /** + * Host clock when the current setting took effect, present exactly while + * enabled. No task's clock starts before it. + */ + readonly enabledAt?: number; +} + +/** + * The archived tasks the policy covers now, counted as Settings › Archived + * tasks counts them: archived, not an Agent Graph operator, in a family no + * member of which is pinned. It is neither bound: families a sweep leaves for + * review are counted, and subtasks a deletion orphans into the archive are + * not. + */ +export interface StorageRetentionPreview { + readonly count: number; + /** + * When the first of them becomes eligible: after this Host instant. Present + * exactly while the setting is enabled and `count` is above 0. Enabling or + * changing the days restarts every clock, so a Client previews that change + * as `count` tasks eligible after its own now plus the new days. + */ + readonly eligibleAt?: number; +} + +export type StorageRetentionQueryInput = Record; + +export interface StorageRetentionQueryResult extends StorageRetentionSetting { + readonly preview: StorageRetentionPreview; + readonly lastSweep?: ArchiveRetentionSweep; + readonly lastDeletion?: ArchiveRetentionDeletion; + /** + * Present while deletions are held because the wall clock moved ahead + * further than a sweep expects; they resume once the Host clock reaches + * `until`, and any setting change clears it. + */ + readonly hold?: ArchiveRetentionHold; +} + +export interface StorageRetentionSetInput { + readonly expectedRevision: number; + readonly enabled: boolean; + readonly days: ArchiveRetentionDays; +} + +export type StorageRetentionSetResult = + | { readonly kind: 'committed'; readonly setting: StorageRetentionSetting } + | { + readonly kind: 'revision_conflict'; + readonly expectedRevision: number; + readonly actualRevision: number; + }; + +const QUERY_ERRORS = [ + 'host_not_ready', + 'host_draining', + 'operation_unavailable', + 'persistence_failed', + 'internal_failure', +] as const; + +const SET_ERRORS = [...QUERY_ERRORS, 'commit_outcome_unknown'] as const; + +export const STORAGE_RETENTION_OPERATION_SPECS = { + 'storage.retention.query': defineOperation< + StorageRetentionQueryInput, + StorageRetentionQueryResult, + (typeof QUERY_ERRORS)[number] + >({ + mode: 'query', + availability: 'ready', + errors: QUERY_ERRORS, + decodeInput: decodeStorageRetentionQueryInput, + decodeOutput: decodeStorageRetentionQueryResult, + }), + 'storage.retention.set': defineOperation< + StorageRetentionSetInput, + StorageRetentionSetResult, + (typeof SET_ERRORS)[number] + >({ + mode: 'command', + availability: 'ready', + errors: SET_ERRORS, + decodeInput: decodeStorageRetentionSetInput, + decodeOutput: decodeStorageRetentionSetResult, + assertOutputForInput: (input, output) => { + if (output.kind === 'revision_conflict') { + if (output.expectedRevision !== input.expectedRevision) { + throw invalidProtocolFrame('Retention revision conflict does not match the request'); + } + return; + } + if ( + output.setting.enabled !== input.enabled || + output.setting.days !== input.days || + output.setting.revision < input.expectedRevision + ) { + throw invalidProtocolFrame('Retention setting does not match the request'); + } + }, + }), +} as const; + +export function decodeStorageRetentionQueryInput(value: unknown): StorageRetentionQueryInput { + requireShapedRecord(value, 'storage retention input', [], []); + return {}; +} + +export function decodeStorageRetentionQueryResult(value: unknown): StorageRetentionQueryResult { + const result = requireShapedRecord( + value, + 'storage retention result', + ['revision', 'enabled', 'days', 'preview'], + ['enabledAt', 'lastSweep', 'lastDeletion', 'hold'], + ); + const setting = decodeSettingFields(result); + return { + ...setting, + preview: decodePreview(result.preview, setting.enabled), + ...(result.lastSweep === undefined + ? {} + : { lastSweep: decodeArchiveRetentionSweep(result.lastSweep, invalidProtocolFrame) }), + ...(result.lastDeletion === undefined + ? {} + : { + lastDeletion: decodeArchiveRetentionDeletion(result.lastDeletion, invalidProtocolFrame), + }), + ...(result.hold === undefined + ? {} + : { hold: decodeArchiveRetentionHold(result.hold, invalidProtocolFrame) }), + }; +} + +export function decodeStorageRetentionSetInput(value: unknown): StorageRetentionSetInput { + const input = requireShapedRecord( + value, + 'storage retention set input', + ['expectedRevision', 'enabled', 'days'], + [], + ); + return { + expectedRevision: requireCount(input.expectedRevision, 'retention expectedRevision'), + enabled: requireBoolean(input.enabled, 'retention enabled'), + days: requireDays(input.days), + }; +} + +export function decodeStorageRetentionSetResult(value: unknown): StorageRetentionSetResult { + const result = requireShapedRecord( + value, + 'storage retention set result', + ['kind'], + ['setting', 'expectedRevision', 'actualRevision'], + ); + if (result.kind === 'committed') { + requireShapedRecord(result, 'storage retention set result', ['kind', 'setting'], []); + const setting = requireShapedRecord( + result.setting, + 'storage retention setting', + ['revision', 'enabled', 'days'], + ['enabledAt'], + ); + return { kind: 'committed', setting: decodeSettingFields(setting) }; + } + if (result.kind === 'revision_conflict') { + requireShapedRecord( + result, + 'storage retention revision conflict', + ['kind', 'expectedRevision', 'actualRevision'], + [], + ); + return { + kind: 'revision_conflict', + expectedRevision: requireCount(result.expectedRevision, 'retention expectedRevision'), + actualRevision: requireCount(result.actualRevision, 'retention actualRevision'), + }; + } + throw invalidProtocolFrame('Invalid storage retention set result'); +} + +function decodeSettingFields(record: Record): StorageRetentionSetting { + const enabled = requireBoolean(record.enabled, 'retention enabled'); + if (enabled !== (record.enabledAt !== undefined)) { + throw invalidProtocolFrame('Retention enabledAt must be present exactly while enabled'); + } + return { + revision: requireCount(record.revision, 'retention revision'), + enabled, + days: requireDays(record.days), + ...(record.enabledAt === undefined + ? {} + : { enabledAt: requireCount(record.enabledAt, 'retention enabledAt') }), + }; +} + +function decodePreview(value: unknown, enabled: boolean): StorageRetentionPreview { + const preview = requireShapedRecord(value, 'retention preview', ['count'], ['eligibleAt']); + const count = requireCount(preview.count, 'retention preview count'); + if ((enabled && count > 0) !== (preview.eligibleAt !== undefined)) { + throw invalidProtocolFrame('Retention preview eligibleAt must be present exactly when due'); + } + return { + count, + ...(preview.eligibleAt === undefined + ? {} + : { eligibleAt: requireCount(preview.eligibleAt, 'retention preview eligibleAt') }), + }; +} + +function requireDays(value: unknown): ArchiveRetentionDays { + if (!isArchiveRetentionDays(value)) throw invalidProtocolFrame('Invalid retention days'); + return value; +} + +function requireBoolean(value: unknown, label: string): boolean { + if (typeof value !== 'boolean') throw invalidProtocolFrame(`Invalid ${label}`); + return value; +} diff --git a/packages/runtime-host/src/server/archive-retention-coordinator.ts b/packages/runtime-host/src/server/archive-retention-coordinator.ts new file mode 100644 index 0000000000..efdbdde7e2 --- /dev/null +++ b/packages/runtime-host/src/server/archive-retention-coordinator.ts @@ -0,0 +1,640 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + ARCHIVE_RETENTION_DAY_MS, + ARCHIVE_RETENTION_HOLD_MS, + type ArchiveRetentionDeletion, + type ArchiveRetentionSweep, + archiveRetentionClockStart, + archiveRetentionDeadline, + archiveRetentionGapThreshold, +} from '@maka/core/archive-retention'; +import { generalizedErrorMessage } from '@maka/core/redaction'; +import { sessionRevisionFamilyId } from '@maka/core/session'; +import type { + ArchiveRetentionDocument, + InteractiveArchiveRetentionStore, +} from '@maka/storage/archive-retention-store'; +import type { ExecutionSessionWriter } from '@maka/storage/execution-stores'; +import { RuntimePolicyStoreError } from '@maka/storage/runtime-policy-stores'; +import type { + OperationOutcome, + StorageRetentionSetInput, + StorageRetentionSetting, +} from '../protocol/index.js'; +import type { StorageRetentionOperationHandlerMap } from './operation-dispatcher.js'; +import type { + HostSessionRetirementCoordinator, + RetentionHoldReason, + RetentionRemovalOutcome, + RetentionRemovalPlan, +} from './session-retirement-coordinator.js'; + +/** Revision families a sweep tick may delete. */ +export const ARCHIVE_RETENTION_FAMILIES_PER_TICK = 8; +const CANDIDATE_PAGE = 64; + +const DISABLED: ArchiveRetentionDocument = Object.freeze({ + version: 1, + revision: 0, + enabled: false, + days: 30, +}); + +type RetentionCatalog = Pick< + ExecutionSessionWriter, + | 'listArchiveRetentionCandidates' + | 'countArchiveRetentionCandidates' + | 'readLatestSessionMetadataTime' + | 'readCatalogRecord' +>; + +export interface HostArchiveRetentionCoordinatorOptions { + readonly document: InteractiveArchiveRetentionStore; + readonly catalog: RetentionCatalog; + readonly retirement: Pick; + /** The Host wall clock. */ + readonly now?: () => number; + /** Diagnostics. Counts only, never a task's name. */ + readonly log?: (message: string) => void; +} + +interface EnabledSetting { + readonly revision: number; + readonly days: ArchiveRetentionDocument['days']; + readonly enabledAt: number; +} + +interface SweepPass { + /** The setting revision the pass started under; its results belong to it alone. */ + readonly revision: number; + /** Resume the candidate order strictly after this row. */ + cursor?: { readonly archivedAt?: number; readonly sessionId: string }; + /** Families (and undecodable rows) this pass already tried; none is retried before the next pass. */ + readonly seen: Set; + deleted: number; + skippedBusy: number; + needsReview: number; + failed: number; + /** Undefined once any deleted family could not be measured. */ + bytes: number | undefined; +} + +/** + * The opt-in retention for archived tasks (#5899). One Host document holds the + * setting and the latest results; the `storage.retention.*` operations are the + * only writer of the setting, and a maintenance lane runs the sweep. + * + * A sweep deletes nothing before `enabledAt + days`, pauses while the wall + * clock reads earlier than a time the Host has already seen, and deletes each + * family through `session.remove`'s own path with a guard that rechecks the + * policy, the archive and pin state, the elapsed time and the plan under the + * removal admission. A setting change waits for the family in flight, so once + * `storage.retention.set` answers, no deletion admitted under the old setting + * is still running. + */ +export class HostArchiveRetentionCoordinator { + readonly handlers: StorageRetentionOperationHandlerMap = { + 'storage.retention.query': () => this.#query(), + 'storage.retention.set': (input) => this.#set(input), + }; + + readonly #store: InteractiveArchiveRetentionStore; + readonly #catalog: RetentionCatalog; + readonly #retirement: HostArchiveRetentionCoordinatorOptions['retirement']; + readonly #now: () => number; + readonly #log: (message: string) => void; + #document: ArchiveRetentionDocument | undefined; + #loading: Promise | undefined; + /** Serializes every read-modify-write of the document. */ + #writes: Promise = Promise.resolve(); + /** Setting changes in flight; while one is pending a sweep admits and starts nothing. */ + #changing = 0; + /** The sweep step running now, which a setting change waits for. */ + #ticking: Promise | undefined; + /** + * The latest Host time observed, in memory only. After a restart the floor + * is `enabledAt`, the last sweep's time and, on every sweep, the newest time + * Session metadata recorded. + */ + #observedAt = 0; + /** Whether `#observedAt` is a time this process saw, rather than the persisted floor. */ + #observedThisRun = false; + #pass: SweepPass | undefined; + #draining = false; + + constructor(options: HostArchiveRetentionCoordinatorOptions) { + this.#store = options.document; + this.#catalog = options.catalog; + this.#retirement = options.retirement; + this.#now = options.now ?? Date.now; + this.#log = options.log ?? ((message) => console.info(`[runtime-host] ${message}`)); + } + + beginDrain(): void { + this.#draining = true; + } + + /** One bounded sweep step; true while candidates remain. */ + async sweep(): Promise { + if (this.#draining) return false; + // A setting change is waiting for this lane; look again shortly. + if (this.#changing > 0) return true; + const tick = this.#tick(); + this.#ticking = tick; + try { + return await tick; + } finally { + if (this.#ticking === tick) this.#ticking = undefined; + } + } + + async #tick(): Promise { + const document = await this.#load(); + if (!document.enabled || document.enabledAt === undefined) { + this.#pass = undefined; + return false; + } + const setting: EnabledSetting = { + revision: document.revision, + days: document.days, + enabledAt: document.enabledAt, + }; + const now = this.#now(); + const previous = this.#observedAt; + const observedThisRun = this.#observedThisRun; + this.#observedAt = Math.max(previous, now); + this.#observedThisRun = true; + const deadline = archiveRetentionDeadline(setting.enabledAt, setting.days); + const gap = archiveRetentionGapThreshold(setting.days); + // A hold ends when its day is over, before or after the deadline alike. + let hold = document.latest?.hold; + let heartbeatCleared = false; + if (hold && now >= hold.until) { + heartbeatCleared = this.#heartbeatDue(document, now); + await this.#clearHold(heartbeatCleared ? now : undefined); + hold = undefined; + } + // A forward jump is caught even when it lands short of the deadline. A + // running Host measures from what it saw; a fresh process from the + // persisted floor and the newest metadata time, which says when the Host + // last ran: one MAX query, once per process. + const since = observedThisRun + ? previous + : Math.max(previous, (await this.#catalog.readLatestSessionMetadataTime()) ?? 0); + if (now - since > gap) return this.#hold(since, now); + // A Host can run for weeks without writing session metadata. Persist a + // coarse heartbeat so a later restart can tell that idle time was spent + // while the Host was running, rather than treating it as a clock jump. + const heartbeat = + !heartbeatCleared && now >= previous && this.#heartbeatDue(this.#document ?? document, now); + // Nothing can be eligible before the policy itself is `days` old, and no + // candidate is read before then. + if (now <= deadline) { + if (heartbeat) await this.#heartbeat(now); + return false; + } + if (now < previous) return this.#pause(now, heartbeat); + const newest = await this.#catalog.readLatestSessionMetadataTime(); + if (newest !== undefined && now < newest) return this.#pause(now, heartbeat); + if (hold) return false; + + if (this.#pass?.revision !== setting.revision) { + this.#pass = { + revision: setting.revision, + seen: new Set(), + deleted: 0, + skippedBusy: 0, + needsReview: 0, + failed: 0, + bytes: 0, + }; + } + const pass = this.#pass; + const page = await this.#catalog.listArchiveRetentionCandidates({ + archivedBefore: now - setting.days * ARCHIVE_RETENTION_DAY_MS, + ...(pass.cursor ? { after: pass.cursor } : {}), + limit: CANDIDATE_PAGE, + }); + const deletedBefore = pass.deleted; + let tried = 0; + let stopped = false; + for (const row of page) { + // Draining or a pending setting change: stop before the next family. + if (this.#draining || this.#changing > 0) { + stopped = true; + break; + } + if ('undecodable' in row) { + // A row that no longer decodes is counted once and passed over. + if (!pass.seen.has(`row:${row.sessionId}`)) { + pass.seen.add(`row:${row.sessionId}`); + pass.failed += 1; + } + } else { + const family = sessionRevisionFamilyId(row.header); + if (!pass.seen.has(family)) { + if (tried === ARCHIVE_RETENTION_FAMILIES_PER_TICK) { + stopped = true; + break; + } + tried += 1; + pass.seen.add(family); + tally( + pass, + await this.#retirement.removeForRetention( + { sessionId: row.header.id, expectedRevision: row.revision }, + (plan) => this.#guard(plan, setting), + ), + ); + } + } + const sessionId = 'undecodable' in row ? row.sessionId : row.header.id; + pass.cursor = { + ...(row.archivedAt === undefined ? {} : { archivedAt: row.archivedAt }), + sessionId, + }; + } + if (stopped || page.length === CANDIDATE_PAGE) { + // A deletion is recorded as it happens, not only when the pass ends. + if (pass.deleted > deletedBefore) await this.#record(now, pass, heartbeat); + return !this.#draining; + } + if (this.#pass === pass) this.#pass = undefined; + await this.#finishPass(now, pass, heartbeat); + return false; + } + + /** + * Runs under the removal admission, after the plan is stable and before any + * retirement work: anything that changed since the sweep read the task keeps + * it for this pass. Every setting change moves the revision. + */ + async #guard( + plan: RetentionRemovalPlan, + setting: EnabledSetting, + ): Promise { + if (this.#changing > 0 || this.#document?.revision !== setting.revision) { + return 'ineligible'; + } + if (plan.remove.some(({ header }) => !header.isArchived || header.isFlagged)) { + return 'ineligible'; + } + // The archive time through the reader `requireArchivedForMs` judges by. + const archivedAt = await Promise.all( + plan.remove.map( + async ({ header }) => (await this.#catalog.readCatalogRecord(header.id)).summary.archivedAt, + ), + ); + const now = this.#now(); + if (now < this.#observedAt) return 'ineligible'; + // A family is as young as its most recently archived member. + const start = Math.max( + ...archivedAt.map((time) => archiveRetentionClockStart(time, setting.enabledAt)), + ); + if (now <= archiveRetentionDeadline(start, setting.days)) return 'ineligible'; + // Unattended cleanup is more conservative than a manual delete. + if (plan.archiveSessionIds.length > 0 || plan.worktreeCount > 0) return 'needs_review'; + return undefined; + } + + /** + * The wall clock moved ahead further than a sweep expects, whether set wrong + * or after a long time offline: delete nothing for a day, so a wrong clock + * can be noticed and the setting turned off. A further jump re-arms it. + */ + async #hold(since: number, now: number): Promise { + this.#pass = undefined; + this.#log('archive retention held: the clock moved ahead further than a sweep expects'); + await this.#update((document) => ({ + ...document, + latest: { + ...document.latest, + hold: { since, detectedAt: now, until: now + ARCHIVE_RETENTION_HOLD_MS }, + }, + })); + return false; + } + + #heartbeatDue(document: ArchiveRetentionDocument, now: number): boolean { + if (!document.enabled || document.enabledAt === undefined) return false; + const previous = document.latest?.observedAt ?? document.enabledAt; + return now > previous && now - previous >= ARCHIVE_RETENTION_DAY_MS; + } + + async #heartbeat(now: number): Promise { + await this.#serialized(async () => { + const document = await this.#load(); + if (!document.enabled || document.enabledAt === undefined) return; + const previous = document.latest?.observedAt ?? document.enabledAt; + if (now <= previous || now - previous < ARCHIVE_RETENTION_DAY_MS) return; + await this.#write({ + ...document, + latest: { ...document.latest, observedAt: now }, + }); + }); + } + + #clearHold(observedAt?: number): Promise { + return this.#update(({ latest, ...rest }) => { + const { hold: _hold, ...kept } = latest ?? {}; + return { + ...rest, + ...(observedAt === undefined ? {} : { latest: { ...kept, observedAt } }), + ...(observedAt === undefined && Object.keys(kept).length > 0 ? { latest: kept } : {}), + }; + }); + } + + async #pause(now: number, heartbeat = false): Promise { + this.#pass = undefined; + // Recorded once: a clock that stays behind writes nothing further. + if (this.#document?.latest?.lastSweep?.paused) return false; + this.#log('archive retention paused: the clock reads earlier than a time already recorded'); + await this.#update((document) => ({ + ...document, + latest: { + ...document.latest, + ...(heartbeat ? { observedAt: now } : {}), + lastSweep: { at: now, deleted: 0, skippedBusy: 0, needsReview: 0, failed: 0, paused: true }, + }, + })); + return false; + } + + async #finishPass(now: number, pass: SweepPass, heartbeat: boolean): Promise { + const previous = this.#document?.latest?.lastSweep; + const unchanged = + pass.deleted === 0 && + pass.skippedBusy === (previous?.skippedBusy ?? 0) && + pass.needsReview === (previous?.needsReview ?? 0) && + pass.failed === (previous?.failed ?? 0) && + previous?.paused !== true; + if (unchanged) { + // A heartbeat alone does not change the latest cleanup result. + if (heartbeat) await this.#heartbeat(now); + return; + } + if (pass.deleted + pass.skippedBusy + pass.needsReview + pass.failed > 0) { + this.#log( + `archive retention deleted ${pass.deleted} tasks; kept ${pass.skippedBusy} busy and ` + + `${pass.needsReview} for review; ${pass.failed} failed`, + ); + } + await this.#record(now, pass, heartbeat); + } + + #record(now: number, pass: SweepPass, heartbeat = false): Promise { + const lastSweep: ArchiveRetentionSweep = { + at: now, + deleted: pass.deleted, + skippedBusy: pass.skippedBusy, + needsReview: pass.needsReview, + failed: pass.failed, + }; + const lastDeletion: ArchiveRetentionDeletion | undefined = + pass.deleted > 0 + ? { + at: now, + count: pass.deleted, + ...(pass.bytes === undefined ? {} : { bytes: pass.bytes }), + } + : undefined; + return this.#serialized(async () => { + const document = await this.#load(); + // A pass belongs to the setting it started under; a newer one starts afresh. + if (document.revision !== pass.revision) return; + await this.#write({ + ...document, + latest: { + ...document.latest, + ...(heartbeat ? { observedAt: now } : {}), + lastSweep, + ...(lastDeletion ? { lastDeletion } : {}), + }, + }); + }); + } + + async #query(): Promise> { + if (this.#draining) return draining(); + try { + const document = await this.#load(); + const enabledAt = document.enabled ? document.enabledAt : undefined; + const { families, firstStart } = await this.#catalog.countArchiveRetentionCandidates( + enabledAt ?? this.#now(), + ); + return { + ok: true, + result: { + ...settingOf(document), + preview: { + count: families, + ...(enabledAt !== undefined && families > 0 && firstStart !== undefined + ? { eligibleAt: archiveRetentionDeadline(firstStart, document.days) } + : {}), + }, + ...(document.latest?.lastSweep ? { lastSweep: document.latest.lastSweep } : {}), + ...(document.latest?.lastDeletion ? { lastDeletion: document.latest.lastDeletion } : {}), + // A hold whose day is over is not reported, cleared or not. + ...(document.latest?.hold && this.#now() < document.latest.hold.until + ? { hold: document.latest.hold } + : {}), + }, + }; + } catch (error) { + reportFailure('read', error); + return { + ok: false, + error: { code: 'persistence_failed', message: 'Retention setting could not be read' }, + }; + } + } + + async #set(input: StorageRetentionSetInput): Promise> { + if (this.#draining) return draining(); + this.#changing += 1; + try { + // A family admitted under the current setting finishes, and is recorded, + // before the setting changes; nothing new is admitted meanwhile. + await this.#ticking?.catch(() => undefined); + return await this.#serialized(async () => { + const current = await this.#load(); + if (current.revision !== input.expectedRevision) { + return { + ok: true, + result: { + kind: 'revision_conflict', + expectedRevision: input.expectedRevision, + actualRevision: current.revision, + }, + } as const; + } + if (current.enabled === input.enabled && current.days === input.days) { + return { ok: true, result: { kind: 'committed', setting: settingOf(current) } } as const; + } + // Any change restarts the clock: enabling, or new days while enabled. + // A clock behind a time already recorded never backdates the deadline. + const newest = input.enabled + ? await this.#catalog.readLatestSessionMetadataTime() + : undefined; + const now = this.#now(); + const enabledAt = Math.max(now, this.#observedAt, newest ?? 0); + this.#observedAt = Math.max(this.#observedAt, now); + this.#observedThisRun = true; + const { enabledAt: _previous, latest, ...rest } = current; + const lastSweep = latest?.lastSweep && withoutPause(latest.lastSweep); + const nextLatest = { + ...(input.enabled ? { observedAt: now } : {}), + ...(lastSweep ? { lastSweep } : {}), + ...(latest?.lastDeletion ? { lastDeletion: latest.lastDeletion } : {}), + }; + const next: ArchiveRetentionDocument = { + ...rest, + revision: current.revision + 1, + enabled: input.enabled, + days: input.days, + ...(input.enabled ? { enabledAt } : {}), + ...(Object.keys(nextLatest).length > 0 ? { latest: nextLatest } : {}), + }; + await this.#write(next); + this.#pass = undefined; + return { ok: true, result: { kind: 'committed', setting: settingOf(next) } } as const; + }); + } catch (error) { + if (error instanceof RuntimePolicyStoreError && error.code === 'commit_outcome_unknown') { + return { + ok: false, + error: { + code: 'commit_outcome_unknown', + message: 'Retention setting commit outcome is unknown', + }, + }; + } + reportFailure('save', error); + return { + ok: false, + error: { code: 'persistence_failed', message: 'Retention setting could not be saved' }, + }; + } finally { + this.#changing -= 1; + } + } + + #update(change: (document: ArchiveRetentionDocument) => ArchiveRetentionDocument): Promise { + return this.#serialized(async () => this.#write(change(await this.#load()))); + } + + async #write(next: ArchiveRetentionDocument): Promise { + try { + await this.#store.write(next); + } catch (error) { + // The file may or may not hold `next`; read it again before trusting either. + this.#document = undefined; + throw error; + } + this.#document = next; + } + + #serialized(operation: () => Promise): Promise { + const run = this.#writes.then(operation, operation); + this.#writes = run.catch(() => undefined); + return run; + } + + #load(): Promise { + if (this.#document) return Promise.resolve(this.#document); + this.#loading ??= this.#read().finally(() => { + this.#loading = undefined; + }); + return this.#loading; + } + + async #read(): Promise { + const read = await this.#store.read(); + let document: ArchiveRetentionDocument; + if (read.kind === 'valid') { + document = read.document; + } else { + // Never act on a document that cannot be fully validated: off, until + // the setting is written again. + if (read.kind === 'invalid') { + this.#log('archive retention is off: its setting document could not be read'); + } + document = DISABLED; + } + this.#observedAt = Math.max( + this.#observedAt, + document.enabledAt ?? 0, + document.latest?.observedAt ?? 0, + document.latest?.lastSweep?.at ?? 0, + document.latest?.hold?.detectedAt ?? 0, + ); + this.#document = document; + return document; + } +} + +function withoutPause(sweep: ArchiveRetentionSweep): ArchiveRetentionSweep { + const { paused: _paused, ...rest } = sweep; + return rest; +} + +function settingOf(document: ArchiveRetentionDocument): StorageRetentionSetting { + return { + revision: document.revision, + enabled: document.enabled, + days: document.days, + ...(document.enabledAt === undefined ? {} : { enabledAt: document.enabledAt }), + }; +} + +function tally(pass: SweepPass, outcome: RetentionRemovalOutcome): void { + switch (outcome.kind) { + case 'removed': + pass.deleted += 1; + pass.bytes = + pass.bytes === undefined || outcome.bytes === undefined + ? undefined + : pass.bytes + outcome.bytes; + return; + case 'held': + if (outcome.reason === 'needs_review') pass.needsReview += 1; + return; + case 'busy': + pass.skippedBusy += 1; + return; + case 'failed': + pass.failed += 1; + return; + case 'skipped': + return; + } +} + +function reportFailure(action: 'read' | 'save', error: unknown): void { + console.error( + `[runtime-host] archive retention could not ${action} its setting: ${generalizedErrorMessage(error)}`, + ); +} + +function draining(): { ok: false; error: { code: 'host_draining'; message: string } } { + return { ok: false, error: { code: 'host_draining', message: 'Runtime Host is draining' } }; +} diff --git a/packages/runtime-host/src/server/execution-composition.ts b/packages/runtime-host/src/server/execution-composition.ts index d751ed9ae3..907e088866 100644 --- a/packages/runtime-host/src/server/execution-composition.ts +++ b/packages/runtime-host/src/server/execution-composition.ts @@ -243,6 +243,7 @@ import { HostWorkspaceResolver } from './workspace-resolver.js'; import { HostSessionRetirementCoordinator } from './session-retirement-coordinator.js'; import { HostStorageMaintenance } from './storage-maintenance.js'; import { HostStorageUsageCoordinator } from './storage-usage-coordinator.js'; +import { HostArchiveRetentionCoordinator } from './archive-retention-coordinator.js'; import { HostSessionRevisionCoordinator } from './session-revision-coordinator.js'; import { HostSessionEffectCoordinator } from './session-effect-coordinator.js'; import { SessionContinuityCoordinator } from './session-continuity-coordinator.js'; @@ -2858,9 +2859,15 @@ export async function createExecutionRuntimeHostComposition( ); let recoverySessions: Awaited> = []; const storageUsage = new HostStorageUsageCoordinator({ footprint: storage.footprint }); + const archiveRetention = new HostArchiveRetentionCoordinator({ + document: storage.archiveRetention, + catalog: stores.sessionStore, + retirement: sessionRetirement, + }); const storageMaintenance = new HostStorageMaintenance({ artifacts: openedArtifactStore, contextOffload: openedContextOffloadStore, + retention: archiveRetention, onError: (name, error) => console.error(`[runtime-host] ${name} will retry: ${generalizedErrorMessage(error)}`), }); @@ -2875,6 +2882,11 @@ export async function createExecutionRuntimeHostComposition( handlers: [storageUsage.handlers], drain: [() => storageUsage.beginDrain()], }), + createRuntimeHostDomainModule({ + id: 'archive-retention', + handlers: [archiveRetention.handlers], + drain: [() => archiveRetention.beginDrain()], + }), createRuntimeHostDomainModule({ id: 'plugin-platform', handlers: [pluginPlatformCoordinator.handlers], diff --git a/packages/runtime-host/src/server/operation-dispatcher.ts b/packages/runtime-host/src/server/operation-dispatcher.ts index c87bc7058b..9b3289575c 100644 --- a/packages/runtime-host/src/server/operation-dispatcher.ts +++ b/packages/runtime-host/src/server/operation-dispatcher.ts @@ -68,6 +68,7 @@ import { SESSION_TODO_OPERATION_SPECS } from '../protocol/session-todo.js'; import { SESSION_TRANSCRIPT_OPERATION_SPECS } from '../protocol/session-transcript.js'; import { SESSION_TURNS_OPERATION_SPECS } from '../protocol/session-turns.js'; import { SKILL_CATALOG_OPERATION_SPECS } from '../protocol/skill-catalog.js'; +import { STORAGE_RETENTION_OPERATION_SPECS } from '../protocol/storage-retention.js'; import { STORAGE_USAGE_OPERATION_SPECS } from '../protocol/storage-usage.js'; import { TURN_OPERATION_SPECS } from '../protocol/turn.js'; import { USAGE_PRICING_OPERATION_SPECS } from '../protocol/usage-pricing.js'; @@ -155,6 +156,7 @@ export type ArtifactOperationKey = keyof typeof ARTIFACT_OPERATION_SPECS; export type SkillCatalogOperationKey = keyof typeof SKILL_CATALOG_OPERATION_SPECS; export type UsagePricingOperationKey = keyof typeof USAGE_PRICING_OPERATION_SPECS; export type StorageUsageOperationKey = keyof typeof STORAGE_USAGE_OPERATION_SPECS; +export type StorageRetentionOperationKey = keyof typeof STORAGE_RETENTION_OPERATION_SPECS; export type MemoryOperationKey = keyof typeof MEMORY_OPERATION_SPECS; export type OAuthOperationKey = keyof typeof OAUTH_OPERATION_SPECS; export type RuntimeResourceOperationKey = keyof typeof RUNTIME_RESOURCE_OPERATION_SPECS; @@ -215,6 +217,10 @@ export type ArtifactOperationHandlerMap = Pick; export type UsagePricingOperationHandlerMap = Pick; export type StorageUsageOperationHandlerMap = Pick; +export type StorageRetentionOperationHandlerMap = Pick< + OperationHandlerMap, + StorageRetentionOperationKey +>; export type MemoryOperationHandlerMap = Pick; export type OAuthOperationHandlerMap = Pick; export type RuntimeResourceOperationHandlerMap = Pick< diff --git a/packages/runtime-host/src/server/session-retirement-coordinator.ts b/packages/runtime-host/src/server/session-retirement-coordinator.ts index 9c23a46304..3a3b35b5cb 100644 --- a/packages/runtime-host/src/server/session-retirement-coordinator.ts +++ b/packages/runtime-host/src/server/session-retirement-coordinator.ts @@ -182,6 +182,37 @@ class SessionRetirementBusyError extends Error { readonly name = 'SessionRetirementBusyError'; } +/** What an archive-retention sweep's guard sees of a removal plan, read under its admission. */ +export interface RetentionRemovalPlan { + /** Every Session the removal deletes, as admitted. */ + readonly remove: readonly SessionHeaderSnapshot[]; + /** Still-active subtasks the removal would move to the archive. */ + readonly archiveSessionIds: readonly string[]; + /** Worktrees cleanup would retire, counted exactly as the removal preview counts them. */ + readonly worktreeCount: number; +} + +/** Why a sweep's guard kept a task: not (or no longer) eligible, or left for manual review. */ +export type RetentionHoldReason = 'ineligible' | 'needs_review'; + +export type RetentionRemovalOutcome = + | { readonly kind: 'removed'; readonly bytes?: number } + | { readonly kind: 'held'; readonly reason: RetentionHoldReason } + | { readonly kind: 'busy' } + /** The task changed or went away after the sweep read it. */ + | { readonly kind: 'skipped' } + | { readonly kind: 'failed' }; + +class RetentionHold extends Error { + readonly name = 'RetentionHold'; + + constructor(readonly reason: RetentionHoldReason) { + super(`Archive retention kept the task: ${reason}`); + } +} + +type RemovalAdmissionGuard = (plan: StableRemovalPlan) => Promise; + /** Host-owned archive, unarchive, remove, and revision-family commit authority. */ export class HostSessionRetirementCoordinator { readonly handlers: SessionRetirementOperationHandlerMap = { @@ -316,7 +347,61 @@ export class HostSessionRetirementCoordinator { } } - async #remove(input: SessionRemoveInput): Promise> { + /** + * Archive retention's removal: `session.remove` for the task the sweep read, + * with `guard` run inside the removal admission — after the plan is stable, + * before any retirement work — so a task whose policy, archive or pin state, + * age or plan changed since the sweep read it is kept rather than deleted. + * The size is measured once the guard admits the plan, as the batch preview + * measures it. + */ + async removeForRetention( + target: { readonly sessionId: string; readonly expectedRevision: number }, + guard: (plan: RetentionRemovalPlan) => Promise, + ): Promise { + let admitted = false; + let bytes: number | undefined; + let outcome: OperationOutcome<'session.remove'>; + try { + outcome = await this.#removeUnder(target, async (plan) => { + const remove = plan.remove.sessionIds.map((id) => requireFamilyRecord(plan.remove, id)); + const reason = await guard({ + remove, + archiveSessionIds: plan.archive.sessionIds, + worktreeCount: this.#reclaimedWorktreeCount(remove.map(({ header }) => header)), + }); + if (reason) throw new RetentionHold(reason); + admitted = true; + bytes = await this.#measureRemoved(plan.remove.sessionIds); + }); + } catch (error) { + if (error instanceof RetentionHold) return { kind: 'held', reason: error.reason }; + return { kind: 'failed' }; + } + if (outcome.ok) { + // Removed without this guard admitting it — already gone — is not this sweep's deletion. + if (outcome.result.kind !== 'removed' || !admitted) return { kind: 'skipped' }; + return { kind: 'removed', ...(bytes === undefined ? {} : { bytes }) }; + } + switch (outcome.error.code) { + case 'session_busy': + return { kind: 'busy' }; + case 'not_found': + case 'operation_conflict': + return { kind: 'skipped' }; + default: + return { kind: 'failed' }; + } + } + + #remove(input: SessionRemoveInput): Promise> { + return this.#removeUnder(input); + } + + async #removeUnder( + input: SessionRemoveInput, + guard?: RemovalAdmissionGuard, + ): Promise> { let probe; try { probe = await this.#stores.probeSessionRemoval(input.sessionId); @@ -353,6 +438,7 @@ export class HostSessionRetirementCoordinator { return removeOutcome({ kind: 'too_recent', sessionId: input.sessionId }); } } + await guard?.(plan); let removeHandles: RetirementHandles | undefined; let archiveHandles: RetirementHandles | undefined; @@ -402,6 +488,7 @@ export class HostSessionRetirementCoordinator { } }); } catch (error) { + if (error instanceof RetentionHold) throw error; return this.#removeFailure(error, input); } } @@ -460,17 +547,23 @@ export class HostSessionRetirementCoordinator { .filter((header) => header.subagentParent?.graph !== undefined) .map(sessionRevisionFamilyId), ).size, - // What cleanup retires: one binding per removed Session, and only - // through a worktree executor. A subagent Session carrying a binding - // cannot be a revision, so no two removed Sessions share one. - worktreeCount: this.#worktrees - ? removedHeaders.filter((header) => header.subagentWorkspace !== undefined).length - : 0, + worktreeCount: this.#reclaimedWorktreeCount(removedHeaders), ...(bytes === undefined ? {} : { bytes }), }, }; } + /** + * What cleanup retires: one binding per removed Session, and only through a + * worktree executor. A subagent Session carrying a binding cannot be a + * revision, so no two removed Sessions share one. + */ + #reclaimedWorktreeCount(removed: readonly SessionHeader[]): number { + return this.#worktrees + ? removed.filter((header) => header.subagentWorkspace !== undefined).length + : 0; + } + /** * Sizes removed Sessions in storage-usage pages, yielding between them. A * failed measurement is left out rather than failing the counts beside it. diff --git a/packages/runtime-host/src/server/storage-maintenance.ts b/packages/runtime-host/src/server/storage-maintenance.ts index 629380f565..e08489ba32 100644 --- a/packages/runtime-host/src/server/storage-maintenance.ts +++ b/packages/runtime-host/src/server/storage-maintenance.ts @@ -24,10 +24,17 @@ const ACTIVE_DELAY_MS = 100; const IDLE_DELAY_MS = 60_000; const MAX_BATCH_ITEMS = 64; const MAX_BATCH_BYTES = 16 * 1024 * 1024; +// Archive retention deletes user data, so it moves slower than reclamation: +// one bounded batch a second while work remains, otherwise a check every +// quarter hour. +const RETENTION_ACTIVE_DELAY_MS = 1000; +const RETENTION_IDLE_DELAY_MS = 15 * 60_000; interface MaintenanceLane { readonly name: string; readonly run: () => Promise; + readonly activeDelay: number; + readonly idleDelay: number; timer?: ReturnType; pending?: Promise; failures: number; @@ -43,6 +50,8 @@ export class HostStorageMaintenance { constructor(input: { artifacts: Pick; contextOffload?: Pick; + /** The opt-in archived-task retention sweep; true while candidates remain. */ + retention?: { sweep(): Promise }; onError: (name: string, error: unknown) => void; }) { this.#onError = input.onError; @@ -51,6 +60,8 @@ export class HostStorageMaintenance { { name: 'artifact upgrade cleanup', failures: 0, + activeDelay: ACTIVE_DELAY_MS, + idleDelay: IDLE_DELAY_MS, run: async () => { const result = await input.artifacts.reclaimUpgradeResidue({ after, @@ -72,6 +83,8 @@ export class HostStorageMaintenance { this.#lanes.push({ name: 'context garbage collection', failures: 0, + activeDelay: ACTIVE_DELAY_MS, + idleDelay: IDLE_DELAY_MS, run: async () => ( await context.collectGarbage({ @@ -81,12 +94,21 @@ export class HostStorageMaintenance { }) ).hasMore, }); + const retention = input.retention; + if (retention) + this.#lanes.push({ + name: 'archive retention', + failures: 0, + activeDelay: RETENTION_ACTIVE_DELAY_MS, + idleDelay: RETENTION_IDLE_DELAY_MS, + run: () => retention.sweep(), + }); } start(): void { if (this.#started || this.#draining) return; this.#started = true; - for (const lane of this.#lanes) this.#schedule(lane, ACTIVE_DELAY_MS); + for (const lane of this.#lanes) this.#schedule(lane, lane.activeDelay); } beginDrain(): void { @@ -113,10 +135,10 @@ export class HostStorageMaintenance { try { const more = await lane.run(); lane.failures = 0; - delay = more ? ACTIVE_DELAY_MS : IDLE_DELAY_MS; + delay = more ? lane.activeDelay : lane.idleDelay; } catch (error) { lane.failures = Math.min(lane.failures + 1, 7); - delay = Math.min(IDLE_DELAY_MS, 1000 * 2 ** (lane.failures - 1)); + delay = Math.min(lane.idleDelay, 1000 * 2 ** (lane.failures - 1)); this.#report(lane.name, error); } this.#schedule(lane, delay); diff --git a/packages/storage/package.json b/packages/storage/package.json index 43b40dc190..e66602e74b 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -6,6 +6,7 @@ "type": "module", "exports": { "./activation-secret-injector": "./dist/activation-secret-injector.js", + "./archive-retention-store": "./dist/archive-retention-store.js", "./agent-graph-control-store": "./dist/agent-graph-control-store.js", "./agent-run-store": "./dist/agent-run-store.js", "./artifact-stores": "./dist/artifact-stores.js", diff --git a/packages/storage/src/__tests__/archive-retention-store.test.ts b/packages/storage/src/__tests__/archive-retention-store.test.ts new file mode 100644 index 0000000000..6cccce2e24 --- /dev/null +++ b/packages/storage/src/__tests__/archive-retention-store.test.ts @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import assert from 'node:assert/strict'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; +import { describe, test } from 'node:test'; +import type { SessionHeader } from '@maka/core/session'; +import { + type ArchiveRetentionDocument, + openArchiveRetentionStore, + readArchiveRetentionDocument, +} from '../archive-retention-store.js'; +import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '../root-authority.js'; +import { createSqliteSessionMetadataStore } from '../sqlite-session-metadata-store.js'; + +const DOCUMENT: ArchiveRetentionDocument = { + version: 1, + revision: 4, + enabled: true, + days: 60, + enabledAt: 1_000, + latest: { + observedAt: 8_000, + lastSweep: { at: 9_000, deleted: 2, skippedBusy: 1, needsReview: 0, failed: 0 }, + lastDeletion: { at: 9_000, count: 2, bytes: 512 }, + hold: { since: 1_000, detectedAt: 9_000, until: 95_400 }, + }, +}; + +describe('archive retention document', () => { + test('round-trips through the State Root and reads a missing file as absent', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-archive-retention-')); + const owner = await tryAcquireInteractiveRootOwner( + await resolveStorageRoot({ path: root, kind: 'interactive' }), + ); + assert.ok(owner); + try { + const store = openArchiveRetentionStore(owner.lease); + assert.deepEqual(await store.read(), { kind: 'absent' }); + await store.write(DOCUMENT); + assert.deepEqual(await store.read(), { kind: 'valid', document: DOCUMENT }); + const disabled: ArchiveRetentionDocument = { + version: 1, + revision: 5, + enabled: false, + days: 30, + }; + await store.write(disabled); + assert.deepEqual(await store.read(), { kind: 'valid', document: disabled }); + // A document the reader would refuse is never published. + await assert.rejects(store.write({ ...disabled, enabledAt: 1 })); + assert.deepEqual( + JSON.parse(await readFile(join(root, 'archive-retention.json'), 'utf8')), + disabled, + ); + } finally { + await owner.close(); + await rm(root, { recursive: true, force: true }); + } + }); + + test('reads anything it cannot fully validate as invalid, never as a setting', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-archive-retention-invalid-')); + const write = (value: string) => writeFile(join(root, 'archive-retention.json'), value); + try { + for (const value of [ + 'not json', + JSON.stringify({ ...DOCUMENT, version: 2 }), + JSON.stringify({ ...DOCUMENT, days: 45 }), + JSON.stringify({ ...DOCUMENT, enabledAt: undefined }), + JSON.stringify({ ...DOCUMENT, enabled: false }), + JSON.stringify({ ...DOCUMENT, revision: -1 }), + JSON.stringify({ ...DOCUMENT, extra: true }), + JSON.stringify({ ...DOCUMENT, latest: { lastSweep: { at: 1 } } }), + JSON.stringify({ ...DOCUMENT, latest: { lastDeletion: { at: 1, count: 1, bytes: 1.5 } } }), + JSON.stringify({ ...DOCUMENT, latest: { hold: { since: 5, detectedAt: 4, until: 6 } } }), + JSON.stringify({ ...DOCUMENT, latest: { observedAt: 1.5 } }), + ]) { + await write(value); + assert.equal((await readArchiveRetentionDocument(root)).kind, 'invalid', value); + } + } finally { + await rm(root, { recursive: true, force: true }); + } + }); +}); + +describe('archive retention candidates', () => { + test('are the archived, unpinned rows Settings shows, oldest first, resumable', async () => { + await withStore(async ({ store, setArchivedAt, sql }) => { + const create = (overrides: Partial) => store.create(header(overrides)); + await create({ id: 'legacy', isArchived: true }); + await create({ id: 'old', isArchived: true }); + await create({ id: 'recent', isArchived: true }); + await create({ + id: 'recent-revision', + isArchived: true, + revisionRootSessionId: 'recent', + revisionParentSessionId: 'recent', + revisionOfTurnId: 'turn-1', + revisionIndex: 2, + revisionState: 'committed', + }); + await create({ id: 'active' }); + await create({ id: 'pinned-root', isArchived: true }); + await create({ + id: 'pinned-revision', + isArchived: true, + isFlagged: true, + revisionRootSessionId: 'pinned-root', + revisionParentSessionId: 'pinned-root', + revisionOfTurnId: 'turn-1', + revisionIndex: 2, + revisionState: 'committed', + }); + await create({ id: 'live-parent', isArchived: true }); + await create({ + id: 'child', + isArchived: true, + subagentParent: subagentParent('live-parent'), + }); + await create({ id: 'orphan', isArchived: true, subagentParent: subagentParent('gone') }); + await create({ + id: 'operator', + isArchived: true, + subagentParent: { + ...subagentParent('gone'), + graph: { graphId: 'graph', workId: 'work', operatorId: 'operator' }, + }, + }); + await create({ id: 'preparing', isArchived: true }); + sql( + `UPDATE session_metadata + SET payload_json = json_set(payload_json, '$.conversationCopy', json(?)) + WHERE session_id = ?`, + JSON.stringify({ + kind: 'branch', + sourceSessionId: 'legacy', + sourceTurnId: 'turn-1', + requestFingerprint: `sha256:${'a'.repeat(64)}`, + state: 'preparing', + }), + 'preparing', + ); + await create({ id: 'unprojected', isArchived: true }); + sql('DELETE FROM session_catalog_projection WHERE session_id = ?', 'unprojected'); + setArchivedAt({ + legacy: null, + old: 100, + recent: 900, + 'live-parent': 200, + orphan: 300, + 'recent-revision': 950, + }); + + const ids = async (query: Parameters[0]) => + (await store.listArchiveRetentionCandidates(query)).map((record) => + 'undecodable' in record ? record.sessionId : record.header.id, + ); + // A pinned revision keeps its whole family; a child whose parent exists is + // that parent's row; a graph operator retires with its root; a preparing + // copy and a row without a catalog projection are not on the page. + assert.deepEqual(await ids({ limit: 50 }), [ + 'legacy', + 'old', + 'live-parent', + 'orphan', + 'recent', + 'recent-revision', + ]); + // The cutoff keeps every unknown time and drops later ones. + assert.deepEqual(await ids({ archivedBefore: 300, limit: 50 }), [ + 'legacy', + 'old', + 'live-parent', + ]); + assert.deepEqual(await ids({ limit: 2 }), ['legacy', 'old']); + assert.deepEqual(await ids({ after: { archivedAt: 100, sessionId: 'old' }, limit: 2 }), [ + 'live-parent', + 'orphan', + ]); + assert.deepEqual(await ids({ after: { sessionId: 'legacy' }, limit: 1 }), ['old']); + const [legacy, old] = await store.listArchiveRetentionCandidates({ limit: 2 }); + assert.equal(legacy?.archivedAt, undefined); + assert.equal(old?.archivedAt, 100); + + // The preview's count: one per family, and the earliest family start + // under a policy enabled at 150 (unknown and earlier times count from it). + assert.deepEqual(await store.countArchiveRetentionCandidates(150), { + families: 5, + firstStart: 150, + }); + assert.deepEqual(await store.countArchiveRetentionCandidates(1_000), { + families: 5, + firstStart: 1_000, + }); + }); + }); + + test('a row that no longer decodes is returned as such, in its place', async () => { + await withStore(async ({ store, setArchivedAt, sql }) => { + await store.create(header({ id: 'a-good', isArchived: true })); + await store.create(header({ id: 'b-bad', isArchived: true })); + setArchivedAt({ 'a-good': 10, 'b-bad': 20 }); + sql( + "UPDATE session_metadata SET payload_json = json_set(payload_json, '$.createdAt', 'never') WHERE session_id = ?", + 'b-bad', + ); + const rows = await store.listArchiveRetentionCandidates({ limit: 10 }); + assert.equal(rows.length, 2); + assert.equal('undecodable' in rows[0]! ? 'bad' : rows[0]!.header.id, 'a-good'); + assert.deepEqual(rows[1], { undecodable: true, sessionId: 'b-bad', archivedAt: 20 }); + }); + }); + + test('the newest metadata time comes from the committed and archive columns', async () => { + await withStore(async ({ store, setArchivedAt, setClock }) => { + setClock(50); + await store.create(header({ id: 'a' })); + await store.create(header({ id: 'b' })); + setClock(70); + await store.setArchivedVersioned([{ sessionId: 'a', expectedVersion: 1 }], true); + assert.equal(await store.readLatestSessionMetadataTime(), 70); + setArchivedAt({ a: 5_000 }); + assert.equal(await store.readLatestSessionMetadataTime(), 5_000); + }); + }); +}); + +async function withStore( + operation: (rig: { + store: ReturnType; + setArchivedAt(times: Record): void; + sql(statement: string, ...parameters: Array): void; + setClock(value: number): void; + }) => Promise, +): Promise { + const root = await mkdtemp(join(tmpdir(), 'maka-archive-retention-candidates-')); + const path = join(root, 'state.sqlite'); + let clock = 10; + const store = createSqliteSessionMetadataStore(path, { now: () => clock }); + try { + await operation({ + store, + setArchivedAt: (times) => { + const database = new DatabaseSync(path); + try { + const update = database.prepare( + 'UPDATE session_metadata SET archived_at = ? WHERE session_id = ?', + ); + for (const [id, archivedAt] of Object.entries(times)) update.run(archivedAt, id); + } finally { + database.close(); + } + }, + sql: (statement, ...parameters) => { + const database = new DatabaseSync(path); + try { + database.prepare(statement).run(...parameters); + } finally { + database.close(); + } + }, + setClock: (value) => { + clock = value; + }, + }); + } finally { + store.close(); + await rm(root, { recursive: true, force: true }); + } +} + +function subagentParent(parentSessionId: string): NonNullable { + return { + kind: 'subagent', + parentSessionId, + spawnedBy: { parentRunId: 'run', parentTurnId: 'turn', toolCallId: 'call' }, + lifecycle: 'foreground', + }; +} + +function header(overrides: Partial): SessionHeader { + return { + id: 'session', + workspaceRoot: '/workspace', + cwd: '/workspace/repo', + createdAt: 1, + lastMessageAt: 3, + name: 'Session', + titleIsManual: true, + isFlagged: false, + labels: [], + isArchived: false, + status: 'active', + statusUpdatedAt: 4, + hasUnread: false, + backend: 'ai-sdk', + llmConnectionSlug: 'openai', + connectionLocked: true, + model: 'gpt-5', + toolProfile: 'headless-coding-v1', + thinkingLevel: 'high', + permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: 'swarm', + schemaVersion: 1, + ...overrides, + }; +} diff --git a/packages/storage/src/__tests__/session-bundle-policy.test.ts b/packages/storage/src/__tests__/session-bundle-policy.test.ts index d58ec04d5c..868e37fbfa 100644 --- a/packages/storage/src/__tests__/session-bundle-policy.test.ts +++ b/packages/storage/src/__tests__/session-bundle-policy.test.ts @@ -26,7 +26,7 @@ import { test } from 'node:test'; import type { CreateSessionInput } from '@maka/core/runtime-inputs'; import type { RuntimeEvent } from '@maka/core/runtime-event'; import { createSessionStore } from '../session-store.js'; -import { exportSessionBundleState } from '../session-bundle-policy.js'; +import { exportSessionBundleState, importSessionBundleState } from '../session-bundle-policy.js'; import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; test('exports one Session as filtered SQLite', async () => { @@ -86,6 +86,73 @@ test('exports one Session as filtered SQLite', async () => { } }); +test('an imported archived Session starts its archive clock at the import', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-session-bundle-archived-')); + const configRoot = join(base, 'config'); + await mkdir(configRoot, { recursive: true }); + try { + // One archived before the time was recorded, one archived long ago. + const exported: Array<{ id: string; bundle: string }> = []; + for (const [name, archivedAt] of [ + ['unknown', null], + ['old', 5], + ] as const) { + const stateRoot = join(base, `source-${name}`); + const sessions = createSessionStore(stateRoot); + const session = await sessions.create(input(name)); + await sessions.appendMessage(session.id, message(`${name}-message`)); + await sessions.setSessionsArchivedVersioned( + [ + { + sessionId: session.id, + expectedVersion: (await sessions.readHeaderRecordSnapshot(session.id)).revision, + }, + ], + true, + ); + await sessions.close?.(); + const source = new DatabaseSync(join(stateRoot, 'runtime.sqlite')); + source + .prepare('UPDATE session_metadata SET archived_at = ? WHERE session_id = ?') + .run(archivedAt, session.id); + source.close(); + const bundle = join(base, `bundle-${name}`); + await exportSessionBundleState({ + stateRoot, + configRoot, + destinationRoot: bundle, + sessionId: session.id, + }); + exported.push({ id: session.id, bundle }); + } + + const target = join(base, 'target'); + await mkdir(target, { recursive: true }); + const before = Date.now(); + for (const { bundle } of exported) { + await importSessionBundleState({ stateRoot: target, bundleStateRoot: bundle }); + } + const after = Date.now(); + const database = new DatabaseSync(join(target, 'runtime.sqlite'), { readOnly: true }); + try { + for (const { id } of exported) { + const row = database + .prepare('SELECT is_archived, archived_at FROM session_metadata WHERE session_id = ?') + .get(id) as { is_archived: number; archived_at: number | null }; + assert.equal(row.is_archived, 1); + assert.ok( + row.archived_at !== null && row.archived_at >= before && row.archived_at <= after, + `archived_at ${row.archived_at} is the import time`, + ); + } + } finally { + database.close(); + } + } finally { + await rm(base, { recursive: true, force: true }); + } +}); + test('retains only the selected Session partial stream segments', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-session-bundle-partials-')); const stateRoot = join(base, 'state'); diff --git a/packages/storage/src/archive-retention-store.ts b/packages/storage/src/archive-retention-store.ts new file mode 100644 index 0000000000..ea3abe7f94 --- /dev/null +++ b/packages/storage/src/archive-retention-store.ts @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { + type ArchiveRetentionDays, + type ArchiveRetentionDeletion, + type ArchiveRetentionHold, + type ArchiveRetentionSweep, + decodeArchiveRetentionDeletion, + decodeArchiveRetentionHold, + decodeArchiveRetentionSweep, + isArchiveRetentionDays, +} from '@maka/core/archive-retention'; +import { runWithStorageRootLease, type StorageRootLease } from './root-authority.js'; +import { readBoundedJsonDocument, writeJsonDocument } from './runtime-policy/document-io.js'; +import { RuntimePolicyStoreError } from './runtime-policy/errors.js'; + +const FILE = 'archive-retention.json'; +const VERSION = 1 as const; +const MAX_BYTES = 16 * 1024; + +/** + * The archived-task retention setting and its latest results: a Host document + * of its own in the State Root. It is deliberately not part of the runtime + * policy, which has several writers and travels with config export and + * import; the only writer of this one is the Host's retention command and its + * sweep. + */ +export interface ArchiveRetentionDocument { + readonly version: typeof VERSION; + /** Moves with every setting change, never with a sweep result. */ + readonly revision: number; + readonly enabled: boolean; + readonly days: ArchiveRetentionDays; + /** Host clock when the current setting took effect; present exactly while enabled. */ + readonly enabledAt?: number; + readonly latest?: { + /** Coarse Host heartbeat used to distinguish an idle restart from a clock jump. */ + readonly observedAt?: number; + readonly lastSweep?: ArchiveRetentionSweep; + readonly lastDeletion?: ArchiveRetentionDeletion; + /** Deletions held after a forward clock gap; any setting change clears it. */ + readonly hold?: ArchiveRetentionHold; + }; +} + +/** + * `invalid` is a document that exists but cannot be fully validated, including + * one written by a newer Host. Retention treats it as disabled. + */ +export type ArchiveRetentionDocumentRead = + | { readonly kind: 'absent' } + | { readonly kind: 'valid'; readonly document: ArchiveRetentionDocument } + | { readonly kind: 'invalid'; readonly reason: string }; + +export interface InteractiveArchiveRetentionStore { + /** Throws only when the document cannot be read at all. */ + read(): Promise; + /** Atomically replaces the document. */ + write(document: ArchiveRetentionDocument): Promise; +} + +export function openArchiveRetentionStore( + lease: StorageRootLease<'interactive', 'write'>, +): InteractiveArchiveRetentionStore { + return Object.freeze({ + read: () => + runWithStorageRootLease(lease, 'interactive', 'write', (root) => + readArchiveRetentionDocument(root), + ), + write: (document: ArchiveRetentionDocument) => + runWithStorageRootLease(lease, 'interactive', 'write', (root) => + writeJsonDocument(root, FILE, encodeArchiveRetentionDocument(document), MAX_BYTES), + ), + }); +} + +export async function readArchiveRetentionDocument( + root: string, +): Promise { + let value: unknown; + try { + value = await readBoundedJsonDocument(root, FILE, MAX_BYTES); + } catch (error) { + if (error instanceof RuntimePolicyStoreError && error.code === 'invalid_document') { + return { kind: 'invalid', reason: error.message }; + } + throw error; + } + if (value === undefined) return { kind: 'absent' }; + try { + return { kind: 'valid', document: decodeArchiveRetentionDocument(value) }; + } catch (error) { + return { kind: 'invalid', reason: error instanceof Error ? error.message : String(error) }; + } +} + +function decodeArchiveRetentionDocument(value: unknown): ArchiveRetentionDocument { + const document = exactRecord( + value, + FILE, + ['version', 'revision', 'enabled', 'days'], + ['enabledAt', 'latest'], + ); + if (document.version !== VERSION) throw new Error(`${FILE} has an unsupported version`); + if (typeof document.enabled !== 'boolean') throw new Error(`${FILE} has an invalid enabled`); + if (!isArchiveRetentionDays(document.days)) throw new Error(`${FILE} has invalid days`); + if (document.enabled !== (document.enabledAt !== undefined)) { + throw new Error(`${FILE} must carry enabledAt exactly while enabled`); + } + const latest = + document.latest === undefined + ? undefined + : exactRecord( + document.latest, + `${FILE}.latest`, + [], + ['observedAt', 'lastSweep', 'lastDeletion', 'hold'], + ); + return { + version: VERSION, + revision: count(document.revision, 'revision'), + enabled: document.enabled, + days: document.days, + ...(document.enabledAt === undefined + ? {} + : { enabledAt: count(document.enabledAt, 'enabledAt') }), + ...(latest === undefined + ? {} + : { + latest: { + ...(latest.observedAt === undefined + ? {} + : { observedAt: count(latest.observedAt, 'latest observedAt') }), + ...(latest.lastSweep === undefined + ? {} + : { lastSweep: decodeArchiveRetentionSweep(latest.lastSweep, documentError) }), + ...(latest.lastDeletion === undefined + ? {} + : { + lastDeletion: decodeArchiveRetentionDeletion(latest.lastDeletion, documentError), + }), + ...(latest.hold === undefined + ? {} + : { hold: decodeArchiveRetentionHold(latest.hold, documentError) }), + }, + }), + }; +} + +function encodeArchiveRetentionDocument(document: ArchiveRetentionDocument): unknown { + // Never publish what the reader would refuse. + return decodeArchiveRetentionDocument(JSON.parse(JSON.stringify(document))); +} + +function exactRecord( + value: unknown, + label: string, + required: readonly string[], + optional: readonly string[], +): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error(`${label} must be an object`); + } + const record = value as Record; + for (const key of Object.keys(record)) { + if (!required.includes(key) && !optional.includes(key)) { + throw new Error(`${label} has an unknown key`); + } + } + for (const key of required) { + if (!Object.hasOwn(record, key)) throw new Error(`${label} is missing ${key}`); + } + return record; +} + +function count(value: unknown, label: string): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0) { + throw new Error(`${FILE} has an invalid ${label}`); + } + return value; +} + +function documentError(message: string): Error { + return new Error(`${FILE}: ${message}`); +} diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index ee57dd52e7..73e8b5a978 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -142,6 +142,10 @@ export type { export { submittedTurnIntentsEqual } from './submitted-turn-intent.js'; export type { SubmittedTurnIntent } from './submitted-turn-intent.js'; export type { + ArchiveRetentionCandidate, + ArchiveRetentionCandidateCount, + ArchiveRetentionCandidateQuery, + ArchiveRetentionCandidateRow, CreateStableSessionRequest, ProbeSessionRemovalResult, ExternalSessionImportLookupResult, @@ -698,6 +702,11 @@ async function createExecutionStoresForWrite( run(() => sessionStore.listPendingSessionRetirementCleanupIds(sessionId)), completeSessionRetirementCleanup: (sessionId) => run(() => sessionStore.completeSessionRetirementCleanup(sessionId)), + listArchiveRetentionCandidates: (query) => + run(() => sessionStore.listArchiveRetentionCandidates(query)), + countArchiveRetentionCandidates: (enabledAt) => + run(() => sessionStore.countArchiveRetentionCandidates(enabledAt)), + readLatestSessionMetadataTime: () => run(() => sessionStore.readLatestSessionMetadataTime()), close, }, agentRunStore: { diff --git a/packages/storage/src/runtime-policy/document-io.ts b/packages/storage/src/runtime-policy/document-io.ts index 508dedd327..0218644f59 100644 --- a/packages/storage/src/runtime-policy/document-io.ts +++ b/packages/storage/src/runtime-policy/document-io.ts @@ -35,7 +35,7 @@ export const VAULT_DOCUMENT_MAX_BYTES = 2 * 1024 * 1024; const READ_CHUNK_BYTES = 64 * 1024; const RUNTIME_POLICY_TEMP_PATTERN = - /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding|runtime-policy-oauth-login-receipts|model-facts)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; + /^(?:runtime-policy|connection-catalog|credential-vault|runtime-policy-onboarding|runtime-policy-oauth-login-receipts|model-facts|archive-retention)\.json\.[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}\.tmp$/; export async function cleanupRuntimePolicyDocumentTemps(root: string): Promise { let failure: unknown; diff --git a/packages/storage/src/session-bundle-policy.ts b/packages/storage/src/session-bundle-policy.ts index c2054532c4..5035cbf701 100644 --- a/packages/storage/src/session-bundle-policy.ts +++ b/packages/storage/src/session-bundle-policy.ts @@ -1301,6 +1301,17 @@ function mergeAttachedBundle(target: DatabaseSync): string[] { const quoted = quoteIdentifier(name); target.exec(`INSERT INTO main.${quoted} SELECT * FROM bundle.${quoted}`); } + // An archived Session arrives with the archive time (or no time) of the + // machine it left. Archive retention counts from that time, so an import + // could be deleted at once; its clock starts at the import instead. + target + .prepare( + `UPDATE main.session_metadata + SET archived_at = ? + WHERE is_archived = 1 + AND session_id IN (SELECT session_id FROM bundle.session_metadata)`, + ) + .run(Date.now()); const violation = target.prepare('PRAGMA foreign_key_check').get(); if (violation) { throw new SessionBundleImportError( diff --git a/packages/storage/src/session-store-contract.ts b/packages/storage/src/session-store-contract.ts index 1becfa40b6..7f42fd1c8b 100644 --- a/packages/storage/src/session-store-contract.ts +++ b/packages/storage/src/session-store-contract.ts @@ -149,6 +149,42 @@ export type ProbeSessionRemovalResult = | { readonly kind: 'removed' } | { readonly kind: 'absent' }; +/** + * A task row archive retention may delete: what Settings › Archived tasks lists + * — an archived root, or an archived subtask whose parent is gone — that no + * member of its revision family pins. + */ +export interface ArchiveRetentionCandidate extends SessionHeaderSnapshot { + /** Absent for a Session archived before the Host recorded the time. */ + readonly archivedAt?: number; +} + +/** A candidate row whose metadata no longer decodes: counted as failed and passed over. */ +export interface ArchiveRetentionUndecodableRow { + readonly undecodable: true; + readonly sessionId: string; + readonly archivedAt?: number; +} + +export type ArchiveRetentionCandidateRow = + | ArchiveRetentionCandidate + | ArchiveRetentionUndecodableRow; + +/** Candidate families and the earliest clock start among them under one `enabledAt`. */ +export interface ArchiveRetentionCandidateCount { + readonly families: number; + readonly firstStart?: number; +} + +/** Candidates come oldest archive first; an unknown time sorts before every known one. */ +export interface ArchiveRetentionCandidateQuery { + /** Only rows archived before this instant, plus every row whose time is unknown. */ + readonly archivedBefore?: number; + /** Resume strictly after this row of the same order. */ + readonly after?: { readonly archivedAt?: number; readonly sessionId: string }; + readonly limit: number; +} + export interface SessionCatalogRecord extends SessionHeaderSnapshot { readonly activityAt: number; readonly summary: SessionSummary; @@ -550,4 +586,10 @@ export interface SessionAuthorityStore extends SessionStore, MessageAdmissionSto reconcileOrphanedAgentGraphRetirements(): Promise; listPendingSessionRetirementCleanupIds(sessionId?: string): Promise; completeSessionRetirementCleanup(sessionId: string): Promise; + listArchiveRetentionCandidates( + query: ArchiveRetentionCandidateQuery, + ): Promise; + countArchiveRetentionCandidates(enabledAt: number): Promise; + /** The newest time any Session metadata write or archive recorded; undefined with no Sessions. */ + readLatestSessionMetadataTime(): Promise; } diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 615f1b320b..e42014643f 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -51,6 +51,9 @@ import { type CoordinationTranscriptIndexRecord, type CoordinationTranscriptIndexState, type SessionAuthorityStore, + type ArchiveRetentionCandidateCount, + type ArchiveRetentionCandidateQuery, + type ArchiveRetentionCandidateRow, } from './session-store-contract.js'; export { isSafeSessionId, @@ -905,6 +908,32 @@ class SqliteSessionStore implements SessionAuthorityStore { await this.metadata.completeSessionRetirementCleanup(sessionId); } + async listArchiveRetentionCandidates( + query: ArchiveRetentionCandidateQuery, + ): Promise { + await this.ensureReady(); + return (await this.metadata.listArchiveRetentionCandidates(query)).map((record) => + 'undecodable' in record + ? record + : { + ...projectHeaderSnapshot(record), + ...(record.archivedAt === undefined ? {} : { archivedAt: record.archivedAt }), + }, + ); + } + + async countArchiveRetentionCandidates( + enabledAt: number, + ): Promise { + await this.ensureReady(); + return this.metadata.countArchiveRetentionCandidates(enabledAt); + } + + async readLatestSessionMetadataTime(): Promise { + await this.ensureReady(); + return this.metadata.readLatestSessionMetadataTime(); + } + async setFlagged(sessionId: string, isFlagged: boolean): Promise { await this.updateHeader(sessionId, { isFlagged }); } diff --git a/packages/storage/src/sqlite-session-catalog-query.ts b/packages/storage/src/sqlite-session-catalog-query.ts index 25de16eeef..f5a94e9a8b 100644 --- a/packages/storage/src/sqlite-session-catalog-query.ts +++ b/packages/storage/src/sqlite-session-catalog-query.ts @@ -30,19 +30,67 @@ export interface SqliteSessionCatalogPageQuery { readonly parameters: readonly (string | number)[]; } +export interface SqliteSessionPredicate { + readonly sql: string; + readonly parameters: readonly string[]; +} + +/** + * A Session row the catalog lists: ordinary, not a copy still being prepared, + * and not a transcript-less shell. `alias` names the `session_metadata` row; + * the caller joins `session_catalog_projection` for it. + */ +export function sqliteCatalogVisibleSessionPredicate(alias = 'metadata'): SqliteSessionPredicate { + const role = sqliteOrdinarySessionRolePredicate(alias); + return { + sql: `( + COALESCE(json_extract(${alias}.payload_json, '$.conversationCopy.state'), '') <> 'preparing' + AND ${role.sql} + AND COALESCE(json_extract(${alias}.payload_json, '$.transcriptLedgerVersion'), 1) <> 0 + )`, + parameters: [...role.parameters], + }; +} + +/** + * A row of Settings › Archived tasks, as the rail derives it from the catalog: + * an archived catalog-visible Session that is not a linked subtask of another + * catalog-visible Session. A subtask whose parent is gone is a row of its own. + * Revision families still collapse to one row; callers group by family. + * Requires `metadata` joined with its `session_catalog_projection`. + */ +export function sqliteArchivedTaskRowPredicate(): SqliteSessionPredicate { + const row = sqliteCatalogVisibleSessionPredicate('metadata'); + const parent = sqliteCatalogVisibleSessionPredicate('parent'); + return { + sql: `( + metadata.is_archived = 1 + AND ${row.sql} + AND ( + metadata.subagent_parent_session_id IS NULL + OR NOT EXISTS ( + SELECT 1 + FROM session_metadata parent + JOIN session_catalog_projection parent_projection + ON parent_projection.session_id = parent.session_id + WHERE parent.session_id = metadata.subagent_parent_session_id + AND ${parent.sql} + ) + ) + )`, + parameters: [...row.parameters, ...parent.parameters], + }; +} + export function buildSqliteSessionCatalogPageQuery( filter: SessionListFilter, cursor: SqliteSessionCatalogCursor | undefined, ): SqliteSessionCatalogPageQuery { const where: string[] = []; const parameters: Array = []; - const role = sqliteOrdinarySessionRolePredicate(); - where.push( - "COALESCE(json_extract(metadata.payload_json, '$.conversationCopy.state'), '') <> 'preparing'", - ); - where.push(role.sql); - parameters.push(...role.parameters); - where.push("COALESCE(json_extract(metadata.payload_json, '$.transcriptLedgerVersion'), 1) <> 0"); + const visible = sqliteCatalogVisibleSessionPredicate(); + where.push(visible.sql); + parameters.push(...visible.parameters); if (filter.subagentParentSessionId !== undefined) { where.push('projection.subagent_parent_session_id = ?'); parameters.push(filter.subagentParentSessionId); diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index b4715adf32..27d7221390 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -186,6 +186,7 @@ import { } from './recall-fold.js'; import { buildSqliteSessionCatalogPageQuery, + sqliteArchivedTaskRowPredicate, type SqliteSessionCatalogCursor, } from './sqlite-session-catalog-query.js'; import { @@ -1284,6 +1285,121 @@ export class SqliteSessionMetadataStore { return (rows as unknown as Array<{ readonly sessionId: string }>).map((row) => row.sessionId); } + /** + * Archive-retention candidates, oldest archive first: the rows Settings › + * Archived tasks shows (the catalog's own archived-row predicate), less + * Agent Graph operators, which retire only with their root, and less any + * family a pinned member keeps. A row that no longer decodes is returned as + * such, so a sweep can count it and move past it. + */ + async listArchiveRetentionCandidates(query: { + readonly archivedBefore?: number; + readonly after?: { readonly archivedAt?: number; readonly sessionId: string }; + readonly limit: number; + }): Promise< + Array< + | (SessionMetadataRecord & { readonly archivedAt?: number }) + | { readonly undecodable: true; readonly sessionId: string; readonly archivedAt?: number } + > + > { + this.assertOpen(); + if (!Number.isSafeInteger(query.limit) || query.limit < 1 || query.limit > 256) { + throw new Error('Archive retention candidate limit must be between 1 and 256'); + } + const candidate = archiveRetentionCandidatePredicate(); + const where = [candidate.sql]; + const parameters: Array = [...candidate.parameters]; + if (query.archivedBefore !== undefined) { + where.push('(metadata.archived_at IS NULL OR metadata.archived_at < ?)'); + parameters.push(query.archivedBefore); + } + if (query.after) { + assertSafeSessionId(query.after.sessionId); + where.push('(COALESCE(metadata.archived_at, -1), metadata.session_id) > (?, ?)'); + parameters.push(query.after.archivedAt ?? -1, query.after.sessionId); + } + const rows = this.db + .prepare( + ` + SELECT + metadata.session_id, + metadata.payload_json, + metadata.metadata_version, + metadata.committed_at, + metadata.archived_at + FROM session_metadata metadata + JOIN session_catalog_projection projection + ON projection.session_id = metadata.session_id + WHERE ${where.join(' AND ')} + ORDER BY COALESCE(metadata.archived_at, -1), metadata.session_id + LIMIT ? + `, + ) + .all(...parameters, query.limit) as unknown as Array< + SessionMetadataRow & { archived_at: number | null } + >; + return rows.map((row) => { + const position = typeof row.archived_at === 'number' ? { archivedAt: row.archived_at } : {}; + try { + const archivedAt = decodeCatalogArchivedAt(row.archived_at, row.session_id); + return { ...decodeRecord(row), ...(archivedAt === undefined ? {} : { archivedAt }) }; + } catch { + return { undecodable: true as const, sessionId: row.session_id, ...position }; + } + }); + } + + /** + * How many families `listArchiveRetentionCandidates` would offer, and the + * earliest clock start among them under a policy enabled at `enabledAt`: a + * family starts when its most recently archived member did, and never before + * enablement. + */ + async countArchiveRetentionCandidates( + enabledAt: number, + ): Promise<{ readonly families: number; readonly firstStart?: number }> { + this.assertOpen(); + const candidate = archiveRetentionCandidatePredicate(); + const row = this.db + .prepare( + ` + SELECT COUNT(*) AS families, MIN(start) AS first_start + FROM ( + SELECT MAX(MAX(COALESCE(metadata.archived_at, ?), ?)) AS start + FROM session_metadata metadata + JOIN session_catalog_projection projection + ON projection.session_id = metadata.session_id + WHERE ${candidate.sql} + GROUP BY COALESCE(metadata.revision_root_session_id, metadata.session_id) + ) + `, + ) + .get(enabledAt, enabledAt, ...candidate.parameters) as { + families: number; + first_start: number | null; + }; + return { + families: row.families, + ...(typeof row.first_start === 'number' ? { firstStart: row.first_start } : {}), + }; + } + + async readLatestSessionMetadataTime(): Promise { + this.assertOpen(); + const row = this.db + .prepare( + ` + SELECT MAX(committed_at) AS committed_at, MAX(archived_at) AS archived_at + FROM session_metadata + `, + ) + .get() as { committed_at: number | null; archived_at: number | null } | undefined; + const times = [row?.committed_at, row?.archived_at].filter( + (value): value is number => typeof value === 'number' && Number.isFinite(value), + ); + return times.length === 0 ? undefined : Math.max(...times); + } + async reconcileOrphanedAgentGraphRetirements(): Promise { this.assertOpen(); return this.transaction(() => { @@ -6373,6 +6489,28 @@ function agentGraphScheduleUpdateRequest( return request; } +function archiveRetentionCandidatePredicate(): { sql: string; parameters: readonly string[] } { + const row = sqliteArchivedTaskRowPredicate(); + return { + // The pinned-family exclusion is deliberately uncorrelated: SQLite + // evaluates the pinned family roots once, instead of rescanning + // session_metadata for every candidate. The correlated form was quadratic + // (about 1 s at 10k Sessions) and runs synchronously on the Host thread. + // No index covers is_flagged/is_archived (migration 26 dropped them). + sql: `( + metadata.is_flagged = 0 + AND ${row.sql} + AND json_type(metadata.payload_json, '$.subagentParent.graph') IS NULL + AND COALESCE(metadata.revision_root_session_id, metadata.session_id) NOT IN ( + SELECT COALESCE(pinned.revision_root_session_id, pinned.session_id) + FROM session_metadata pinned + WHERE pinned.is_flagged = 1 + ) + )`, + parameters: row.parameters, + }; +} + function decodeRecord(row: SessionMetadataRow): SessionMetadataRecord { const parsed = JSON.parse(row.payload_json) as SessionHeader; if ( diff --git a/packages/storage/src/sqlite-session-role-scope.ts b/packages/storage/src/sqlite-session-role-scope.ts index e3721279e4..fbede8db88 100644 --- a/packages/storage/src/sqlite-session-role-scope.ts +++ b/packages/storage/src/sqlite-session-role-scope.ts @@ -32,11 +32,11 @@ export interface SqliteSessionRolePredicate { * metadata is corrupt or missing. This keeps damaged authority state out of * ordinary catalogs and write paths until the Host can be repaired. */ -export function sqliteOrdinarySessionRolePredicate(): SqliteSessionRolePredicate { +export function sqliteOrdinarySessionRolePredicate(alias = 'metadata'): SqliteSessionRolePredicate { return { sql: `( - metadata.session_id <> ? - AND json_type(metadata.payload_json, '$.role') IS NULL + ${alias}.session_id <> ? + AND json_type(${alias}.payload_json, '$.role') IS NULL )`, parameters: [WORKHUB_COORDINATION_SESSION_ID], }; diff --git a/packages/storage/src/storage-writer-composition.ts b/packages/storage/src/storage-writer-composition.ts index d63e14ca1d..85d4e31832 100644 --- a/packages/storage/src/storage-writer-composition.ts +++ b/packages/storage/src/storage-writer-composition.ts @@ -17,6 +17,10 @@ * under the License. */ +import { + openArchiveRetentionStore, + type InteractiveArchiveRetentionStore, +} from './archive-retention-store.js'; import { openInteractiveArtifactStoreForWrite } from './artifact-stores.js'; import type { ContextOffloadLimits } from '@maka/core/context-offload'; import { openInteractiveContextOffloadStoreForWrite } from './context-offload-store.js'; @@ -80,6 +84,8 @@ export interface StorageWriterComposition { readonly usage: Awaited>; readonly shellRuns: Awaited>; readonly footprint: InteractiveStorageFootprintReader; + /** The archived-task retention document, bound to the composition's write lease. */ + readonly archiveRetention: InteractiveArchiveRetentionStore; close(): Promise; } @@ -233,6 +239,7 @@ async function createComposition( usage, shellRuns, footprint, + archiveRetention: openArchiveRetentionStore(lease), close, }); } diff --git a/packages/storage/src/test-only/memory-execution-session.ts b/packages/storage/src/test-only/memory-execution-session.ts index 3df960d17a..14fb652cc9 100644 --- a/packages/storage/src/test-only/memory-execution-session.ts +++ b/packages/storage/src/test-only/memory-execution-session.ts @@ -61,6 +61,7 @@ import { type SessionCatalogRecord, type CreateStableSessionRequest, type CoordinationTranscriptIndexRecord, + type ArchiveRetentionCandidate, } from '../session-store-contract.js'; import { buildSessionHeader, normalizeSessionHeader, toSummary } from '../session-store-values.js'; import { isValidConversationCopyTransition } from '../session-conversation-copy.js'; @@ -248,6 +249,25 @@ function setArchived( else rows(s, 'archivedAt').delete(id); return next; } +/** The rows the archived-task page lists, less graph operators and pinned families. */ +function archiveRetentionCandidates(s: MemoryState): ArchiveRetentionCandidate[] { + const all = [...headers(s).values()]; + const family = (h: SessionHeader) => h.revisionRootSessionId ?? h.id; + const pinned = new Set(all.filter((r) => r.header.isFlagged).map((r) => family(r.header))); + return all + .filter(({ header: h }) => h.isArchived && !h.isFlagged && h.id !== HUB && !h.role) + .filter(({ header: h }) => h.conversationCopy?.state !== 'preparing') + .filter(({ header: h }) => h.transcriptLedgerVersion !== 0 && !h.subagentParent?.graph) + .filter( + ({ header: h }) => !h.subagentParent || !headers(s).has(h.subagentParent.parentSessionId), + ) + .filter(({ header: h }) => !pinned.has(family(h))) + .map((r) => { + const archivedAt = rows(s, 'archivedAt').get(r.header.id); + return archivedAt === undefined ? r : { ...r, archivedAt }; + }); +} + function catalog(s: MemoryState, id: string): SessionCatalogRecord { const record = requireHeader(s, id); const preview = rows(s, 'previews').get(id); @@ -724,6 +744,50 @@ export function createMemorySessionStore( rows(s, 'cleanup').delete(id); }); }, + listArchiveRetentionCandidates: async (query) => + read((s) => { + const order = (r: ArchiveRetentionCandidate) => r.archivedAt ?? -1; + const after = query.after; + return archiveRetentionCandidates(s) + .filter( + (r) => + query.archivedBefore === undefined || + r.archivedAt === undefined || + r.archivedAt < query.archivedBefore, + ) + .sort( + (a, b) => + order(a) - order(b) || + (a.header.id < b.header.id ? -1 : a.header.id > b.header.id ? 1 : 0), + ) + .filter( + (r) => + !after || + order(r) > (after.archivedAt ?? -1) || + (order(r) === (after.archivedAt ?? -1) && r.header.id > after.sessionId), + ) + .slice(0, query.limit); + }), + countArchiveRetentionCandidates: async (enabledAt) => + read((s) => { + const starts = new Map(); + for (const r of archiveRetentionCandidates(s)) { + const family = r.header.revisionRootSessionId ?? r.header.id; + const start = Math.max(r.archivedAt ?? enabledAt, enabledAt); + starts.set(family, Math.max(starts.get(family) ?? start, start)); + } + return starts.size === 0 + ? { families: 0 } + : { families: starts.size, firstStart: Math.min(...starts.values()) }; + }), + readLatestSessionMetadataTime: async () => + read((s) => { + const times = [ + ...[...headers(s).values()].map((r) => r.committedAt), + ...rows(s, 'archivedAt').values(), + ]; + return times.length === 0 ? undefined : Math.max(...times); + }), reconcileOrphanedAgentGraphRetirements: async () => write('session.reconcileRetirement', (s) => { const ids = [...headers(s).values()]