Move integration tests and evals out of packaged module

README.md

@@ -121,6 +121,8 @@ $ npm install
 $ npm run test:watch
 ```
 
+See [`TESTING.md`](TESTING.md) for test placement and boundary rules.
+
 Run the wizard locally against a project with
 
 ```
@@ -157,8 +159,6 @@ The wizard is an [Ink] application under `src/lib`:
 - `app.tsx` is the state machine driving the run,
   one phase per step, and holds all of the rendering.
 - `steps/` holds the logic for each step, with no UI in it.
-- `util/` holds the Seam API client, dotenv handling,
-  and the subprocess runner.
 - `version.ts` holds the package version reported by `--version`.
   It ships a `0.0.0` placeholder that `prepack.ts` replaces with the
   version from `package.json` when the package is packed,

TESTING.md

@@ -0,0 +1,104 @@
+# Testing the Seam Wizard
+
+Use this guide to decide what kind of test to write, where it belongs, and what
+the test can replace.
+
+## Test types
+
+### Unit tests
+
+A unit test checks the public behavior of one source module. Its collaborators
+stay real. The test can use Node.js APIs, external packages, and type-only
+imports, but it imports runtime behavior only from the module with the same
+name.
+
+Keep a unit test beside its module:
+
+```text
+src/lib/api-key.ts
+src/lib/api-key.test.ts
+```
+
+A test that uses a temporary directory can still be a unit test when the file
+system is part of that module's public behavior. Use the real file system in a
+temporary directory.
+
+### Integration tests
+
+An integration test checks multiple wizard modules together. This includes a
+test that imports a memory adapter, store, or other sibling module to exercise
+its subject. It also includes an orchestration module that runs several real
+collaborators.
+
+Put integration tests under top-level `test/`, mirroring `src/lib/`:
+
+```text
+src/lib/steps/connection.ts
+src/lib/steps/connection.test.ts      # unit behavior
+test/steps/connection.test.ts         # connection + adapter + store
+```
+
+### End-to-end tests
+
+An end-to-end test starts the real package or host CLI and checks observable
+process behavior. Put it under `test/`. Keep each user-visible flow in one
+end-to-end test; test its branches in smaller module or integration tests.
+
+### Evals
+
+An eval is a manual, paid test of probabilistic agent behavior. It reports
+quality, cost, and time instead of supplying a deterministic CI pass/fail gate.
+Eval code and fixtures belong under top-level `eval/`; deterministic tests of
+the eval harness follow the same unit and integration placement rules above.
+
+## Fixtures
+
+Reusable or on-disk test data belongs under `test/fixtures/`. A small value used
+by one test stays in that test file. Eval sample applications belong under
+`eval/fixtures/` because they are eval inputs, not test fixtures.
+
+## Boundaries
+
+Use classical assertions: call the subject, then assert on its return value or
+on data captured at a process boundary.
+
+The wizard has these process boundaries:
+
+- **Host state and authentication:** use `createMemoryAdapter()`, install it
+  with `setAdapter()`, and restore it with `resetAdapter()`.
+- **Terminal:** render Ink components with `ink-testing-library`, or capture
+  writes to stdout. At the package entrypoint, the renderer can be replaced so
+  a test does not take over the terminal.
+- **Wire:** replace `fetch` or use a local HTTP server, capture the request, and
+  return a real `Response`. Never call a live service from a deterministic test.
+- **Disk:** use the real file system in a temporary directory. Do not fake
+  `node:fs`.
+- **Environment:** use `vi.stubEnv()` and restore it after the test.
+- **Agent execution:** inject the narrow runner or harness seam and capture its
+  events. Real model calls belong only in an eval.
+
+Prefer an injected value over module-path substitution. A module replacement is
+acceptable only at a process edge that has no value-level injection seam, such
+as stopping the package entrypoint before Ink takes over the terminal.
+
+## Assertions
+
+Assert on observable behavior:
+
+- rendered terminal text;
+- returned values;
+- stored adapter state;
+- files written in a temporary project;
+- HTTP requests sent and responses handled;
+- process exit status;
+- agent events and resulting diffs.
+
+An assertion that one internal helper called another tests implementation
+structure. Keep it only when the call crosses one of the process boundaries
+above and the captured message is itself the behavior.
+
+## Rule of thumb
+
+> A test for one module stays beside it. A test for cooperating modules goes in
+> `test/`. Fake only the edge where data leaves the wizard, and assert on the
+> data that crossed it.

eslint.config.ts

@@ -90,7 +90,7 @@ export default [
             ['^node:'],
             ['^@?\\w'],
             ['@seamapi/wizard'],
-            ['^lib/', '^test/'],
+            ['^eval/', '^lib/', '^test/'],
             ['^'],
             ['^\\.'],
           ],

eval/README.md

@@ -17,9 +17,8 @@ For each fixture × mode (`full_api`, `customer_portal`):
 3. Capture the diff, cost, and elapsed time.
 4. Apply deterministic **gates** (build-free): `.env` untouched, `seam`
    imported, no standalone Seam-only page.
-5. Print an A/B-ready table.
-
-Quality **scoring** (LLM-judge over the diff) is layered on next.
+5. Score the diff against the mode's rubric with an LLM judge.
+6. Print an A/B-ready table.
 
 ## Running it
 

src/eval/real-runner.ts → eval/real-runner.ts

@@ -1,6 +1,6 @@
+import { getInferenceBaseUrl } from 'lib/seam-api.js'
 import { buildIntegrationSteps } from 'lib/steps/build-plan.js'
 import { runIntegration } from 'lib/steps/integrate.js'
-import { getInferenceBaseUrl } from 'lib/util/seam-api.js'
 
 import type { CaseRunner } from './run-case.js'
 

src/eval/run.ts → eval/run.ts

@@ -1,11 +1,11 @@
 import { existsSync, readdirSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 
-import type { BuildMode } from 'lib/steps/build-plan.js'
 import {
   exchangeWizardInferenceToken,
   getInferenceBaseUrl,
-} from 'lib/util/seam-api.js'
+} from 'lib/seam-api.js'
+import type { BuildMode } from 'lib/steps/build-plan.js'
 
 import { createRealRunner } from './real-runner.js'
 import { formatReport } from './report.js'

src/eval/score.ts → eval/score.ts

@@ -1,5 +1,5 @@
+import { callInferenceForText } from 'lib/seam-api.js'
 import type { BuildMode } from 'lib/steps/build-plan.js'
-import { callInferenceForText } from 'lib/util/seam-api.js'
 
 import { getRubric, type RubricDimension } from './rubric.js'
 import type { ScoreResult } from './types.js'

package.json

@@ -60,7 +60,7 @@
     "preformat": "eslint --fix .",
     "report": "vitest run --coverage",
     "screen": "tsx src/bin/screen.ts",
-    "eval": "tsx src/eval/run.ts"
+    "eval": "tsx eval/run.ts"
   },
   "engines": {
     "node": ">=22.12.0",

src/lib/app.tsx

@@ -11,6 +11,8 @@ import {
 } from 'react'
 
 import { getAuth } from './adapter.js'
+import { ensureProjectEnvConventions, findExistingApiKey } from './env-file.js'
+import { runInstall } from './run-install.js'
 import { AnalyzeScreen } from './screens/analyze.js'
 import { DoneScreen, type IntegrationOutcome } from './screens/done.js'
 import { Header } from './screens/header.js'
@@ -22,6 +24,14 @@ import { IntegrationModeScreen } from './screens/integration-mode.js'
 import { NoteScreen } from './screens/note.js'
 import { SetupProgress } from './screens/setup-progress.js'
 import { WelcomeScreen } from './screens/welcome.js'
+import {
+  ApiKeyError,
+  exchangeWizardInferenceToken,
+  getInferenceBaseUrl,
+  looksLikeSeamApiKey,
+  type SeamWorkspace,
+  type WizardInferenceSession,
+} from './seam-api.js'
 import {
   analyzeProject,
   type ProjectAnalysis,
@@ -67,19 +77,6 @@ import {
   recordResult,
   writePreferredSdk,
 } from './store/index.js'
-import {
-  ensureProjectEnvConventions,
-  findExistingApiKey,
-} from './util/env-file.js'
-import { runInstall } from './util/run-install.js'
-import {
-  ApiKeyError,
-  exchangeWizardInferenceToken,
-  getInferenceBaseUrl,
-  looksLikeSeamApiKey,
-  type SeamWorkspace,
-  type WizardInferenceSession,
-} from './util/seam-api.js'
 
 const MAX_ATTEMPTS = 3
 

src/lib/steps/analyze-project.ts

@@ -1,11 +1,8 @@
 import { existsSync, readFileSync } from 'node:fs'
 import { join } from 'node:path'
 
-import { findExistingApiKey } from 'lib/util/env-file.js'
-import {
-  callInferenceForText,
-  type WizardOnboarding,
-} from 'lib/util/seam-api.js'
+import { findExistingApiKey } from 'lib/env-file.js'
+import { callInferenceForText, type WizardOnboarding } from 'lib/seam-api.js'
 
 import type { BuildMode } from './build-plan.js'
 import type { ProjectInfo, Sdk } from './detect-project.js'

src/lib/steps/authenticate.ts

@@ -1,6 +1,6 @@
 import { getAuth } from 'lib/adapter.js'
-import { findExistingApiKey, saveProjectApiKey } from 'lib/util/env-file.js'
-import { getWorkspaceForApiKey, type SeamWorkspace } from 'lib/util/seam-api.js'
+import { findExistingApiKey, saveProjectApiKey } from 'lib/env-file.js'
+import { getWorkspaceForApiKey, type SeamWorkspace } from 'lib/seam-api.js'
 
 export interface AuthResult {
   workspace: SeamWorkspace

src/lib/steps/connect-web.ts

@@ -3,8 +3,8 @@ import { createServer, type ServerResponse } from 'node:http'
 
 import open from 'open'
 
-import { saveProjectApiKey } from 'lib/util/env-file.js'
-import { getWorkspaceForApiKey, type SeamWorkspace } from 'lib/util/seam-api.js'
+import { saveProjectApiKey } from 'lib/env-file.js'
+import { getWorkspaceForApiKey, type SeamWorkspace } from 'lib/seam-api.js'
 
 // The dashboard "wizard" page mints a key and posts it back to the local
 // callback. Override the console host with SEAM_CONSOLE_URL for dev.

src/lib/steps/connection.test.ts

@@ -1,44 +1,33 @@
-import { afterEach, beforeEach, expect, test } from 'vitest'
-
-import { createMemoryAdapter, resetAdapter, setAdapter } from 'lib/adapter.js'
-import { type ProjectConnection, readProjectRecord } from 'lib/store/index.js'
-import { fingerprintApiKey } from 'lib/util/api-key.js'
-import type { SeamWorkspace } from 'lib/util/seam-api.js'
+import { expect, test } from 'vitest'
 
 import {
   compareConnection,
+  type CurrentConnection,
   describeChange,
-  saveConnection,
 } from './connection.js'
 
 const apiKey = 'seam_apikey1_first_key'
-const workspace: SeamWorkspace = {
+const workspace: CurrentConnection['workspace'] = {
   workspace_id: 'workspace-1',
   name: 'Acme',
   is_sandbox: false,
 }
 
-const recorded: ProjectConnection = {
+const recorded: Parameters<typeof compareConnection>[0] = {
   endpoint: 'https://connect.getseam.com',
   workspace_id: workspace.workspace_id,
   workspace_name: workspace.name,
-  api_key: fingerprintApiKey(apiKey),
+  api_key: { digest: '1068caa4a195dd39', hint: '_key' },
   api_key_source: 'project',
   api_key_location: '.env',
 }
 
-const current = {
+const current: CurrentConnection = {
   endpoint: recorded.endpoint,
   workspace,
   api_key: apiKey,
 }
 
-beforeEach(() => {
-  setAdapter(createMemoryAdapter())
-})
-
-afterEach(resetAdapter)
-
 test('compareConnection: nothing changed is nothing to ask about', () => {
   expect(compareConnection(recorded, current)).toEqual([])
 })
@@ -109,37 +98,3 @@ test('describeChange: reads as what moved, and where to', () => {
     }),
   ).toBe('Endpoint: https://connect.getseam.com → https://connect.example.com')
 })
-
-test('saveConnection: records what the project talks to, never the key', async () => {
-  await saveConnection('/projects/app', {
-    workspace,
-    api_key: apiKey,
-    source: 'project',
-    location: '.env.local',
-  })
-
-  const record = await readProjectRecord('/projects/app')
-  expect(record?.connection).toEqual({
-    endpoint: 'https://connect.getseam.com',
-    workspace_id: 'workspace-1',
-    workspace_name: 'Acme',
-    api_key: fingerprintApiKey(apiKey),
-    api_key_source: 'project',
-    api_key_location: '.env.local',
-  })
-  expect(JSON.stringify(record)).not.toContain(apiKey)
-})
-
-test('saveConnection: what it records reads back as unchanged', async () => {
-  await saveConnection('/projects/app', {
-    workspace,
-    api_key: apiKey,
-    source: 'browser',
-  })
-
-  const record = await readProjectRecord('/projects/app')
-  expect(record?.connection).not.toBeNull()
-  expect(
-    compareConnection(record?.connection as ProjectConnection, current),
-  ).toEqual([])
-})

src/lib/steps/connection.ts

@@ -1,11 +1,11 @@
 import { getAuth } from 'lib/adapter.js'
+import { fingerprintApiKey } from 'lib/api-key.js'
+import type { SeamWorkspace } from 'lib/seam-api.js'
 import {
   type ConnectionSource,
   type ProjectConnection,
   recordConnection,
 } from 'lib/store/index.js'
-import { fingerprintApiKey } from 'lib/util/api-key.js'
-import type { SeamWorkspace } from 'lib/util/seam-api.js'
 
 export type ConnectionChange =
   | { what: 'api_key'; from: string; to: string }

src/lib/store/project-store.ts

@@ -2,8 +2,8 @@ import { createHash } from 'node:crypto'
 import { basename, resolve } from 'node:path'
 
 import { getAdapter } from 'lib/adapter.js'
+import type { ApiKeyFingerprint } from 'lib/api-key.js'
 import type { BuildMode } from 'lib/steps/build-plan.js'
-import type { ApiKeyFingerprint } from 'lib/util/api-key.js'
 
 export type ConnectionSource = 'project' | 'cli' | 'browser' | 'pasted'
 

src/lib/app.test.tsx → test/app.test.tsx

@@ -1,8 +1,8 @@
 import { render } from 'ink-testing-library'
 import { afterEach, beforeEach, expect, test, vi } from 'vitest'
 
-import { createMemoryAdapter, resetAdapter, setAdapter } from './adapter.js'
-import { App } from './app.js'
+import { createMemoryAdapter, resetAdapter, setAdapter } from 'lib/adapter.js'
+import { App } from 'lib/app.js'
 
 // A project root that does not exist, with no key in the environment, keeps the
 // render offline: the wizard opens on the welcome splash and only leaves it on a

src/eval/run-case.test.ts → test/eval/run-case.test.ts

@@ -4,8 +4,8 @@ import { join } from 'node:path'
 
 import { afterEach, expect, test } from 'vitest'
 
-import { runCase } from './run-case.js'
-import type { FixtureConfig } from './types.js'
+import { runCase } from 'eval/run-case.js'
+import type { FixtureConfig } from 'eval/types.js'
 
 const config: FixtureConfig = {
   name: 'demo',