From 83edc9feedb0776c44a312a8fdc6989e28785358 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 6 Oct 2026 23:59:21 +0000 Subject: [PATCH 01/10] Add default-off telemetry switch, shared contract, and adapter interface Checkpoint for splitting PR #13. TELEMETRY_ENABLED is false and nothing turns it on; the generator writes telemetry hooks only when it is true. The contract no longer holds client tool prefixes or hook activation; each client's adapter in hooks/telemetry-adapters/ supplies those. Co-authored-by: Teal Larson --- hooks/hook-hosts.mjs | 47 ++++- hooks/telemetry-adapter.mjs | 86 +++++++++ hooks/telemetry-config.mjs | 38 ++++ hooks/telemetry-contract.mjs | 309 +++++++++++++++++++++++++++++++++ package-lock.json | 35 +++- package.json | 7 +- scripts/generate-manifests.mjs | 73 ++++++-- test/telemetry-off.test.mjs | 44 +++++ tsconfig.json | 14 ++ 9 files changed, 629 insertions(+), 24 deletions(-) create mode 100644 hooks/telemetry-adapter.mjs create mode 100644 hooks/telemetry-config.mjs create mode 100644 hooks/telemetry-contract.mjs create mode 100644 test/telemetry-off.test.mjs create mode 100644 tsconfig.json diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 0fd5f33..1c3c4a9 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -4,18 +4,16 @@ * `--host ` so the script prints the output format that client reads. */ +import { loadTelemetryAdapter } from "./telemetry-adapter.mjs"; +import { TELEMETRY_ENABLED } from "./telemetry-config.mjs"; + export const HOOK_TIMEOUT_SEC = 5; /** - * One entry per hook script. The event is Claude Code's name for it; a client - * with an `events` map uses its own names and only gets the events it lists. + * Each client that runs plugin hooks. `runOnlyIfScriptExists` makes each + * command check for its script first. `telemetry` names the client's adapter + * in hooks/telemetry-adapters/ (see hooks/telemetry-adapter.mjs). */ -export const HOOKS = [ - { script: "session-start.mjs", event: "SessionStart" }, - { script: "user-prompt-submit.mjs", event: "UserPromptSubmit" }, - { script: "subagent-start.mjs", event: "SubagentStart" }, -]; - export const HOSTS = { "claude-code": { // Not hooks/hooks.json: Cursor falls back to that default path and would @@ -56,6 +54,39 @@ export const HOSTS = { }, }; +/** + * The telemetry hook entries of every client with a `telemetry` adapter, + * whether or not telemetry is enabled. Tests use this to check the wiring. + */ +export const telemetryHookRows = async () => { + const rows = []; + for (const [hostName, host] of Object.entries(HOSTS)) { + if (!host.telemetry) continue; + const adapter = await loadTelemetryAdapter(host.telemetry); + for (const row of adapter.hookRows) { + rows.push({ script: "telemetry.mjs", ...row, hosts: [hostName] }); + } + } + return rows; +}; + +/** + * One entry per hook command. The event is Claude Code's name for it; a client + * with an `events` map uses its own names and only gets the events it lists. + * `hosts` limits an entry to some clients. `matcher` picks the tools a tool + * event runs for. `if` (a Claude Code permission rule such as "Bash(gh *)" + * that keeps the hook from starting for other commands) and `extraArgs` + * (added to the command) work only in the nested format. + */ +export const HOOKS = [ + { script: "session-start.mjs", event: "SessionStart" }, + // Copilot CLI drops prompt-hook output, so only Claude Code runs this one. + { script: "user-prompt-submit.mjs", event: "UserPromptSubmit", hosts: ["claude-code"] }, + { script: "subagent-start.mjs", event: "SubagentStart" }, + // No telemetry hooks are written while telemetry is off (telemetry-config.mjs). + ...(TELEMETRY_ENABLED ? await telemetryHookRows() : []), +]; + /** The client named by `--host`, or null if it's missing or unknown. */ export const hostFromArgs = (argv) => { const flag = argv.indexOf("--host"); diff --git a/hooks/telemetry-adapter.mjs b/hooks/telemetry-adapter.mjs new file mode 100644 index 0000000..657579f --- /dev/null +++ b/hooks/telemetry-adapter.mjs @@ -0,0 +1,86 @@ +// @ts-check +/** + * The interface between the shared telemetry code and each client. A client + * runs telemetry when its HOSTS entry in hook-hosts.mjs names an adapter in + * `telemetry`; the adapter lives in hooks/telemetry-adapters/.mjs and + * default-exports a TelemetryAdapter. Removing that file and that HOSTS field + * removes the client without touching the shared code or other clients. + */ + +import { TELEMETRY_HOSTS } from "./telemetry-contract.mjs"; + +/** + * Hook input in Claude Code's field names. Adapters translate their client's + * input into this shape; the shared code reads nothing else. + * @typedef {object} HookInput + * @property {string} [hook_event_name] Claude Code's name for the hook. + * @property {string} [session_id] + * @property {string} [prompt_id] + * @property {string} [source] SessionStart source, e.g. "compact". + * @property {unknown} [prompt] + * @property {unknown} [tool_name] + * @property {Record} [tool_input] + * @property {unknown} [tool_response] The tool's result, read only to classify sign-in answers. + * @property {unknown} [error] + * @property {unknown} [is_interrupt] + * @property {unknown} [agent_type] + * @property {unknown} [agent_id] + * @property {unknown} [last_assistant_message] + */ + +/** + * One telemetry hook entry the generator writes for this client. `event` is + * Claude Code's name; the client's `events` map in hook-hosts.mjs translates + * it. `if` and `extraArgs` work only in the nested (Claude Code) format. + * @typedef {object} HookRow + * @property {string} event + * @property {string} [matcher] + * @property {string} [if] + * @property {string[]} [extraArgs] + */ + +/** + * @typedef {{ name: string, anyValue: boolean }} OptOutSwitch + * A client's own off switch. `anyValue: true`: any non-empty value turns + * telemetry off. `anyValue: false`: any value except empty, 0, false, off, + * or no does. + */ + +/** + * @typedef {Record} ToolProperties + * Contract properties for an MCP tool event (`server`, `tool`, `service`), or + * for a built-in tool event (`tool`, `cli`, `service`). + */ + +/** + * @typedef {object} TelemetryAdapter + * @property {typeof TELEMETRY_HOSTS[number]} host The `host` property on every event. + * @property {string} dataVariable Environment variable naming the plugin's data folder. + * @property {OptOutSwitch[]} optOutSwitches + * @property {boolean} requiresTurn Whether tool events count as app work only + * with the same prompt ID as the prompt that opened scope. Without it, a + * tool event inherits its session's scope. + * @property {boolean} promptReminder Whether the client runs user-prompt-submit.mjs. + * @property {boolean} subagentSession Whether SubagentStop events carry `subagent_session`. + * @property {HookRow[]} hookRows + * @property {(raw: Record) => HookInput} normalize + * @property {(toolName: unknown, toolInput: Record | undefined) => ToolProperties | null} toolProperties + * @property {(toolName: unknown, toolInput: Record | undefined) => ToolProperties | null} [attemptProperties] + * Only for clients with a PreToolUse row. + * @property {(toolName: unknown, toolInput: Record | undefined, cli: string | undefined) => ToolProperties | null} [builtinToolProperties] + * Only for clients that report built-in CLI and web tools. + */ + +/** + * Loads the adapter for a contract host. Throws for any other name. + * @param {string} host + * @returns {Promise} + */ +export const loadTelemetryAdapter = async (host) => { + if (!(/** @type {readonly string[]} */ (TELEMETRY_HOSTS)).includes(host)) { + throw new Error(`${host} is not a telemetry host in hooks/telemetry-contract.mjs`); + } + const adapter = (await import(new URL(`./telemetry-adapters/${host}.mjs`, import.meta.url).href)).default; + if (adapter?.host !== host) throw new Error(`hooks/telemetry-adapters/${host}.mjs does not export the ${host} adapter`); + return adapter; +}; diff --git a/hooks/telemetry-config.mjs b/hooks/telemetry-config.mjs new file mode 100644 index 0000000..45569b2 --- /dev/null +++ b/hooks/telemetry-config.mjs @@ -0,0 +1,38 @@ +// @ts-check +/** Telemetry settings: where events go, the opt-out variable, and the local file names. */ + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const HOOKS_DIR = path.dirname(fileURLToPath(import.meta.url)); + +// Off in every build until collection is separately approved. No environment +// variable or setting turns it on. While false, `npm run generate` writes no +// telemetry hooks and telemetry.mjs exits before reading its input. +export const TELEMETRY_ENABLED = false; + +// Arcade's PostHog proxy, the same one arcade.dev and identity-ui use. +export const POSTHOG_HOST = + process.env.ARCADE_PLUGIN_TELEMETRY_HOST || "https://p.arcade.dev"; + +// Public, client-side key for the Production project. +export const POSTHOG_KEY = "phc_zNHKkPFsrKVSpd7y85jnxW8jNVW6AQD6AwqE4nWjwpXg"; + +export const OPT_OUT_ENV = "ARCADE_PLUGIN_TELEMETRY"; + +// Holds "true" once an Arcade tool call has succeeded for this client plugin installation. +export const ARCADE_USED_FILE = "arcade-used"; + +// telemetry.mjs passes the event to telemetry-send.mjs in this variable. +export const EVENT_ENV = "ARCADE_PLUGIN_TELEMETRY_EVENT"; + +const readPluginVersion = () => { + try { + return readFileSync(path.join(HOOKS_DIR, "..", "VERSION"), "utf8").trim(); + } catch { + return "unknown"; + } +}; + +export const PLUGIN_VERSION = readPluginVersion(); diff --git a/hooks/telemetry-contract.mjs b/hooks/telemetry-contract.mjs new file mode 100644 index 0000000..db4aa5e --- /dev/null +++ b/hooks/telemetry-contract.mjs @@ -0,0 +1,309 @@ +// @ts-check +/** + * Every telemetry event the plugin sends, every property on it, and the values + * each property may take. telemetry-events.mjs sends nothing that isn't listed + * here, `npm run generate` writes the tables in docs/telemetry.md from it, and + * the tests check every built event against eventSchema(). + */ + +export const SERVICE_CATEGORIES = /** @type {const} */ ([ + "email", + "calendar", + "chat", + "issues", + "docs", + "meetings", + "crm", + "code_hosting", + "analytics", + "storage", +]); + +export const TELEMETRY_HOSTS = /** @type {const} */ (["claude-code", "copilot-cli"]); +export const OS_NAMES = /** @type {const} */ (["darwin", "linux", "win32", "other"]); +export const SERVERS = /** @type {const} */ (["arcade", "other_arcade", "other"]); +export const AGENTS = /** @type {const} */ (["arcade-operator"]); +export const OPERATOR_STATUSES = /** @type {const} */ ([ + "completed", + "needs_auth", + "needs_confirmation", + "needs_clarification", + "failed", + "unknown", +]); +export const FAILURE_KINDS = /** @type {const} */ ([ + "auth_required", + "session_expired", + "unreachable", + "timeout", + "http_error", + "interrupted", + "tool_error", +]); +export const BUILTIN_TOOLS = /** @type {const} */ (["Bash", "WebFetch", "WebSearch"]); + +// Programs that reach an outside service from Bash. A Bash call sends an event +// only when one of its commands starts with one of these. +export const BASH_CLIS = /** @type {const} */ (["gh", "glab", "curl", "wget", "http", "osascript"]); + +/** @type {Record} */ +export const CLI_SERVICES = { gh: "code_hosting", glab: "code_hosting" }; + +// Tools every Arcade gateway exposes. Seeing one on another server means the +// model used a different Arcade connection than this plugin's. +export const GATEWAY_TOOLS = /** @type {const} */ ([ + "Arcade_ListApps", + "Arcade_SelectTools", + "Arcade_UseTool", + "System_ManageAuthorization", +]); + +// Arcade toolkit name (the part of a tool name before "_" or ".", lowercased) +// to its service category. Tool suffixes are never sent. +/** @type {Record} */ +export const TOOLKIT_SERVICES = { + gmail: "email", outlookmail: "email", + googlecalendar: "calendar", outlookcalendar: "calendar", microsoftoutlookcalendar: "calendar", calendly: "calendar", + slack: "chat", discord: "chat", discordbot: "chat", microsoftteams: "chat", + linear: "issues", jira: "issues", asana: "issues", clickup: "issues", trello: "issues", + notion: "docs", googledocs: "docs", confluence: "docs", + googlesheets: "docs", microsoftword: "docs", microsoftexcel: "docs", + granola: "meetings", zoom: "meetings", fireflies: "meetings", + hubspot: "crm", salesforce: "crm", attio: "crm", + github: "code_hosting", gitlab: "code_hosting", bitbucket: "code_hosting", + posthog: "analytics", + googledrive: "storage", dropbox: "storage", sharepoint: "storage", onedrive: "storage", +}; + +/** + * @typedef {object} Property + * @property {object} schema JSON Schema for the value. + * @property {string} doc What the value is, for docs/telemetry.md. + */ + +const HASH = { type: "string", pattern: "^[0-9a-f]{16}$" }; +// VERSION, or "unknown" when it can't be read. +const PLUGIN_VERSION_PATTERN = "^(unknown|[0-9]+\\.[0-9]+\\.[0-9]+(-[0-9A-Za-z.-]+)?)$"; +const enumOf = (/** @type {readonly string[]} */ values) => ({ enum: [...values] }); +const list = (/** @type {readonly string[]} */ values) => values.map((value) => `\`${value}\``).join(" \\| "); + +/** Properties every event may carry. @type {Record} */ +export const COMMON_PROPERTIES = { + session: { + schema: HASH, + doc: "`sha256(session_id)`, first 16 hex characters, where `session_id` is the client's random ID for the session", + }, + turn: { + schema: HASH, + doc: '`sha256(session_id + ":" + prompt_id)`, first 16 hex characters; Claude Code only, because Copilot CLI has no prompt ID', + }, + arcade_used_before: { + schema: { type: "boolean" }, + doc: "whether an Arcade tool call had succeeded for this client plugin installation before this event (from the `arcade-used` file)", + }, + host: { schema: enumOf(TELEMETRY_HOSTS), doc: list(TELEMETRY_HOSTS) }, + telemetry_version: { schema: { const: 2 }, doc: "`2`, the scoped event contract; earlier events have no version" }, + plugin_version: { schema: { type: "string", pattern: PLUGIN_VERSION_PATTERN }, doc: "from `VERSION`" }, + os: { schema: enumOf(OS_NAMES), doc: list(OS_NAMES) }, + $process_person_profile: { schema: { const: false }, doc: "`false`" }, + $geoip_disable: { schema: { const: true }, doc: "`true`" }, + $ip: { schema: { const: "0.0.0.0" }, doc: "`0.0.0.0`, so PostHog stores this instead of your real IP address" }, +}; + +/** Common properties present on every event. */ +const ALWAYS_SENT = ["session", "arcade_used_before", "host", "plugin_version", "telemetry_version", "os", "$process_person_profile", "$geoip_disable", "$ip"]; + +const TOOL_PROPERTIES = { + server: { + schema: enumOf(SERVERS), + doc: "`arcade` (this plugin's gateway) \\| `other_arcade` (another connection exposing Arcade's gateway tools) \\| `other`", + }, + tool: { + schema: enumOf(["other", "app_tool", ...GATEWAY_TOOLS]), + doc: "only for `arcade` and `other_arcade`: an exact gateway tool name, `app_tool` for a recognized service category, or `other`; app tool names are never sent", + }, + service: { + schema: enumOf(SERVICE_CATEGORIES), + doc: "the service category, when the tool, the app tool passed to `Arcade_UseTool`, or the server name matches one", + }, +}; + +// Tool values are sent only for Arcade connections. +const TOOL_RULES = [ + { if: { properties: { server: { const: "other" } } }, then: { not: { required: ["tool"] } } }, + { if: { properties: { server: enumOf(["arcade", "other_arcade"]) } }, then: { required: ["tool"] } }, +]; + +const TOOL_CALLED_PROPERTIES = { + ...TOOL_PROPERTIES, + auth_needed: { + schema: { type: "boolean" }, + doc: "only for `System_ManageAuthorization`: whether its answer says a service still needs sign-in", + }, +}; + +const TOOL_ATTEMPTED_PROPERTIES = { + ...TOOL_PROPERTIES, + server: { + schema: enumOf(["arcade", "other_arcade"]), + doc: "`arcade` (this plugin's gateway) \\| `other_arcade` (the claude.ai Arcade connection)", + }, +}; + +const TOOL_CALLED_RULES = [ + ...TOOL_RULES, + { + if: { required: ["tool"], properties: { tool: { const: "System_ManageAuthorization" } } }, + then: { required: ["auth_needed"] }, + else: { not: { required: ["auth_needed"] } }, + }, +]; + +const TOOL_FAILED_PROPERTIES = { + ...TOOL_PROPERTIES, + failure_kind: { + schema: enumOf(FAILURE_KINDS), + doc: `picked on your machine from the error message; the message is not sent: ${list(FAILURE_KINDS)}`, + }, +}; + +const CLI_SERVICE_VALUES = [...new Set(Object.values(CLI_SERVICES))]; + +const BUILTIN_TOOL_PROPERTIES = { + tool: { schema: enumOf(BUILTIN_TOOLS), doc: list(BUILTIN_TOOLS) }, + cli: { + schema: enumOf(BASH_CLIS), + doc: `only for \`Bash\`: the program the command runs, ${list(BASH_CLIS)}`, + }, + service: { + schema: enumOf(CLI_SERVICE_VALUES), + doc: `the program's service category, only for ${list(Object.keys(CLI_SERVICES))}: ${list(CLI_SERVICE_VALUES)}`, + }, +}; + +// A Bash call always names its program, and nothing else does. The service +// comes only from CLI_SERVICES. +const BUILTIN_TOOL_RULES = [ + { + if: { properties: { tool: { const: "Bash" } } }, + then: { required: ["cli"] }, + else: { not: { required: ["cli"] } }, + }, + ...Object.entries(CLI_SERVICES).map(([cli, service]) => ({ + if: { required: ["cli"], properties: { cli: { const: cli } } }, + then: { required: ["service"], properties: { service: { const: service } } }, + })), + { + if: { required: ["cli"], properties: { cli: enumOf(Object.keys(CLI_SERVICES)) } }, + else: { not: { required: ["service"] } }, + }, +]; + +const BUILTIN_TOOL_WHEN = `Claude Code only, on \`WebFetch\` and \`WebSearch\`, and on \`Bash\` commands that run ${list(BASH_CLIS)}`; + +/** + * @typedef {object} EventSpec + * @property {string} hook The Claude Code name of the hook that sends it. + * Which clients run that hook, and on which tools, is up to each client's + * adapter in hooks/telemetry-adapters/. + * @property {string} when Extra conditions, for docs/telemetry.md. + * @property {Record} properties Properties beyond COMMON_PROPERTIES. + * @property {string[]} required + * @property {object[]} [rules] Extra JSON Schema rules for this event. + */ + +/** @type {Record} */ +export const EVENTS = { + "Plugin prompt submitted": { + hook: "UserPromptSubmit", + when: "only for locally classified app work and short confirmations of that work; background task results are excluded. In Copilot CLI a relevant subagent prompt also sends it", + properties: { + could_use_arcade: { schema: { type: "boolean" }, doc: "boolean, a local keyword guess (see below)" }, + service_hints: { + schema: { type: "array", items: enumOf(SERVICE_CATEGORIES), uniqueItems: true }, + doc: "service categories the prompt mentions", + }, + reminder_sent: { schema: { type: "boolean" }, doc: "boolean, whether the routing reminder was added (always `false` in Copilot CLI)" }, + }, + required: ["could_use_arcade", "service_hints", "reminder_sent"], + }, + "Plugin tool attempted": { + hook: "PreToolUse", + when: "in Claude Code, before an MCP call through this plugin's Arcade gateway or the claude.ai Arcade connection; an attempt does not show whether the tool finished", + properties: TOOL_ATTEMPTED_PROPERTIES, + required: ["server", "tool"], + }, + "Plugin tool called": { + hook: "PostToolUse", + when: "on Arcade tools, or alternative MCP tools while the current turn concerns app work", + properties: TOOL_CALLED_PROPERTIES, + required: ["server"], + rules: TOOL_CALLED_RULES, + }, + "Plugin tool failed": { + hook: "PostToolUseFailure", + when: "on Arcade tools, or alternative MCP tools while the current turn concerns app work", + properties: TOOL_FAILED_PROPERTIES, + required: ["server", "failure_kind"], + rules: TOOL_RULES, + }, + "Plugin built-in tool called": { + hook: "PostToolUse", + when: `${BUILTIN_TOOL_WHEN}; only while the current turn concerns app work`, + properties: BUILTIN_TOOL_PROPERTIES, + required: ["tool"], + rules: BUILTIN_TOOL_RULES, + }, + "Plugin built-in tool failed": { + hook: "PostToolUseFailure", + when: `${BUILTIN_TOOL_WHEN}; only while the current turn concerns app work`, + properties: BUILTIN_TOOL_PROPERTIES, + required: ["tool"], + rules: BUILTIN_TOOL_RULES, + }, + "Plugin subagent stopped": { + hook: "SubagentStop", + when: "only for arcade-operator", + properties: { + agent: { schema: enumOf(AGENTS), doc: list(AGENTS) }, + status: { + schema: enumOf(OPERATOR_STATUSES), + doc: `only for \`arcade-operator\`: the status line of its final report, ${list(OPERATOR_STATUSES)}`, + }, + subagent_session: { + schema: HASH, + doc: "`sha256(agent_id)`, first 16 hex characters. In Copilot CLI the subagent's own events carry this as `session`", + }, + }, + required: ["agent", "status"], + }, +}; + +/** Property names an event may carry, common ones included. */ +export const allowedProperties = (/** @type {string} */ eventName) => [ + ...Object.keys(COMMON_PROPERTIES), + ...Object.keys(EVENTS[eventName].properties), +]; + +/** JSON Schema (2020-12) for one event as sent to PostHog: `{ event, distinct_id, properties }`. */ +export const eventSchema = () => ({ + $schema: "https://json-schema.org/draft/2020-12/schema", + oneOf: Object.entries(EVENTS).map(([name, spec]) => ({ + type: "object", + additionalProperties: false, + required: ["event", "distinct_id", "properties"], + properties: { + event: { const: name }, + distinct_id: HASH, + properties: { + type: "object", + additionalProperties: false, + required: [...ALWAYS_SENT, ...spec.required], + properties: Object.fromEntries( + allowedProperties(name).map((key) => [key, { ...COMMON_PROPERTIES, ...spec.properties }[key].schema]), + ), + ...(spec.rules ? { allOf: spec.rules } : {}), + }, + }, + })), +}); diff --git a/package-lock.json b/package-lock.json index a551451..e907470 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,9 @@ "@anthropic-ai/claude-code": "2.1.258", "@github/copilot": "1.0.88", "@openai/codex": "0.156.1", - "ajv": "8.20.0" + "@types/node": "22.20.4", + "ajv": "8.20.0", + "typescript": "6.0.3" }, "engines": { "node": "22.23.2" @@ -463,6 +465,16 @@ "node": ">=16" } }, + "node_modules/@types/node": { + "version": "22.20.4", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.4.tgz", + "integrity": "sha512-zJRE40jpHtKqE/C4fgHrAKQLJuSpzEnP9ff9Y7YtoR3Wd2pwqzlekDeEuUQXjRd+QCYnVnNwuJYmhdk9XV8gvA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -530,6 +542,27 @@ "engines": { "node": ">=0.10.0" } + }, + "node_modules/typescript": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" } } } diff --git a/package.json b/package.json index 864f477..58b615b 100644 --- a/package.json +++ b/package.json @@ -9,13 +9,16 @@ "verify:cursor": "node scripts/verify-cursor.mjs", "verify:codex": "node scripts/verify-codex.mjs", "verify:copilot": "node scripts/verify-copilot.mjs", - "verify": "npm test && npm run verify:claude && npm run verify:cursor && npm run verify:codex && npm run verify:copilot" + "verify": "npm run typecheck && npm test && npm run verify:claude && npm run verify:cursor && npm run verify:codex && npm run verify:copilot", + "typecheck": "tsc -p tsconfig.json" }, "devDependencies": { "@anthropic-ai/claude-code": "2.1.258", "@github/copilot": "1.0.88", "@openai/codex": "0.156.1", - "ajv": "8.20.0" + "@types/node": "22.20.4", + "ajv": "8.20.0", + "typescript": "6.0.3" }, "engines": { "node": "22.23.2" diff --git a/scripts/generate-manifests.mjs b/scripts/generate-manifests.mjs index cb9bb7f..8eb99e2 100644 --- a/scripts/generate-manifests.mjs +++ b/scripts/generate-manifests.mjs @@ -53,9 +53,17 @@ export const FILE_SOURCES = { // The hook manifest paths in the list come from hook-hosts.mjs. ".gitattributes": ["hooks/hook-hosts.mjs", "scripts/generate-manifests.mjs"], // hook-hosts.mjs supplies the events, scripts, and timeout; the script writes - // the version, entry type, nesting, and command template. + // the version, entry type, nesting, and command template. With telemetry + // enabled, the client's adapter supplies its telemetry entries. ...Object.fromEntries( - Object.values(HOSTS).map((h) => [h.manifest, ["hooks/hook-hosts.mjs", "scripts/generate-manifests.mjs"]]), + Object.values(HOSTS).map((h) => [ + h.manifest, + [ + "hooks/hook-hosts.mjs", + ...(h.telemetry ? ["hooks/telemetry-config.mjs", `hooks/telemetry-adapters/${h.telemetry}.mjs`] : []), + "scripts/generate-manifests.mjs", + ], + ]), ), "skills/scale-arcade/references/arcade-docs.md": ["skills/try-arcade/references/arcade-docs.md"], // A copy of the operator with its rules block filled in. @@ -113,21 +121,60 @@ const fillRulesBlock = (text, rules, path) => { return `${text.slice(0, begin + RULES_BLOCK_BEGIN.length)}\n${rules}\n${text.slice(end)}`; }; -const hookCommand = (hostName, script) => - `node "\${${HOSTS[hostName].rootVariable}}/hooks/${script}" --host ${hostName}`; - +/** + * The command fields of one hook entry. With `runOnlyIfScriptExists`, the + * command runs the script only if the file is there, so a client that leaves + * the root variable unset (VS Code) gets exit 0 and no output. `powershell` + * is intended for Windows (not verified); it has no double quotes because VS + * Code is expected to pass it as one argument to `powershell.exe -Command`. + */ +const hookCommandFields = (hostName, script) => { + const { rootVariable, runOnlyIfScriptExists } = HOSTS[hostName]; + const scriptPath = `\${${rootVariable}}/hooks/${script}`; + const command = `node "${scriptPath}" --host ${hostName}`; + if (!runOnlyIfScriptExists) return { command }; + const psPath = `($env:${rootVariable} + '/hooks/${script}')`; + return { + command: `if [ -f "${scriptPath}" ]; then ${command}; fi`, + powershell: `if ($env:${rootVariable} -and (Test-Path -LiteralPath ${psPath})) { node ${psPath} --host ${hostName} }`, + }; +}; -// Claude Code nests each command in a group: { hooks: { Event: [{ hooks: [entry] }] } }. -// Cursor, Copilot CLI, and VS Code take the entries directly and need version 1. -const buildHookManifest = (hostName) => { +// Claude Code nests each command in a group: { hooks: { Event: [{ matcher, hooks: [entry] }] } }, +// with one group per event and matcher. Cursor, Copilot CLI, and VS Code take +// the entries directly and need version 1. +export const buildHookManifest = (hostName, hookRows = HOOKS) => { + const host = HOSTS[hostName]; + const nested = host.format === "nested"; const hooks = {}; - for (const { event, script } of HOOKS) { - const name = HOSTS[hostName].events ? HOSTS[hostName].events[event] : event; + for (const hook of hookRows) { + if (hook.hosts && !hook.hosts.includes(hostName)) continue; + const name = host.events ? host.events[hook.event] : hook.event; if (!name) continue; - const entry = { type: "command", command: hookCommand(hostName, script), timeout: HOOK_TIMEOUT_SEC }; - hooks[name] = HOSTS[hostName].format === "nested" ? [{ hooks: [entry] }] : [entry]; + hooks[name] ??= []; + if (!nested) { + if (hook.if || hook.extraArgs) { + throw new Error(`${hook.script} entry for ${hostName} has if or extra args, which the flat format does not support`); + } + const entry = { type: "command", ...hookCommandFields(hostName, hook.script), timeout: HOOK_TIMEOUT_SEC }; + hooks[name].push({ ...entry, ...(hook.matcher ? { matcher: hook.matcher } : {}) }); + continue; + } + const { command } = hookCommandFields(hostName, hook.script); + const entry = { + type: "command", + ...(hook.if ? { if: hook.if } : {}), + command: [command, ...(hook.extraArgs ?? [])].join(" "), + timeout: HOOK_TIMEOUT_SEC, + }; + let group = hooks[name].find((existing) => existing.matcher === hook.matcher); + if (!group) { + group = { ...(hook.matcher ? { matcher: hook.matcher } : {}), hooks: [] }; + hooks[name].push(group); + } + group.hooks.push(entry); } - return HOSTS[hostName].format === "nested" ? { hooks } : { version: 1, hooks }; + return nested ? { hooks } : { version: 1, hooks }; }; /** Every generated file and its contents, from the sources in `root`. */ diff --git a/test/telemetry-off.test.mjs b/test/telemetry-off.test.mjs new file mode 100644 index 0000000..be33587 --- /dev/null +++ b/test/telemetry-off.test.mjs @@ -0,0 +1,44 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { HOOKS, HOSTS } from "../hooks/hook-hosts.mjs"; +import { TELEMETRY_ENABLED } from "../hooks/telemetry-config.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile } from "./helpers.mjs"; + +// Turning collection on needs its own approval after ingestion, opt-outs, and +// the privacy and distribution requirements are verified. Change this test +// only with that approval. +test("telemetry is off in this build", () => { + assert.equal(TELEMETRY_ENABLED, false); +}); + +test("no generated hooks.json runs the telemetry hook while telemetry is off", () => { + assert.equal(HOOKS.some((hook) => hook.script === "telemetry.mjs"), false); + for (const { manifest } of Object.values(HOSTS)) { + assert.equal(readRepoFile(manifest).includes("telemetry.mjs"), false, manifest); + } +}); + +test("nested manifests group entries by event and matcher and keep if and extra args", () => { + const rows = [ + { script: "a.mjs", event: "PostToolUse", matcher: "mcp__.*" }, + { script: "b.mjs", event: "PostToolUse", matcher: "Bash", if: "Bash(gh *)", extraArgs: ["--cli", "gh"] }, + { script: "c.mjs", event: "PostToolUse", matcher: "mcp__.*" }, + { script: "d.mjs", event: "PostToolUse", hosts: ["copilot"] }, + ]; + const { hooks } = buildHookManifest("claude-code", rows); + assert.deepEqual(hooks.PostToolUse.map((group) => group.matcher), ["mcp__.*", "Bash"]); + assert.equal(hooks.PostToolUse[0].hooks.length, 2); + const [bash] = hooks.PostToolUse[1].hooks; + assert.equal(bash.if, "Bash(gh *)"); + assert.match(bash.command, /\/hooks\/b\.mjs" --host claude-code --cli gh$/); +}); + +test("flat manifests keep matchers and reject if and extra args", () => { + const { hooks } = buildHookManifest("cursor", [{ script: "a.mjs", event: "SessionStart", matcher: "x" }]); + assert.equal(hooks.sessionStart[0].matcher, "x"); + assert.throws( + () => buildHookManifest("cursor", [{ script: "a.mjs", event: "SessionStart", if: "Bash(gh *)" }]), + /flat format does not support/, + ); +}); diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..3568e8d --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,14 @@ +{ + "compilerOptions": { + "allowJs": true, + "checkJs": false, + "noEmit": true, + "strict": true, + "target": "es2023", + "module": "nodenext", + "moduleResolution": "nodenext", + "types": ["node"], + "skipLibCheck": true + }, + "include": ["hooks/*.mjs", "hooks/telemetry-adapters/*.mjs", "scripts/*.mjs"] +} From 083892f776228fb0d929a96d81f7b5125d4100d0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:03:09 +0000 Subject: [PATCH 02/10] Keep hook manifest sources independent of telemetry wiring Co-authored-by: Teal Larson --- scripts/generate-manifests.mjs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/scripts/generate-manifests.mjs b/scripts/generate-manifests.mjs index 8eb99e2..bb1a89a 100644 --- a/scripts/generate-manifests.mjs +++ b/scripts/generate-manifests.mjs @@ -54,16 +54,9 @@ export const FILE_SOURCES = { ".gitattributes": ["hooks/hook-hosts.mjs", "scripts/generate-manifests.mjs"], // hook-hosts.mjs supplies the events, scripts, and timeout; the script writes // the version, entry type, nesting, and command template. With telemetry - // enabled, the client's adapter supplies its telemetry entries. + // enabled, the adapter that hook-hosts.mjs names supplies telemetry entries. ...Object.fromEntries( - Object.values(HOSTS).map((h) => [ - h.manifest, - [ - "hooks/hook-hosts.mjs", - ...(h.telemetry ? ["hooks/telemetry-config.mjs", `hooks/telemetry-adapters/${h.telemetry}.mjs`] : []), - "scripts/generate-manifests.mjs", - ], - ]), + Object.values(HOSTS).map((h) => [h.manifest, ["hooks/hook-hosts.mjs", "scripts/generate-manifests.mjs"]]), ), "skills/scale-arcade/references/arcade-docs.md": ["skills/try-arcade/references/arcade-docs.md"], // A copy of the operator with its rules block filled in. From 058e7c8303fb07f040a6c7f5b3b2b715c594532a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:15:46 +0000 Subject: [PATCH 03/10] test(telemetry): add foundation coverage with fake adapter Add telemetry-helpers, hook-scope/classify/classifier tests, generic hook manifest guard, and telemetry-foundation integration tests that exercise runTelemetry with enabled fixtures while production telemetry stays off. Co-authored-by: Teal Larson --- hooks/hook-scope.mjs | 109 ++++++ hooks/prompt-filters.mjs | 10 +- hooks/routing-guidance.mjs | 7 + hooks/session-start.mjs | 21 +- hooks/subagent-start.mjs | 11 +- hooks/telemetry-classify.mjs | 109 ++++++ hooks/telemetry-commands.mjs | 31 ++ hooks/telemetry-events.mjs | 226 +++++++++++ hooks/telemetry-failures.mjs | 70 ++++ hooks/telemetry-run.mjs | 113 ++++++ hooks/telemetry-send.mjs | 26 ++ hooks/telemetry.mjs | 26 ++ .../hook-inputs/auth-required-arcade.json | 4 + .../hook-inputs/auth-required-connect.json | 4 + .../hook-inputs/auth-required-reauth.json | 4 + .../auth-required-sign-in-again.json | 4 + .../hook-inputs/auth-required-url.json | 4 + .../hook-inputs/auth-required-use-tool.json | 4 + test/fixtures/hook-inputs/http-error-500.json | 4 + .../hook-inputs/http-error-streamable.json | 4 + test/fixtures/hook-inputs/interrupted.json | 4 + .../fixtures/hook-inputs/session-expired.json | 4 + .../fixtures/hook-inputs/timeout-gateway.json | 4 + .../hook-inputs/timeout-mcp-request.json | 4 + .../hook-inputs/timeout-no-response.json | 4 + .../hook-inputs/tool-error-exit-code.json | 4 + .../hook-inputs/tool-error-rate-limit.json | 4 + test/fixtures/hook-inputs/tool-error-rpc.json | 4 + .../unreachable-connection-closed.json | 4 + .../unreachable-mcp-connection-closed.json | 4 + .../hook-inputs/unreachable-socket.json | 4 + .../unreachable-transport-dropped.json | 4 + .../unreachable-unable-to-connect.json | 4 + test/fixtures/routing-prompts.json | 114 ++++++ test/fixtures/telemetry-fake-adapter.mjs | 69 ++++ test/hook-scope.test.mjs | 95 +++++ test/telemetry-classifiers.test.mjs | 306 +++++++++++++++ test/telemetry-classify.test.mjs | 84 ++++ test/telemetry-foundation.test.mjs | 370 ++++++++++++++++++ test/telemetry-helpers.mjs | 109 ++++++ test/telemetry-hooks.test.mjs | 8 + 41 files changed, 1988 insertions(+), 10 deletions(-) create mode 100644 hooks/hook-scope.mjs create mode 100644 hooks/telemetry-classify.mjs create mode 100644 hooks/telemetry-commands.mjs create mode 100644 hooks/telemetry-events.mjs create mode 100644 hooks/telemetry-failures.mjs create mode 100644 hooks/telemetry-run.mjs create mode 100644 hooks/telemetry-send.mjs create mode 100644 hooks/telemetry.mjs create mode 100644 test/fixtures/hook-inputs/auth-required-arcade.json create mode 100644 test/fixtures/hook-inputs/auth-required-connect.json create mode 100644 test/fixtures/hook-inputs/auth-required-reauth.json create mode 100644 test/fixtures/hook-inputs/auth-required-sign-in-again.json create mode 100644 test/fixtures/hook-inputs/auth-required-url.json create mode 100644 test/fixtures/hook-inputs/auth-required-use-tool.json create mode 100644 test/fixtures/hook-inputs/http-error-500.json create mode 100644 test/fixtures/hook-inputs/http-error-streamable.json create mode 100644 test/fixtures/hook-inputs/interrupted.json create mode 100644 test/fixtures/hook-inputs/session-expired.json create mode 100644 test/fixtures/hook-inputs/timeout-gateway.json create mode 100644 test/fixtures/hook-inputs/timeout-mcp-request.json create mode 100644 test/fixtures/hook-inputs/timeout-no-response.json create mode 100644 test/fixtures/hook-inputs/tool-error-exit-code.json create mode 100644 test/fixtures/hook-inputs/tool-error-rate-limit.json create mode 100644 test/fixtures/hook-inputs/tool-error-rpc.json create mode 100644 test/fixtures/hook-inputs/unreachable-connection-closed.json create mode 100644 test/fixtures/hook-inputs/unreachable-mcp-connection-closed.json create mode 100644 test/fixtures/hook-inputs/unreachable-socket.json create mode 100644 test/fixtures/hook-inputs/unreachable-transport-dropped.json create mode 100644 test/fixtures/hook-inputs/unreachable-unable-to-connect.json create mode 100644 test/fixtures/routing-prompts.json create mode 100644 test/fixtures/telemetry-fake-adapter.mjs create mode 100644 test/hook-scope.test.mjs create mode 100644 test/telemetry-classifiers.test.mjs create mode 100644 test/telemetry-classify.test.mjs create mode 100644 test/telemetry-foundation.test.mjs create mode 100644 test/telemetry-helpers.mjs create mode 100644 test/telemetry-hooks.test.mjs diff --git a/hooks/hook-scope.mjs b/hooks/hook-scope.mjs new file mode 100644 index 0000000..a289c40 --- /dev/null +++ b/hooks/hook-scope.mjs @@ -0,0 +1,109 @@ +// @ts-check +/** Limits observations to app work in the current session and turn. */ + +import { createHash, randomUUID } from "node:crypto"; +import { mkdirSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { isConfirmation, isTaskNotification } from "./prompt-filters.mjs"; +import { classifyPrompt } from "./telemetry-classify.mjs"; + +export const SCOPE_TTL_MS = 30 * 60 * 1000; +export const SCOPE_DIRECTORY = "prompt-scope"; +export const MAX_SCOPE_FILES = 256; + +const hash = (/** @type {string} */ value) => createHash("sha256").update(value).digest("hex"); +const turnHash = (/** @type {Record} */ input) => + typeof input.prompt_id === "string" && input.prompt_id !== "" ? hash(input.prompt_id) : undefined; + +/** @typedef {{ relevant: boolean, expiresAt: number, turn?: string }} ScopeState */ + +/** + * Pure prompt decision; confirmations keep the original app prompt's deadline. + * @param {unknown} prompt + * @param {ScopeState | null} previous + * @param {number} now + * @returns {ScopeState} + */ +export const classifyAppWork = (prompt, previous = null, now = Date.now()) => { + const direct = classifyPrompt(prompt).couldUseArcade + || (typeof prompt === "string" && /\barcade\b/i.test(prompt)); + if (direct) return { relevant: true, expiresAt: now + SCOPE_TTL_MS }; + if (previous?.relevant && previous.expiresAt > now && isConfirmation(prompt)) { + return { relevant: true, expiresAt: previous.expiresAt }; + } + return { relevant: false, expiresAt: now + SCOPE_TTL_MS }; +}; + +const readState = (/** @type {string} */ file, /** @type {number} */ now) => { + try { + const state = JSON.parse(readFileSync(file, "utf8")); + if (Object.keys(state).some((key) => !["relevant", "expiresAt", "turn"].includes(key)) + || typeof state.relevant !== "boolean" || !Number.isFinite(state.expiresAt) + || state.expiresAt <= now || state.expiresAt > now + SCOPE_TTL_MS + || (state.turn !== undefined && (typeof state.turn !== "string" || !/^[a-f0-9]{64}$/.test(state.turn)))) return null; + return /** @type {ScopeState} */ (state); + } catch { + return null; + } +}; + +const writeState = (/** @type {string} */ file, /** @type {ScopeState} */ state, /** @type {number} */ now) => { + const dir = path.dirname(file); + mkdirSync(dir, { recursive: true, mode: 0o700 }); + const files = readdirSync(dir).filter((name) => /^[a-f0-9]{64}\.json$/.test(name)); + const live = []; + for (const name of files) { + const entry = path.join(dir, name); + if (!readState(entry, now)) rmSync(entry, { force: true }); + else live.push({ file: entry, modified: statSync(entry).mtimeMs }); + } + live.sort((a, b) => a.modified - b.modified); + for (const entry of live.slice(0, Math.max(0, live.length - MAX_SCOPE_FILES + 1))) { + if (entry.file !== file) rmSync(entry.file, { force: true }); + } + const temporary = `${file}.${randomUUID()}.tmp`; + try { + writeFileSync(temporary, JSON.stringify(state), { mode: 0o600 }); + renameSync(temporary, file); + } finally { + rmSync(temporary, { force: true }); + } +}; + +/** + * Whether this hook input is part of app work. Prompt hooks record the + * decision; other hooks read it. + * @param {Record} input + * @param {{ host: string, requiresTurn: boolean, dir?: string, now?: number }} options + */ +export const scopeForInput = (input, { host, requiresTurn, dir, now = Date.now() }) => { + const fallback = { appWork: false }; + const promptHook = input.hook_event_name === "UserPromptSubmit"; + if (promptHook && isTaskNotification(input.prompt)) return fallback; + const session = typeof input.session_id === "string" && input.session_id !== "" ? input.session_id : null; + const file = dir && path.isAbsolute(dir) && session + ? path.join(dir, SCOPE_DIRECTORY, `${hash(`${host}:${session}`)}.json`) : null; + if (input.hook_event_name === "SessionStart") { + if (file && input.source !== "compact") rmSync(file, { force: true }); + return fallback; + } + const previous = file ? readState(file, now) : null; + const turn = turnHash(input); + if (!promptHook) { + const matchesTurn = requiresTurn + ? turn !== undefined && previous?.turn === turn + : turn === undefined || previous?.turn === turn; + const appWork = previous?.relevant === true && matchesTurn; + return { appWork }; + } + const state = turn && previous?.turn === turn + ? previous : { ...classifyAppWork(input.prompt, previous, now), ...(turn ? { turn } : {}) }; + if (file) { + try { + writeState(file, state, now); + } catch { + try { rmSync(file, { force: true }); } catch {} + } + } + return { appWork: state.relevant }; +}; diff --git a/hooks/prompt-filters.mjs b/hooks/prompt-filters.mjs index 5e8ac80..fac9a6c 100644 --- a/hooks/prompt-filters.mjs +++ b/hooks/prompt-filters.mjs @@ -10,7 +10,7 @@ const CONTINUATION_WORDS = new Set([ const MAX_CONTINUATION_WORDS = 2; -const isBareContinuation = (prompt) => { +export const isBareContinuation = (prompt) => { const words = prompt .toLowerCase() .replace(/[^a-z\s]/g, " ") @@ -25,6 +25,14 @@ const isBareContinuation = (prompt) => { export const isTaskNotification = (prompt) => typeof prompt === "string" && prompt.trimStart().startsWith(""); +export const isConfirmation = (prompt) => { + if (typeof prompt !== "string") return false; + if (isBareContinuation(prompt)) return true; + const text = prompt.toLowerCase().replace(/[^a-z\s]/g, " ").replace(/\s+/g, " ").trim(); + return /^(?:(?:yes|yeah|yep|ok|okay|sure|please) )?(?:go ahead(?: and)? )?(?:send|post|create|schedule|book|reply|submit|do)(?: it| that| them| the draft| the message)(?: please)?$/.test(text) + || text === "go ahead"; +}; + export const shouldRemind = (prompt) => typeof prompt === "string" && prompt.trim() !== "" && diff --git a/hooks/routing-guidance.mjs b/hooks/routing-guidance.mjs index 317342d..dfe40d6 100644 --- a/hooks/routing-guidance.mjs +++ b/hooks/routing-guidance.mjs @@ -112,3 +112,10 @@ export const SUBAGENT_CONTEXT = join( line("Routing", "For external service tasks, use try-arcade."), line("Privacy", PRIVACY), ); + +const OPERATOR_AGENT = "arcade-operator"; + +/** True for arcade-operator, bare or plugin-scoped (e.g. "arcade:arcade-operator"). */ +export const isOperatorAgentType = (agentType) => + typeof agentType === "string" && + (agentType === OPERATOR_AGENT || agentType.endsWith(`:${OPERATOR_AGENT}`)); diff --git a/hooks/session-start.mjs b/hooks/session-start.mjs index 5817518..3d2be69 100644 --- a/hooks/session-start.mjs +++ b/hooks/session-start.mjs @@ -1,11 +1,30 @@ #!/usr/bin/env node // Adds the Arcade routing rules at session start. Always exits 0. -import { hostFromArgs, printContext } from "./hook-hosts.mjs"; +import path from "node:path"; +import { hostFromArgs, printContext, readInput } from "./hook-hosts.mjs"; +import { scopeForInput } from "./hook-scope.mjs"; +import { TELEMETRY_ENABLED } from "./telemetry-config.mjs"; +import { loadTelemetryAdapter } from "./telemetry-adapter.mjs"; import { SESSION_CONTEXT } from "./routing-guidance.mjs"; const host = hostFromArgs(process.argv); try { + if (TELEMETRY_ENABLED && host?.telemetry) { + try { + const adapter = await loadTelemetryAdapter(host.telemetry); + const dir = process.env[adapter.dataVariable]; + if (dir && path.isAbsolute(dir)) { + const input = await readInput(); + scopeForInput( + { ...adapter.normalize(input), hook_event_name: "SessionStart" }, + { host: adapter.host, requiresTurn: adapter.requiresTurn, dir }, + ); + } + } catch { + // Clearing local scope must not suppress the routing context. + } + } if (host) printContext(host, "SessionStart", SESSION_CONTEXT); } catch { // Never block session startup. diff --git a/hooks/subagent-start.mjs b/hooks/subagent-start.mjs index a74bd5b..de6bbc9 100644 --- a/hooks/subagent-start.mjs +++ b/hooks/subagent-start.mjs @@ -3,18 +3,13 @@ // own instructions, so it is skipped. Always exits 0. import { hostFromArgs, printContext, readInput } from "./hook-hosts.mjs"; -import { SUBAGENT_CONTEXT } from "./routing-guidance.mjs"; - -// Plugin agents can arrive scoped, e.g. "arcade:arcade-operator". -// Claude Code sends the name as agent_type, Copilot CLI as agentName. -const isOperator = (name) => - typeof name === "string" && - (name === "arcade-operator" || name.endsWith(":arcade-operator")); +import { isOperatorAgentType, SUBAGENT_CONTEXT } from "./routing-guidance.mjs"; const host = hostFromArgs(process.argv); try { const input = await readInput(); - if (host && !isOperator(input.agent_type ?? input.agentName)) { + // Claude Code sends the name as agent_type, Copilot CLI as agentName. + if (host && !isOperatorAgentType(input.agent_type ?? input.agentName)) { printContext(host, "SubagentStart", SUBAGENT_CONTEXT); } } catch { diff --git a/hooks/telemetry-classify.mjs b/hooks/telemetry-classify.mjs new file mode 100644 index 0000000..2977b37 --- /dev/null +++ b/hooks/telemetry-classify.mjs @@ -0,0 +1,109 @@ +// @ts-check +/** Guesses, on the user's machine, whether a prompt is a task Arcade could do. */ + +import { SERVICE_CATEGORIES, TOOLKIT_SERVICES } from "./telemetry-contract.mjs"; + +// Words like "issue", "PR", "branch", "schedule", "event", "channel", and +// "docs" are common in coding prompts, so generic words only count inside a +// phrase that points at a person's own apps ("my calendar", "in linear"). +/** Keyword phrases per category in SERVICE_CATEGORIES. @type {Record} */ +export const KEYWORDS = { + email: [ + "gmail", "outlook", "my inbox", "check my email", "my emails", + "unread email", "unread emails", "send an email", "draft an email", + "write an email", "an email to", "reply to the email", "draft a reply", + "email thread", + ], + calendar: [ + "my calendar", "our calendar", "team calendar", "google calendar", + "outlook calendar", "calendar invite", "calendar event", "on the calendar", + "schedule a meeting", "schedule a call", "set up a meeting", "book a meeting", + "find a time for", "find a time to", "my meetings", "meetings do i have", + "my schedule", "am i free", + ], + chat: [ + "slack", "discord", "microsoft teams", "teams channel", "teams chat", + "dm me", "post in #", "post to #", "posted in #", "in the #", + ], + issues: [ + "jira", "asana", "clickup", "trello", "linear ticket", "linear tickets", + "linear issue", "linear issues", "linear project", "in linear", "on linear", + "file a ticket", "open a ticket", "create a ticket", "my tickets", + "issue tracker", + ], + docs: [ + "notion page", "notion doc", "notion database", "in notion", "on notion", + "google doc", "google docs", "confluence", + ], + meetings: [ + "granola", "fireflies", "zoom call", "zoom meeting", "zoom recording", + "teams meeting", "google meet", "meeting notes", "meeting transcript", + "call notes", "notes from my", "action items from", "my 1:1", + ], + crm: ["hubspot", "salesforce", "attio", "pipedrive", "our crm", "in the crm"], + code_hosting: ["github", "gitlab", "bitbucket"], + analytics: [ + "posthog", "mixpanel", "amplitude", "google analytics", "our analytics", + "product analytics", "weekly active users", "daily active users", + ], + storage: [ + "google drive", "dropbox", "onedrive", "sharepoint", "shared drive", + "my drive", "on drive", "in drive", + ], +}; + +// Removed before matching: they contain a service keyword but mean something +// else in a coding prompt. +const NOT_SERVICES = [ + "github actions", "github action", "linear time", "linear space", + "linear order", "linear algebra", +]; + +// "Slack-style", "notion-like": a comparison, not a request to use the app. +const STYLE_WORD = /\b\w+-(?:style|like)\b/g; + +const escapeRegex = (/** @type {string} */ text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + +const wordsRegex = (/** @type {string[]} */ phrases, flags = "") => + new RegExp(`\\b(?:${phrases.map(escapeRegex).join("|")})\\b`, flags); + +const NOT_SERVICES_REGEX = wordsRegex(NOT_SERVICES, "g"); + +/** @type {[string, RegExp][]} */ +const CATEGORY_REGEXES = SERVICE_CATEGORIES.map((category) => [ + category, + wordsRegex(KEYWORDS[category]), +]); + +/** @param {unknown} prompt */ +export const classifyPrompt = (prompt) => { + if (typeof prompt !== "string") { + return { couldUseArcade: false, serviceHints: [] }; + } + const text = prompt + .toLowerCase() + .replace(/\s+/g, " ") + .replace(STYLE_WORD, " ") + .replace(NOT_SERVICES_REGEX, " "); + const serviceHints = CATEGORY_REGEXES.filter(([, regex]) => regex.test(text)) + .map(([category]) => category) + .sort(); + return { couldUseArcade: serviceHints.length > 0, serviceHints }; +}; + +/** @param {unknown} toolkit */ +export const serviceForToolkit = (toolkit) => { + if (typeof toolkit !== "string") return null; + const name = toolkit.toLowerCase(); + return Object.hasOwn(TOOLKIT_SERVICES, name) ? TOOLKIT_SERVICES[name] : null; +}; + +// Accepts "Gmail_ListEmails" (MCP tool names) and "Gmail.ListEmails" +// (the tool_name passed to Arcade_UseTool). +/** @param {unknown} toolName */ +export const serviceForToolName = (toolName) => { + if (typeof toolName !== "string") return null; + const separator = toolName.search(/[_.]/); + if (separator <= 0) return null; + return serviceForToolkit(toolName.slice(0, separator)); +}; diff --git a/hooks/telemetry-commands.mjs b/hooks/telemetry-commands.mjs new file mode 100644 index 0000000..7082deb --- /dev/null +++ b/hooks/telemetry-commands.mjs @@ -0,0 +1,31 @@ +// @ts-check +/** + * Check whether a shell command segment invokes the named CLI. + * Used to guard against clients that ignore the `if` field on hook entries. + */ + +// Split on shell segment separators: &&, ||, ;, |, newline, ( +const SEGMENT_SPLIT = /&&|\|\||[;|\n(]/; + +// Shell variable assignment: NAME=value at the start of a word +const ASSIGNMENT = /^[A-Za-z_][A-Za-z0-9_]*=/; + +/** + * Returns true when `command` contains a segment whose first non-assignment + * word is exactly `cli`. Absolute paths and substrings do not match. + * + * @param {unknown} command + * @param {string} cli + * @returns {boolean} + */ +export const commandUsesCli = (command, cli) => { + if (typeof command !== "string") return false; + const segments = command.split(SEGMENT_SPLIT); + for (const segment of segments) { + const words = segment.trim().split(/\s+/); + let i = 0; + while (i < words.length && ASSIGNMENT.test(words[i])) i++; + if (i < words.length && words[i] === cli) return true; + } + return false; +}; diff --git a/hooks/telemetry-events.mjs b/hooks/telemetry-events.mjs new file mode 100644 index 0000000..d5ddef7 --- /dev/null +++ b/hooks/telemetry-events.mjs @@ -0,0 +1,226 @@ +// @ts-check +/** + * Turns normalized hook input into a telemetry event. Pure: no I/O. + * What may be sent is defined in telemetry-contract.mjs. + */ + +import { createHash } from "node:crypto"; +import { isTaskNotification, shouldRemind } from "./prompt-filters.mjs"; +import { isOperatorAgentType } from "./routing-guidance.mjs"; +import { + classifyPrompt, + serviceForToolkit, + serviceForToolName, +} from "./telemetry-classify.mjs"; +import { commandUsesCli } from "./telemetry-commands.mjs"; +import { + allowedProperties, + BASH_CLIS, + CLI_SERVICES, + GATEWAY_TOOLS, + OPERATOR_STATUSES, + OS_NAMES, +} from "./telemetry-contract.mjs"; +import { PLUGIN_VERSION } from "./telemetry-config.mjs"; +import { authNeeded, failureKind } from "./telemetry-failures.mjs"; + +/** @typedef {import("./telemetry-adapter.mjs").TelemetryAdapter} TelemetryAdapter */ +/** @typedef {import("./telemetry-adapter.mjs").HookInput} HookInput */ + +// Matches the operator's report line, e.g. "status: needs_auth", with or +// without markdown around it. "unknown" is what we send when none matches. +const OPERATOR_STATUS = new RegExp( + `^[^\\w\\n]*status[^\\w\\n]*(${OPERATOR_STATUSES.filter((s) => s !== "unknown").join("|")})\\b`, + "im", +); + +// The client's session ID is random, new for each session, and never sent, +// so it works as the salt: nothing links one session's hashes to another's. +export const shortHash = (/** @type {string} */ text) => + createHash("sha256").update(text).digest("hex").slice(0, 16); + +/** + * @param {unknown} value + * @param {readonly string[]} allowed + * @param {string} fallback + */ +const oneOf = (value, allowed, fallback) => + typeof value === "string" && allowed.includes(value) ? value : fallback; + +/** + * @param {Record} properties + * @param {string | null | undefined} service + */ +const withService = (properties, service) => + service ? { ...properties, service } : properties; + +/** + * @param {string} server + * @param {string} tool + * @param {Record | undefined} toolInput + */ +export const arcadeToolProperties = (server, tool, toolInput) => { + const service = + tool === "Arcade_UseTool" + ? serviceForToolName(toolInput?.tool_name) + : serviceForToolName(tool); + const isGateway = /** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool); + const category = isGateway ? tool : service !== null ? "app_tool" : "other"; + return withService({ server, tool: category }, service); +}; + +/** + * @param {string | null | undefined} service + */ +export const otherServerProperties = (service) => withService({ server: "other" }, service); + +/** + * Properties for a WebFetch, WebSearch, or listed-CLI Bash call, or null to + * send nothing. Only the tool name and the CLI name are read into the event; + * the command, its description, URLs, and queries never are. + * @param {unknown} toolName + * @param {Record | undefined} toolInput + * @param {string | undefined} cli + */ +export const builtinToolProperties = (toolName, toolInput, cli) => { + if (toolName === "WebFetch" || toolName === "WebSearch") return { tool: toolName }; + if (toolName !== "Bash" || typeof cli !== "string") return null; + if (!(/** @type {readonly string[]} */ (BASH_CLIS).includes(cli))) return null; + if (!commandUsesCli(toolInput?.command, cli)) return null; + return withService({ tool: "Bash", cli }, CLI_SERVICES[cli]); +}; + +const operatorStatus = (/** @type {unknown} */ message) => { + const match = typeof message === "string" && message.match(OPERATOR_STATUS); + return match ? match[1].toLowerCase() : "unknown"; +}; + +/** + * @param {unknown} prompt + * @param {TelemetryAdapter} adapter + */ +const promptProperties = (prompt, adapter) => { + const { couldUseArcade, serviceHints } = classifyPrompt(prompt); + return { + could_use_arcade: couldUseArcade, + service_hints: serviceHints, + reminder_sent: adapter.promptReminder && shouldRemind(prompt), + }; +}; + +const subagentSession = (/** @type {unknown} */ agentId) => + typeof agentId === "string" && agentId !== "" ? { subagent_session: shortHash(agentId) } : {}; + +/** + * Returns [event name, extra properties], or null for untracked input. + * @param {HookInput} input + * @param {TelemetryAdapter} adapter + * @param {string | undefined} cli + * @param {boolean} appWork + * @returns {[string, Record] | null} + */ +const eventFor = (input, adapter, cli, appWork) => { + switch (input.hook_event_name) { + case "SessionStart": + return null; + case "UserPromptSubmit": + if (isTaskNotification(input.prompt) || !appWork) return null; + return ["Plugin prompt submitted", promptProperties(input.prompt, adapter)]; + case "PreToolUse": { + const extra = adapter.attemptProperties?.(input.tool_name, input.tool_input); + return extra ? ["Plugin tool attempted", extra] : null; + } + case "PostToolUse": { + const builtin = adapter.builtinToolProperties?.(input.tool_name, input.tool_input, cli); + if (builtin && appWork) return ["Plugin built-in tool called", builtin]; + const extra = adapter.toolProperties(input.tool_name, input.tool_input); + if (!extra || (extra.server === "other" && !appWork)) return null; + if (extra.tool === "System_ManageAuthorization") { + return ["Plugin tool called", { ...extra, auth_needed: authNeeded(input.tool_response) }]; + } + return ["Plugin tool called", extra]; + } + case "PostToolUseFailure": { + const builtin = adapter.builtinToolProperties?.(input.tool_name, input.tool_input, cli); + if (builtin && appWork) return ["Plugin built-in tool failed", builtin]; + const extra = adapter.toolProperties(input.tool_name, input.tool_input); + if (!extra || (extra.server === "other" && !appWork)) return null; + return ["Plugin tool failed", { ...extra, failure_kind: failureKind(input.error, input.is_interrupt) }]; + } + case "SubagentStop": { + const session = adapter.subagentSession ? subagentSession(input.agent_id) : {}; + if (!isOperatorAgentType(input.agent_type)) { + return null; + } + return [ + "Plugin subagent stopped", + { + agent: "arcade-operator", + status: operatorStatus(input.last_assistant_message), + ...session, + }, + ]; + } + default: + return null; + } +}; + +/** + * @param {string} event + * @param {Record} properties + */ +const keepAllowed = (event, properties) => { + /** @type {Record} */ + const kept = {}; + for (const key of allowedProperties(event)) { + if (properties[key] !== undefined) kept[key] = properties[key]; + } + return kept; +}; + +/** + * Builds `{ event, distinct_id, properties }` from hook stdin, or returns null + * when the input is not something the contract tracks. + * @param {HookInput | null | undefined} input + * @param {{ adapter: TelemetryAdapter, os: string, arcadeUsedBefore: boolean, cli?: string, appWork: boolean }} options + */ +export const buildEvent = (input, { adapter, os, arcadeUsedBefore, cli, appWork }) => { + if (!input || typeof input !== "object") return null; + if (typeof input.session_id !== "string" || input.session_id === "") return null; + const found = eventFor(input, adapter, cli, appWork); + if (!found) return null; + const [event, extra] = found; + + /** @type {Record} */ + const properties = { + ...extra, + host: adapter.host, + plugin_version: PLUGIN_VERSION, + telemetry_version: 2, + os: oneOf(os, OS_NAMES, "other"), + $process_person_profile: false, + $geoip_disable: true, + $ip: "0.0.0.0", + arcade_used_before: arcadeUsedBefore === true, + }; + const session = shortHash(input.session_id); + properties.session = session; + if (typeof input.prompt_id === "string") { + properties.turn = shortHash(`${input.session_id}:${input.prompt_id}`); + } + + return { + event, + distinct_id: session, + properties: keepAllowed(event, properties), + }; +}; + +/** + * True for a successful call to this plugin's Arcade gateway or another + * Arcade connection. + * @param {{ event: string, properties: Record }} event + */ +export const isArcadeCall = ({ event, properties }) => + event === "Plugin tool called" && (properties.server === "arcade" || properties.server === "other_arcade"); diff --git a/hooks/telemetry-failures.mjs b/hooks/telemetry-failures.mjs new file mode 100644 index 0000000..6a4f242 --- /dev/null +++ b/hooks/telemetry-failures.mjs @@ -0,0 +1,70 @@ +// @ts-check +/** Classifiers for failure_kind and auth_needed from hook inputs. */ + +/** @typedef {"auth_required"|"session_expired"|"unreachable"|"timeout"|"http_error"|"interrupted"|"tool_error"} FailureKind */ + +// Claude Code 2.1.246 says "requires re-authorization". 2.1.278 says "needs +// you to sign in again", "needs additional permissions", or "rejected the +// credential from its headersHelper" / "rejected the Authorization header". +const AUTH_REQUIRED_RE = + /requires authorization|authorization required|"authorization_url"|requires re-authorization|needs to be connected in claude\.ai|needs you to sign in again|needs additional permissions \(scope:|rejected the credential from its headersHelper|rejected the Authorization header in its config/i; +const SESSION_EXPIRED_RE = /session expired/; +const TIMEOUT_RE = /sent no response or progress for|timed out after|MCP error -32001: Request timed out/; +const UNREACHABLE_RE = + /Unable to connect|socket connection was closed unexpectedly|^Connection closed$|MCP error -32000: Connection closed|ECONNREFUSED|ENOTFOUND|ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|transport dropped mid-call|transport closed before the tool responded/; +const HTTP_ERROR_RE = /Error POSTing to endpoint/; + +/** + * Returns the failure_kind for a PostToolUseFailure hook input. + * First matching rule wins. + * + * @param {unknown} error + * @param {unknown} isInterrupt + * @returns {FailureKind} + */ +export const failureKind = (error, isInterrupt) => { + if (isInterrupt === true) return "interrupted"; + const msg = typeof error === "string" ? error : ""; + if (AUTH_REQUIRED_RE.test(msg)) return "auth_required"; + if (SESSION_EXPIRED_RE.test(msg)) return "session_expired"; + if (TIMEOUT_RE.test(msg)) return "timeout"; + if (UNREACHABLE_RE.test(msg)) return "unreachable"; + if (HTTP_ERROR_RE.test(msg)) return "http_error"; + return "tool_error"; +}; + +/** + * Returns true when a System_ManageAuthorization tool_response indicates that + * at least one provider still needs sign-in. The tool_response is the content + * array [{type:"text",text}] as Claude Code delivers it, or a plain string. + * Parses the JSON text and checks providers[].status === "authorization_required". + * + * @param {unknown} toolResponse + * @returns {boolean} + */ +export const authNeeded = (toolResponse) => { + if (toolResponse === null || toolResponse === undefined) return false; + try { + /** @type {string|undefined} */ + let text; + if (typeof toolResponse === "string") { + text = toolResponse; + } else if (Array.isArray(toolResponse)) { + const first = toolResponse[0]; + if (first && typeof first === "object" && "text" in first && typeof first.text === "string") { + text = first.text; + } + } + if (text === undefined) return false; + const parsed = JSON.parse(text); + if (!parsed || !Array.isArray(parsed.providers)) return false; + return parsed.providers.some( + (/** @type {unknown} */ p) => + p !== null && + typeof p === "object" && + /** @type {Record} */ (p).status === "authorization_required", + ); + } catch { + return false; + } +}; diff --git a/hooks/telemetry-run.mjs b/hooks/telemetry-run.mjs new file mode 100644 index 0000000..087ace5 --- /dev/null +++ b/hooks/telemetry-run.mjs @@ -0,0 +1,113 @@ +// @ts-check +/** Opt-out checks and the shared telemetry hook body. Never throws. */ + +import { spawn } from "node:child_process"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { scopeForInput } from "./hook-scope.mjs"; +import { ARCADE_USED_FILE, EVENT_ENV, OPT_OUT_ENV, TELEMETRY_ENABLED } from "./telemetry-config.mjs"; +import { buildEvent, isArcadeCall } from "./telemetry-events.mjs"; + +/** @typedef {import("./telemetry-adapter.mjs").TelemetryAdapter} TelemetryAdapter */ + +const SENDER = path.join( + path.dirname(fileURLToPath(import.meta.url)), + "telemetry-send.mjs", +); + +const OFF_VALUES = ["0", "false", "off", "no"]; + +const isSet = (/** @type {string} */ value) => value !== "" && !OFF_VALUES.includes(value.toLowerCase()); + +/** + * A switch with `anyValue` is on for any non-empty value, even "0" or "false", + * because that is how Claude Code reads its own switches. + * @param {{ name: string, anyValue: boolean }[]} optOutSwitches + * @param {NodeJS.ProcessEnv} env + */ +export const isOptedOut = (optOutSwitches, env = process.env) => { + if (OFF_VALUES.includes((env[OPT_OUT_ENV] ?? "").toLowerCase())) return true; + if (isSet(env.DO_NOT_TRACK ?? "")) return true; + return optOutSwitches.some(({ name, anyValue }) => { + const value = env[name] ?? ""; + return anyValue ? value !== "" : isSet(value); + }); +}; + +const readArcadeUsed = (/** @type {string} */ dir) => { + try { + return readFileSync(path.join(dir, ARCADE_USED_FILE), "utf8").trim() === "true"; + } catch { + return false; + } +}; + +const cliFromArgs = (/** @type {string[]} */ argv) => { + const flag = argv.indexOf("--cli"); + return flag === -1 ? undefined : argv[flag + 1]; +}; + +const markArcadeUsed = (/** @type {string} */ dir) => { + mkdirSync(dir, { recursive: true }); + writeFileSync(path.join(dir, ARCADE_USED_FILE), "true", { mode: 0o600 }); +}; + +const defaultSend = (/** @type {object} */ event, env = process.env) => { + const child = spawn(process.execPath, [SENDER], { + detached: true, + stdio: "ignore", + windowsHide: true, + env: { ...env, [EVENT_ENV]: JSON.stringify(event) }, + }); + child.on("error", () => {}); + child.unref(); +}; + +/** + * @param {object} options + * @param {Record} options.input Raw hook stdin. + * @param {TelemetryAdapter} options.adapter + * @param {string[]} [options.argv] + * @param {NodeJS.ProcessEnv} [options.env] + * @param {number} [options.now] + * @param {boolean} [options.enabled] When true, runs even if TELEMETRY_ENABLED is false (tests). + * @param {(event: object) => void} [options.send] + */ +export const runTelemetry = async ({ + input, + adapter, + argv = process.argv, + env = process.env, + now = Date.now(), + enabled = false, + send = (event) => defaultSend(event, env), +}) => { + try { + if (!enabled && !TELEMETRY_ENABLED) return; + const dir = env[adapter.dataVariable]; + if (!dir || !path.isAbsolute(dir)) return; + if (isOptedOut(adapter.optOutSwitches, env)) return; + const normalized = adapter.normalize(input ?? {}); + const { appWork } = scopeForInput(normalized, { + host: adapter.host, + requiresTurn: adapter.requiresTurn, + dir, + now, + }); + + const arcadeUsedBefore = readArcadeUsed(dir); + const event = buildEvent(normalized, { + adapter, + os: process.platform, + arcadeUsedBefore, + cli: cliFromArgs(argv), + appWork, + }); + if (!event) return; + if (!arcadeUsedBefore && isArcadeCall(event)) markArcadeUsed(dir); + send(event); + } catch { + // Telemetry must never affect the session. + } +}; diff --git a/hooks/telemetry-send.mjs b/hooks/telemetry-send.mjs new file mode 100644 index 0000000..9bfd03f --- /dev/null +++ b/hooks/telemetry-send.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// @ts-check +// Detached sender started by telemetry.mjs. Posts one event to PostHog, no +// retries. Always exit 0. + +import { EVENT_ENV, POSTHOG_HOST, POSTHOG_KEY } from "./telemetry-config.mjs"; + +try { + const { event, distinct_id, properties } = JSON.parse(process.env[EVENT_ENV] ?? ""); + await fetch(`${POSTHOG_HOST}/i/v0/e/`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + api_key: POSTHOG_KEY, + event, + distinct_id, + properties, + timestamp: new Date().toISOString(), + }), + signal: AbortSignal.timeout(1000), + }); +} catch { + // Telemetry is best effort. +} + +process.exit(0); diff --git a/hooks/telemetry.mjs b/hooks/telemetry.mjs new file mode 100644 index 0000000..d28d0a4 --- /dev/null +++ b/hooks/telemetry.mjs @@ -0,0 +1,26 @@ +#!/usr/bin/env node +// @ts-check +// Telemetry hook for each client in hook-hosts.mjs with a `telemetry` entry. +// Sends the usage events described in docs/telemetry.md. Always exit 0. + +import { TELEMETRY_ENABLED } from "./telemetry-config.mjs"; + +if (!TELEMETRY_ENABLED) process.exit(0); + +import { hostFromArgs, readInput } from "./hook-hosts.mjs"; +import { loadTelemetryAdapter } from "./telemetry-adapter.mjs"; +import { runTelemetry } from "./telemetry-run.mjs"; + +try { + const adapterHost = hostFromArgs(process.argv)?.telemetry; + if (!adapterHost) { + process.exit(0); + } + const adapter = await loadTelemetryAdapter(adapterHost); + const input = await readInput(); + await runTelemetry({ input, adapter, enabled: true }); +} catch { + // Telemetry must never affect the session. +} + +process.exit(0); diff --git a/test/fixtures/hook-inputs/auth-required-arcade.json b/test/fixtures/hook-inputs/auth-required-arcade.json new file mode 100644 index 0000000..fd4ab78 --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-arcade.json @@ -0,0 +1,4 @@ +{ + "error": "The tool was not executed because it requires authorization. This is not an error; the tool requires permission before it can run.", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/auth-required-connect.json b/test/fixtures/hook-inputs/auth-required-connect.json new file mode 100644 index 0000000..5f6a0ea --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-connect.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"arcade\" needs to be connected in claude.ai (run /mcp to connect it)", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/auth-required-reauth.json b/test/fixtures/hook-inputs/auth-required-reauth.json new file mode 100644 index 0000000..1042e6e --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-reauth.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"arcade\" requires re-authorization (token expired)", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/auth-required-sign-in-again.json b/test/fixtures/hook-inputs/auth-required-sign-in-again.json new file mode 100644 index 0000000..c35854e --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-sign-in-again.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"arcade\" needs you to sign in again (run /mcp to re-authenticate)", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/auth-required-url.json b/test/fixtures/hook-inputs/auth-required-url.json new file mode 100644 index 0000000..6532796 --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-url.json @@ -0,0 +1,4 @@ +{ + "error": "âš  Authorization required\n\n Tool 'Gmail.ListEmails' needs your permission to access your account.\n\n To authorize:\n 1. Click this link: https://example.invalid/oauth/abc\n{\"llm_instructions\":\"show link\",\"authorization_url\":\"https://example.invalid/oauth/abc\"}", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/auth-required-use-tool.json b/test/fixtures/hook-inputs/auth-required-use-tool.json new file mode 100644 index 0000000..0a710cc --- /dev/null +++ b/test/fixtures/hook-inputs/auth-required-use-tool.json @@ -0,0 +1,4 @@ +{ + "error": "this tool requires authorization. Please visit this URL to authorize: https://example.invalid/oauth/abc", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/http-error-500.json b/test/fixtures/hook-inputs/http-error-500.json new file mode 100644 index 0000000..6b8f955 --- /dev/null +++ b/test/fixtures/hook-inputs/http-error-500.json @@ -0,0 +1,4 @@ +{ + "error": "Error POSTing to endpoint http://127.0.0.1:12345/mcp: upstream exploded", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/http-error-streamable.json b/test/fixtures/hook-inputs/http-error-streamable.json new file mode 100644 index 0000000..efb35e6 --- /dev/null +++ b/test/fixtures/hook-inputs/http-error-streamable.json @@ -0,0 +1,4 @@ +{ + "error": "Streamable HTTP error: Error POSTing to endpoint http://127.0.0.1:12345/mcp: upstream exploded", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/interrupted.json b/test/fixtures/hook-inputs/interrupted.json new file mode 100644 index 0000000..3c7d21d --- /dev/null +++ b/test/fixtures/hook-inputs/interrupted.json @@ -0,0 +1,4 @@ +{ + "error": "Tool call interrupted", + "is_interrupt": true +} diff --git a/test/fixtures/hook-inputs/session-expired.json b/test/fixtures/hook-inputs/session-expired.json new file mode 100644 index 0000000..f9065cc --- /dev/null +++ b/test/fixtures/hook-inputs/session-expired.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"probe\" session expired", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/timeout-gateway.json b/test/fixtures/hook-inputs/timeout-gateway.json new file mode 100644 index 0000000..bd936c5 --- /dev/null +++ b/test/fixtures/hook-inputs/timeout-gateway.json @@ -0,0 +1,4 @@ +{ + "error": "tool execution timed out after 30s", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/timeout-mcp-request.json b/test/fixtures/hook-inputs/timeout-mcp-request.json new file mode 100644 index 0000000..abaa896 --- /dev/null +++ b/test/fixtures/hook-inputs/timeout-mcp-request.json @@ -0,0 +1,4 @@ +{ + "error": "MCP error -32001: Request timed out", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/timeout-no-response.json b/test/fixtures/hook-inputs/timeout-no-response.json new file mode 100644 index 0000000..a66a1fa --- /dev/null +++ b/test/fixtures/hook-inputs/timeout-no-response.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"probe\" tool \"slow_tool\" sent no response or progress for 30s; aborting. You can increase the timeout in settings.", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/tool-error-exit-code.json b/test/fixtures/hook-inputs/tool-error-exit-code.json new file mode 100644 index 0000000..b835d17 --- /dev/null +++ b/test/fixtures/hook-inputs/tool-error-exit-code.json @@ -0,0 +1,4 @@ +{ + "error": "Exit code 1", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/tool-error-rate-limit.json b/test/fixtures/hook-inputs/tool-error-rate-limit.json new file mode 100644 index 0000000..763f634 --- /dev/null +++ b/test/fixtures/hook-inputs/tool-error-rate-limit.json @@ -0,0 +1,4 @@ +{ + "error": "Rate limited by upstream", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/tool-error-rpc.json b/test/fixtures/hook-inputs/tool-error-rpc.json new file mode 100644 index 0000000..0b5f9ba --- /dev/null +++ b/test/fixtures/hook-inputs/tool-error-rpc.json @@ -0,0 +1,4 @@ +{ + "error": "Invalid params: q must be int", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/unreachable-connection-closed.json b/test/fixtures/hook-inputs/unreachable-connection-closed.json new file mode 100644 index 0000000..91cefaf --- /dev/null +++ b/test/fixtures/hook-inputs/unreachable-connection-closed.json @@ -0,0 +1,4 @@ +{ + "error": "Connection closed", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/unreachable-mcp-connection-closed.json b/test/fixtures/hook-inputs/unreachable-mcp-connection-closed.json new file mode 100644 index 0000000..db979bf --- /dev/null +++ b/test/fixtures/hook-inputs/unreachable-mcp-connection-closed.json @@ -0,0 +1,4 @@ +{ + "error": "MCP error -32000: Connection closed", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/unreachable-socket.json b/test/fixtures/hook-inputs/unreachable-socket.json new file mode 100644 index 0000000..be6b995 --- /dev/null +++ b/test/fixtures/hook-inputs/unreachable-socket.json @@ -0,0 +1,4 @@ +{ + "error": "The socket connection was closed unexpectedly. Please try again.", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/unreachable-transport-dropped.json b/test/fixtures/hook-inputs/unreachable-transport-dropped.json new file mode 100644 index 0000000..883d7f3 --- /dev/null +++ b/test/fixtures/hook-inputs/unreachable-transport-dropped.json @@ -0,0 +1,4 @@ +{ + "error": "MCP server \"arcade\" transport dropped mid-call; response for tool \"Slack_SendMessage\" was lost", + "is_interrupt": false +} diff --git a/test/fixtures/hook-inputs/unreachable-unable-to-connect.json b/test/fixtures/hook-inputs/unreachable-unable-to-connect.json new file mode 100644 index 0000000..edf68e6 --- /dev/null +++ b/test/fixtures/hook-inputs/unreachable-unable-to-connect.json @@ -0,0 +1,4 @@ +{ + "error": "Unable to connect. Is the computer able to access the url?", + "is_interrupt": false +} diff --git a/test/fixtures/routing-prompts.json b/test/fixtures/routing-prompts.json new file mode 100644 index 0000000..44d4d8a --- /dev/null +++ b/test/fixtures/routing-prompts.json @@ -0,0 +1,114 @@ +[ + { "prompt": "pull up the granola notes from my 1:1 with valerie yesterday", "couldUseArcade": true, "categories": ["meetings"] }, + { "prompt": "what's on my calendar tomorrow", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "draft a reply to the email from Sam", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "file a linear ticket for this bug", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "post in #eng that the deploy is done", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "check posthog for signups last week", "couldUseArcade": true, "categories": ["analytics"] }, + { "prompt": "summarize my unread emails from this morning", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "search my gmail for the invoice from AWS", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "find the thread in outlook where legal approved the contract", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "check my inbox for anything from the recruiter", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "reply to Tom's email saying I'll be there at 4", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "forward the receipt email to finance@acme.com", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "send an email to the team with these release notes", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "did anyone reply to my email about the offsite?", "couldUseArcade": true, "categories": ["email"] }, + { "prompt": "block two hours on my calendar friday afternoon for focus time", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "schedule a meeting with Priya and Dan next week about the migration", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "move my 3pm to thursday", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "am I free at 2 tomorrow?", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "invite alex to the planning meeting on thursday", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "what meetings do I have today", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "check my calendar and find a time for a 30 minute sync with Kim", "couldUseArcade": true, "categories": ["calendar"] }, + { "prompt": "send a slack DM to jordan asking for the Q3 numbers", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "what did people say in the #incidents channel today", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "message the design team on slack that the mocks are ready", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "catch me up on what I missed in slack while I was out", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "ask in the teams channel whether anyone has the VPN config", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "reply in the slack thread that I'm looking into it", "couldUseArcade": true, "categories": ["chat"] }, + { "prompt": "create a jira ticket for the flaky login test", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "what linear issues are assigned to me", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "move GRO-353 to in review in linear", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "create an asana task to follow up with legal on the DPA", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "what's the status of the onboarding epic in jira", "couldUseArcade": true, "categories": ["issues"] }, + { "prompt": "find the notion page for the Q4 roadmap", "couldUseArcade": true, "categories": ["docs"] }, + { "prompt": "add these notes to our onboarding doc in notion", "couldUseArcade": true, "categories": ["docs"] }, + { "prompt": "open the google doc Sarah shared about pricing and summarize it", "couldUseArcade": true, "categories": ["docs"] }, + { "prompt": "update the confluence page for the on-call runbook", "couldUseArcade": true, "categories": ["docs"] }, + { "prompt": "what were the action items from yesterday's standup meeting", "couldUseArcade": true, "categories": ["meetings"] }, + { "prompt": "get the transcript from my zoom call with Acme", "couldUseArcade": true, "categories": ["meetings"] }, + { "prompt": "what did we decide on the planning call? check fireflies", "couldUseArcade": true, "categories": ["meetings"] }, + { "prompt": "summarize my last three granola meetings", "couldUseArcade": true, "categories": ["meetings"] }, + { "prompt": "pull the notes from my meeting with the Acme team and draft a follow-up email", "couldUseArcade": true, "categories": ["email", "meetings"] }, + { "prompt": "update the Acme deal stage in hubspot to closed won", "couldUseArcade": true, "categories": ["crm"] }, + { "prompt": "look up the account owner for Globex in salesforce", "couldUseArcade": true, "categories": ["crm"] }, + { "prompt": "add Maria Lopez as a contact in our CRM", "couldUseArcade": true, "categories": ["crm"] }, + { "prompt": "which deals in attio are closing this month", "couldUseArcade": true, "categories": ["crm"] }, + { "prompt": "prep me for my 10am: pull the hubspot notes on Initech and any recent emails with them", "couldUseArcade": true, "categories": ["calendar", "crm", "email"] }, + { "prompt": "open a github issue on arcade-mcp for the auth bug", "couldUseArcade": true, "categories": ["code_hosting"] }, + { "prompt": "list my open pull requests on github", "couldUseArcade": true, "categories": ["code_hosting"] }, + { "prompt": "summarize the review comments on PR #412 on GitHub", "couldUseArcade": true, "categories": ["code_hosting"] }, + { "prompt": "pull the weekly active users trend from posthog", "couldUseArcade": true, "categories": ["analytics"] }, + { "prompt": "how many people hit the pricing page yesterday according to our analytics", "couldUseArcade": true, "categories": ["analytics"] }, + { "prompt": "show me the signup funnel conversion for last month", "couldUseArcade": true, "categories": ["analytics"] }, + { "prompt": "find the Q3 board deck in google drive", "couldUseArcade": true, "categories": ["storage"] }, + { "prompt": "upload this csv to the shared dropbox folder", "couldUseArcade": true, "categories": ["storage"] }, + { "prompt": "share the design folder on drive with the contractor", "couldUseArcade": true, "categories": ["storage"] }, + + { "prompt": "fix the issue in auth.ts", "couldUseArcade": false, "categories": [] }, + { "prompt": "open a PR for this branch", "couldUseArcade": false, "categories": [] }, + { "prompt": "why does the event handler fire twice", "couldUseArcade": false, "categories": [] }, + { "prompt": "update the docs folder with the new CLI flags", "couldUseArcade": false, "categories": [] }, + { "prompt": "schedule the cron job to run nightly at 2am", "couldUseArcade": false, "categories": [] }, + { "prompt": "add a Slack-style toast component", "couldUseArcade": false, "categories": [] }, + { "prompt": "refactor the calendar component to use date-fns", "couldUseArcade": false, "categories": [] }, + { "prompt": "email regex is wrong, it rejects plus addresses", "couldUseArcade": false, "categories": [] }, + { "prompt": "add a unit test for the calendar date parsing", "couldUseArcade": false, "categories": [] }, + { "prompt": "rename the Channel type to ChatChannel in the websocket server", "couldUseArcade": false, "categories": [] }, + { "prompt": "the slack webhook integration test is failing, fix it", "couldUseArcade": false, "categories": [] }, + { "prompt": "commit these changes with a good message", "couldUseArcade": false, "categories": [] }, + { "prompt": "rebase my branch onto main and fix the conflicts", "couldUseArcade": false, "categories": [] }, + { "prompt": "write a migration to add an email_verified column to users", "couldUseArcade": false, "categories": [] }, + { "prompt": "why is the meeting room booking page slow to render", "couldUseArcade": false, "categories": [] }, + { "prompt": "add pagination to the issues list endpoint", "couldUseArcade": false, "categories": [] }, + { "prompt": "implement linear interpolation for the animation easing", "couldUseArcade": false, "categories": [] }, + { "prompt": "make the dedupe step run in linear time", "couldUseArcade": false, "categories": [] }, + { "prompt": "add a zoom control to the image viewer", "couldUseArcade": false, "categories": [] }, + { "prompt": "fix the broken links in docs/", "couldUseArcade": false, "categories": [] }, + { "prompt": "add an analytics event when the user clicks checkout", "couldUseArcade": false, "categories": [] }, + { "prompt": "write a parser for ICS calendar files", "couldUseArcade": false, "categories": [] }, + { "prompt": "set up a GitHub Actions workflow that runs npm test", "couldUseArcade": false, "categories": [] }, + { "prompt": "add the channel id to the kafka consumer config", "couldUseArcade": false, "categories": [] }, + { "prompt": "debug why the email template renders blank in outlook", "couldUseArcade": false, "categories": [] }, + { "prompt": "write a docstring for the schedule() function", "couldUseArcade": false, "categories": [] }, + { "prompt": "bump the node version in package.json", "couldUseArcade": false, "categories": [] }, + { "prompt": "the ticket booking flow throws on submit", "couldUseArcade": false, "categories": [] }, + { "prompt": "add retry logic to the notion API client", "couldUseArcade": false, "categories": [] }, + { "prompt": "explain the notion of idempotency used in this handler", "couldUseArcade": false, "categories": [] }, + { "prompt": "clean up unused imports in the drive sync module", "couldUseArcade": false, "categories": [] }, + { "prompt": "implement a CRM-style contact list page with filtering", "couldUseArcade": false, "categories": [] }, + { "prompt": "the log shows a weird merge commit, what happened", "couldUseArcade": false, "categories": [] }, + { "prompt": "create a new branch called feature/login", "couldUseArcade": false, "categories": [] }, + { "prompt": "list all the TODO comments in the repo", "couldUseArcade": false, "categories": [] }, + { "prompt": "the calendar picker shows the wrong month in Safari", "couldUseArcade": false, "categories": [] }, + { "prompt": "add a Chat component with message bubbles", "couldUseArcade": false, "categories": [] }, + { "prompt": "write tests for the sendEmail helper", "couldUseArcade": false, "categories": [] }, + { "prompt": "why does the meeting scheduler algorithm pick overlapping slots", "couldUseArcade": false, "categories": [] }, + { "prompt": "document the public API in docs/api.md", "couldUseArcade": false, "categories": [] }, + { "prompt": "the PR description template needs a checklist section", "couldUseArcade": false, "categories": [] }, + { "prompt": "run the test suite and fix any failures", "couldUseArcade": false, "categories": [] }, + { "prompt": "add an index on the events table's created_at column", "couldUseArcade": false, "categories": [] }, + { "prompt": "migrate the storage layer from S3 to GCS", "couldUseArcade": false, "categories": [] }, + { "prompt": "profile the dashboard page load and cut the bundle size", "couldUseArcade": false, "categories": [] }, + { "prompt": "handle the case where the inbox query returns null", "couldUseArcade": false, "categories": [] }, + { "prompt": "rename getMeetings to listMeetings across the codebase", "couldUseArcade": false, "categories": [] }, + { "prompt": "the notification channel dropdown is empty on first load", "couldUseArcade": false, "categories": [] }, + { "prompt": "make the email input required on the signup form", "couldUseArcade": false, "categories": [] }, + { "prompt": "extract the email from the jwt claims in the middleware", "couldUseArcade": false, "categories": [] }, + { "prompt": "add posthog capture calls to the signup form", "couldUseArcade": false, "categories": [] }, + { "prompt": "push this branch to github", "couldUseArcade": false, "categories": [] }, + { "prompt": "the dashboard metrics component re-renders too often", "couldUseArcade": false, "categories": [] }, + { "prompt": "why does the pre-commit hook hang on every commit", "couldUseArcade": false, "categories": [] }, + { "prompt": "add a --schedule flag to the deploy CLI", "couldUseArcade": false, "categories": [] }, + { "prompt": "write a function that finds a free time slot given a list of busy intervals", "couldUseArcade": false, "categories": [] } +] diff --git a/test/fixtures/telemetry-fake-adapter.mjs b/test/fixtures/telemetry-fake-adapter.mjs new file mode 100644 index 0000000..5f66c09 --- /dev/null +++ b/test/fixtures/telemetry-fake-adapter.mjs @@ -0,0 +1,69 @@ +// @ts-check +/** Test-only telemetry adapter; not loaded by production hook-hosts.mjs. */ + +import { serviceForToolName } from "../../hooks/telemetry-classify.mjs"; +import { + arcadeToolProperties, + builtinToolProperties, + otherServerProperties, +} from "../../hooks/telemetry-events.mjs"; + +export const FAKE_ARCADE_PREFIX = "mcp__fake_arcade__"; +export const FAKE_OTHER_PREFIX = "mcp__fake_other__"; + +/** @type {import("../../hooks/telemetry-adapter.mjs").TelemetryAdapter} */ +const fakeAdapter = { + host: "claude-code", + dataVariable: "ARCADE_TEST_PLUGIN_DATA", + optOutSwitches: [ + { name: "FAKE_DISABLE_TELEMETRY", anyValue: true }, + { name: "FAKE_OFFLINE", anyValue: false }, + ], + requiresTurn: true, + promptReminder: true, + subagentSession: false, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PreToolUse", matcher: "mcp__fake_arcade__.*" }, + { event: "PostToolUse", matcher: "mcp__.*" }, + { event: "PostToolUseFailure", matcher: "mcp__.*" }, + { event: "PostToolUse", matcher: "WebFetch|WebSearch" }, + { event: "PostToolUseFailure", matcher: "WebFetch|WebSearch" }, + { event: "PostToolUse", matcher: "Bash", if: "Bash(gh *)", extraArgs: ["--cli", "gh"] }, + { event: "PostToolUseFailure", matcher: "Bash", if: "Bash(gh *)", extraArgs: ["--cli", "gh"] }, + { event: "SubagentStop" }, + ], + normalize: (raw) => ({ + hook_event_name: raw.hook_event_name, + session_id: raw.session_id, + prompt_id: raw.prompt_id, + source: raw.source, + prompt: raw.prompt, + tool_name: raw.tool_name, + tool_input: raw.tool_input, + tool_response: raw.tool_response, + error: raw.error, + is_interrupt: raw.is_interrupt, + agent_type: raw.agent_type, + agent_id: raw.agent_id, + last_assistant_message: raw.last_assistant_message, + }), + toolProperties(toolName, toolInput) { + if (typeof toolName !== "string") return null; + if (toolName.startsWith(FAKE_ARCADE_PREFIX)) { + return arcadeToolProperties("arcade", toolName.slice(FAKE_ARCADE_PREFIX.length), toolInput); + } + if (toolName.startsWith(FAKE_OTHER_PREFIX)) { + const tool = toolName.slice(FAKE_OTHER_PREFIX.length); + return otherServerProperties(serviceForToolName(tool)); + } + return null; + }, + attemptProperties(toolName, toolInput) { + if (typeof toolName !== "string" || !toolName.startsWith(FAKE_ARCADE_PREFIX)) return null; + return arcadeToolProperties("arcade", toolName.slice(FAKE_ARCADE_PREFIX.length), toolInput); + }, + builtinToolProperties, +}; + +export default fakeAdapter; diff --git a/test/hook-scope.test.mjs b/test/hook-scope.test.mjs new file mode 100644 index 0000000..cf886d7 --- /dev/null +++ b/test/hook-scope.test.mjs @@ -0,0 +1,95 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { after, test } from "node:test"; +import { classifyAppWork, MAX_SCOPE_FILES, SCOPE_DIRECTORY, SCOPE_TTL_MS, scopeForInput } from "../hooks/hook-scope.mjs"; +import { buildEvent } from "../hooks/telemetry-events.mjs"; +import fakeAdapter, { FAKE_ARCADE_PREFIX } from "./fixtures/telemetry-fake-adapter.mjs"; + +const root = mkdtempSync(path.join(os.tmpdir(), "arcade-scope-")); +after(() => rmSync(root, { recursive: true, force: true })); +const makeDir = () => mkdtempSync(path.join(root, "data-")); +const prompt = (text, turn = "turn-1", session = "session-1") => ({ + hook_event_name: "UserPromptSubmit", prompt: text, prompt_id: turn, session_id: session, +}); +const tool = (turn = "turn-1", session = "session-1") => ({ + hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: turn, session_id: session, +}); +const options = (dir, now = 1000, requiresTurn = true) => ({ host: "claude-code", requiresTurn, dir, now }); + +test("confirmation relevance expires from the original app prompt and unrelated work closes it", () => { + const initial = classifyAppWork("Check my calendar", null, 1000); + assert.equal(initial.relevant, true); + for (const text of ["ok", "yes, send it", "go ahead and send it", "do it"]) { + assert.deepEqual(classifyAppWork(text, initial, 2000), initial, text); + } + assert.equal(classifyAppWork("yes, send it", initial, initial.expiresAt).relevant, false); + assert.equal(classifyAppWork("Fix the parser", initial, 2000).relevant, false); + assert.equal(classifyAppWork("continue the implementation", initial, 2000).relevant, false); + assert.equal(classifyAppWork("yes, send it", null, 2000).relevant, false); +}); + +test("scope isolates sessions and Claude turns, preserves notifications, and resets on session start", () => { + const dir = makeDir(); + assert.equal(scopeForInput(tool(), options(dir)).appWork, false); + assert.equal(scopeForInput(prompt("Check my calendar"), options(dir)).appWork, true); + assert.equal(scopeForInput(tool(), options(dir)).appWork, true); + assert.equal(scopeForInput(tool("different-turn"), options(dir)).appWork, false); + assert.equal(scopeForInput(tool("turn-1", "different-session"), options(dir)).appWork, false); + assert.equal(scopeForInput({ ...tool(), prompt_id: undefined }, options(dir)).appWork, false); + assert.equal(scopeForInput(prompt("Fix the parser", "notification"), options(dir)).appWork, false); + assert.equal(scopeForInput(tool(), options(dir)).appWork, true); + const confirmation = prompt("yes, send it", "turn-2"); + assert.equal(scopeForInput(confirmation, options(dir, 2000)).appWork, true); + const file = path.join(dir, SCOPE_DIRECTORY, readdirSync(path.join(dir, SCOPE_DIRECTORY))[0]); + const state = JSON.parse(readFileSync(file, "utf8")); + assert.deepEqual(Object.keys(state).sort(), ["expiresAt", "relevant", "turn"]); + assert.equal(state.expiresAt, 1000 + SCOPE_TTL_MS); + assert.doesNotMatch(readFileSync(file, "utf8"), /calendar|send|session-1|turn-2/); + scopeForInput(confirmation, options(dir, 3000)); + assert.equal(JSON.parse(readFileSync(file, "utf8")).expiresAt, state.expiresAt, "hook order does not extend scope"); + assert.equal(scopeForInput(tool("turn-2"), options(dir, state.expiresAt)).appWork, false); + scopeForInput({ hook_event_name: "SessionStart", session_id: "session-1" }, options(dir)); + assert.equal(scopeForInput(tool("turn-2"), options(dir)).appWork, false); + scopeForInput(prompt("Check my calendar", "turn-3"), options(dir)); + scopeForInput({ hook_event_name: "SessionStart", session_id: "session-1", source: "compact" }, options(dir)); + assert.equal(scopeForInput(tool("turn-3"), options(dir)).appWork, true, "compaction keeps scope"); + scopeForInput(prompt("Fix the parser", "turn-4"), options(dir)); + assert.equal(scopeForInput(tool("turn-4"), options(dir)).appWork, false); +}); + +test("Copilot uses bounded session scope without a prompt id and corrupt state fails closed", () => { + const dir = makeDir(); + const config = { ...options(dir), host: "copilot-cli", requiresTurn: false }; + scopeForInput({ ...prompt("Check my calendar"), prompt_id: undefined }, config); + assert.equal(scopeForInput({ ...tool(), prompt_id: undefined }, config).appWork, true); + const file = path.join(dir, SCOPE_DIRECTORY, readdirSync(path.join(dir, SCOPE_DIRECTORY))[0]); + for (const bad of ["not json", '{"relevant":true}', JSON.stringify({ relevant: true, expiresAt: 1000 + SCOPE_TTL_MS, prompt: "private" })]) { + writeFileSync(file, bad); + assert.equal(scopeForInput({ ...tool(), prompt_id: undefined }, config).appWork, false); + } +}); + +test("expired session state is removed and stored sessions are bounded", () => { + const dir = makeDir(); + scopeForInput(prompt("Check my calendar", "old-turn", "old-session"), options(dir)); + const oldFile = readdirSync(path.join(dir, SCOPE_DIRECTORY))[0]; + for (let i = 0; i <= MAX_SCOPE_FILES; i++) { + scopeForInput(prompt("Check my calendar", `turn-${i}`, `session-${i}`), options(dir, 1000 + SCOPE_TTL_MS)); + } + const files = readdirSync(path.join(dir, SCOPE_DIRECTORY)); + assert.equal(files.includes(oldFile), false); + assert.equal(files.length, MAX_SCOPE_FILES); +}); + +test("unscoped alternatives and unrelated hooks are silent, while direct Arcade use stays observable", () => { + const eventOptions = { adapter: fakeAdapter, os: "darwin", arcadeUsedBefore: false, appWork: false }; + for (const input of [tool(), prompt("Fix the parser"), { ...tool(), tool_name: "mcp__fake_other__Granola_ListMeetings" }, + { ...tool(), hook_event_name: "SessionStart" }, { ...tool(), hook_event_name: "SubagentStop", agent_type: "general-purpose" }]) { + assert.equal(buildEvent(input, eventOptions), null); + } + for (const hook of ["PreToolUse", "PostToolUse", "PostToolUseFailure"]) { + assert.ok(buildEvent({ ...tool(), hook_event_name: hook, tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails` }, eventOptions)); + } +}); diff --git a/test/telemetry-classifiers.test.mjs b/test/telemetry-classifiers.test.mjs new file mode 100644 index 0000000..75ca2e6 --- /dev/null +++ b/test/telemetry-classifiers.test.mjs @@ -0,0 +1,306 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { authNeeded, failureKind } from "../hooks/telemetry-failures.mjs"; +import { commandUsesCli } from "../hooks/telemetry-commands.mjs"; + +const ROOT = path.join(path.dirname(new URL(import.meta.url).pathname), ".."); +const fixture = (name) => { + const data = JSON.parse(readFileSync(path.join(ROOT, "test/fixtures/hook-inputs", name), "utf8")); + return data; +}; + +// --------------------------------------------------------------------------- +// failureKind +// --------------------------------------------------------------------------- + +test("failureKind: interrupted wins when is_interrupt is true", () => { + const f = fixture("interrupted.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "interrupted"); +}); + +test("failureKind: interrupted requires strict true, not any truthy value", () => { + assert.equal(failureKind("tool execution timed out after 5s", 1), "timeout"); + assert.equal(failureKind("Connection closed", "true"), "unreachable"); +}); + +test("failureKind: auth_required for hosted Arcade direct tool text", () => { + const f = fixture("auth-required-arcade.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required for fake-server authorization_url in error", () => { + const f = fixture("auth-required-url.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required for Arcade_UseTool requires authorization text", () => { + const f = fixture("auth-required-use-tool.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required for Claude Code 2.1.246 re-authorization error", () => { + const f = fixture("auth-required-reauth.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required for Claude Code 2.1.278 sign-in-again error", () => { + const f = fixture("auth-required-sign-in-again.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required for Claude Code scope and rejected-credential errors", () => { + for (const msg of [ + 'MCP server "arcade" needs additional permissions (scope: "calendar.read") \u2014 run /mcp to re-authenticate', + 'MCP server "arcade" rejected the credential from its headersHelper (check the helper and run /mcp to reconnect, or to authenticate if the server also uses OAuth)', + 'MCP server "arcade" rejected the Authorization header in its config (update it, then run /mcp to reconnect)', + ]) { + assert.equal(failureKind(msg, false), "auth_required", msg); + } +}); + +test("failureKind: sign-in and permission wording in a tool's own error stays tool_error", () => { + assert.equal(failureKind("Please sign in again to continue", false), "tool_error"); + assert.equal(failureKind("The app needs additional permissions to read this file", false), "tool_error"); + assert.equal(failureKind("Upstream API rejected the credential", false), "tool_error"); +}); + +test("failureKind: auth_required for Claude Code needs-to-be-connected error", () => { + const f = fixture("auth-required-connect.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "auth_required"); +}); + +test("failureKind: auth_required matches case-insensitively", () => { + assert.equal(failureKind("tool requires Authorization", false), "auth_required"); + assert.equal(failureKind("Authorization Required: dropbox", false), "auth_required"); +}); + +test("failureKind: session_expired for session expired message", () => { + const f = fixture("session-expired.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "session_expired"); +}); + +test("failureKind: timeout for no-response for 30s message", () => { + const f = fixture("timeout-no-response.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "timeout"); +}); + +test("failureKind: timeout for gateway-side timed out after", () => { + const f = fixture("timeout-gateway.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "timeout"); +}); + +test("failureKind: timeout for MCP SDK request timeout", () => { + const f = fixture("timeout-mcp-request.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "timeout"); +}); + +test("failureKind: unreachable for stdio server Connection closed", () => { + const f = fixture("unreachable-connection-closed.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "unreachable"); +}); + +test("failureKind: unreachable for MCP SDK Connection closed", () => { + const f = fixture("unreachable-mcp-connection-closed.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "unreachable"); +}); + +test("failureKind: unreachable for socket connection closed unexpectedly", () => { + const f = fixture("unreachable-socket.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "unreachable"); +}); + +test("failureKind: unreachable for unable to connect", () => { + const f = fixture("unreachable-unable-to-connect.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "unreachable"); +}); + +test("failureKind: unreachable for transport dropped mid-call", () => { + const f = fixture("unreachable-transport-dropped.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "unreachable"); +}); + +test("failureKind: unreachable for Node.js error codes", () => { + for (const code of ["ECONNREFUSED", "ENOTFOUND", "ECONNRESET", "ETIMEDOUT", "EAI_AGAIN", "ENETUNREACH"]) { + assert.equal(failureKind(`connect ${code} 127.0.0.1:9`, false), "unreachable", code); + } +}); + +test("failureKind: Connection closed matches only alone or after the MCP SDK error code", () => { + assert.equal(failureKind("Connection closed unexpectedly", false), "tool_error"); + assert.equal(failureKind("Error: Connection closed", false), "tool_error"); + assert.equal(failureKind("The upstream connection closed before the file was saved", false), "tool_error"); + assert.equal(failureKind("MCP error -32603: Connection closed by upstream API", false), "tool_error"); +}); + +test("failureKind: http_error for HTTP 500 response", () => { + const f = fixture("http-error-500.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "http_error"); +}); + +test("failureKind: http_error for Streamable HTTP error prefix", () => { + const f = fixture("http-error-streamable.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "http_error"); +}); + +test("failureKind: tool_error for JSON-RPC error message", () => { + const f = fixture("tool-error-rpc.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "tool_error"); +}); + +test("failureKind: tool_error for rate limit message", () => { + const f = fixture("tool-error-rate-limit.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "tool_error"); +}); + +test("failureKind: tool_error for non-zero exit code", () => { + const f = fixture("tool-error-exit-code.json"); + assert.equal(failureKind(f.error, f.is_interrupt), "tool_error"); +}); + +test("failureKind: non-string error falls through to tool_error", () => { + assert.equal(failureKind(null, false), "tool_error"); + assert.equal(failureKind(undefined, false), "tool_error"); + assert.equal(failureKind(42, false), "tool_error"); + assert.equal(failureKind({ message: "Connection closed" }, false), "tool_error"); +}); + +// --------------------------------------------------------------------------- +// authNeeded +// --------------------------------------------------------------------------- + +test("authNeeded: true when providers array contains authorization_required status", () => { + const response = [ + { + type: "text", + text: JSON.stringify({ + message: "Not yet authorized: dropbox.", + providers: [ + { provider: "dropbox", status: "authorization_required" }, + { provider: "gmail", status: "authorized" }, + ], + }), + }, + ]; + assert.equal(authNeeded(response), true); +}); + +test("authNeeded: false when all providers are authorized", () => { + const response = [ + { + type: "text", + text: JSON.stringify({ + message: "All authorized.", + providers: [{ provider: "gmail", status: "authorized" }], + }), + }, + ]; + assert.equal(authNeeded(response), false); +}); + +test("authNeeded: false for prose text containing authorization_required outside providers", () => { + assert.equal( + authNeeded('{"message":"status is authorization_required","providers":[{"provider":"x","status":"authorized"}]}'), + false + ); + assert.equal(authNeeded("This service has authorization_required status"), false); +}); + +test("authNeeded: false for plain non-JSON text", () => { + assert.equal(authNeeded([{ type: "text", text: "ok" }]), false); + assert.equal(authNeeded("some plain text"), false); +}); + +test("authNeeded: false for null and undefined", () => { + assert.equal(authNeeded(null), false); + assert.equal(authNeeded(undefined), false); +}); + +test("authNeeded: false for non-string non-array inputs", () => { + assert.equal(authNeeded(42), false); + assert.equal(authNeeded(true), false); +}); + +test("authNeeded: false when JSON has no providers field", () => { + assert.equal(authNeeded('{"status":"authorization_required"}'), false); + assert.equal(authNeeded(JSON.stringify({ message: "ok" })), false); +}); + +// --------------------------------------------------------------------------- +// commandUsesCli +// --------------------------------------------------------------------------- + +test("commandUsesCli: matches bare CLI name", () => { + assert.equal(commandUsesCli("gh --version", "gh"), true); +}); + +test("commandUsesCli: matches CLI after && in compound command", () => { + assert.equal(commandUsesCli("cd . && gh --version", "gh"), true); +}); + +test("commandUsesCli: matches CLI after leading NAME=value assignment", () => { + assert.equal(commandUsesCli("GH_PAGER=cat gh --version", "gh"), true); +}); + +test("commandUsesCli: matches CLI after multiple assignments", () => { + assert.equal(commandUsesCli("GH_PAGER=cat GH_NO_UPDATE_NOTIFIER=1 gh pr list", "gh"), true); +}); + +test("commandUsesCli: matches CLI after || separator", () => { + assert.equal(commandUsesCli("false || gh --version", "gh"), true); +}); + +test("commandUsesCli: matches CLI after ; separator", () => { + assert.equal(commandUsesCli("echo hi; gh status", "gh"), true); +}); + +test("commandUsesCli: matches CLI after | separator", () => { + // after the |, "gh" is the first word of the next segment + assert.equal(commandUsesCli("echo text | gh gist create -", "gh"), true); + assert.equal(commandUsesCli("cat file | gh gist create -", "gh"), true); + // "gh" as an argument of echo, before any separator, does not match + assert.equal(commandUsesCli("echo gh", "gh"), false); +}); + +test("commandUsesCli: matches CLI after ( separator", () => { + assert.equal(commandUsesCli("(gh --version)", "gh"), true); +}); + +test("commandUsesCli: matches CLI after newline separator", () => { + assert.equal(commandUsesCli("echo hi\ngh --version", "gh"), true); +}); + +test("commandUsesCli: does not match CLI appearing as argument", () => { + assert.equal(commandUsesCli("echo gh", "gh"), false); +}); + +test("commandUsesCli: does not match absolute path to CLI", () => { + assert.equal(commandUsesCli("/opt/homebrew/bin/gh --version", "gh"), false); +}); + +test("commandUsesCli: does not match a different command that starts with CLI name", () => { + assert.equal(commandUsesCli("ghost list", "gh"), false); +}); + +test("commandUsesCli: does not match CLI name inside a string argument", () => { + assert.equal(commandUsesCli('python -c "gh"', "gh"), false); +}); + +test("commandUsesCli: returns false for non-string command", () => { + assert.equal(commandUsesCli(null, "gh"), false); + assert.equal(commandUsesCli(undefined, "gh"), false); + assert.equal(commandUsesCli(42, "gh"), false); + assert.equal(commandUsesCli(["gh", "--version"], "gh"), false); +}); + +test("commandUsesCli: works for all listed CLIs", () => { + for (const cli of ["gh", "glab", "curl", "wget", "http", "osascript"]) { + assert.equal(commandUsesCli(`${cli} --version`, cli), true, cli); + assert.equal(commandUsesCli(`echo ${cli}`, cli), false, `echo ${cli}`); + } +}); + +test("commandUsesCli: curl after && compound command", () => { + assert.equal(commandUsesCli("cd . && curl --version", "curl"), true); +}); diff --git a/test/telemetry-classify.test.mjs b/test/telemetry-classify.test.mjs new file mode 100644 index 0000000..d029914 --- /dev/null +++ b/test/telemetry-classify.test.mjs @@ -0,0 +1,84 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { classifyPrompt, KEYWORDS, serviceForToolName } from "../hooks/telemetry-classify.mjs"; +import { SERVICE_CATEGORIES, TOOLKIT_SERVICES } from "../hooks/telemetry-contract.mjs"; +import { readRepoFile } from "./helpers.mjs"; + +const fixtures = JSON.parse(readRepoFile("test/fixtures/routing-prompts.json")); + +test("classifier tables and labeled prompts use only the contract's categories", () => { + assert.deepEqual(Object.keys(KEYWORDS).sort(), [...SERVICE_CATEGORIES].sort()); + for (const [toolkit, category] of Object.entries(TOOLKIT_SERVICES)) { + assert.ok(SERVICE_CATEGORIES.includes(category), `${toolkit}: ${category}`); + } + for (const row of fixtures) { + assert.deepEqual(Object.keys(row).sort(), ["categories", "couldUseArcade", "prompt"], row.prompt); + assert.equal(typeof row.prompt, "string"); + assert.equal(typeof row.couldUseArcade, "boolean"); + for (const category of row.categories) { + assert.ok(SERVICE_CATEGORIES.includes(category), `${row.prompt}: ${category}`); + } + } +}); + +// A keyword like "microsoft teams" and the toolkit MicrosoftTeams name the +// same app, so a prompt and a tool call for it must get the same category. +test("an app in both the keyword list and the toolkit list gets one category", () => { + for (const [category, phrases] of Object.entries(KEYWORDS)) { + for (const phrase of phrases) { + const toolkit = phrase.replaceAll(" ", ""); + if (Object.hasOwn(TOOLKIT_SERVICES, toolkit)) { + assert.equal(TOOLKIT_SERVICES[toolkit], category, phrase); + } + } + } +}); + +test("classifyPrompt returns sorted known categories and never throws", () => { + assert.deepEqual(classifyPrompt("check slack and my calendar, then slack again"), { + couldUseArcade: true, + serviceHints: ["calendar", "chat"], + }); + const odd = ["", " ", undefined, null, 42, {}, Symbol("x"), "[".repeat(1000), "\u0000\ud800"]; + for (const input of odd) { + assert.deepEqual(classifyPrompt(input), { couldUseArcade: false, serviceHints: [] }); + } +}); + +test("serviceForToolName maps toolkit prefixes, case-insensitive", () => { + const cases = [ + ["Gmail_SendEmail", "email"], ["GoogleCalendar.ListEvents", "calendar"], + ["SLACK_SENDMESSAGE", "chat"], ["Linear_CreateIssue", "issues"], + ["Notion_SearchPages", "docs"], ["Granola_ListMeetings", "meetings"], + ["Hubspot_GetContact", "crm"], ["GitHub_ListRepositories", "code_hosting"], + ["Posthog_GetTrends", "analytics"], ["GoogleDrive_SearchFiles", "storage"], + ["Arcade_ListApps", null], ["Unknown_DoThing", null], ["Gmail", null], + ["_SendEmail", null], ["constructor_x", null], [undefined, null], + ]; + for (const [toolName, expected] of cases) { + assert.equal(serviceForToolName(toolName), expected, String(toolName)); + } +}); + +test("classifier accuracy on labeled prompts", () => { + const positives = fixtures.filter((row) => row.couldUseArcade); + const negatives = fixtures.filter((row) => !row.couldUseArcade); + const misses = positives.filter((row) => !classifyPrompt(row.prompt).couldUseArcade); + const falseAlarms = negatives.filter((row) => classifyPrompt(row.prompt).couldUseArcade); + const wrongCategory = positives.filter((row) => { + const { serviceHints } = classifyPrompt(row.prompt); + return serviceHints.length > 0 && !serviceHints.some((hint) => row.categories.includes(hint)); + }); + + const percent = (part, whole) => `${((part.length / whole.length) * 100).toFixed(1)}%`; + console.log(`miss rate: ${percent(misses, positives)} (${misses.length}/${positives.length})`); + for (const row of misses) console.log(` missed: ${row.prompt}`); + console.log( + `false-alarm rate: ${percent(falseAlarms, negatives)} (${falseAlarms.length}/${negatives.length})`, + ); + for (const row of falseAlarms) console.log(` false alarm: ${row.prompt}`); + + assert.ok(falseAlarms.length / negatives.length <= 0.1, "false-alarm rate above 10%"); + assert.ok(misses.length / positives.length <= 0.2, "miss rate above 20%"); + assert.deepEqual(wrongCategory.map((row) => row.prompt), [], "wrong category"); +}); diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs new file mode 100644 index 0000000..174a8b9 --- /dev/null +++ b/test/telemetry-foundation.test.mjs @@ -0,0 +1,370 @@ +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import net from "node:net"; +import os from "node:os"; +import path from "node:path"; +import { after, test } from "node:test"; +import { EVENTS, eventSchema } from "../hooks/telemetry-contract.mjs"; +import { EVENT_ENV, PLUGIN_VERSION, POSTHOG_KEY, TELEMETRY_ENABLED } from "../hooks/telemetry-config.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; +import { ROOT, runHook } from "./helpers.mjs"; +import fakeAdapter, { FAKE_ARCADE_PREFIX, FAKE_OTHER_PREFIX } from "./fixtures/telemetry-fake-adapter.mjs"; +import { + assertMatchesContract, + assertNoLeak, + expectedEvent, + hash16, + hookInput, + runTelemetryScript, + sleep, + startServer, + telemetryEnv, + waitForRequests, +} from "./telemetry-helpers.mjs"; +import Ajv2020 from "ajv/dist/2020.js"; + +const SESSION_ID = "raw-session-id-123"; +const PROMPT_ID = "raw-prompt-id-456"; +const OPERATOR = "arcade:arcade-operator"; +const OPTIONS = { adapter: fakeAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + +const TEMP_ROOT = mkdtempSync(path.join(os.tmpdir(), "arcade-telemetry-foundation-")); +const makeTempDir = () => mkdtempSync(path.join(TEMP_ROOT, "data-")); +after(() => rmSync(TEMP_ROOT, { recursive: true, force: true })); + +const HOOK_EVENTS = [...new Set(Object.values(EVENTS).map((spec) => spec.hook))]; + +test("telemetry is off in this build", () => { + assert.equal(TELEMETRY_ENABLED, false); +}); + +test("telemetry.mjs exits before work when telemetry is disabled", async () => { + const server = await startServer(); + try { + const dataDir = makeTempDir(); + const env = telemetryEnv(fakeAdapter, dataDir, server.url); + for (const host of ["claude-code", "copilot"]) { + for (const hook_event_name of HOOK_EVENTS) { + const input = JSON.stringify(hookInput({ hook_event_name, prompt: "calendar" })); + const result = runTelemetryScript(input, env, host); + assert.equal(result.status, 0, `${host} ${hook_event_name}: ${result.stderr}`); + assert.equal(result.stdout, "", `${host} ${hook_event_name}`); + assert.equal(result.stderr, "", `${host} ${hook_event_name}`); + } + const bad = runTelemetryScript("not-json", env, host); + assert.equal(bad.status, 0); + assert.equal(bad.stdout, ""); + } + await sleep(300); + assert.deepEqual(server.requests, []); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + await server.close(); + } +}); + +test("session-start with telemetry off creates no scope dir and still prints routing context", () => { + const dataDir = makeTempDir(); + const result = runHook( + "session-start.mjs", + { session_id: SESSION_ID, source: "startup" }, + ["--host", "claude-code"], + ); + assert.equal(result.status, 0, result.stderr); + assert.ok(result.stdout.includes("Gateway")); + assert.deepEqual(readdirSync(dataDir), []); +}); + +test("routing hooks still exit 0 with malformed stdin", () => { + for (const script of ["session-start.mjs", "subagent-start.mjs", "user-prompt-submit.mjs"]) { + const result = spawnSync(process.execPath, [path.join(ROOT, "hooks", script), "--host", "claude-code"], { + input: "not-json", + encoding: "utf8", + }); + assert.equal(result.status, 0, `${script}: ${result.stderr}`); + } +}); + +test("buildEvent maps fake adapter hook input to contract events", () => { + const ATTEMPTED = "Plugin tool attempted"; + const CALLED = "Plugin tool called"; + const FAILED = "Plugin tool failed"; + const cases = [ + ["UserPromptSubmit", { prompt: "What is on my calendar tomorrow?" }, "Plugin prompt submitted", + { could_use_arcade: true, service_hints: ["calendar"], reminder_sent: true }], + ["PreToolUse", { tool_name: `${FAKE_ARCADE_PREFIX}Arcade_SelectTools` }, ATTEMPTED, + { server: "arcade", tool: "Arcade_SelectTools" }], + ["PostToolUse", { tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails` }, CALLED, + { server: "arcade", tool: "app_tool", service: "email" }], + ["PostToolUseFailure", { tool_name: `${FAKE_OTHER_PREFIX}DoThing`, error: "Error POSTing to endpoint: x" }, FAILED, + { server: "other", failure_kind: "http_error" }], + ["PreToolUse", { tool_name: `${FAKE_OTHER_PREFIX}search` }, null], + ]; + for (const [hook, fields, event, extra] of cases) { + const built = buildEvent(hookInput({ hook_event_name: hook, ...fields }), OPTIONS); + if (event === null) { + assert.equal(built, null); + continue; + } + assert.deepEqual(built, expectedEvent(event, extra, fakeAdapter), `${hook}`); + assertMatchesContract(built); + } +}); + +test("buildEvent never leaks raw ids or prompt text", () => { + const secrets = /SECRET|private-repo|raw-session|raw-prompt/i; + const built = buildEvent(hookInput({ + hook_event_name: "UserPromptSubmit", + prompt: "SECRET calendar", + session_id: "SECRET-session", + prompt_id: "SECRET-prompt", + }), OPTIONS); + assertNoLeak(JSON.stringify(built), secrets); + assertMatchesContract(built); +}); + +test("contract schema rejects extra properties and values", () => { + const valid = buildEvent(hookInput({ + hook_event_name: "UserPromptSubmit", + prompt: "Check my calendar", + }), OPTIONS); + assertMatchesContract(valid); + const validate = new Ajv2020({ allErrors: true }).compile(eventSchema()); + const withExtra = { ...valid, properties: { ...valid.properties, extra_field: "x" } }; + assert.equal(validate(withExtra), false); + const badEnum = { + ...valid, + properties: { ...valid.properties, os: "freebsd" }, + }; + assert.equal(validate(badEnum), false); +}); + +test("isArcadeCall is true only for successful arcade gateway calls", () => { + const arcade = buildEvent(hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + }), OPTIONS); + assert.equal(isArcadeCall(arcade), true); + const attempt = buildEvent(hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + }), OPTIONS); + assert.equal(isArcadeCall(attempt), false); +}); + +test("runTelemetry opt-out matrix sends nothing and writes no files", async () => { + const server = await startServer(); + try { + const sent = []; + const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); + const cases = [ + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["ARCADE_PLUGIN_TELEMETRY=OFF", { ARCADE_PLUGIN_TELEMETRY: "OFF" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["FAKE_DISABLE_TELEMETRY=1", { FAKE_DISABLE_TELEMETRY: "1" }], + ["FAKE_DISABLE_TELEMETRY=0", { FAKE_DISABLE_TELEMETRY: "0" }], + ["FAKE_OFFLINE=true", { FAKE_OFFLINE: "true" }], + ["FAKE_OFFLINE=1", { FAKE_OFFLINE: "1" }], + ["missing data dir", { ARCADE_TEST_PLUGIN_DATA: "" }], + ["relative data dir", { ARCADE_TEST_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, extra] of cases) { + const dataDir = makeTempDir(); + const env = telemetryEnv(fakeAdapter, dataDir, server.url, extra); + await runTelemetry({ + input: base, + adapter: fakeAdapter, + env, + enabled: true, + send: (event) => sent.push(event), + }); + assert.deepEqual(readdirSync(dataDir), [], label); + } + await sleep(200); + assert.deepEqual(server.requests, []); + assert.deepEqual(sent, []); + } finally { + await server.close(); + } +}); + +test("runTelemetry enabled fixture emits allowed fields and sets arcade-used once", async () => { + const server = await startServer(); + try { + const dataDir = makeTempDir(); + const env = telemetryEnv(fakeAdapter, dataDir, server.url); + const send = []; + await runTelemetry({ + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }), + adapter: fakeAdapter, + env, + enabled: true, + send: (event) => send.push(event), + }); + await runTelemetry({ + input: hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + prompt_id: PROMPT_ID, + }), + adapter: fakeAdapter, + env, + enabled: true, + send: (event) => send.push(event), + }); + await runTelemetry({ + input: hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + prompt_id: PROMPT_ID, + }), + adapter: fakeAdapter, + env, + enabled: true, + send: (event) => send.push(event), + }); + assert.equal(send.length, 3); + for (const event of send) assertMatchesContract(event); + assert.equal(send[1].properties.arcade_used_before, false); + assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); + assert.equal(send[2].properties.arcade_used_before, true); + assert.equal(isArcadeCall(send[2]), false); + } finally { + await server.close(); + } +}); + +test("runTelemetry survives malformed input and send failures", async () => { + const dataDir = makeTempDir(); + const env = telemetryEnv(fakeAdapter, dataDir, "http://127.0.0.1:9"); + await runTelemetry({ input: null, adapter: fakeAdapter, env, enabled: true, send: () => {} }); + await runTelemetry({ input: { hook_event_name: "UserPromptSubmit" }, adapter: fakeAdapter, env, enabled: true, send: () => {} }); + const scopeDir = path.join(dataDir, "prompt-scope"); + mkdirSync(scopeDir, { recursive: true }); + writeFileSync(path.join(scopeDir, "bad.json"), "not json"); + await runTelemetry({ + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails` }), + adapter: fakeAdapter, + env, + enabled: true, + send: () => { throw new Error("send failed"); }, + }); +}); + +test("isOptedOut respects case on ARCADE_PLUGIN_TELEMETRY", () => { + assert.equal(isOptedOut([], { ARCADE_PLUGIN_TELEMETRY: "OFF" }), true); + assert.equal(isOptedOut([], { ARCADE_PLUGIN_TELEMETRY: "off" }), true); + assert.equal(isOptedOut(fakeAdapter.optOutSwitches, { FAKE_OFFLINE: "FALSE" }), false); +}); + +test("telemetry-send posts one event and gives up on a silent host", async () => { + const server = await startServer(); + const silent = net.createServer(() => {}); + await new Promise((resolve) => silent.listen(0, "127.0.0.1", resolve)); + const silentUrl = `http://127.0.0.1:${silent.address().port}`; + try { + const event = buildEvent(hookInput({ hook_event_name: "UserPromptSubmit", prompt: "calendar" }), OPTIONS); + const started = Date.now(); + const child = spawn(process.execPath, [path.join(ROOT, "hooks", "telemetry-send.mjs")], { + env: { ...process.env, ARCADE_PLUGIN_TELEMETRY_HOST: server.url, [EVENT_ENV]: JSON.stringify(event) }, + }); + assert.equal(await new Promise((resolve) => child.on("exit", resolve)), 0); + await waitForRequests(server.requests, 1); + const body = JSON.parse(server.requests[0].body); + assert.equal(body.api_key, POSTHOG_KEY); + assert.equal(body.event, "Plugin prompt submitted"); + + const hangStart = Date.now(); + const hang = spawn(process.execPath, [path.join(ROOT, "hooks", "telemetry-send.mjs")], { + env: { ...process.env, ARCADE_PLUGIN_TELEMETRY_HOST: silentUrl, [EVENT_ENV]: JSON.stringify(event) }, + }); + assert.equal(await new Promise((resolve) => hang.on("exit", resolve)), 0); + const elapsed = Date.now() - hangStart; + assert.ok(elapsed >= 900 && elapsed < 3000, `timeout was ${elapsed} ms`); + assert.ok(Date.now() - started < 3000); + } finally { + await server.close(); + silent.close(); + } +}); + +test("only telemetry-send.mjs uses network APIs in hooks", () => { + const hookFiles = readdirSync(path.join(ROOT, "hooks")).filter((file) => file.endsWith(".mjs")); + for (const file of hookFiles) { + if (file === "telemetry-send.mjs") continue; + const source = readFileSync(path.join(ROOT, "hooks", file), "utf8"); + assert.doesNotMatch( + source, + /\bfetch\b|\b(?:from|import|require)\s*\(?\s*["'](?:node:)?(?:http|https|http2|net|tls|dgram|dns)["']/, + file, + ); + } +}); + +test("telemetry.mjs prints nothing", () => { + const source = readFileSync(path.join(ROOT, "hooks", "telemetry.mjs"), "utf8"); + assert.doesNotMatch(source, /\bconsole\.(log|info|warn|error)\b/); +}); + +test("every telemetry source file starts with @ts-check", () => { + const files = readdirSync(path.join(ROOT, "hooks")) + .filter((file) => file.startsWith("telemetry")) + .map((file) => `hooks/${file}`); + const adapterDir = path.join(ROOT, "hooks", "telemetry-adapters"); + try { + for (const file of readdirSync(adapterDir).filter((f) => f.endsWith(".mjs"))) { + files.push(`hooks/telemetry-adapters/${file}`); + } + } catch { + // no adapters yet on this slice + } + for (const file of files) { + assert.match(readFileSync(path.join(ROOT, file), "utf8"), /^(#!.*\n)?\/\/ @ts-check\n/, file); + } +}); + +test("scope expiry and confirmation flow via runTelemetry", async () => { + const dataDir = makeTempDir(); + const env = telemetryEnv(fakeAdapter, dataDir, "http://127.0.0.1:9"); + const now = 1_000_000; + const events = []; + await runTelemetry({ + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: "turn-1" }), + adapter: fakeAdapter, + env, + enabled: true, + now, + send: (e) => events.push(e), + }); + await runTelemetry({ + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "yes, send it", prompt_id: "turn-2" }), + adapter: fakeAdapter, + env, + enabled: true, + now: now + 1000, + send: (e) => events.push(e), + }); + assert.equal(events.length, 2); + assert.equal(events[1].properties.could_use_arcade, false); + await runTelemetry({ + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: "turn-2" }), + adapter: fakeAdapter, + env, + enabled: true, + now: now + 2000, + send: (e) => events.push(e), + }); + assert.equal(events.length, 3); +}); + +test("subagent-stop builds operator status for fake adapter", () => { + const built = buildEvent(hookInput({ + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + last_assistant_message: "status: needs_auth", + }), OPTIONS); + assert.equal(built.properties.status, "needs_auth"); + assertMatchesContract(built); +}); diff --git a/test/telemetry-helpers.mjs b/test/telemetry-helpers.mjs new file mode 100644 index 0000000..aac23b1 --- /dev/null +++ b/test/telemetry-helpers.mjs @@ -0,0 +1,109 @@ +// @ts-check +/** + * Shared helpers for telemetry foundation tests. API kept stable for slices B/C. + */ + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import http from "node:http"; +import path from "node:path"; +import Ajv2020 from "ajv/dist/2020.js"; +import { eventSchema } from "../hooks/telemetry-contract.mjs"; +import { PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { ROOT } from "./helpers.mjs"; + +const validateEvent = new Ajv2020({ allErrors: true }).compile(eventSchema()); + +/** @param {object} event */ +export const assertMatchesContract = (event, label = JSON.stringify(event)) => { + assert.equal(validateEvent(event), true, `${label}: ${JSON.stringify(validateEvent.errors)}`); +}; + +/** + * @param {string} serialized + * @param {RegExp | RegExp[]} patterns + */ +export const assertNoLeak = (serialized, patterns) => { + const list = Array.isArray(patterns) ? patterns : [patterns]; + for (const pattern of list) { + assert.doesNotMatch(serialized, pattern, serialized); + } +}; + +export const hash16 = (text) => createHash("sha256").update(text).digest("hex").slice(0, 16); + +export const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const waitForRequests = async (requests, count) => { + const deadline = Date.now() + 3000; + while (requests.length < count && Date.now() < deadline) await sleep(50); +}; + +export const startServer = async () => { + const requests = []; + const server = http.createServer((req, res) => { + let body = ""; + req.on("data", (chunk) => (body += chunk)); + req.on("end", () => { + requests.push({ url: req.url, body }); + res.end("{}"); + }); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const close = () => { + server.closeAllConnections(); + return new Promise((resolve) => server.close(resolve)); + }; + return { url: `http://127.0.0.1:${server.address().port}`, requests, close }; +}; + +/** + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter + */ +export const telemetryEnv = (adapter, dataDir, serverUrl, extra = {}) => ({ + [adapter.dataVariable]: dataDir, + ARCADE_PLUGIN_TELEMETRY_HOST: serverUrl, + ARCADE_PLUGIN_TELEMETRY: "", + DO_NOT_TRACK: "", + FAKE_DISABLE_TELEMETRY: "", + FAKE_OFFLINE: "", + ...extra, +}); + +export const hookInput = (fields) => ({ + session_id: "raw-session-id-123", + prompt_id: "raw-prompt-id-456", + cwd: "/Users/someone/private-repo", + ...fields, +}); + +/** + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter + */ +export const expectedEvent = (event, extra, adapter, sessionId = "raw-session-id-123", promptId = "raw-prompt-id-456") => { + const session = hash16(sessionId); + const properties = { + ...extra, + host: adapter.host, + plugin_version: PLUGIN_VERSION, + telemetry_version: 2, + os: "darwin", + $process_person_profile: false, + $geoip_disable: true, + $ip: "0.0.0.0", + arcade_used_before: false, + session, + }; + if (adapter.requiresTurn && typeof promptId === "string") { + properties.turn = hash16(`${sessionId}:${promptId}`); + } + return { event, distinct_id: session, properties }; +}; + +export const runTelemetryScript = (stdin, env, hostName = "claude-code", extraArgs = []) => + spawnSync(process.execPath, [path.join(ROOT, "hooks", "telemetry.mjs"), "--host", hostName, ...extraArgs], { + input: stdin, + encoding: "utf8", + env: { ...process.env, ...env }, + }); diff --git a/test/telemetry-hooks.test.mjs b/test/telemetry-hooks.test.mjs new file mode 100644 index 0000000..59ca2bd --- /dev/null +++ b/test/telemetry-hooks.test.mjs @@ -0,0 +1,8 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; + +test("a flat-format client can't get an entry with if or extra args", () => { + const row = { script: "telemetry.mjs", event: "SessionStart", if: "Bash(gh *)", extraArgs: ["--cli", "gh"] }; + assert.throws(() => buildHookManifest("copilot", [row]), /flat format does not support/); +}); From 462d3f0ade5924b4d02c6684787e554a29eff9e6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:18:30 +0000 Subject: [PATCH 04/10] fix(telemetry): lazy-load telemetry.mjs and respect opt-outs on session start Use dynamic import after the TELEMETRY_ENABLED guard so the OFF path only loads telemetry-config. Export clearSessionScope from telemetry-run for session-start scope clearing with isOptedOut checks and foundation tests. Co-authored-by: Teal Larson --- hooks/session-start.mjs | 18 ++++----- hooks/telemetry-run.mjs | 28 +++++++++++++ hooks/telemetry.mjs | 6 +-- test/telemetry-foundation.test.mjs | 63 +++++++++++++++++++++++++++++- 4 files changed, 101 insertions(+), 14 deletions(-) diff --git a/hooks/session-start.mjs b/hooks/session-start.mjs index 3d2be69..39b8334 100644 --- a/hooks/session-start.mjs +++ b/hooks/session-start.mjs @@ -1,11 +1,10 @@ #!/usr/bin/env node // Adds the Arcade routing rules at session start. Always exits 0. -import path from "node:path"; import { hostFromArgs, printContext, readInput } from "./hook-hosts.mjs"; -import { scopeForInput } from "./hook-scope.mjs"; import { TELEMETRY_ENABLED } from "./telemetry-config.mjs"; import { loadTelemetryAdapter } from "./telemetry-adapter.mjs"; +import { clearSessionScope } from "./telemetry-run.mjs"; import { SESSION_CONTEXT } from "./routing-guidance.mjs"; const host = hostFromArgs(process.argv); @@ -13,14 +12,13 @@ try { if (TELEMETRY_ENABLED && host?.telemetry) { try { const adapter = await loadTelemetryAdapter(host.telemetry); - const dir = process.env[adapter.dataVariable]; - if (dir && path.isAbsolute(dir)) { - const input = await readInput(); - scopeForInput( - { ...adapter.normalize(input), hook_event_name: "SessionStart" }, - { host: adapter.host, requiresTurn: adapter.requiresTurn, dir }, - ); - } + const input = await readInput(); + clearSessionScope({ + adapter, + env: process.env, + input, + enabled: TELEMETRY_ENABLED, + }); } catch { // Clearing local scope must not suppress the routing context. } diff --git a/hooks/telemetry-run.mjs b/hooks/telemetry-run.mjs index 087ace5..3e5872f 100644 --- a/hooks/telemetry-run.mjs +++ b/hooks/telemetry-run.mjs @@ -53,6 +53,34 @@ const markArcadeUsed = (/** @type {string} */ dir) => { writeFileSync(path.join(dir, ARCADE_USED_FILE), "true", { mode: 0o600 }); }; +/** + * Clears prompt scope on session start when telemetry is active and not opted out. + * @param {object} options + * @param {TelemetryAdapter} options.adapter + * @param {NodeJS.ProcessEnv} [options.env] + * @param {Record} [options.input] + * @param {boolean} [options.enabled] + */ +export const clearSessionScope = ({ + adapter, + env = process.env, + input = {}, + enabled = false, +}) => { + try { + if (!enabled && !TELEMETRY_ENABLED) return; + if (isOptedOut(adapter.optOutSwitches, env)) return; + const dir = env[adapter.dataVariable]; + if (!dir || !path.isAbsolute(dir)) return; + scopeForInput( + { ...adapter.normalize(input), hook_event_name: "SessionStart" }, + { host: adapter.host, requiresTurn: adapter.requiresTurn, dir }, + ); + } catch { + // Scope clearing must never affect the session. + } +}; + const defaultSend = (/** @type {object} */ event, env = process.env) => { const child = spawn(process.execPath, [SENDER], { detached: true, diff --git a/hooks/telemetry.mjs b/hooks/telemetry.mjs index d28d0a4..cb6440f 100644 --- a/hooks/telemetry.mjs +++ b/hooks/telemetry.mjs @@ -7,9 +7,9 @@ import { TELEMETRY_ENABLED } from "./telemetry-config.mjs"; if (!TELEMETRY_ENABLED) process.exit(0); -import { hostFromArgs, readInput } from "./hook-hosts.mjs"; -import { loadTelemetryAdapter } from "./telemetry-adapter.mjs"; -import { runTelemetry } from "./telemetry-run.mjs"; +const { hostFromArgs, readInput } = await import("./hook-hosts.mjs"); +const { loadTelemetryAdapter } = await import("./telemetry-adapter.mjs"); +const { runTelemetry } = await import("./telemetry-run.mjs"); try { const adapterHost = hostFromArgs(process.argv)?.telemetry; diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index 174a8b9..186ed22 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -8,7 +8,8 @@ import { after, test } from "node:test"; import { EVENTS, eventSchema } from "../hooks/telemetry-contract.mjs"; import { EVENT_ENV, PLUGIN_VERSION, POSTHOG_KEY, TELEMETRY_ENABLED } from "../hooks/telemetry-config.mjs"; import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; -import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; +import { SCOPE_DIRECTORY, scopeForInput } from "../hooks/hook-scope.mjs"; +import { clearSessionScope, isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; import { ROOT, runHook } from "./helpers.mjs"; import fakeAdapter, { FAKE_ARCADE_PREFIX, FAKE_OTHER_PREFIX } from "./fixtures/telemetry-fake-adapter.mjs"; import { @@ -308,6 +309,66 @@ test("telemetry.mjs prints nothing", () => { assert.doesNotMatch(source, /\bconsole\.(log|info|warn|error)\b/); }); +test("telemetry.mjs statically imports only telemetry-config", () => { + const source = readFileSync(path.join(ROOT, "hooks", "telemetry.mjs"), "utf8"); + const staticImports = [...source.matchAll(/^import\s.+from\s+["'](.+?)["']/gm)].map((match) => match[1]); + assert.deepEqual(staticImports, ["./telemetry-config.mjs"]); +}); + +test("clearSessionScope respects enabled, opt-out, and SessionStart source", () => { + const seedScope = (dir) => { + scopeForInput( + hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", session_id: "scope-session" }), + { host: fakeAdapter.host, requiresTurn: fakeAdapter.requiresTurn, dir, now: 1000 }, + ); + const scopeDir = path.join(dir, SCOPE_DIRECTORY); + assert.equal(readdirSync(scopeDir).length, 1); + return scopeDir; + }; + const baseEnv = (dir) => telemetryEnv(fakeAdapter, dir, "http://127.0.0.1:9"); + + const offDir = makeTempDir(); + const offScope = seedScope(offDir); + clearSessionScope({ + adapter: fakeAdapter, + env: baseEnv(offDir), + input: { session_id: "scope-session", source: "startup" }, + enabled: false, + }); + assert.equal(readdirSync(offScope).length, 1, "disabled telemetry leaves scope files"); + + const optedDir = makeTempDir(); + const optedScope = seedScope(optedDir); + clearSessionScope({ + adapter: fakeAdapter, + env: { ...baseEnv(optedDir), ARCADE_PLUGIN_TELEMETRY: "0" }, + input: { session_id: "scope-session", source: "startup" }, + enabled: true, + }); + assert.equal(readdirSync(optedScope).length, 1, "opt-out leaves scope files"); + + const compactDir = makeTempDir(); + const compactScope = seedScope(compactDir); + clearSessionScope({ + adapter: fakeAdapter, + env: baseEnv(compactDir), + input: { session_id: "scope-session", source: "compact" }, + enabled: true, + }); + assert.equal(readdirSync(compactScope).length, 1, "compact session start keeps scope"); + + const freshDir = makeTempDir(); + const freshScope = seedScope(freshDir); + assert.equal(readdirSync(freshScope).length, 1); + clearSessionScope({ + adapter: fakeAdapter, + env: baseEnv(freshDir), + input: { session_id: "scope-session", source: "startup" }, + enabled: true, + }); + assert.equal(readdirSync(freshScope).length, 0, "normal session start clears scope"); +}); + test("every telemetry source file starts with @ts-check", () => { const files = readdirSync(path.join(ROOT, "hooks")) .filter((file) => file.startsWith("telemetry")) From ae61598487978ecec97b74dc62904206d7bad40e Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:21:27 +0000 Subject: [PATCH 05/10] test(telemetry): add tempDataDir and captureTelemetry helpers Expose the agreed helper API for slices B/C/D and refactor a foundation test to exercise scoped prompt-then-tool capture with the fake adapter. Co-authored-by: Teal Larson --- hooks/telemetry-events.mjs | 2 +- test/telemetry-foundation.test.mjs | 79 +++++++++++++----------------- test/telemetry-helpers.mjs | 77 +++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+), 45 deletions(-) diff --git a/hooks/telemetry-events.mjs b/hooks/telemetry-events.mjs index d5ddef7..0a5f323 100644 --- a/hooks/telemetry-events.mjs +++ b/hooks/telemetry-events.mjs @@ -70,7 +70,7 @@ export const arcadeToolProperties = (server, tool, toolInput) => { }; /** - * @param {string | null | undefined} service + * @param {string | null | undefined} service Contract service category (e.g. `email`), when known. */ export const otherServerProperties = (service) => withService({ server: "other" }, service); diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index 186ed22..56bf711 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -15,6 +15,7 @@ import fakeAdapter, { FAKE_ARCADE_PREFIX, FAKE_OTHER_PREFIX } from "./fixtures/t import { assertMatchesContract, assertNoLeak, + captureTelemetry, expectedEvent, hash16, hookInput, @@ -191,50 +192,40 @@ test("runTelemetry opt-out matrix sends nothing and writes no files", async () = } }); -test("runTelemetry enabled fixture emits allowed fields and sets arcade-used once", async () => { - const server = await startServer(); - try { - const dataDir = makeTempDir(); - const env = telemetryEnv(fakeAdapter, dataDir, server.url); - const send = []; - await runTelemetry({ - input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }), - adapter: fakeAdapter, - env, - enabled: true, - send: (event) => send.push(event), - }); - await runTelemetry({ - input: hookInput({ - hook_event_name: "PostToolUse", - tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, - prompt_id: PROMPT_ID, - }), - adapter: fakeAdapter, - env, - enabled: true, - send: (event) => send.push(event), - }); - await runTelemetry({ - input: hookInput({ - hook_event_name: "PreToolUse", - tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, - prompt_id: PROMPT_ID, - }), - adapter: fakeAdapter, - env, - enabled: true, - send: (event) => send.push(event), - }); - assert.equal(send.length, 3); - for (const event of send) assertMatchesContract(event); - assert.equal(send[1].properties.arcade_used_before, false); - assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); - assert.equal(send[2].properties.arcade_used_before, true); - assert.equal(isArcadeCall(send[2]), false); - } finally { - await server.close(); - } +test("captureTelemetry carries scope and sets arcade-used across prompt then tool events", async () => { + const { sent: promptSent, dataDir } = await captureTelemetry({ + adapter: fakeAdapter, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }), + }); + assert.equal(promptSent.length, 1); + assertMatchesContract(promptSent[0]); + + const { sent: toolSent } = await captureTelemetry({ + adapter: fakeAdapter, + dataDir, + input: hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + prompt_id: PROMPT_ID, + }), + }); + assert.equal(toolSent.length, 1); + assertMatchesContract(toolSent[0]); + assert.equal(toolSent[0].properties.arcade_used_before, false); + assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); + + const { sent: attemptSent } = await captureTelemetry({ + adapter: fakeAdapter, + dataDir, + input: hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${FAKE_ARCADE_PREFIX}Gmail_ListEmails`, + prompt_id: PROMPT_ID, + }), + }); + assert.equal(attemptSent.length, 1); + assert.equal(attemptSent[0].properties.arcade_used_before, true); + assert.equal(isArcadeCall(attemptSent[0]), false); }); test("runTelemetry survives malformed input and send failures", async () => { diff --git a/test/telemetry-helpers.mjs b/test/telemetry-helpers.mjs index aac23b1..fca1824 100644 --- a/test/telemetry-helpers.mjs +++ b/test/telemetry-helpers.mjs @@ -6,13 +6,90 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; import http from "node:http"; +import os from "node:os"; import path from "node:path"; +import { after } from "node:test"; import Ajv2020 from "ajv/dist/2020.js"; import { eventSchema } from "../hooks/telemetry-contract.mjs"; import { PLUGIN_VERSION } from "../hooks/telemetry-config.mjs"; +import { runTelemetry } from "../hooks/telemetry-run.mjs"; import { ROOT } from "./helpers.mjs"; +const TELEMETRY_ENV_BLOCKLIST = [ + "ARCADE_PLUGIN_TELEMETRY", + "DO_NOT_TRACK", + "DISABLE_TELEMETRY", + "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", + "COPILOT_OFFLINE", + "CLAUDE_PLUGIN_DATA", + "COPILOT_PLUGIN_DATA", +]; + +/** @type {string[]} */ +const tempDataDirs = []; +let tempDataCleanupRegistered = false; + +/** Absolute path of a fresh temp directory; removed after the test run finishes. */ +export const tempDataDir = () => { + if (!tempDataCleanupRegistered) { + after(() => { + for (const dir of tempDataDirs) rmSync(dir, { recursive: true, force: true }); + tempDataDirs.length = 0; + }); + tempDataCleanupRegistered = true; + } + const dir = mkdtempSync(path.join(os.tmpdir(), "arcade-telemetry-")); + tempDataDirs.push(dir); + return dir; +}; + +/** + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} adapter + * @param {string | null} dataDir + * @param {NodeJS.ProcessEnv} overrides + */ +const captureEnv = (adapter, dataDir, overrides) => { + const env = { ...process.env }; + for (const key of TELEMETRY_ENV_BLOCKLIST) delete env[key]; + if (dataDir !== null) env[adapter.dataVariable] = dataDir; + return { ...env, ...overrides }; +}; + +/** + * @param {object} options + * @param {import("../hooks/telemetry-adapter.mjs").TelemetryAdapter} options.adapter + * @param {Record} options.input + * @param {NodeJS.ProcessEnv} [options.env] + * @param {string[]} [options.argv] + * @param {string | null} [options.dataDir] + * @param {number} [options.now] + * @returns {Promise<{ sent: object[], dataDir: string | null }>} + */ +export const captureTelemetry = async ({ + adapter, + input, + env: envOverrides = {}, + argv = [], + dataDir, + now, +}) => { + const resolvedDir = dataDir === undefined ? tempDataDir() : dataDir; + /** @type {object[]} */ + const sent = []; + await runTelemetry({ + input, + adapter, + argv, + env: captureEnv(adapter, resolvedDir, envOverrides), + enabled: true, + now, + send: (event) => sent.push(event), + }); + return { sent, dataDir: resolvedDir }; +}; + const validateEvent = new Ajv2020({ allErrors: true }).compile(eventSchema()); /** @param {object} event */ From f58fba73bf91531acead13027f4443a0c550e751 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:30:33 +0000 Subject: [PATCH 06/10] test(telemetry): ignore telemetry-adapters dir in hooks listing Filter hooks/ to telemetry *.mjs files only so readdir does not treat telemetry-adapters as a file when the directory exists. Co-authored-by: Teal Larson --- test/telemetry-foundation.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index 56bf711..ad798aa 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -362,7 +362,7 @@ test("clearSessionScope respects enabled, opt-out, and SessionStart source", () test("every telemetry source file starts with @ts-check", () => { const files = readdirSync(path.join(ROOT, "hooks")) - .filter((file) => file.startsWith("telemetry")) + .filter((file) => file.startsWith("telemetry") && file.endsWith(".mjs")) .map((file) => `hooks/${file}`); const adapterDir = path.join(ROOT, "hooks", "telemetry-adapters"); try { From b0acca474c327220a17d2dda5bb9d3298fa3cefd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:47:24 +0000 Subject: [PATCH 07/10] Skip telemetry commands when running the routing hooks They print nothing by design, so the routing-output check would fail on them once telemetry hooks are generated. Co-authored-by: Teal Larson --- test/hooks.test.mjs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/hooks.test.mjs b/test/hooks.test.mjs index d28d948..aac0f40 100644 --- a/test/hooks.test.mjs +++ b/test/hooks.test.mjs @@ -36,6 +36,8 @@ test("every command in every generated hooks.json runs and prints what its clien for (const [hostName, { manifest, rootVariable }] of Object.entries(HOSTS)) { for (const [event, entries] of Object.entries(JSON.parse(readRepoFile(manifest)).hooks)) { for (const { command } of entries.flatMap((entry) => entry.hooks ?? [entry])) { + // Telemetry prints nothing; the telemetry tests run its commands. + if (command.includes("/hooks/telemetry.mjs")) continue; const label = `${hostName} ${event}`; const result = spawnSync(command.replaceAll(`\${${rootVariable}}`, ROOT), { shell: true, From c7e452a5ab7d271d0a852e07c204d8475e5c3c86 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:49:20 +0000 Subject: [PATCH 08/10] chore(telemetry): restore PR #13 comments and tighten foundation tests Re-add security and behavior notes from PR #13 in the shared pipeline, scan telemetry-adapters for network imports when present, and relax the sender timeout ceiling for slow CI runners. Co-authored-by: Teal Larson --- hooks/hook-scope.mjs | 3 +++ hooks/telemetry-events.mjs | 10 +++++++++- hooks/telemetry-run.mjs | 7 +++++++ test/telemetry-foundation.test.mjs | 24 ++++++++++++++++-------- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/hooks/hook-scope.mjs b/hooks/hook-scope.mjs index a289c40..b790ea5 100644 --- a/hooks/hook-scope.mjs +++ b/hooks/hook-scope.mjs @@ -84,6 +84,8 @@ export const scopeForInput = (input, { host, requiresTurn, dir, now = Date.now() const file = dir && path.isAbsolute(dir) && session ? path.join(dir, SCOPE_DIRECTORY, `${hash(`${host}:${session}`)}.json`) : null; if (input.hook_event_name === "SessionStart") { + // Claude Code also sends SessionStart after compacting a conversation. The + // session and turn continue, so their scope does too. if (file && input.source !== "compact") rmSync(file, { force: true }); return fallback; } @@ -102,6 +104,7 @@ export const scopeForInput = (input, { host, requiresTurn, dir, now = Date.now() try { writeState(file, state, now); } catch { + // Remove stale relevance when a new prompt cannot be stored. try { rmSync(file, { force: true }); } catch {} } } diff --git a/hooks/telemetry-events.mjs b/hooks/telemetry-events.mjs index 0a5f323..411ff14 100644 --- a/hooks/telemetry-events.mjs +++ b/hooks/telemetry-events.mjs @@ -9,7 +9,6 @@ import { isTaskNotification, shouldRemind } from "./prompt-filters.mjs"; import { isOperatorAgentType } from "./routing-guidance.mjs"; import { classifyPrompt, - serviceForToolkit, serviceForToolName, } from "./telemetry-classify.mjs"; import { commandUsesCli } from "./telemetry-commands.mjs"; @@ -60,10 +59,12 @@ const withService = (properties, service) => * @param {Record | undefined} toolInput */ export const arcadeToolProperties = (server, tool, toolInput) => { + // Arcade_UseTool names the app tool in its input, e.g. "Gmail.ListEmails". const service = tool === "Arcade_UseTool" ? serviceForToolName(toolInput?.tool_name) : serviceForToolName(tool); + // A recognized toolkit prefix does not establish that its tool name is public. const isGateway = /** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool); const category = isGateway ? tool : service !== null ? "app_tool" : "other"; return withService({ server, tool: category }, service); @@ -86,6 +87,9 @@ export const builtinToolProperties = (toolName, toolInput, cli) => { if (toolName === "WebFetch" || toolName === "WebSearch") return { tool: toolName }; if (toolName !== "Bash" || typeof cli !== "string") return null; if (!(/** @type {readonly string[]} */ (BASH_CLIS).includes(cli))) return null; + // `cli` comes from the hook entry's `if` condition. Checking the command as + // well keeps a client that ignores `if` from reporting every CLI on every + // Bash call. if (!commandUsesCli(toolInput?.command, cli)) return null; return withService({ tool: "Bash", cli }, CLI_SERVICES[cli]); }; @@ -108,6 +112,7 @@ const promptProperties = (prompt, adapter) => { }; }; +// The parent's SubagentStop names the subagent's session ID as agent_id. const subagentSession = (/** @type {unknown} */ agentId) => typeof agentId === "string" && agentId !== "" ? { subagent_session: shortHash(agentId) } : {}; @@ -184,6 +189,7 @@ const keepAllowed = (event, properties) => { * when the input is not something the contract tracks. * @param {HookInput | null | undefined} input * @param {{ adapter: TelemetryAdapter, os: string, arcadeUsedBefore: boolean, cli?: string, appWork: boolean }} options + * `cli` is the hook command's `--cli` argument, set only on Bash entries. */ export const buildEvent = (input, { adapter, os, arcadeUsedBefore, cli, appWork }) => { if (!input || typeof input !== "object") return null; @@ -201,6 +207,8 @@ export const buildEvent = (input, { adapter, os, arcadeUsedBefore, cli, appWork os: oneOf(os, OS_NAMES, "other"), $process_person_profile: false, $geoip_disable: true, + // PostHog stores the request's IP unless the event sets one. Null and "" + // are replaced; a fixed placeholder is kept. $ip: "0.0.0.0", arcade_used_before: arcadeUsedBefore === true, }; diff --git a/hooks/telemetry-run.mjs b/hooks/telemetry-run.mjs index 3e5872f..422e1f0 100644 --- a/hooks/telemetry-run.mjs +++ b/hooks/telemetry-run.mjs @@ -43,6 +43,7 @@ const readArcadeUsed = (/** @type {string} */ dir) => { } }; +// Bash hook entries pass `--cli `; buildEvent checks the value. const cliFromArgs = (/** @type {string[]} */ argv) => { const flag = argv.indexOf("--cli"); return flag === -1 ? undefined : argv[flag + 1]; @@ -82,6 +83,10 @@ export const clearSessionScope = ({ }; const defaultSend = (/** @type {object} */ event, env = process.env) => { + // Claude Code kills hook processes when the session exits, so the network + // call runs in a detached child that can outlive this hook. The event goes + // in an environment variable, not an argument, because other users on the + // machine can read a process's arguments but not its environment. const child = spawn(process.execPath, [SENDER], { detached: true, stdio: "ignore", @@ -113,6 +118,8 @@ export const runTelemetry = async ({ }) => { try { if (!enabled && !TELEMETRY_ENABLED) return; + // The client always sets this. Without an absolute data folder there is + // nowhere to keep the arcade-used flag, so nothing is sent. const dir = env[adapter.dataVariable]; if (!dir || !path.isAbsolute(dir)) return; if (isOptedOut(adapter.optOutSwitches, env)) return; diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index ad798aa..10682f1 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -274,24 +274,32 @@ test("telemetry-send posts one event and gives up on a silent host", async () => }); assert.equal(await new Promise((resolve) => hang.on("exit", resolve)), 0); const elapsed = Date.now() - hangStart; - assert.ok(elapsed >= 900 && elapsed < 3000, `timeout was ${elapsed} ms`); - assert.ok(Date.now() - started < 3000); + assert.ok(elapsed >= 900 && elapsed < 5000, `timeout was ${elapsed} ms`); + assert.ok(Date.now() - started < 5000); } finally { await server.close(); silent.close(); } }); -test("only telemetry-send.mjs uses network APIs in hooks", () => { +test("only the detached sender does network I/O", () => { + const networkImport = + /\bfetch\b|\b(?:from|import|require)\s*\(?\s*["'](?:node:)?(?:http|https|http2|net|tls|dgram|dns)["']/; const hookFiles = readdirSync(path.join(ROOT, "hooks")).filter((file) => file.endsWith(".mjs")); for (const file of hookFiles) { if (file === "telemetry-send.mjs") continue; const source = readFileSync(path.join(ROOT, "hooks", file), "utf8"); - assert.doesNotMatch( - source, - /\bfetch\b|\b(?:from|import|require)\s*\(?\s*["'](?:node:)?(?:http|https|http2|net|tls|dgram|dns)["']/, - file, - ); + assert.doesNotMatch(source, networkImport, file); + } + const adapterDir = path.join(ROOT, "hooks", "telemetry-adapters"); + try { + for (const file of readdirSync(adapterDir).filter((f) => f.endsWith(".mjs"))) { + const label = `telemetry-adapters/${file}`; + const source = readFileSync(path.join(adapterDir, file), "utf8"); + assert.doesNotMatch(source, networkImport, label); + } + } catch { + // adapters ship in client slices } }); From 26710300b4c989431b87085f6cb2ad57bb0b79f3 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:59:14 +0000 Subject: [PATCH 09/10] test(telemetry): let adapter network guard surface failures Use existsSync instead of a broad try/catch so adapter fetch assertions are not swallowed, and name the clearSessionScope disabled case in the test title. Co-authored-by: Teal Larson --- test/telemetry-foundation.test.mjs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index 10682f1..70056a4 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawn, spawnSync } from "node:child_process"; -import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; @@ -292,14 +292,12 @@ test("only the detached sender does network I/O", () => { assert.doesNotMatch(source, networkImport, file); } const adapterDir = path.join(ROOT, "hooks", "telemetry-adapters"); - try { + if (existsSync(adapterDir)) { for (const file of readdirSync(adapterDir).filter((f) => f.endsWith(".mjs"))) { const label = `telemetry-adapters/${file}`; const source = readFileSync(path.join(adapterDir, file), "utf8"); assert.doesNotMatch(source, networkImport, label); } - } catch { - // adapters ship in client slices } }); @@ -314,7 +312,7 @@ test("telemetry.mjs statically imports only telemetry-config", () => { assert.deepEqual(staticImports, ["./telemetry-config.mjs"]); }); -test("clearSessionScope respects enabled, opt-out, and SessionStart source", () => { +test("clearSessionScope off-state (enabled false) respects opt-out and SessionStart source", () => { const seedScope = (dir) => { scopeForInput( hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", session_id: "scope-session" }), From 9d1e8fb69c25918f8170db70d905ab4977fcb850 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 19:52:25 +0000 Subject: [PATCH 10/10] test(telemetry): resolve classifier fixtures with the shared ROOT helper URL.pathname keeps percent-encoding, so fixture reads failed with ENOENT in checkouts whose path has spaces or encoded characters. Co-authored-by: Teal Larson --- test/telemetry-classifiers.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/telemetry-classifiers.test.mjs b/test/telemetry-classifiers.test.mjs index 75ca2e6..d132c0a 100644 --- a/test/telemetry-classifiers.test.mjs +++ b/test/telemetry-classifiers.test.mjs @@ -4,8 +4,8 @@ import path from "node:path"; import { test } from "node:test"; import { authNeeded, failureKind } from "../hooks/telemetry-failures.mjs"; import { commandUsesCli } from "../hooks/telemetry-commands.mjs"; +import { ROOT } from "./helpers.mjs"; -const ROOT = path.join(path.dirname(new URL(import.meta.url).pathname), ".."); const fixture = (name) => { const data = JSON.parse(readFileSync(path.join(ROOT, "test/fixtures/hook-inputs", name), "utf8")); return data;