From ab9ea4aef8441c8ee4adc7666ab313d3919319f1 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:03:18 +0000 Subject: [PATCH 1/5] Name the Claude Code telemetry adapter in the hook table Inactive while TELEMETRY_ENABLED is false; the adapter itself comes in hooks/telemetry-adapters/claude-code.mjs. Co-authored-by: Teal Larson --- hooks/hook-hosts.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 1c3c4a9..74ee29d 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -21,6 +21,7 @@ export const HOSTS = { manifest: ".claude-plugin/hooks.json", format: "nested", rootVariable: "CLAUDE_PLUGIN_ROOT", + telemetry: "claude-code", contextOutput: (eventName, text) => ({ hookSpecificOutput: { hookEventName: eventName, additionalContext: text }, }), From a398c055da2d70ad3ff7d29d03c0cfbc40f5dfe6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:28:44 +0000 Subject: [PATCH 2/5] Add Claude Code telemetry adapter, fixtures, and tests Port PR #13 Claude hook mapping into hooks/telemetry-adapters/claude-code.mjs with contract event fixtures and captureTelemetry coverage. Telemetry remains off; generated hooks.json is unchanged. Co-authored-by: Teal Larson --- hooks/telemetry-adapters/claude-code.mjs | 117 ++++++ .../claude-code/auth-status-response.mjs | 10 + .../telemetry/claude-code/event-cases.mjs | 154 +++++++ .../telemetry/claude-code/hooks.enabled.json | 191 +++++++++ test/telemetry-claude.test.mjs | 390 ++++++++++++++++++ 5 files changed, 862 insertions(+) create mode 100644 hooks/telemetry-adapters/claude-code.mjs create mode 100644 test/fixtures/telemetry/claude-code/auth-status-response.mjs create mode 100644 test/fixtures/telemetry/claude-code/event-cases.mjs create mode 100644 test/fixtures/telemetry/claude-code/hooks.enabled.json create mode 100644 test/telemetry-claude.test.mjs diff --git a/hooks/telemetry-adapters/claude-code.mjs b/hooks/telemetry-adapters/claude-code.mjs new file mode 100644 index 0000000..7296326 --- /dev/null +++ b/hooks/telemetry-adapters/claude-code.mjs @@ -0,0 +1,117 @@ +// @ts-check + +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { serviceForToolkit, serviceForToolName } from "../telemetry-classify.mjs"; +import { BASH_CLIS, GATEWAY_TOOLS } from "../telemetry-contract.mjs"; +import { + arcadeToolProperties, + builtinToolProperties as sharedBuiltinToolProperties, + otherServerProperties, +} from "../telemetry-events.mjs"; + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const plugin = JSON.parse(readFileSync(path.join(ROOT, "plugin.json"), "utf8")); +const [mcpServerKey] = Object.keys(JSON.parse(readFileSync(path.join(ROOT, "mcp.json"), "utf8")).mcpServers); + +export const ARCADE_TOOL_PREFIX = `mcp__plugin_${plugin.name}_${mcpServerKey}__`; +export const CLAUDE_AI_ARCADE_TOOL_PREFIX = "mcp__claude_ai_arcade__"; + +const preToolMatcher = `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeToolProperties = (toolName, toolInput) => { + if (typeof toolName !== "string" || !toolName.startsWith("mcp__")) { + return null; + } + const splitAt = toolName.lastIndexOf("__"); + const tool = toolName.slice(splitAt + 2); + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", tool, toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + const serverParts = toolName.slice("mcp__".length, splitAt).split(/[^A-Za-z0-9]+/); + const service = + serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); + return otherServerProperties(service ?? undefined); +}; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const claudeAttemptProperties = (toolName, toolInput) => { + if (typeof toolName !== "string") return null; + if (toolName.startsWith(ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("arcade", toolName.slice(ARCADE_TOOL_PREFIX.length), toolInput); + } + if (toolName.startsWith(CLAUDE_AI_ARCADE_TOOL_PREFIX)) { + return arcadeToolProperties("other_arcade", toolName.slice(CLAUDE_AI_ARCADE_TOOL_PREFIX.length), toolInput); + } + return null; +}; + +/** @param {Record} raw */ +const normalize = (raw) => ({ + hook_event_name: raw.hook_event_name, + session_id: raw.session_id, + prompt_id: raw.prompt_id, + source: raw.source, + prompt: raw.prompt, + tool_name: raw.tool_name, + tool_input: raw.tool_input, + tool_response: raw.tool_response, + error: raw.error, + is_interrupt: raw.is_interrupt, + agent_type: raw.agent_type, + agent_id: raw.agent_id, + last_assistant_message: raw.last_assistant_message, +}); + +/** @param {"PostToolUse" | "PostToolUseFailure"} event */ +const bashHookRows = (event) => + BASH_CLIS.map((cli) => ({ + event, + matcher: "Bash", + if: `Bash(${cli} *)`, + extraArgs: ["--cli", cli], + })); + +/** @type {import("../telemetry-adapter.mjs").TelemetryAdapter} */ +const claudeCodeAdapter = { + host: "claude-code", + dataVariable: "CLAUDE_PLUGIN_DATA", + optOutSwitches: [ + { name: "DISABLE_TELEMETRY", anyValue: true }, + { name: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", anyValue: true }, + ], + requiresTurn: true, + promptReminder: true, + subagentSession: false, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PreToolUse", matcher: preToolMatcher }, + { event: "PostToolUse", matcher: "mcp__.*" }, + { event: "PostToolUse", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUse"), + { event: "PostToolUseFailure", matcher: "mcp__.*" }, + { event: "PostToolUseFailure", matcher: "WebFetch|WebSearch" }, + ...bashHookRows("PostToolUseFailure"), + { event: "SubagentStop" }, + ], + normalize, + toolProperties: claudeToolProperties, + attemptProperties: claudeAttemptProperties, + builtinToolProperties: sharedBuiltinToolProperties, +}; + +export default claudeCodeAdapter; diff --git a/test/fixtures/telemetry/claude-code/auth-status-response.mjs b/test/fixtures/telemetry/claude-code/auth-status-response.mjs new file mode 100644 index 0000000..cac1f49 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/auth-status-response.mjs @@ -0,0 +1,10 @@ +/** Text of a System_ManageAuthorization status answer, as tool_response. */ +export const authStatusResponse = (statuses) => [ + { + type: "text", + text: JSON.stringify({ + message: statuses.includes("authorization_required") ? "Not yet authorized: dropbox." : "All authorized.", + providers: statuses.map((status, index) => ({ provider: `provider${index}`, status })), + }), + }, +]; diff --git a/test/fixtures/telemetry/claude-code/event-cases.mjs b/test/fixtures/telemetry/claude-code/event-cases.mjs new file mode 100644 index 0000000..1e1ae57 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/event-cases.mjs @@ -0,0 +1,154 @@ +import { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../../../../hooks/telemetry-adapters/claude-code.mjs"; +import { authStatusResponse } from "./auth-status-response.mjs"; + +const SIGN_IN = `${ARCADE_TOOL_PREFIX}System_ManageAuthorization`; +const OPERATOR = "arcade:arcade-operator"; + +const tool = (name, toolInput) => ({ hook_event_name: "PostToolUse", tool_name: name, tool_input: toolInput }); +const failed = (name, error, fields = {}) => ({ + hook_event_name: "PostToolUseFailure", + tool_name: name, + error, + ...fields, +}); +const signInCheck = (name, statuses) => ({ + hook_event_name: "PostToolUse", + tool_name: name, + tool_response: authStatusResponse(statuses), +}); +const stop = (message) => ({ + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + last_assistant_message: message, +}); +const operator = (status) => ({ agent: "arcade-operator", status }); + +/** + * @typedef {object} EventCase + * @property {string} id + * @property {Record} input + * @property {null | { event: string, extra: Record }} expected + * @property {string[]} [argv] + * @property {boolean} [seedAppWork] + */ + +/** @returns {EventCase[]} */ +export const eventCases = () => { + const ATTEMPTED = "Plugin tool attempted"; + const CALLED = "Plugin tool called"; + const FAILED = "Plugin tool failed"; + const STOPPED = "Plugin subagent stopped"; + /** @type {EventCase[]} */ + const cases = [ + { id: "session-start-startup", input: { hook_event_name: "SessionStart", source: "startup" }, expected: null }, + { id: "prompt-app-work", input: { hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: true, service_hints: ["calendar"], reminder_sent: true } } }, + { id: "prompt-confirmation-style", input: { hook_event_name: "UserPromptSubmit", prompt: "ok", prompt_id: "confirm-turn" }, + expected: { event: "Plugin prompt submitted", extra: { could_use_arcade: false, service_hints: [], reminder_sent: false } }, + seedAppWork: true }, + { id: "prompt-unrelated", input: { hook_event_name: "UserPromptSubmit", prompt: "Fix the parser" }, expected: null }, + { id: "prompt-task-notification", input: { + hook_event_name: "UserPromptSubmit", + prompt: "\ncompleted calendar", + }, expected: null }, + { id: "pre-arcade-select", input: { hook_event_name: "PreToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-select", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}Arcade_SelectTools` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "Arcade_SelectTools" } } }, + { id: "pre-claude-ai-calendar", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "pre-use-tool", input: { + hook_event_name: "PreToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Arcade_UseTool`, + tool_input: { tool_name: "GoogleCalendar.ListEvents" }, + }, expected: { event: ATTEMPTED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "pre-other-arcade-tool", input: { hook_event_name: "PreToolUse", tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll` }, + expected: { event: ATTEMPTED, extra: { server: "other_arcade", tool: "other" } } }, + { id: "pre-gmail-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Gmail__search_threads" }, expected: null }, + { id: "pre-wrong-arcade-connector", input: { hook_event_name: "PreToolUse", tool_name: "mcp__claude_ai_Arcade__Arcade_SelectTools" }, expected: null }, + { id: "post-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`), + expected: { event: CALLED, extra: { server: "arcade", tool: "app_tool", service: "email" } } }, + { id: "post-claude-ai-calendar", input: tool(`${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "app_tool", service: "calendar" } } }, + { id: "post-sign-in-no-auth", input: tool(SIGN_IN), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-sign-in-auth-needed", input: signInCheck(SIGN_IN, ["authorized", "authorization_required"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-sign-in-authorized", input: signInCheck(SIGN_IN, ["authorized"]), + expected: { event: CALLED, extra: { server: "arcade", tool: "System_ManageAuthorization", auth_needed: false } } }, + { id: "post-claude-ai-sign-in", input: signInCheck("mcp__claude_ai_Arcade__System_ManageAuthorization", ["authorization_required"]), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "System_ManageAuthorization", auth_needed: true } } }, + { id: "post-use-tool-gmail", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "Gmail.ListEmails" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email" } } }, + { id: "post-use-tool-calendar", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "GoogleCalendar.ListEvents" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "calendar" } } }, + { id: "post-use-tool-unknown", input: tool(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, { tool_name: "AcmeHR.RunPayroll" }), + expected: { event: CALLED, extra: { server: "arcade", tool: "Arcade_UseTool" } } }, + { id: "post-other-arcade-app", input: tool(`${ARCADE_TOOL_PREFIX}AcmeHR_RunPayroll`), + expected: { event: CALLED, extra: { server: "arcade", tool: "other" } } }, + { id: "post-claude-ai-use-tool", input: tool("mcp__claude_ai_Arcade_Production__Arcade_UseTool", { tool_name: "Slack_SendMessage" }), + expected: { event: CALLED, extra: { server: "other_arcade", tool: "Arcade_UseTool", service: "chat" } } }, + { id: "post-other-server-granola", input: tool("mcp__granola__Granola_ListMeetings"), + expected: { event: CALLED, extra: { server: "other", service: "meetings" } }, seedAppWork: true }, + { id: "post-gmail-connector", input: tool("mcp__claude_ai_Gmail__search_threads"), + expected: { event: CALLED, extra: { server: "other", service: "email" } }, seedAppWork: true }, + { id: "post-other-server-unknown", input: tool("mcp__secret-server__DoThing"), + expected: { event: CALLED, extra: { server: "other" } }, seedAppWork: true }, + { id: "post-read-builtin", input: tool("Read"), expected: null }, + { id: "failure-slack", input: { hook_event_name: "PostToolUseFailure", tool_name: `${ARCADE_TOOL_PREFIX}Slack_SendMessage` }, + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "chat", failure_kind: "tool_error" } } }, + { id: "failure-auth", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + '{"message":"The tool was not executed because it requires authorization.","authorization_url":"https://example.com/auth"}'), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "auth_required" } } }, + { id: "failure-session-expired", input: failed(`${ARCADE_TOOL_PREFIX}Arcade_UseTool`, 'MCP server "plugin:arcade:arcade" session expired', + { tool_input: { tool_name: "Gmail.ListEmails" } }), + expected: { event: FAILED, extra: { server: "arcade", tool: "Arcade_UseTool", service: "email", failure_kind: "session_expired" } } }, + { id: "failure-unreachable", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed"), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "unreachable" } } }, + { id: "failure-interrupted", input: failed(`${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, "Connection closed", { is_interrupt: true }), + expected: { event: FAILED, extra: { server: "arcade", tool: "app_tool", service: "email", failure_kind: "interrupted" } } }, + { id: "failure-other-http", input: failed("mcp__secret-server__DoThing", "Error POSTing to endpoint: internal error"), + expected: { event: FAILED, extra: { server: "other", failure_kind: "http_error" } }, seedAppWork: true }, + { id: "failure-read", input: failed("Read", "File does not exist."), expected: null }, + { id: "subagent-operator-needs-auth", input: stop("Done.\n\nstatus: needs_auth\nsummary: sign in"), + expected: { event: STOPPED, extra: operator("needs_auth") } }, + { id: "subagent-operator-needs-confirmation", input: stop("**status:** needs_confirmation"), + expected: { event: STOPPED, extra: operator("needs_confirmation") } }, + { id: "subagent-operator-needs-clarification", input: stop("- status: `needs_clarification`"), + expected: { event: STOPPED, extra: operator("needs_clarification") } }, + { id: "subagent-operator-failed", input: stop("STATUS: FAILED"), + expected: { event: STOPPED, extra: operator("failed") } }, + { id: "subagent-operator-unknown", input: stop("status: exploded"), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-operator-no-message", input: stop(undefined), + expected: { event: STOPPED, extra: operator("unknown") } }, + { id: "subagent-non-operator", input: { + hook_event_name: "SubagentStop", + agent_type: "general-purpose", + last_assistant_message: "status: completed", + }, expected: null }, + { id: "subagent-operator-completed", input: { + hook_event_name: "SubagentStop", + agent_type: OPERATOR, + agent_id: "agent-1", + last_assistant_message: "status: completed", + }, expected: { event: STOPPED, extra: operator("completed") } }, + { id: "builtin-webfetch", input: { hook_event_name: "PostToolUse", tool_name: "WebFetch", tool_input: { url: "https://example.com" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebFetch" } }, seedAppWork: true }, + { id: "builtin-websearch", input: { hook_event_name: "PostToolUse", tool_name: "WebSearch", tool_input: { query: "weather" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "WebSearch" } }, seedAppWork: true }, + { id: "builtin-bash-gh", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "gh pr list" } }, + expected: { event: "Plugin built-in tool called", extra: { tool: "Bash", cli: "gh", service: "code_hosting" } }, + argv: ["--cli", "gh"], seedAppWork: true }, + { id: "builtin-bash-curl", input: { hook_event_name: "PostToolUseFailure", tool_name: "Bash", tool_input: { command: "curl x" } }, + expected: { event: "Plugin built-in tool failed", extra: { tool: "Bash", cli: "curl" } }, + argv: ["--cli", "curl"], seedAppWork: true }, + { id: "builtin-bash-non-listed", input: { hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "echo hi" } }, + expected: null, argv: ["--cli", "gh"], seedAppWork: true }, + { id: "session-start-compact", input: { hook_event_name: "SessionStart", source: "compact" }, expected: null }, + ]; + return cases; +}; diff --git a/test/fixtures/telemetry/claude-code/hooks.enabled.json b/test/fixtures/telemetry/claude-code/hooks.enabled.json new file mode 100644 index 0000000..71cd3f7 --- /dev/null +++ b/test/fixtures/telemetry/claude-code/hooks.enabled.json @@ -0,0 +1,191 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/session-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "UserPromptSubmit": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/user-prompt-submit.mjs\" --host claude-code", + "timeout": 5 + }, + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "SubagentStart": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PreToolUse": [ + { + "matcher": "^(?:mcp__plugin_arcade_arcade__|mcp__claude_ai_arcade__)", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ], + "PostToolUse": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "PostToolUseFailure": [ + { + "matcher": "mcp__.*", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "WebFetch|WebSearch", + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + }, + { + "matcher": "Bash", + "hooks": [ + { + "type": "command", + "if": "Bash(gh *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli gh", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(glab *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli glab", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(curl *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli curl", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(wget *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli wget", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(http *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli http", + "timeout": 5 + }, + { + "type": "command", + "if": "Bash(osascript *)", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code --cli osascript", + "timeout": 5 + } + ] + } + ], + "SubagentStop": [ + { + "hooks": [ + { + "type": "command", + "command": "node \"${CLAUDE_PLUGIN_ROOT}/hooks/telemetry.mjs\" --host claude-code", + "timeout": 5 + } + ] + } + ] + } +} diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs new file mode 100644 index 0000000..c45ba85 --- /dev/null +++ b/test/telemetry-claude.test.mjs @@ -0,0 +1,390 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { readFileSync, readdirSync, statSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { BASH_CLIS } from "../hooks/telemetry-contract.mjs"; +import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; +import { SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import { runTelemetry } from "../hooks/telemetry-run.mjs"; +import adapter, { + ARCADE_TOOL_PREFIX, + CLAUDE_AI_ARCADE_TOOL_PREFIX, +} from "../hooks/telemetry-adapters/claude-code.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile, ROOT } from "./helpers.mjs"; +import { eventCases } from "./fixtures/telemetry/claude-code/event-cases.mjs"; +import { + assertMatchesContract, + assertNoLeak, + captureTelemetry, + expectedEvent, + hookInput, + runTelemetryScript, + sleep, + startServer, + telemetryEnv, + tempDataDir, + waitForRequests, +} from "./telemetry-helpers.mjs"; + +const OPERATOR = "arcade:arcade-operator"; +const SESSION_ID = "raw-session-id-123"; +const PROMPT_ID = "raw-prompt-id-456"; +const LEAK_RE = /SECRET|private-repo|raw-session|raw-prompt|authorization_url|example\.com/i; + +const seedAppWork = async (dataDir, promptId = PROMPT_ID, now) => { + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ + hook_event_name: "UserPromptSubmit", + prompt: "Check my calendar", + prompt_id: promptId, + }), + }); +}; + +test("Arcade tool prefixes match plugin.json and mcp.json", () => { + const plugin = JSON.parse(readRepoFile("plugin.json")); + const [server] = Object.keys(JSON.parse(readRepoFile("mcp.json")).mcpServers); + assert.equal(ARCADE_TOOL_PREFIX, `mcp__plugin_${plugin.name}_${server}__`); + assert.equal(CLAUDE_AI_ARCADE_TOOL_PREFIX, "mcp__claude_ai_arcade__"); +}); + +test("every fixture maps through captureTelemetry to the expected contract event", async () => { + for (const { id, input, expected, argv = [], seedAppWork: seed } of eventCases()) { + const dataDir = tempDataDir(); + if (seed) await seedAppWork(dataDir, PROMPT_ID); + const { sent } = await captureTelemetry({ adapter, dataDir, input: hookInput(input), argv }); + if (expected === null) { + assert.equal(sent.length, 0, id); + continue; + } + assert.equal(sent.length, 1, id); + const promptId = typeof input.prompt_id === "string" ? input.prompt_id : PROMPT_ID; + const want = expectedEvent(expected.event, expected.extra, adapter, SESSION_ID, promptId); + want.properties.os = process.platform; + assert.deepEqual(sent[0], want, id); + assertMatchesContract(sent[0], id); + assertNoLeak(JSON.stringify(sent[0]), LEAK_RE); + } +}); + +test("scope transitions across a sequence sharing one data dir", async () => { + const dataDir = tempDataDir(); + const now = 1_000_000; + const calendar = "calendar-turn"; + const unrelated = "unrelated-turn"; + const confirm = "confirm-turn"; + + let r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: unrelated }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1); + + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: "other-turn" }), + }); + assert.equal(r.sent.length, 0); + + await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "yes, send it", prompt_id: confirm }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + 1000, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: confirm }), + }); + assert.equal(r.sent.length, 1); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: calendar }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "compact", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 1, "compact SessionStart keeps scope"); + + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: "after-compact" }), + }); + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "SessionStart", source: "startup", session_id: SESSION_ID }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: calendar }), + }); + assert.equal(r.sent.length, 0, "normal SessionStart clears scope"); + + const expTurn = "expiry-turn"; + await captureTelemetry({ + adapter, + dataDir, + now, + input: hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar", prompt_id: expTurn }), + }); + r = await captureTelemetry({ + adapter, + dataDir, + now: now + SCOPE_TTL_MS + 1, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch", prompt_id: expTurn }), + }); + assert.equal(r.sent.length, 0, "scope expires after 30 minutes"); +}); + +test("direct Arcade calls are observed without prompt scope; other and built-in tools need scope", async () => { + const fresh = tempDataDir(); + const arcade = await captureTelemetry({ + adapter, + dataDir: fresh, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + }); + assert.equal(arcade.sent.length, 1); + + const scoped = tempDataDir(); + const other = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(other.sent.length, 0); + await seedAppWork(scoped); + const otherInScope = await captureTelemetry({ + adapter, + dataDir: scoped, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + }); + assert.equal(otherInScope.sent.length, 1); + + const builtinDir = tempDataDir(); + const builtin = await captureTelemetry({ + adapter, + dataDir: builtinDir, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch" }), + }); + assert.equal(builtin.sent.length, 0); +}); + +test("arcade-used is set only by a successful Arcade gateway call", async () => { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); + const send = []; + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: "mcp__granola__Granola_ListMeetings" }), + send: (e) => send.push(e), + }); + assert.deepEqual(readdirSync(dataDir), []); + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ + hook_event_name: "PreToolUse", + tool_name: `${CLAUDE_AI_ARCADE_TOOL_PREFIX}GoogleCalendar_ListEvents`, + tool_input: { query: "SECRET private meeting" }, + }), + send: (e) => send.push(e), + }); + assert.deepEqual(readdirSync(dataDir), []); + await runTelemetry({ + adapter, + env, + enabled: true, + input: hookInput({ hook_event_name: "PostToolUse", tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails` }), + send: (e) => send.push(e), + }); + assert.equal(readFileSync(path.join(dataDir, "arcade-used"), "utf8"), "true"); + if (process.platform !== "win32") { + assert.equal(statSync(path.join(dataDir, "arcade-used")).mode & 0o777, 0o600); + } + const attempt = send.find((e) => e.event === "Plugin tool attempted"); + assert.ok(attempt); + assertNoLeak(JSON.stringify(attempt), /SECRET|private meeting/); + const flags = send.map((e) => e.properties.arcade_used_before).sort(); + assert.deepEqual(flags, [false, false]); +}); + +test("Claude opt-out switches send nothing and write no data dir files", async () => { + const server = await startServer(); + try { + const sent = []; + const base = hookInput({ hook_event_name: "UserPromptSubmit", prompt: "Check my calendar" }); + const sessionStart = hookInput({ hook_event_name: "SessionStart", source: "startup" }); + const cases = [ + ["DISABLE_TELEMETRY=1", { DISABLE_TELEMETRY: "1" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "1" }], + ["DISABLE_TELEMETRY=0", { DISABLE_TELEMETRY: "0" }], + ["CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=false", { CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: "false" }], + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["missing CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "" }], + ["relative CLAUDE_PLUGIN_DATA", { CLAUDE_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, extra] of cases) { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, server.url, extra); + for (const input of [sessionStart, base]) { + await runTelemetry({ adapter, input, env, enabled: true, send: (e) => sent.push(e) }); + } + assert.deepEqual(readdirSync(dataDir), [], label); + } + await sleep(200); + assert.deepEqual(server.requests, []); + assert.deepEqual(sent, []); + } finally { + await server.close(); + } +}); + +test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { + const rows = [ + ...HOOKS, + ...(await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")), + ]; + const built = buildHookManifest("claude-code", rows); + const expected = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + assert.deepEqual(built, expected); + assert.equal(readRepoFile(HOSTS["claude-code"].manifest).includes("telemetry.mjs"), false); +}); + +test("PreToolUse and PostToolUse telemetry groups match PR #13 Claude wiring", () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )).hooks; + const [pre] = enabled.PreToolUse; + assert.equal(pre.matcher, `^(?:${ARCADE_TOOL_PREFIX}|${CLAUDE_AI_ARCADE_TOOL_PREFIX})`); + for (const event of ["PostToolUse", "PostToolUseFailure"]) { + const groups = enabled[event]; + assert.equal(groups.length, 3); + assert.equal(groups[0].matcher, "mcp__.*"); + assert.equal(groups[1].matcher, "WebFetch|WebSearch"); + assert.equal(groups[2].matcher, "Bash"); + assert.equal(groups[2].hooks.length, BASH_CLIS.length); + for (const hook of groups[2].hooks) { + assert.ok(hook.if); + assert.ok(hook.command.endsWith(`--cli ${hook.if.match(/^Bash\((\w+) \*\)$/)[1]}`)); + } + } +}); + +test("every enabled telemetry command exits quietly with telemetry off", async () => { + const enabled = JSON.parse(readFileSync( + path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), + "utf8", + )); + const commands = Object.values(enabled.hooks) + .flatMap((groups) => groups.flatMap((group) => group.hooks)) + .map((hook) => hook.command) + .filter((command) => command.includes("/hooks/telemetry.mjs")); + const rows = (await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")); + assert.equal(commands.length, rows.length); + const server = await startServer(); + try { + const dataDir = tempDataDir(); + const stdin = JSON.stringify(hookInput({ hook_event_name: "SessionStart" })); + const env = telemetryEnv(adapter, dataDir, server.url); + for (const row of rows) { + const result = runTelemetryScript(stdin, env, "claude-code", row.extraArgs ?? []); + assert.equal(result.status, 0, `${row.event} ${row.matcher ?? ""}: ${result.stderr}`); + assert.equal(result.stdout, "", row.event); + assert.equal(result.stderr, "", row.event); + } + await sleep(300); + assert.deepEqual(server.requests, []); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + await server.close(); + } +}); + +test("invalid telemetry inputs are ignored without affecting hooks", async () => { + const dataDir = tempDataDir(); + const env = telemetryEnv(adapter, dataDir, "http://127.0.0.1:9"); + const cases = [ + "not-json", + JSON.stringify({ hook_event_name: "UserPromptSubmit" }), + JSON.stringify(hookInput({ hook_event_name: "UnknownHook" })), + JSON.stringify(hookInput({ hook_event_name: "PreToolUse", tool_name: "not-mcp" })), + ]; + for (const stdin of cases) { + const result = runTelemetryScript(stdin, env); + assert.equal(result.status, 0, stdin); + assert.equal(result.stdout, ""); + } + const routing = spawnSync(process.execPath, [path.join(ROOT, "hooks", "session-start.mjs"), "--host", "claude-code"], { + input: "not-json", + encoding: "utf8", + }); + assert.equal(routing.status, 0); +}); + +test("buildEvent uses adapter tool properties for malformed tool names", () => { + const options = { adapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + assert.equal(buildEvent(hookInput({ hook_event_name: "PreToolUse", tool_name: 42 }), options), null); + assert.equal(isArcadeCall(buildEvent(hookInput({ + hook_event_name: "PostToolUse", + tool_name: `${ARCADE_TOOL_PREFIX}Gmail_ListEmails`, + }), options)), true); +}); From 9bc5280083f6718d92ec330a14b23114aa6a9c59 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:28:47 +0000 Subject: [PATCH 3/5] Fix telemetry @ts-check scan when adapters directory exists Ignore hooks/telemetry-adapters/ in the top-level telemetry filename filter. Co-authored-by: Teal Larson --- test/telemetry-foundation.test.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/telemetry-foundation.test.mjs b/test/telemetry-foundation.test.mjs index 56bf711..ad798aa 100644 --- a/test/telemetry-foundation.test.mjs +++ b/test/telemetry-foundation.test.mjs @@ -362,7 +362,7 @@ test("clearSessionScope respects enabled, opt-out, and SessionStart source", () test("every telemetry source file starts with @ts-check", () => { const files = readdirSync(path.join(ROOT, "hooks")) - .filter((file) => file.startsWith("telemetry")) + .filter((file) => file.startsWith("telemetry") && file.endsWith(".mjs")) .map((file) => `hooks/${file}`); const adapterDir = path.join(ROOT, "hooks", "telemetry-adapters"); try { From 11ff8719559bc0054b3f07712d9f8e73e858d753 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:49:04 +0000 Subject: [PATCH 4/5] Restore Claude adapter comments and tighten enabled-wiring test Document connector service inference, Bash hook if/extraArgs, and Claude opt-out switch semantics. Build manifest rows without duplicate telemetry entries; rename the hook-group test to describe behavior. Co-authored-by: Teal Larson --- hooks/telemetry-adapters/claude-code.mjs | 5 +++++ test/telemetry-claude.test.mjs | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/hooks/telemetry-adapters/claude-code.mjs b/hooks/telemetry-adapters/claude-code.mjs index 7296326..abff60c 100644 --- a/hooks/telemetry-adapters/claude-code.mjs +++ b/hooks/telemetry-adapters/claude-code.mjs @@ -39,6 +39,8 @@ const claudeToolProperties = (toolName, toolInput) => { if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { return arcadeToolProperties("other_arcade", tool, toolInput); } + // Some connectors name the service in the server, not the tool, e.g. + // mcp__claude_ai_Gmail__search_threads. Only the category is sent. const serverParts = toolName.slice("mcp__".length, splitAt).split(/[^A-Za-z0-9]+/); const service = serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); @@ -77,6 +79,8 @@ const normalize = (raw) => ({ last_assistant_message: raw.last_assistant_message, }); +// `if` is a Claude Code permission rule so the hook does not start for other commands; +// `extraArgs` pass the CLI name through to builtinToolProperties. /** @param {"PostToolUse" | "PostToolUseFailure"} event */ const bashHookRows = (event) => BASH_CLIS.map((cli) => ({ @@ -90,6 +94,7 @@ const bashHookRows = (event) => const claudeCodeAdapter = { host: "claude-code", dataVariable: "CLAUDE_PLUGIN_DATA", + // Claude Code reads these as set for any non-empty value, even "0" or "false". optOutSwitches: [ { name: "DISABLE_TELEMETRY", anyValue: true }, { name: "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", anyValue: true }, diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs index c45ba85..59ff0ba 100644 --- a/test/telemetry-claude.test.mjs +++ b/test/telemetry-claude.test.mjs @@ -296,7 +296,7 @@ test("Claude opt-out switches send nothing and write no data dir files", async ( test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { const rows = [ - ...HOOKS, + ...HOOKS.filter((hook) => hook.script !== "telemetry.mjs"), ...(await telemetryHookRows()).filter((row) => row.hosts.includes("claude-code")), ]; const built = buildHookManifest("claude-code", rows); @@ -308,7 +308,7 @@ test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { assert.equal(readRepoFile(HOSTS["claude-code"].manifest).includes("telemetry.mjs"), false); }); -test("PreToolUse and PostToolUse telemetry groups match PR #13 Claude wiring", () => { +test("PreToolUse and PostToolUse nest MCP, web, and per-CLI Bash telemetry groups", () => { const enabled = JSON.parse(readFileSync( path.join(ROOT, "test/fixtures/telemetry/claude-code/hooks.enabled.json"), "utf8", From a7625a3644da7d3862904b60bbac8fce1a7c7e7a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:58:55 +0000 Subject: [PATCH 5/5] Split Claude off-state manifest check from enabled-wiring test Co-authored-by: Teal Larson --- test/telemetry-claude.test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/telemetry-claude.test.mjs b/test/telemetry-claude.test.mjs index 59ff0ba..3247549 100644 --- a/test/telemetry-claude.test.mjs +++ b/test/telemetry-claude.test.mjs @@ -305,6 +305,9 @@ test("enabled Claude manifest wiring matches hooks.enabled.json", async () => { "utf8", )); assert.deepEqual(built, expected); +}); + +test("checked-in Claude manifest has no telemetry rows", () => { assert.equal(readRepoFile(HOSTS["claude-code"].manifest).includes("telemetry.mjs"), false); });