From f8fe6a6a6be7cb4bcf3a8ffe711c01e992c079f9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:04:05 +0000 Subject: [PATCH 1/5] Wire Copilot hooks for VS Code and name the Copilot CLI telemetry adapter Every Copilot hook command now checks that its script exists, so VS Code, which reads the same hooks.json without setting PLUGIN_ROOT, exits 0 with no output. The extra event names and adapter name are inactive while TELEMETRY_ENABLED is false. Co-authored-by: Teal Larson --- com.github.copilot/hooks/hooks.json | 6 +- hooks/hook-hosts.mjs | 21 +++++-- test/hooks.test.mjs | 96 ++++++++++++++++++++++++++--- 3 files changed, 109 insertions(+), 14 deletions(-) diff --git a/com.github.copilot/hooks/hooks.json b/com.github.copilot/hooks/hooks.json index 7b5c3b0..7550a40 100644 --- a/com.github.copilot/hooks/hooks.json +++ b/com.github.copilot/hooks/hooks.json @@ -4,14 +4,16 @@ "SessionStart": [ { "type": "command", - "command": "node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/session-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/session-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/session-start.mjs') --host copilot }", "timeout": 5 } ], "SubagentStart": [ { "type": "command", - "command": "node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs') --host copilot }", "timeout": 5 } ] diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 1c3c4a9..f915840 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -38,15 +38,26 @@ export const HOSTS = { }, // Copilot CLI. PascalCase names put it in its VS Code-compatible mode // (snake_case SessionStart input; SubagentStart still sends camelCase - // agentName in 1.0.88); it drops prompt-hook output, so there's no prompt hook. - // VS Code reads this file too but doesn't expand ${PLUGIN_ROOT} for Agent - // Plugins hooks or pass their output to the model (pluginParsers.ts, - // copilotPluginConverters.ts on microsoft/vscode main, 2026-09). + // agentName in 1.0.88); it drops prompt-hook output, so it has no prompt + // reminder hook. The other events it lists run only telemetry. + // VS Code reads this file too but doesn't set or expand ${PLUGIN_ROOT} for + // Agent Plugins hooks or pass their output to the model (pluginParsers.ts, + // copilotPluginConverters.ts on microsoft/vscode main, 2026-09), so each + // command checks that its script exists and otherwise exits 0 without output. copilot: { manifest: "com.github.copilot/hooks/hooks.json", format: "flat", rootVariable: "PLUGIN_ROOT", - events: { SessionStart: "SessionStart", SubagentStart: "SubagentStart" }, + runOnlyIfScriptExists: true, + telemetry: "copilot-cli", + events: { + SessionStart: "SessionStart", + SubagentStart: "SubagentStart", + UserPromptSubmit: "UserPromptSubmit", + PostToolUse: "PostToolUse", + PostToolUseFailure: "PostToolUseFailure", + SubagentStop: "SubagentStop", + }, contextOutput: (eventName, text) => ({ additionalContext: text, hookSpecificOutput: { hookEventName: eventName, additionalContext: text }, diff --git a/test/hooks.test.mjs b/test/hooks.test.mjs index d28d948..2f5d79a 100644 --- a/test/hooks.test.mjs +++ b/test/hooks.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { test } from "node:test"; import { HOSTS } from "../hooks/hook-hosts.mjs"; import { shouldRemind } from "../hooks/prompt-filters.mjs"; @@ -32,26 +35,105 @@ test("every client in the hook table has an expected output format", () => { assert.deepEqual(Object.keys(EXPECTED_OUTPUT).sort(), Object.keys(HOSTS).sort()); }); +const HOOK_INPUT = JSON.stringify({ prompt: "What's on my calendar?", agent_type: "Explore" }); + +const assertPrintsContext = (hostName, event, result, label) => { + assert.equal(result.status, 0, `${label}: ${result.stderr}`); + const out = JSON.parse(result.stdout); + const text = out.hookSpecificOutput?.additionalContext ?? out.additional_context; + assert.ok(text, `${label}: no context in ${result.stdout}`); + assert.deepEqual(out, EXPECTED_OUTPUT[hostName](event, text), label); +}; + test("every command in every generated hooks.json runs and prints what its client reads", () => { for (const [hostName, { manifest, rootVariable }] of Object.entries(HOSTS)) { for (const [event, entries] of Object.entries(JSON.parse(readRepoFile(manifest)).hooks)) { for (const { command } of entries.flatMap((entry) => entry.hooks ?? [entry])) { - const label = `${hostName} ${event}`; const result = spawnSync(command.replaceAll(`\${${rootVariable}}`, ROOT), { shell: true, - input: JSON.stringify({ prompt: "What's on my calendar?", agent_type: "Explore" }), + input: HOOK_INPUT, encoding: "utf8", }); - assert.equal(result.status, 0, `${label}: ${result.stderr}`); - const out = JSON.parse(result.stdout); - const text = out.hookSpecificOutput?.additionalContext ?? out.additional_context; - assert.ok(text, `${label}: no context in ${result.stdout}`); - assert.deepEqual(out, EXPECTED_OUTPUT[hostName](event, text), label); + assertPrintsContext(hostName, event, result, `${hostName} ${event}`); } } } }); +// VS Code reads com.github.copilot/hooks/hooks.json but runs each command as +// written, through `spawn(command, { shell: true })` from the workspace root, +// without setting or replacing PLUGIN_ROOT (microsoft/vscode main, 2026-09: +// pluginParsers.ts, hookExecutor.ts). Copilot CLI replaces ${PLUGIN_ROOT} in +// the command and also sets it in the environment. +const copilotEntries = () => + Object.entries(JSON.parse(readRepoFile(HOSTS.copilot.manifest)).hooks).flatMap(([event, entries]) => + entries.map((entry) => ({ event, ...entry })), + ); + +const PLUGIN_VARIABLES = ["PLUGIN_ROOT", "COPILOT_PLUGIN_DATA", "CLAUDE_PLUGIN_DATA"]; + +/** process.env without the variables a client sets for plugin hooks, plus `extra`. */ +const hookEnv = (extra = {}) => ({ + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLUGIN_VARIABLES.includes(name))), + ...extra, +}); + +const VSCODE_INPUT = JSON.stringify({ + hook_event_name: "SessionStart", + session_id: "vscode-session", + cwd: "/workspace", + prompt: "What's on my calendar?", + agent_type: "Explore", +}); + +// The `command` field is the macOS and Linux form; cmd.exe can't run it. +const skipOnWindows = { skip: process.platform === "win32" && "the command field is for macOS and Linux" }; + +test("in VS Code every Copilot hook command exits 0 without output", skipOnWindows, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-vscode-")); + for (const extra of [{}, { PLUGIN_ROOT: "/nonexistent" }]) { + for (const { event, command } of copilotEntries()) { + const label = `${event} ${command} ${JSON.stringify(extra)}`; + const result = spawnSync(command, { shell: true, cwd, env: hookEnv(extra), input: VSCODE_INPUT, encoding: "utf8" }); + assert.equal(result.status, 0, `${label}: ${result.stderr}`); + assert.equal(result.stdout, "", label); + assert.equal(result.stderr, "", label); + } + } +}); + +test("in Copilot CLI every Copilot hook command runs with PLUGIN_ROOT set only in the environment", skipOnWindows, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-copilot-")); + for (const { event, command } of copilotEntries()) { + const result = spawnSync(command, { shell: true, cwd, env: hookEnv({ PLUGIN_ROOT: ROOT }), input: HOOK_INPUT, encoding: "utf8" }); + assertPrintsContext("copilot", event, result, `${event} ${command}`); + } +}); + +// GitHub's ubuntu-latest image includes pwsh. +const POWERSHELL = (process.platform === "win32" ? ["pwsh", "powershell.exe"] : ["pwsh"]).find( + (exe) => !spawnSync(exe, ["-NoProfile", "-NonInteractive", "-Command", "exit 0"]).error, +); + +test("every Copilot hook's powershell command skips without PLUGIN_ROOT and runs with it", { skip: !POWERSHELL && "pwsh not found" }, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-powershell-")); + const runPowerShell = (script, env) => + spawnSync(String(POWERSHELL), ["-NoProfile", "-NonInteractive", "-Command", script], { + cwd, + env, + input: HOOK_INPUT, + encoding: "utf8", + }); + for (const { event, command, powershell } of copilotEntries()) { + assert.ok(powershell, `${event} ${command}: no powershell command`); + const skipped = runPowerShell(powershell, hookEnv()); + assert.equal(skipped.status, 0, `${powershell}: ${skipped.stderr}`); + assert.equal(skipped.stdout, "", powershell); + assert.equal(skipped.stderr, "", powershell); + assertPrintsContext("copilot", event, runPowerShell(powershell, hookEnv({ PLUGIN_ROOT: ROOT })), powershell); + } +}); + test("shouldRemind skips acknowledgements and background task results", () => { for (const prompt of ["What's on my calendar?", "fix it", "go ahead and send it"]) { assert.equal(shouldRemind(prompt), true, prompt); From b7aac0024b2723664d44f637a8fc8f52d5661700 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:28:39 +0000 Subject: [PATCH 2/5] Add Copilot CLI telemetry adapter, fixtures, and tests Port PR #13 Copilot mapping into hooks/telemetry-adapters/copilot-cli.mjs with fixture-driven captureTelemetry tests. Telemetry remains off; the checked-in Copilot hooks.json is unchanged. Co-authored-by: Teal Larson --- hooks/telemetry-adapters/copilot-cli.mjs | 101 ++++ .../telemetry/copilot-cli/app-prompt.json | 17 + .../copilot-cli/confirmation-prompt.json | 26 + .../telemetry/copilot-cli/hooks.enabled.json | 55 ++ .../copilot-cli/post-tool-agent.json | 24 + .../post-tool-arcade-list-apps.json | 30 ++ .../telemetry/copilot-cli/post-tool-bash.json | 24 + .../post-tool-failure-auth-required.json | 29 ++ .../post-tool-failure-tool-error.json | 28 + .../post-tool-manage-auth-needed.json | 31 ++ .../post-tool-manage-auth-not-needed.json | 31 ++ .../copilot-cli/post-tool-other-arcade.json | 30 ++ .../copilot-cli/post-tool-other-mcp.json | 30 ++ .../copilot-cli/post-tool-select-tools.json | 30 ++ .../copilot-cli/post-tool-use-tool.json | 31 ++ .../telemetry/copilot-cli/pre-tool-use.json | 20 + .../telemetry/copilot-cli/session-start.json | 10 + .../copilot-cli/subagent-prompt.json | 17 + .../copilot-cli/subagent-start-camelcase.json | 10 + .../subagent-stop-non-operator.json | 12 + .../copilot-cli/subagent-stop-operator.json | 23 + .../copilot-cli/unrelated-prompt.json | 19 + test/telemetry-copilot.test.mjs | 477 ++++++++++++++++++ 23 files changed, 1105 insertions(+) create mode 100644 hooks/telemetry-adapters/copilot-cli.mjs create mode 100644 test/fixtures/telemetry/copilot-cli/app-prompt.json create mode 100644 test/fixtures/telemetry/copilot-cli/confirmation-prompt.json create mode 100644 test/fixtures/telemetry/copilot-cli/hooks.enabled.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-agent.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-bash.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json create mode 100644 test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json create mode 100644 test/fixtures/telemetry/copilot-cli/pre-tool-use.json create mode 100644 test/fixtures/telemetry/copilot-cli/session-start.json create mode 100644 test/fixtures/telemetry/copilot-cli/subagent-prompt.json create mode 100644 test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json create mode 100644 test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json create mode 100644 test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json create mode 100644 test/fixtures/telemetry/copilot-cli/unrelated-prompt.json create mode 100644 test/telemetry-copilot.test.mjs diff --git a/hooks/telemetry-adapters/copilot-cli.mjs b/hooks/telemetry-adapters/copilot-cli.mjs new file mode 100644 index 0000000..28fea7e --- /dev/null +++ b/hooks/telemetry-adapters/copilot-cli.mjs @@ -0,0 +1,101 @@ +// @ts-check + +import { + serviceForToolkit, + serviceForToolName, +} from "../telemetry-classify.mjs"; +import { GATEWAY_TOOLS } from "../telemetry-contract.mjs"; +import { + arcadeToolProperties, + otherServerProperties, +} from "../telemetry-events.mjs"; + +/** @typedef {import("../telemetry-adapter.mjs").TelemetryAdapter} TelemetryAdapter */ +/** @typedef {import("../telemetry-adapter.mjs").HookInput} HookInput */ + +export const COPILOT_ARCADE_SERVER = "arcade"; + +const pickString = (/** @type {unknown} */ value) => + typeof value === "string" && value !== "" ? value : undefined; + +/** + * @param {Record} raw + * @returns {HookInput} + */ +const normalize = (raw) => { + /** @type {HookInput} */ + const input = {}; + const hook_event_name = pickString(raw.hook_event_name); + if (hook_event_name) input.hook_event_name = hook_event_name; + const session_id = pickString(raw.session_id) ?? pickString(raw.sessionId); + if (session_id) input.session_id = session_id; + const source = pickString(raw.source); + if (source) input.source = source; + if (raw.prompt !== undefined) input.prompt = raw.prompt; + const tool_name = pickString(raw.tool_name); + if (tool_name) input.tool_name = tool_name; + if (raw.tool_input !== undefined && raw.tool_input !== null && typeof raw.tool_input === "object") { + input.tool_input = raw.tool_input; + } + let tool_response = raw.tool_response; + if ( + tool_response === undefined && + raw.tool_result !== null && + typeof raw.tool_result === "object" && + typeof raw.tool_result.text_result_for_llm === "string" + ) { + tool_response = raw.tool_result.text_result_for_llm; + } + if (tool_response !== undefined) input.tool_response = tool_response; + if (raw.error !== undefined) input.error = raw.error; + if (raw.is_interrupt !== undefined) input.is_interrupt = raw.is_interrupt; + const agent_type = + pickString(raw.agent_type) ?? pickString(raw.agentName) ?? pickString(raw.agent_name); + if (agent_type) input.agent_type = agent_type; + const agent_id = pickString(raw.agent_id); + if (agent_id) input.agent_id = agent_id; + if (raw.last_assistant_message !== undefined) input.last_assistant_message = raw.last_assistant_message; + return input; +}; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const toolProperties = (toolName, toolInput) => { + if (typeof toolName !== "string") return null; + const splitAt = toolName.lastIndexOf("-"); + if (splitAt <= 0 || splitAt === toolName.length - 1) return null; + const server = toolName.slice(0, splitAt); + const tool = toolName.slice(splitAt + 1); + if (server === COPILOT_ARCADE_SERVER) { + return arcadeToolProperties("arcade", tool, toolInput); + } + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + const serverParts = server.split(/[^A-Za-z0-9]+/); + const service = + serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); + return otherServerProperties(service); +}; + +/** @type {TelemetryAdapter} */ +const copilotCliAdapter = { + host: "copilot-cli", + dataVariable: "COPILOT_PLUGIN_DATA", + optOutSwitches: [{ name: "COPILOT_OFFLINE", anyValue: false }], + requiresTurn: false, + promptReminder: false, + subagentSession: true, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PostToolUse", matcher: ".+-.+" }, + { event: "PostToolUseFailure", matcher: ".+-.+" }, + { event: "SubagentStop" }, + ], + normalize, + toolProperties, +}; + +export default copilotCliAdapter; diff --git a/test/fixtures/telemetry/copilot-cli/app-prompt.json b/test/fixtures/telemetry/copilot-cli/app-prompt.json new file mode 100644 index 0000000..b9483ab --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/app-prompt.json @@ -0,0 +1,17 @@ +{ + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": true, + "service_hints": ["calendar"], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json b/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json new file mode 100644 index 0000000..37ca03d --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json @@ -0,0 +1,26 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "ok" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": false, + "service_hints": [], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/hooks.enabled.json b/test/fixtures/telemetry/copilot-cli/hooks.enabled.json new file mode 100644 index 0000000..d7aa7d3 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/hooks.enabled.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "hooks": { + "SessionStart": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/session-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/session-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/session-start.mjs') --host copilot }", + "timeout": 5 + } + ], + "SubagentStart": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs') --host copilot }", + "timeout": 5 + } + ], + "UserPromptSubmit": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5 + } + ], + "PostToolUse": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5, + "matcher": ".+-.+" + } + ], + "PostToolUseFailure": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5, + "matcher": ".+-.+" + } + ], + "SubagentStop": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5 + } + ] + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-agent.json b/test/fixtures/telemetry/copilot-cli/post-tool-agent.json new file mode 100644 index 0000000..611a9a4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-agent.json @@ -0,0 +1,24 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "Agent", + "tool_input": { "agent_type": "arcade:arcade-operator" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "done" + } + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json b/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json new file mode 100644 index 0000000..66c8d1e --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_ListApps", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[\"Gmail\",\"Slack\",\"Calendar\"]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_ListApps" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-bash.json b/test/fixtures/telemetry/copilot-cli/post-tool-bash.json new file mode 100644 index 0000000..df98da9 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-bash.json @@ -0,0 +1,24 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "Bash", + "tool_input": { "command": "echo done" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "done" + } + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json b/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json new file mode 100644 index 0000000..dc3a937 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json @@ -0,0 +1,29 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUseFailure", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Slack_SendMessage", + "tool_input": {}, + "error": "MCP server 'arcade': Slack requires authorization" + }, + "expected": { + "event": "Plugin tool failed", + "extra": { + "server": "arcade", + "tool": "app_tool", + "service": "chat", + "failure_kind": "auth_required" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json b/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json new file mode 100644 index 0000000..f784035 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json @@ -0,0 +1,28 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUseFailure", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-error_tool", + "tool_input": {}, + "error": "MCP server 'arcade': Something went wrong in the upstream service" + }, + "expected": { + "event": "Plugin tool failed", + "extra": { + "server": "arcade", + "tool": "other", + "failure_kind": "tool_error" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json new file mode 100644 index 0000000..9a1d94f --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-System_ManageAuthorization", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "{\"providers\":[{\"status\":\"authorization_required\"}]}" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "System_ManageAuthorization", + "auth_needed": true + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json new file mode 100644 index 0000000..b2f5f91 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-System_ManageAuthorization", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "{\"providers\":[{\"status\":\"authorized\"}]}" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "System_ManageAuthorization", + "auth_needed": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json b/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json new file mode 100644 index 0000000..3055da4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-staging-Arcade_UseTool", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "ok" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "other_arcade", + "tool": "Arcade_UseTool" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json b/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json new file mode 100644 index 0000000..a0e42c9 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "granola-Granola_ListMeetings", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "other", + "service": "meetings" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json b/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json new file mode 100644 index 0000000..25180f4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_SelectTools", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_SelectTools" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json b/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json new file mode 100644 index 0000000..f017e2f --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_UseTool", + "tool_input": { "tool_name": "GoogleCalendar.ListEvents" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_UseTool", + "service": "calendar" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/pre-tool-use.json b/test/fixtures/telemetry/copilot-cli/pre-tool-use.json new file mode 100644 index 0000000..68baf39 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/pre-tool-use.json @@ -0,0 +1,20 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PreToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-list_apps", + "tool_input": {} + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/session-start.json b/test/fixtures/telemetry/copilot-cli/session-start.json new file mode 100644 index 0000000..374834e --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/session-start.json @@ -0,0 +1,10 @@ +{ + "input": { + "hook_event_name": "SessionStart", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "source": "startup" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-prompt.json b/test/fixtures/telemetry/copilot-cli/subagent-prompt.json new file mode 100644 index 0000000..1c6da05 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-prompt.json @@ -0,0 +1,17 @@ +{ + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "57946be7-1a73-40ca-a042-abb0f68d9445", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "Complete todo listing-arcade-apps" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": false, + "service_hints": [], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json b/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json new file mode 100644 index 0000000..1e5e75d --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json @@ -0,0 +1,10 @@ +{ + "input": { + "sessionId": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": 1790285282793, + "cwd": "/Users/someone/private-repo", + "agentName": "arcade:arcade-operator", + "agentDisplayName": "arcade-operator" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json b/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json new file mode 100644 index 0000000..a66be76 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json @@ -0,0 +1,12 @@ +{ + "input": { + "hook_event_name": "SubagentStop", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "agent_type": "general-purpose", + "agent_name": "general-purpose", + "last_assistant_message": "status: completed" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json b/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json new file mode 100644 index 0000000..3e3d8f8 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json @@ -0,0 +1,23 @@ +{ + "input": { + "hook_event_name": "SubagentStop", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "transcript_path": "/Users/someone/.copilot/session-state/dd3beb80-4471-4513-99a4-a57d3d7c08df/events.jsonl", + "agent_id": "57946be7-1a73-40ca-a042-abb0f68d9445", + "agent_type": "arcade:arcade-operator", + "agent_name": "arcade:arcade-operator", + "agent_display_name": "arcade-operator", + "stop_reason": "end_turn", + "last_assistant_message": "Done.\n\nstatus: completed" + }, + "expected": { + "event": "Plugin subagent stopped", + "extra": { + "agent": "arcade-operator", + "status": "completed", + "subagent_session": "$hash:57946be7-1a73-40ca-a042-abb0f68d9445" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json b/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json new file mode 100644 index 0000000..d5777e8 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json @@ -0,0 +1,19 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "Fix the parser in src/main.ts" + }, + "expected": null +} diff --git a/test/telemetry-copilot.test.mjs b/test/telemetry-copilot.test.mjs new file mode 100644 index 0000000..f2e4d9f --- /dev/null +++ b/test/telemetry-copilot.test.mjs @@ -0,0 +1,477 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; +import { SCOPE_DIRECTORY, SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; +import { ARCADE_USED_FILE } from "../hooks/telemetry-config.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; +import copilotAdapter, { COPILOT_ARCADE_SERVER } from "../hooks/telemetry-adapters/copilot-cli.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile, ROOT } from "./helpers.mjs"; +import { + assertMatchesContract, + assertNoLeak, + captureTelemetry, + expectedEvent, + hash16, + runTelemetryScript, + sleep, + startServer, + tempDataDir, + waitForRequests, +} from "./telemetry-helpers.mjs"; + +const FIXTURE_DIR = path.join(ROOT, "test/fixtures/telemetry/copilot-cli"); +const SESSION_ID = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; +const SUBAGENT_ID = "57946be7-1a73-40ca-a042-abb0f68d9445"; +const OPERATOR = "arcade:arcade-operator"; +const MCP_TOOL_MATCHER = ".+-.+"; + +const copilotInput = (fields) => ({ + session_id: SESSION_ID, + timestamp: "2026-09-24T21:27:02.743Z", + cwd: "/Users/someone/private-repo", + ...fields, +}); + +const fixtureFiles = () => + readdirSync(FIXTURE_DIR) + .filter((name) => name.endsWith(".json") && name !== "hooks.enabled.json") + .sort(); + +const readFixture = (name) => JSON.parse(readFileSync(path.join(FIXTURE_DIR, name), "utf8")); + +const resolveExpected = (fixture, input) => { + if (fixture.expected === null) return null; + const extra = { ...fixture.expected.extra }; + if (extra.subagent_session?.startsWith("$hash:")) { + extra.subagent_session = hash16(extra.subagent_session.slice("$hash:".length)); + } + if (extra.subagent_session === SUBAGENT_ID) { + extra.subagent_session = hash16(SUBAGENT_ID); + } + const sessionId = input.session_id ?? input.sessionId ?? SESSION_ID; + const event = expectedEvent(fixture.expected.event, extra, copilotAdapter, sessionId, undefined); + const os = /** @type {readonly string[]} */ (["darwin", "linux", "win32"]).includes(process.platform) + ? process.platform + : "other"; + event.properties.os = os; + return event; +}; + +const runFixture = async (fixture, dataDir) => { + for (const step of fixture.prelude ?? []) { + await captureTelemetry({ adapter: copilotAdapter, input: step, dataDir }); + } + return captureTelemetry({ adapter: copilotAdapter, input: fixture.input, dataDir }); +}; + +const PLUGIN_VARIABLES = ["PLUGIN_ROOT", "COPILOT_PLUGIN_DATA", "CLAUDE_PLUGIN_DATA"]; +const hookEnv = (extra = {}) => ({ + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLUGIN_VARIABLES.includes(name))), + ...extra, +}); + +const POWERSHELL = (process.platform === "win32" ? ["pwsh", "powershell.exe"] : ["pwsh"]).find( + (exe) => !spawnSync(exe, ["-NoProfile", "-NonInteractive", "-Command", "exit 0"]).error, +); + +const skipOnWindows = { skip: process.platform === "win32" && "the command field is for macOS and Linux" }; + +const enabledManifest = JSON.parse(readFileSync(path.join(FIXTURE_DIR, "hooks.enabled.json"), "utf8")); +const enabledTelemetryEntries = () => + Object.entries(enabledManifest.hooks).flatMap(([event, entries]) => + entries + .filter((entry) => entry.command?.includes("telemetry.mjs")) + .map((entry) => ({ event, ...entry })), + ); + +for (const file of fixtureFiles()) { + test(`fixture ${file} maps through captureTelemetry`, async () => { + const fixture = readFixture(file); + const dataDir = tempDataDir(); + const { sent } = await runFixture(fixture, dataDir); + const want = resolveExpected(fixture, fixture.input); + if (want === null) { + assert.equal(sent.length, 0, file); + return; + } + assert.equal(sent.length, 1, file); + assert.deepEqual(sent[0], want, file); + assertMatchesContract(sent[0], file); + assertNoLeak(JSON.stringify(sent[0]), /private-repo|dd3beb80|57946be7|SECRET/i); + }); +} + +test("Copilot events never carry turn and PreToolUse sends no attempt event", async () => { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + }); + const { sent } = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + }); + assert.equal(sent[0].properties.turn, undefined); + const attempt = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "PreToolUse", tool_name: "arcade-list_apps", tool_input: {} }), + dataDir, + }); + assert.equal(attempt.sent.length, 0); + assert.ok(!attempt.sent.some((event) => event.event === "Plugin tool attempted")); +}); + +test("session scope follows Copilot prompts and tools in one data dir", async () => { + const dataDir = tempDataDir(); + const baseNow = 1_700_000_000_000; + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow, + }); + const tool = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 1000, + }); + assert.equal(tool.sent.length, 1); + assert.equal(tool.sent[0].properties.turn, undefined); + + const unrelated = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts" }), + dataDir, + now: baseNow + 2000, + }); + assert.equal(unrelated.sent.length, 0); + const silentTool = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 3000, + }); + assert.equal(silentTool.sent.length, 0); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow + 4000, + }); + const confirm = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "ok" }), + dataDir, + now: baseNow + 5000, + }); + assert.equal(confirm.sent.length, 1); + const scopeFile = path.join(dataDir, SCOPE_DIRECTORY, readdirSync(path.join(dataDir, SCOPE_DIRECTORY))[0]); + const expiresAt = JSON.parse(readFileSync(scopeFile, "utf8")).expiresAt; + assert.ok(expiresAt <= baseNow + SCOPE_TTL_MS + 5000); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "SessionStart", source: "startup" }), + dataDir, + now: baseNow + 6000, + }); + const afterStart = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 7000, + }); + assert.equal(afterStart.sent.length, 0); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow + 8000, + }); + const expired = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + SCOPE_TTL_MS + 9000, + }); + assert.equal(expired.sent.length, 0); +}); + +test("operator stop subagent_session matches the subagent session hash", async () => { + const subagentPrompt = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "UserPromptSubmit", + session_id: SUBAGENT_ID, + prompt: "Complete todo listing-arcade-apps", + }), + }); + const { sent } = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "SubagentStop", + agent_id: SUBAGENT_ID, + agent_type: OPERATOR, + last_assistant_message: "status: completed", + }), + }); + assert.equal(sent[0].properties.subagent_session, subagentPrompt.sent[0].properties.session); +}); + +test("arcade-used flag is stored in COPILOT_PLUGIN_DATA", async () => { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "UserPromptSubmit", + prompt: "What is on my calendar tomorrow?", + }), + dataDir, + }); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "private inbox" }, + }), + dataDir, + }); + assert.equal(readFileSync(path.join(dataDir, ARCADE_USED_FILE), "utf8"), "true"); + const second = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "private inbox" }, + }), + dataDir, + }); + assert.equal(second.sent[0].properties.arcade_used_before, true); + assert.equal(isArcadeCall(second.sent[0]), true); +}); + +test("opt-out and missing data dir send nothing", async () => { + const input = copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }); + const cases = [ + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["COPILOT_OFFLINE=true", { COPILOT_OFFLINE: "true" }], + ["COPILOT_OFFLINE=1", { COPILOT_OFFLINE: "1" }], + ["no COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "" }], + ["relative COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, env] of cases) { + const dataDir = tempDataDir(); + const { sent } = await captureTelemetry({ adapter: copilotAdapter, input, dataDir, env }); + assert.equal(sent.length, 0, label); + assert.deepEqual(readdirSync(dataDir), [], label); + } + for (const extra of [{ COPILOT_OFFLINE: "false" }, { COPILOT_OFFLINE: "0" }, { DISABLE_TELEMETRY: "1" }]) { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + env: extra, + }); + const { sent } = await captureTelemetry({ adapter: copilotAdapter, input, dataDir, env: extra }); + assert.ok(sent.length > 0, JSON.stringify(extra)); + } + assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "true" }), true); + assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "false" }), false); +}); + +test("malformed stdin sends nothing through runTelemetry", async () => { + const dataDir = tempDataDir(); + const sent = []; + await runTelemetry({ + input: { session_id: SESSION_ID, hook_event_name: "UserPromptSubmit" }, + adapter: copilotAdapter, + env: { ...process.env, COPILOT_PLUGIN_DATA: dataDir }, + enabled: true, + send: (event) => sent.push(event), + }); + assert.equal(sent.length, 0); +}); + +test("enabled Copilot manifest wiring matches hooks.enabled.json", async () => { + const rows = [ + ...HOOKS, + ...(await telemetryHookRows()).filter((row) => row.hosts.includes("copilot")), + ]; + const built = buildHookManifest("copilot", rows); + assert.deepEqual(built, enabledManifest); +}); + +test("checked-in Copilot manifest has no telemetry rows", () => { + assert.equal(readRepoFile(HOSTS.copilot.manifest).includes("telemetry.mjs"), false); +}); + +test("enabled telemetry commands exit quietly while telemetry is off", skipOnWindows, async () => { + const server = await startServer(); + const cwd = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-telemetry-off-")); + const dataDir = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-data-")); + try { + for (const { command } of enabledTelemetryEntries()) { + for (const extra of [{}, { PLUGIN_ROOT: ROOT }]) { + const result = spawnSync(command, { + shell: true, + cwd, + env: hookEnv({ ...extra, COPILOT_PLUGIN_DATA: dataDir, ARCADE_PLUGIN_TELEMETRY: "1" }), + input: JSON.stringify(copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "calendar" })), + encoding: "utf8", + }); + assert.equal(result.status, 0, command); + assert.equal(result.stdout, "", command); + assert.equal(result.stderr, "", command); + } + } + await sleep(200); + assert.deepEqual(server.requests, []); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + rmSync(cwd, { recursive: true, force: true }); + rmSync(dataDir, { recursive: true, force: true }); + await server.close(); + } +}); + +test( + "enabled telemetry powershell commands exit quietly while telemetry is off", + { skip: !POWERSHELL && "pwsh not found" }, + async () => { + const server = await startServer(); + const cwd = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-ps-")); + const dataDir = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-ps-data-")); + try { + const runPs = (script, env, input) => + spawnSync(String(POWERSHELL), ["-NoProfile", "-NonInteractive", "-Command", script], { + cwd, + env: hookEnv(env), + input, + encoding: "utf8", + }); + for (const { powershell } of enabledTelemetryEntries()) { + assert.ok(powershell); + const input = JSON.stringify(copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "calendar" })); + const skipped = runPs(powershell, {}, input); + assert.equal(skipped.status, 0); + assert.equal(skipped.stdout, ""); + const ran = runPs(powershell, { PLUGIN_ROOT: ROOT, COPILOT_PLUGIN_DATA: dataDir }, input); + assert.equal(ran.status, 0); + assert.equal(ran.stdout, ""); + } + await sleep(200); + assert.deepEqual(server.requests, []); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + rmSync(cwd, { recursive: true, force: true }); + rmSync(dataDir, { recursive: true, force: true }); + await server.close(); + } + }, +); + +test("the Copilot MCP tool matcher picks server tools, not built-ins", () => { + const matcher = new RegExp(`^(?:${MCP_TOOL_MATCHER})$`); + for (const name of ["arcade-Arcade_UseTool", "github-mcp-server-get_issue"]) assert.match(name, matcher); + for (const name of ["Bash", "Agent", "view"]) assert.doesNotMatch(name, matcher); +}); + +test("normalize maps tool_result and agentName without leaking raw fields", () => { + const normalized = copilotAdapter.normalize({ + sessionId: SESSION_ID, + agentName: OPERATOR, + tool_name: "arcade-System_ManageAuthorization", + tool_result: { result_type: "success", text_result_for_llm: '{"providers":[]}' }, + transcript_path: "/SECRET/path", + cwd: "/SECRET/cwd", + }); + assert.equal(normalized.session_id, SESSION_ID); + assert.equal(normalized.agent_type, OPERATOR); + assert.equal(normalized.tool_response, '{"providers":[]}'); + assert.equal(normalized.transcript_path, undefined); + const built = buildEvent( + { ...normalized, hook_event_name: "PostToolUse" }, + { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }, + ); + assertNoLeak(JSON.stringify(built), /SECRET/i); +}); + +test("Copilot CLI sends no built-in tool events", () => { + const options = { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + assert.equal( + buildEvent(copilotInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch" }), options), + null, + ); + assert.equal( + buildEvent( + copilotInput({ hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "gh pr list" } }), + { ...options, cli: "gh" }, + ), + null, + ); +}); + +test("telemetry.mjs with --host copilot exits quietly while disabled", () => { + const dataDir = tempDataDir(); + const result = runTelemetryScript( + JSON.stringify(copilotInput({ hook_event_name: "PostToolUse", tool_name: "arcade-Gmail_ListEmails" })), + { COPILOT_PLUGIN_DATA: dataDir }, + "copilot", + ); + assert.equal(result.status, 0); + assert.equal(result.stdout, ""); + assert.deepEqual(readdirSync(dataDir), []); +}); + +test("arcade server prefix matches mcp.json", () => { + const [server] = Object.keys(JSON.parse(readRepoFile("mcp.json")).mcpServers); + assert.equal(COPILOT_ARCADE_SERVER, server); +}); From 21b0cb5039fc1936792142878bcd266537e86ee9 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 00:49:43 +0000 Subject: [PATCH 3/5] Tighten Copilot normalize parity and restore adapter comments Match PR #13 field selection in normalize, filter telemetry rows in the enabled-manifest wiring test, and split normalize tests by behavior. Co-authored-by: Teal Larson --- hooks/telemetry-adapters/copilot-cli.mjs | 16 ++++++------ test/telemetry-copilot.test.mjs | 31 +++++++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/hooks/telemetry-adapters/copilot-cli.mjs b/hooks/telemetry-adapters/copilot-cli.mjs index 28fea7e..7323ebe 100644 --- a/hooks/telemetry-adapters/copilot-cli.mjs +++ b/hooks/telemetry-adapters/copilot-cli.mjs @@ -27,8 +27,10 @@ const normalize = (raw) => { const input = {}; const hook_event_name = pickString(raw.hook_event_name); if (hook_event_name) input.hook_event_name = hook_event_name; - const session_id = pickString(raw.session_id) ?? pickString(raw.sessionId); + const session_id = pickString(raw.session_id); if (session_id) input.session_id = session_id; + const prompt_id = pickString(raw.prompt_id); + if (prompt_id) input.prompt_id = prompt_id; const source = pickString(raw.source); if (source) input.source = source; if (raw.prompt !== undefined) input.prompt = raw.prompt; @@ -37,20 +39,16 @@ const normalize = (raw) => { if (raw.tool_input !== undefined && raw.tool_input !== null && typeof raw.tool_input === "object") { input.tool_input = raw.tool_input; } - let tool_response = raw.tool_response; if ( - tool_response === undefined && raw.tool_result !== null && typeof raw.tool_result === "object" && typeof raw.tool_result.text_result_for_llm === "string" ) { - tool_response = raw.tool_result.text_result_for_llm; + input.tool_response = raw.tool_result.text_result_for_llm; } - if (tool_response !== undefined) input.tool_response = tool_response; if (raw.error !== undefined) input.error = raw.error; if (raw.is_interrupt !== undefined) input.is_interrupt = raw.is_interrupt; - const agent_type = - pickString(raw.agent_type) ?? pickString(raw.agentName) ?? pickString(raw.agent_name); + const agent_type = pickString(raw.agent_type); if (agent_type) input.agent_type = agent_type; const agent_id = pickString(raw.agent_id); if (agent_id) input.agent_id = agent_id; @@ -64,7 +62,9 @@ const normalize = (raw) => { */ const toolProperties = (toolName, toolInput) => { if (typeof toolName !== "string") return null; + // Arcade tool names never contain "-", so the last one ends the server name. const splitAt = toolName.lastIndexOf("-"); + // Built-in tools (Bash, Agent) have no server prefix. if (splitAt <= 0 || splitAt === toolName.length - 1) return null; const server = toolName.slice(0, splitAt); const tool = toolName.slice(splitAt + 1); @@ -84,6 +84,8 @@ const toolProperties = (toolName, toolInput) => { const copilotCliAdapter = { host: "copilot-cli", dataVariable: "COPILOT_PLUGIN_DATA", + // Copilot documents "true" and doesn't say which other values it + // accepts, so any value that isn't clearly off counts. optOutSwitches: [{ name: "COPILOT_OFFLINE", anyValue: false }], requiresTurn: false, promptReminder: false, diff --git a/test/telemetry-copilot.test.mjs b/test/telemetry-copilot.test.mjs index f2e4d9f..418f0fa 100644 --- a/test/telemetry-copilot.test.mjs +++ b/test/telemetry-copilot.test.mjs @@ -56,7 +56,7 @@ const resolveExpected = (fixture, input) => { if (extra.subagent_session === SUBAGENT_ID) { extra.subagent_session = hash16(SUBAGENT_ID); } - const sessionId = input.session_id ?? input.sessionId ?? SESSION_ID; + const sessionId = input.session_id ?? SESSION_ID; const event = expectedEvent(fixture.expected.event, extra, copilotAdapter, sessionId, undefined); const os = /** @type {readonly string[]} */ (["darwin", "linux", "win32"]).includes(process.platform) ? process.platform @@ -109,7 +109,7 @@ for (const file of fixtureFiles()) { }); } -test("Copilot events never carry turn and PreToolUse sends no attempt event", async () => { +test("Copilot-shaped hook input never carries turn and PreToolUse sends no attempt event", async () => { const dataDir = tempDataDir(); await captureTelemetry({ adapter: copilotAdapter, @@ -342,7 +342,7 @@ test("malformed stdin sends nothing through runTelemetry", async () => { test("enabled Copilot manifest wiring matches hooks.enabled.json", async () => { const rows = [ - ...HOOKS, + ...HOOKS.filter((hook) => hook.script !== "telemetry.mjs"), ...(await telemetryHookRows()).filter((row) => row.hosts.includes("copilot")), ]; const built = buildHookManifest("copilot", rows); @@ -424,26 +424,39 @@ test("the Copilot MCP tool matcher picks server tools, not built-ins", () => { for (const name of ["Bash", "Agent", "view"]) assert.doesNotMatch(name, matcher); }); -test("normalize maps tool_result and agentName without leaking raw fields", () => { +test("normalize reads tool_result text_result_for_llm and ignores top-level tool_response", () => { const normalized = copilotAdapter.normalize({ - sessionId: SESSION_ID, - agentName: OPERATOR, + session_id: SESSION_ID, + agent_type: OPERATOR, tool_name: "arcade-System_ManageAuthorization", + tool_response: '{"providers":[{"status":"authorization_required"}]}', tool_result: { result_type: "success", text_result_for_llm: '{"providers":[]}' }, transcript_path: "/SECRET/path", cwd: "/SECRET/cwd", }); - assert.equal(normalized.session_id, SESSION_ID); - assert.equal(normalized.agent_type, OPERATOR); assert.equal(normalized.tool_response, '{"providers":[]}'); - assert.equal(normalized.transcript_path, undefined); + assert.equal(normalized.prompt_id, undefined); + assert.equal(normalized.session_id, SESSION_ID); + const withPromptId = copilotAdapter.normalize({ session_id: SESSION_ID, prompt_id: "turn-1" }); + assert.equal(withPromptId.prompt_id, "turn-1"); const built = buildEvent( { ...normalized, hook_event_name: "PostToolUse" }, { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }, ); + assert.equal(built?.properties.auth_needed, false); assertNoLeak(JSON.stringify(built), /SECRET/i); }); +test("normalize does not map SubagentStart camelCase fields into hook input", () => { + const normalized = copilotAdapter.normalize({ + sessionId: SESSION_ID, + agentName: OPERATOR, + agentDisplayName: "arcade-operator", + }); + assert.equal(normalized.session_id, undefined); + assert.equal(normalized.agent_type, undefined); +}); + test("Copilot CLI sends no built-in tool events", () => { const options = { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; assert.equal( From 665e626b448a531e7eabf768b8fc487d1a96a42a Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 18:54:16 +0000 Subject: [PATCH 4/5] test(telemetry): point Copilot capture tests at the loopback server Both the shell and PowerShell hard-OFF tests now set ARCADE_PLUGIN_TELEMETRY_HOST to the loopback capture server, use a prompt that would produce an event if the gate were open, and finish with a positive control that sends a synthetic event through the same environment and requires the server to receive it. Co-authored-by: Teal Larson --- test/telemetry-copilot.test.mjs | 49 +++++++++++++++++++++++++++------ 1 file changed, 41 insertions(+), 8 deletions(-) diff --git a/test/telemetry-copilot.test.mjs b/test/telemetry-copilot.test.mjs index 418f0fa..4759af2 100644 --- a/test/telemetry-copilot.test.mjs +++ b/test/telemetry-copilot.test.mjs @@ -1,14 +1,14 @@ // @ts-check import assert from "node:assert/strict"; -import { spawnSync } from "node:child_process"; +import { spawn, spawnSync } from "node:child_process"; import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; import os from "node:os"; import path from "node:path"; import { test } from "node:test"; import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; import { SCOPE_DIRECTORY, SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; -import { ARCADE_USED_FILE } from "../hooks/telemetry-config.mjs"; +import { ARCADE_USED_FILE, EVENT_ENV } from "../hooks/telemetry-config.mjs"; import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; import copilotAdapter, { COPILOT_ARCADE_SERVER } from "../hooks/telemetry-adapters/copilot-cli.mjs"; @@ -84,6 +84,32 @@ const POWERSHELL = (process.platform === "win32" ? ["pwsh", "powershell.exe"] : const skipOnWindows = { skip: process.platform === "win32" && "the command field is for macOS and Linux" }; +// Classified as app work, so these hooks would send an event if the hard-OFF gate were open. +const SENDABLE_PROMPT = copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }); + +const SEND_SCRIPT = path.join(ROOT, "hooks", "telemetry-send.mjs"); +// Async spawn keeps this process's event loop free so the in-process server can answer. +const exitCode = (command, args, options) => + new Promise((resolve) => spawn(command, args, { ...options, stdio: "ignore" }).on("exit", resolve)); +const sendWithNode = (env) => exitCode(process.execPath, [SEND_SCRIPT], { env }); + +// Asserts nothing reached the server, then proves the hook environment resolves to it, +// so the empty request list could not come from sending somewhere else. +const assertCaptureObservesSend = async (server, env, send = sendWithNode) => { + assert.deepEqual(server.requests, []); + const event = buildEvent(SENDABLE_PROMPT, { + adapter: copilotAdapter, + os: "linux", + arcadeUsedBefore: false, + appWork: true, + }); + assert.ok(event); + assert.equal(await send({ ...env, [EVENT_ENV]: JSON.stringify(event) }), 0); + await waitForRequests(server.requests, 1); + assert.equal(server.requests.length, 1); + assert.equal(JSON.parse(server.requests[0].body).event, "Plugin prompt submitted"); +}; + const enabledManifest = JSON.parse(readFileSync(path.join(FIXTURE_DIR, "hooks.enabled.json"), "utf8")); const enabledTelemetryEntries = () => Object.entries(enabledManifest.hooks).flatMap(([event, entries]) => @@ -357,14 +383,15 @@ test("enabled telemetry commands exit quietly while telemetry is off", skipOnWin const server = await startServer(); const cwd = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-telemetry-off-")); const dataDir = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-data-")); + const captureEnv = { COPILOT_PLUGIN_DATA: dataDir, ARCADE_PLUGIN_TELEMETRY: "1", ARCADE_PLUGIN_TELEMETRY_HOST: server.url }; try { for (const { command } of enabledTelemetryEntries()) { for (const extra of [{}, { PLUGIN_ROOT: ROOT }]) { const result = spawnSync(command, { shell: true, cwd, - env: hookEnv({ ...extra, COPILOT_PLUGIN_DATA: dataDir, ARCADE_PLUGIN_TELEMETRY: "1" }), - input: JSON.stringify(copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "calendar" })), + env: hookEnv({ ...extra, ...captureEnv }), + input: JSON.stringify(SENDABLE_PROMPT), encoding: "utf8", }); assert.equal(result.status, 0, command); @@ -373,7 +400,7 @@ test("enabled telemetry commands exit quietly while telemetry is off", skipOnWin } } await sleep(200); - assert.deepEqual(server.requests, []); + await assertCaptureObservesSend(server, hookEnv(captureEnv)); assert.deepEqual(readdirSync(dataDir), []); } finally { rmSync(cwd, { recursive: true, force: true }); @@ -393,13 +420,13 @@ test( const runPs = (script, env, input) => spawnSync(String(POWERSHELL), ["-NoProfile", "-NonInteractive", "-Command", script], { cwd, - env: hookEnv(env), + env: hookEnv({ ARCADE_PLUGIN_TELEMETRY_HOST: server.url, ...env }), input, encoding: "utf8", }); for (const { powershell } of enabledTelemetryEntries()) { assert.ok(powershell); - const input = JSON.stringify(copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "calendar" })); + const input = JSON.stringify(SENDABLE_PROMPT); const skipped = runPs(powershell, {}, input); assert.equal(skipped.status, 0); assert.equal(skipped.stdout, ""); @@ -408,7 +435,13 @@ test( assert.equal(ran.stdout, ""); } await sleep(200); - assert.deepEqual(server.requests, []); + await assertCaptureObservesSend(server, hookEnv({ ARCADE_PLUGIN_TELEMETRY_HOST: server.url }), (env) => + exitCode( + String(POWERSHELL), + ["-NoProfile", "-NonInteractive", "-Command", `& '${process.execPath}' '${SEND_SCRIPT}'; exit $LASTEXITCODE`], + { cwd, env }, + ), + ); assert.deepEqual(readdirSync(dataDir), []); } finally { rmSync(cwd, { recursive: true, force: true }); From 2a9265ff11f094bb506dc2276be2ab70fbf7e84f Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Wed, 7 Oct 2026 20:02:38 +0000 Subject: [PATCH 5/5] test(telemetry-copilot): clear inherited opt-outs in hard-OFF boundary tests An inherited DO_NOT_TRACK, COPILOT_OFFLINE or ARCADE_PLUGIN_TELEMETRY made the hook skip on its own, so the shell and PowerShell boundary tests still passed with the TELEMETRY_ENABLED gate open. Co-authored-by: Teal Larson --- test/telemetry-copilot.test.mjs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/test/telemetry-copilot.test.mjs b/test/telemetry-copilot.test.mjs index 4759af2..4287498 100644 --- a/test/telemetry-copilot.test.mjs +++ b/test/telemetry-copilot.test.mjs @@ -8,7 +8,7 @@ import path from "node:path"; import { test } from "node:test"; import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; import { SCOPE_DIRECTORY, SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; -import { ARCADE_USED_FILE, EVENT_ENV } from "../hooks/telemetry-config.mjs"; +import { ARCADE_USED_FILE, EVENT_ENV, OPT_OUT_ENV } from "../hooks/telemetry-config.mjs"; import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; import copilotAdapter, { COPILOT_ARCADE_SERVER } from "../hooks/telemetry-adapters/copilot-cli.mjs"; @@ -72,7 +72,15 @@ const runFixture = async (fixture, dataDir) => { return captureTelemetry({ adapter: copilotAdapter, input: fixture.input, dataDir }); }; -const PLUGIN_VARIABLES = ["PLUGIN_ROOT", "COPILOT_PLUGIN_DATA", "CLAUDE_PLUGIN_DATA"]; +// An inherited opt-out would make the hook skip even with the hard-OFF gate open. +const PLUGIN_VARIABLES = [ + "PLUGIN_ROOT", + "COPILOT_PLUGIN_DATA", + "CLAUDE_PLUGIN_DATA", + OPT_OUT_ENV, + "DO_NOT_TRACK", + ...copilotAdapter.optOutSwitches.map(({ name }) => name), +]; const hookEnv = (extra = {}) => ({ ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLUGIN_VARIABLES.includes(name))), ...extra,