diff --git a/com.github.copilot/hooks/hooks.json b/com.github.copilot/hooks/hooks.json index 7b5c3b0..7550a40 100644 --- a/com.github.copilot/hooks/hooks.json +++ b/com.github.copilot/hooks/hooks.json @@ -4,14 +4,16 @@ "SessionStart": [ { "type": "command", - "command": "node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/session-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/session-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/session-start.mjs') --host copilot }", "timeout": 5 } ], "SubagentStart": [ { "type": "command", - "command": "node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs') --host copilot }", "timeout": 5 } ] diff --git a/hooks/hook-hosts.mjs b/hooks/hook-hosts.mjs index 1c3c4a9..f915840 100644 --- a/hooks/hook-hosts.mjs +++ b/hooks/hook-hosts.mjs @@ -38,15 +38,26 @@ export const HOSTS = { }, // Copilot CLI. PascalCase names put it in its VS Code-compatible mode // (snake_case SessionStart input; SubagentStart still sends camelCase - // agentName in 1.0.88); it drops prompt-hook output, so there's no prompt hook. - // VS Code reads this file too but doesn't expand ${PLUGIN_ROOT} for Agent - // Plugins hooks or pass their output to the model (pluginParsers.ts, - // copilotPluginConverters.ts on microsoft/vscode main, 2026-09). + // agentName in 1.0.88); it drops prompt-hook output, so it has no prompt + // reminder hook. The other events it lists run only telemetry. + // VS Code reads this file too but doesn't set or expand ${PLUGIN_ROOT} for + // Agent Plugins hooks or pass their output to the model (pluginParsers.ts, + // copilotPluginConverters.ts on microsoft/vscode main, 2026-09), so each + // command checks that its script exists and otherwise exits 0 without output. copilot: { manifest: "com.github.copilot/hooks/hooks.json", format: "flat", rootVariable: "PLUGIN_ROOT", - events: { SessionStart: "SessionStart", SubagentStart: "SubagentStart" }, + runOnlyIfScriptExists: true, + telemetry: "copilot-cli", + events: { + SessionStart: "SessionStart", + SubagentStart: "SubagentStart", + UserPromptSubmit: "UserPromptSubmit", + PostToolUse: "PostToolUse", + PostToolUseFailure: "PostToolUseFailure", + SubagentStop: "SubagentStop", + }, contextOutput: (eventName, text) => ({ additionalContext: text, hookSpecificOutput: { hookEventName: eventName, additionalContext: text }, diff --git a/hooks/telemetry-adapters/copilot-cli.mjs b/hooks/telemetry-adapters/copilot-cli.mjs new file mode 100644 index 0000000..7323ebe --- /dev/null +++ b/hooks/telemetry-adapters/copilot-cli.mjs @@ -0,0 +1,103 @@ +// @ts-check + +import { + serviceForToolkit, + serviceForToolName, +} from "../telemetry-classify.mjs"; +import { GATEWAY_TOOLS } from "../telemetry-contract.mjs"; +import { + arcadeToolProperties, + otherServerProperties, +} from "../telemetry-events.mjs"; + +/** @typedef {import("../telemetry-adapter.mjs").TelemetryAdapter} TelemetryAdapter */ +/** @typedef {import("../telemetry-adapter.mjs").HookInput} HookInput */ + +export const COPILOT_ARCADE_SERVER = "arcade"; + +const pickString = (/** @type {unknown} */ value) => + typeof value === "string" && value !== "" ? value : undefined; + +/** + * @param {Record} raw + * @returns {HookInput} + */ +const normalize = (raw) => { + /** @type {HookInput} */ + const input = {}; + const hook_event_name = pickString(raw.hook_event_name); + if (hook_event_name) input.hook_event_name = hook_event_name; + const session_id = pickString(raw.session_id); + if (session_id) input.session_id = session_id; + const prompt_id = pickString(raw.prompt_id); + if (prompt_id) input.prompt_id = prompt_id; + const source = pickString(raw.source); + if (source) input.source = source; + if (raw.prompt !== undefined) input.prompt = raw.prompt; + const tool_name = pickString(raw.tool_name); + if (tool_name) input.tool_name = tool_name; + if (raw.tool_input !== undefined && raw.tool_input !== null && typeof raw.tool_input === "object") { + input.tool_input = raw.tool_input; + } + if ( + raw.tool_result !== null && + typeof raw.tool_result === "object" && + typeof raw.tool_result.text_result_for_llm === "string" + ) { + input.tool_response = raw.tool_result.text_result_for_llm; + } + if (raw.error !== undefined) input.error = raw.error; + if (raw.is_interrupt !== undefined) input.is_interrupt = raw.is_interrupt; + const agent_type = pickString(raw.agent_type); + if (agent_type) input.agent_type = agent_type; + const agent_id = pickString(raw.agent_id); + if (agent_id) input.agent_id = agent_id; + if (raw.last_assistant_message !== undefined) input.last_assistant_message = raw.last_assistant_message; + return input; +}; + +/** + * @param {unknown} toolName + * @param {Record | undefined} toolInput + */ +const toolProperties = (toolName, toolInput) => { + if (typeof toolName !== "string") return null; + // Arcade tool names never contain "-", so the last one ends the server name. + const splitAt = toolName.lastIndexOf("-"); + // Built-in tools (Bash, Agent) have no server prefix. + if (splitAt <= 0 || splitAt === toolName.length - 1) return null; + const server = toolName.slice(0, splitAt); + const tool = toolName.slice(splitAt + 1); + if (server === COPILOT_ARCADE_SERVER) { + return arcadeToolProperties("arcade", tool, toolInput); + } + if (/** @type {readonly string[]} */ (GATEWAY_TOOLS).includes(tool)) { + return arcadeToolProperties("other_arcade", tool, toolInput); + } + const serverParts = server.split(/[^A-Za-z0-9]+/); + const service = + serviceForToolName(tool) ?? serverParts.map(serviceForToolkit).find(Boolean); + return otherServerProperties(service); +}; + +/** @type {TelemetryAdapter} */ +const copilotCliAdapter = { + host: "copilot-cli", + dataVariable: "COPILOT_PLUGIN_DATA", + // Copilot documents "true" and doesn't say which other values it + // accepts, so any value that isn't clearly off counts. + optOutSwitches: [{ name: "COPILOT_OFFLINE", anyValue: false }], + requiresTurn: false, + promptReminder: false, + subagentSession: true, + hookRows: [ + { event: "UserPromptSubmit" }, + { event: "PostToolUse", matcher: ".+-.+" }, + { event: "PostToolUseFailure", matcher: ".+-.+" }, + { event: "SubagentStop" }, + ], + normalize, + toolProperties, +}; + +export default copilotCliAdapter; diff --git a/test/fixtures/telemetry/copilot-cli/app-prompt.json b/test/fixtures/telemetry/copilot-cli/app-prompt.json new file mode 100644 index 0000000..b9483ab --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/app-prompt.json @@ -0,0 +1,17 @@ +{ + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": true, + "service_hints": ["calendar"], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json b/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json new file mode 100644 index 0000000..37ca03d --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/confirmation-prompt.json @@ -0,0 +1,26 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "ok" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": false, + "service_hints": [], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/hooks.enabled.json b/test/fixtures/telemetry/copilot-cli/hooks.enabled.json new file mode 100644 index 0000000..d7aa7d3 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/hooks.enabled.json @@ -0,0 +1,55 @@ +{ + "version": 1, + "hooks": { + "SessionStart": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/session-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/session-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/session-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/session-start.mjs') --host copilot }", + "timeout": 5 + } + ], + "SubagentStart": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/subagent-start.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/subagent-start.mjs') --host copilot }", + "timeout": 5 + } + ], + "UserPromptSubmit": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5 + } + ], + "PostToolUse": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5, + "matcher": ".+-.+" + } + ], + "PostToolUseFailure": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5, + "matcher": ".+-.+" + } + ], + "SubagentStop": [ + { + "type": "command", + "command": "if [ -f \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" ]; then node \"${PLUGIN_ROOT}/hooks/telemetry.mjs\" --host copilot; fi", + "powershell": "if ($env:PLUGIN_ROOT -and (Test-Path -LiteralPath ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs'))) { node ($env:PLUGIN_ROOT + '/hooks/telemetry.mjs') --host copilot }", + "timeout": 5 + } + ] + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-agent.json b/test/fixtures/telemetry/copilot-cli/post-tool-agent.json new file mode 100644 index 0000000..611a9a4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-agent.json @@ -0,0 +1,24 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "Agent", + "tool_input": { "agent_type": "arcade:arcade-operator" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "done" + } + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json b/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json new file mode 100644 index 0000000..66c8d1e --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-arcade-list-apps.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_ListApps", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[\"Gmail\",\"Slack\",\"Calendar\"]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_ListApps" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-bash.json b/test/fixtures/telemetry/copilot-cli/post-tool-bash.json new file mode 100644 index 0000000..df98da9 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-bash.json @@ -0,0 +1,24 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "Bash", + "tool_input": { "command": "echo done" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "done" + } + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json b/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json new file mode 100644 index 0000000..dc3a937 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-failure-auth-required.json @@ -0,0 +1,29 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUseFailure", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Slack_SendMessage", + "tool_input": {}, + "error": "MCP server 'arcade': Slack requires authorization" + }, + "expected": { + "event": "Plugin tool failed", + "extra": { + "server": "arcade", + "tool": "app_tool", + "service": "chat", + "failure_kind": "auth_required" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json b/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json new file mode 100644 index 0000000..f784035 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-failure-tool-error.json @@ -0,0 +1,28 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUseFailure", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-error_tool", + "tool_input": {}, + "error": "MCP server 'arcade': Something went wrong in the upstream service" + }, + "expected": { + "event": "Plugin tool failed", + "extra": { + "server": "arcade", + "tool": "other", + "failure_kind": "tool_error" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json new file mode 100644 index 0000000..9a1d94f --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-needed.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-System_ManageAuthorization", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "{\"providers\":[{\"status\":\"authorization_required\"}]}" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "System_ManageAuthorization", + "auth_needed": true + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json new file mode 100644 index 0000000..b2f5f91 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-manage-auth-not-needed.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-System_ManageAuthorization", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "{\"providers\":[{\"status\":\"authorized\"}]}" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "System_ManageAuthorization", + "auth_needed": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json b/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json new file mode 100644 index 0000000..3055da4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-other-arcade.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-staging-Arcade_UseTool", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "ok" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "other_arcade", + "tool": "Arcade_UseTool" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json b/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json new file mode 100644 index 0000000..a0e42c9 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-other-mcp.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "granola-Granola_ListMeetings", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "other", + "service": "meetings" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json b/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json new file mode 100644 index 0000000..25180f4 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-select-tools.json @@ -0,0 +1,30 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_SelectTools", + "tool_input": {}, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_SelectTools" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json b/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json new file mode 100644 index 0000000..f017e2f --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/post-tool-use-tool.json @@ -0,0 +1,31 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PostToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-Arcade_UseTool", + "tool_input": { "tool_name": "GoogleCalendar.ListEvents" }, + "tool_result": { + "result_type": "success", + "text_result_for_llm": "[]" + } + }, + "expected": { + "event": "Plugin tool called", + "extra": { + "server": "arcade", + "tool": "Arcade_UseTool", + "service": "calendar" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/pre-tool-use.json b/test/fixtures/telemetry/copilot-cli/pre-tool-use.json new file mode 100644 index 0000000..68baf39 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/pre-tool-use.json @@ -0,0 +1,20 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "PreToolUse", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "tool_name": "arcade-list_apps", + "tool_input": {} + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/session-start.json b/test/fixtures/telemetry/copilot-cli/session-start.json new file mode 100644 index 0000000..374834e --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/session-start.json @@ -0,0 +1,10 @@ +{ + "input": { + "hook_event_name": "SessionStart", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "source": "startup" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-prompt.json b/test/fixtures/telemetry/copilot-cli/subagent-prompt.json new file mode 100644 index 0000000..1c6da05 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-prompt.json @@ -0,0 +1,17 @@ +{ + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "57946be7-1a73-40ca-a042-abb0f68d9445", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "Complete todo listing-arcade-apps" + }, + "expected": { + "event": "Plugin prompt submitted", + "extra": { + "could_use_arcade": false, + "service_hints": [], + "reminder_sent": false + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json b/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json new file mode 100644 index 0000000..1e5e75d --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-start-camelcase.json @@ -0,0 +1,10 @@ +{ + "input": { + "sessionId": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": 1790285282793, + "cwd": "/Users/someone/private-repo", + "agentName": "arcade:arcade-operator", + "agentDisplayName": "arcade-operator" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json b/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json new file mode 100644 index 0000000..a66be76 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-stop-non-operator.json @@ -0,0 +1,12 @@ +{ + "input": { + "hook_event_name": "SubagentStop", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "agent_type": "general-purpose", + "agent_name": "general-purpose", + "last_assistant_message": "status: completed" + }, + "expected": null +} diff --git a/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json b/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json new file mode 100644 index 0000000..3e3d8f8 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/subagent-stop-operator.json @@ -0,0 +1,23 @@ +{ + "input": { + "hook_event_name": "SubagentStop", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "transcript_path": "/Users/someone/.copilot/session-state/dd3beb80-4471-4513-99a4-a57d3d7c08df/events.jsonl", + "agent_id": "57946be7-1a73-40ca-a042-abb0f68d9445", + "agent_type": "arcade:arcade-operator", + "agent_name": "arcade:arcade-operator", + "agent_display_name": "arcade-operator", + "stop_reason": "end_turn", + "last_assistant_message": "Done.\n\nstatus: completed" + }, + "expected": { + "event": "Plugin subagent stopped", + "extra": { + "agent": "arcade-operator", + "status": "completed", + "subagent_session": "$hash:57946be7-1a73-40ca-a042-abb0f68d9445" + } + } +} diff --git a/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json b/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json new file mode 100644 index 0000000..d5777e8 --- /dev/null +++ b/test/fixtures/telemetry/copilot-cli/unrelated-prompt.json @@ -0,0 +1,19 @@ +{ + "prelude": [ + { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "What is on my calendar tomorrow?" + } + ], + "input": { + "hook_event_name": "UserPromptSubmit", + "session_id": "dd3beb80-4471-4513-99a4-a57d3d7c08df", + "timestamp": "2026-09-24T21:27:02.743Z", + "cwd": "/Users/someone/private-repo", + "prompt": "Fix the parser in src/main.ts" + }, + "expected": null +} diff --git a/test/hooks.test.mjs b/test/hooks.test.mjs index aac0f40..7f38eae 100644 --- a/test/hooks.test.mjs +++ b/test/hooks.test.mjs @@ -1,5 +1,8 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; import { test } from "node:test"; import { HOSTS } from "../hooks/hook-hosts.mjs"; import { shouldRemind } from "../hooks/prompt-filters.mjs"; @@ -32,28 +35,117 @@ test("every client in the hook table has an expected output format", () => { assert.deepEqual(Object.keys(EXPECTED_OUTPUT).sort(), Object.keys(HOSTS).sort()); }); +const HOOK_INPUT = JSON.stringify({ prompt: "What's on my calendar?", agent_type: "Explore" }); + +const assertPrintsContext = (hostName, event, result, label) => { + assert.equal(result.status, 0, `${label}: ${result.stderr}`); + const out = JSON.parse(result.stdout); + const text = out.hookSpecificOutput?.additionalContext ?? out.additional_context; + assert.ok(text, `${label}: no context in ${result.stdout}`); + assert.deepEqual(out, EXPECTED_OUTPUT[hostName](event, text), label); +}; + test("every command in every generated hooks.json runs and prints what its client reads", () => { for (const [hostName, { manifest, rootVariable }] of Object.entries(HOSTS)) { for (const [event, entries] of Object.entries(JSON.parse(readRepoFile(manifest)).hooks)) { for (const { command } of entries.flatMap((entry) => entry.hooks ?? [entry])) { // Telemetry prints nothing; the telemetry tests run its commands. if (command.includes("/hooks/telemetry.mjs")) continue; - const label = `${hostName} ${event}`; const result = spawnSync(command.replaceAll(`\${${rootVariable}}`, ROOT), { shell: true, - input: JSON.stringify({ prompt: "What's on my calendar?", agent_type: "Explore" }), + input: HOOK_INPUT, encoding: "utf8", }); - assert.equal(result.status, 0, `${label}: ${result.stderr}`); - const out = JSON.parse(result.stdout); - const text = out.hookSpecificOutput?.additionalContext ?? out.additional_context; - assert.ok(text, `${label}: no context in ${result.stdout}`); - assert.deepEqual(out, EXPECTED_OUTPUT[hostName](event, text), label); + assertPrintsContext(hostName, event, result, `${hostName} ${event}`); } } } }); +// VS Code reads com.github.copilot/hooks/hooks.json but runs each command as +// written, through `spawn(command, { shell: true })` from the workspace root, +// without setting or replacing PLUGIN_ROOT (microsoft/vscode main, 2026-09: +// pluginParsers.ts, hookExecutor.ts). Copilot CLI replaces ${PLUGIN_ROOT} in +// the command and also sets it in the environment. +const copilotEntries = () => + Object.entries(JSON.parse(readRepoFile(HOSTS.copilot.manifest)).hooks).flatMap(([event, entries]) => + entries.map((entry) => ({ event, ...entry })), + ); + +const PLUGIN_VARIABLES = ["PLUGIN_ROOT", "COPILOT_PLUGIN_DATA", "CLAUDE_PLUGIN_DATA"]; + +/** process.env without the variables a client sets for plugin hooks, plus `extra`. */ +const hookEnv = (extra = {}) => ({ + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLUGIN_VARIABLES.includes(name))), + ...extra, +}); + +const VSCODE_INPUT = JSON.stringify({ + hook_event_name: "SessionStart", + session_id: "vscode-session", + cwd: "/workspace", + prompt: "What's on my calendar?", + agent_type: "Explore", +}); + +// The `command` field is the macOS and Linux form; cmd.exe can't run it. +const skipOnWindows = { skip: process.platform === "win32" && "the command field is for macOS and Linux" }; + +test("in VS Code every Copilot hook command exits 0 without output", skipOnWindows, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-vscode-")); + for (const extra of [{}, { PLUGIN_ROOT: "/nonexistent" }]) { + for (const { event, command } of copilotEntries()) { + const label = `${event} ${command} ${JSON.stringify(extra)}`; + const result = spawnSync(command, { shell: true, cwd, env: hookEnv(extra), input: VSCODE_INPUT, encoding: "utf8" }); + assert.equal(result.status, 0, `${label}: ${result.stderr}`); + assert.equal(result.stdout, "", label); + assert.equal(result.stderr, "", label); + } + } +}); + +/** Checks the output of a Copilot hook run with PLUGIN_ROOT set to the repo. */ +const assertCopilotHookRan = (event, command, result, label) => { + if (command.includes("/hooks/telemetry.mjs")) { + assert.equal(result.status, 0, `${label}: ${result.stderr}`); + assert.equal(result.stdout, "", label); + return; + } + assertPrintsContext("copilot", event, result, label); +}; + +test("in Copilot CLI every Copilot hook command runs with PLUGIN_ROOT set only in the environment", skipOnWindows, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-copilot-")); + for (const { event, command } of copilotEntries()) { + const result = spawnSync(command, { shell: true, cwd, env: hookEnv({ PLUGIN_ROOT: ROOT }), input: HOOK_INPUT, encoding: "utf8" }); + assertCopilotHookRan(event, command, result, `${event} ${command}`); + } +}); + +// GitHub's ubuntu-latest image includes pwsh. +const POWERSHELL = (process.platform === "win32" ? ["pwsh", "powershell.exe"] : ["pwsh"]).find( + (exe) => !spawnSync(exe, ["-NoProfile", "-NonInteractive", "-Command", "exit 0"]).error, +); + +test("every Copilot hook's powershell command skips without PLUGIN_ROOT and runs with it", { skip: !POWERSHELL && "pwsh not found" }, () => { + const cwd = mkdtempSync(path.join(tmpdir(), "arcade-powershell-")); + const runPowerShell = (script, env) => + spawnSync(String(POWERSHELL), ["-NoProfile", "-NonInteractive", "-Command", script], { + cwd, + env, + input: HOOK_INPUT, + encoding: "utf8", + }); + for (const { event, command, powershell } of copilotEntries()) { + assert.ok(powershell, `${event} ${command}: no powershell command`); + const skipped = runPowerShell(powershell, hookEnv()); + assert.equal(skipped.status, 0, `${powershell}: ${skipped.stderr}`); + assert.equal(skipped.stdout, "", powershell); + assert.equal(skipped.stderr, "", powershell); + assertCopilotHookRan(event, powershell, runPowerShell(powershell, hookEnv({ PLUGIN_ROOT: ROOT })), powershell); + } +}); + test("shouldRemind skips acknowledgements and background task results", () => { for (const prompt of ["What's on my calendar?", "fix it", "go ahead and send it"]) { assert.equal(shouldRemind(prompt), true, prompt); diff --git a/test/telemetry-copilot.test.mjs b/test/telemetry-copilot.test.mjs new file mode 100644 index 0000000..4287498 --- /dev/null +++ b/test/telemetry-copilot.test.mjs @@ -0,0 +1,531 @@ +// @ts-check + +import assert from "node:assert/strict"; +import { spawn, spawnSync } from "node:child_process"; +import { mkdtempSync, readdirSync, readFileSync, rmSync } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { HOOKS, HOSTS, telemetryHookRows } from "../hooks/hook-hosts.mjs"; +import { SCOPE_DIRECTORY, SCOPE_TTL_MS } from "../hooks/hook-scope.mjs"; +import { ARCADE_USED_FILE, EVENT_ENV, OPT_OUT_ENV } from "../hooks/telemetry-config.mjs"; +import { buildEvent, isArcadeCall } from "../hooks/telemetry-events.mjs"; +import { isOptedOut, runTelemetry } from "../hooks/telemetry-run.mjs"; +import copilotAdapter, { COPILOT_ARCADE_SERVER } from "../hooks/telemetry-adapters/copilot-cli.mjs"; +import { buildHookManifest } from "../scripts/generate-manifests.mjs"; +import { readRepoFile, ROOT } from "./helpers.mjs"; +import { + assertMatchesContract, + assertNoLeak, + captureTelemetry, + expectedEvent, + hash16, + runTelemetryScript, + sleep, + startServer, + tempDataDir, + waitForRequests, +} from "./telemetry-helpers.mjs"; + +const FIXTURE_DIR = path.join(ROOT, "test/fixtures/telemetry/copilot-cli"); +const SESSION_ID = "dd3beb80-4471-4513-99a4-a57d3d7c08df"; +const SUBAGENT_ID = "57946be7-1a73-40ca-a042-abb0f68d9445"; +const OPERATOR = "arcade:arcade-operator"; +const MCP_TOOL_MATCHER = ".+-.+"; + +const copilotInput = (fields) => ({ + session_id: SESSION_ID, + timestamp: "2026-09-24T21:27:02.743Z", + cwd: "/Users/someone/private-repo", + ...fields, +}); + +const fixtureFiles = () => + readdirSync(FIXTURE_DIR) + .filter((name) => name.endsWith(".json") && name !== "hooks.enabled.json") + .sort(); + +const readFixture = (name) => JSON.parse(readFileSync(path.join(FIXTURE_DIR, name), "utf8")); + +const resolveExpected = (fixture, input) => { + if (fixture.expected === null) return null; + const extra = { ...fixture.expected.extra }; + if (extra.subagent_session?.startsWith("$hash:")) { + extra.subagent_session = hash16(extra.subagent_session.slice("$hash:".length)); + } + if (extra.subagent_session === SUBAGENT_ID) { + extra.subagent_session = hash16(SUBAGENT_ID); + } + const sessionId = input.session_id ?? SESSION_ID; + const event = expectedEvent(fixture.expected.event, extra, copilotAdapter, sessionId, undefined); + const os = /** @type {readonly string[]} */ (["darwin", "linux", "win32"]).includes(process.platform) + ? process.platform + : "other"; + event.properties.os = os; + return event; +}; + +const runFixture = async (fixture, dataDir) => { + for (const step of fixture.prelude ?? []) { + await captureTelemetry({ adapter: copilotAdapter, input: step, dataDir }); + } + return captureTelemetry({ adapter: copilotAdapter, input: fixture.input, dataDir }); +}; + +// An inherited opt-out would make the hook skip even with the hard-OFF gate open. +const PLUGIN_VARIABLES = [ + "PLUGIN_ROOT", + "COPILOT_PLUGIN_DATA", + "CLAUDE_PLUGIN_DATA", + OPT_OUT_ENV, + "DO_NOT_TRACK", + ...copilotAdapter.optOutSwitches.map(({ name }) => name), +]; +const hookEnv = (extra = {}) => ({ + ...Object.fromEntries(Object.entries(process.env).filter(([name]) => !PLUGIN_VARIABLES.includes(name))), + ...extra, +}); + +const POWERSHELL = (process.platform === "win32" ? ["pwsh", "powershell.exe"] : ["pwsh"]).find( + (exe) => !spawnSync(exe, ["-NoProfile", "-NonInteractive", "-Command", "exit 0"]).error, +); + +const skipOnWindows = { skip: process.platform === "win32" && "the command field is for macOS and Linux" }; + +// Classified as app work, so these hooks would send an event if the hard-OFF gate were open. +const SENDABLE_PROMPT = copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }); + +const SEND_SCRIPT = path.join(ROOT, "hooks", "telemetry-send.mjs"); +// Async spawn keeps this process's event loop free so the in-process server can answer. +const exitCode = (command, args, options) => + new Promise((resolve) => spawn(command, args, { ...options, stdio: "ignore" }).on("exit", resolve)); +const sendWithNode = (env) => exitCode(process.execPath, [SEND_SCRIPT], { env }); + +// Asserts nothing reached the server, then proves the hook environment resolves to it, +// so the empty request list could not come from sending somewhere else. +const assertCaptureObservesSend = async (server, env, send = sendWithNode) => { + assert.deepEqual(server.requests, []); + const event = buildEvent(SENDABLE_PROMPT, { + adapter: copilotAdapter, + os: "linux", + arcadeUsedBefore: false, + appWork: true, + }); + assert.ok(event); + assert.equal(await send({ ...env, [EVENT_ENV]: JSON.stringify(event) }), 0); + await waitForRequests(server.requests, 1); + assert.equal(server.requests.length, 1); + assert.equal(JSON.parse(server.requests[0].body).event, "Plugin prompt submitted"); +}; + +const enabledManifest = JSON.parse(readFileSync(path.join(FIXTURE_DIR, "hooks.enabled.json"), "utf8")); +const enabledTelemetryEntries = () => + Object.entries(enabledManifest.hooks).flatMap(([event, entries]) => + entries + .filter((entry) => entry.command?.includes("telemetry.mjs")) + .map((entry) => ({ event, ...entry })), + ); + +for (const file of fixtureFiles()) { + test(`fixture ${file} maps through captureTelemetry`, async () => { + const fixture = readFixture(file); + const dataDir = tempDataDir(); + const { sent } = await runFixture(fixture, dataDir); + const want = resolveExpected(fixture, fixture.input); + if (want === null) { + assert.equal(sent.length, 0, file); + return; + } + assert.equal(sent.length, 1, file); + assert.deepEqual(sent[0], want, file); + assertMatchesContract(sent[0], file); + assertNoLeak(JSON.stringify(sent[0]), /private-repo|dd3beb80|57946be7|SECRET/i); + }); +} + +test("Copilot-shaped hook input never carries turn and PreToolUse sends no attempt event", async () => { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + }); + const { sent } = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + }); + assert.equal(sent[0].properties.turn, undefined); + const attempt = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "PreToolUse", tool_name: "arcade-list_apps", tool_input: {} }), + dataDir, + }); + assert.equal(attempt.sent.length, 0); + assert.ok(!attempt.sent.some((event) => event.event === "Plugin tool attempted")); +}); + +test("session scope follows Copilot prompts and tools in one data dir", async () => { + const dataDir = tempDataDir(); + const baseNow = 1_700_000_000_000; + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow, + }); + const tool = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 1000, + }); + assert.equal(tool.sent.length, 1); + assert.equal(tool.sent[0].properties.turn, undefined); + + const unrelated = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "Fix the parser in src/main.ts" }), + dataDir, + now: baseNow + 2000, + }); + assert.equal(unrelated.sent.length, 0); + const silentTool = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 3000, + }); + assert.equal(silentTool.sent.length, 0); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow + 4000, + }); + const confirm = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "ok" }), + dataDir, + now: baseNow + 5000, + }); + assert.equal(confirm.sent.length, 1); + const scopeFile = path.join(dataDir, SCOPE_DIRECTORY, readdirSync(path.join(dataDir, SCOPE_DIRECTORY))[0]); + const expiresAt = JSON.parse(readFileSync(scopeFile, "utf8")).expiresAt; + assert.ok(expiresAt <= baseNow + SCOPE_TTL_MS + 5000); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "SessionStart", source: "startup" }), + dataDir, + now: baseNow + 6000, + }); + const afterStart = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + 7000, + }); + assert.equal(afterStart.sent.length, 0); + + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + now: baseNow + 8000, + }); + const expired = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "granola-Granola_ListMeetings", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }), + dataDir, + now: baseNow + SCOPE_TTL_MS + 9000, + }); + assert.equal(expired.sent.length, 0); +}); + +test("operator stop subagent_session matches the subagent session hash", async () => { + const subagentPrompt = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "UserPromptSubmit", + session_id: SUBAGENT_ID, + prompt: "Complete todo listing-arcade-apps", + }), + }); + const { sent } = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "SubagentStop", + agent_id: SUBAGENT_ID, + agent_type: OPERATOR, + last_assistant_message: "status: completed", + }), + }); + assert.equal(sent[0].properties.subagent_session, subagentPrompt.sent[0].properties.session); +}); + +test("arcade-used flag is stored in COPILOT_PLUGIN_DATA", async () => { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "UserPromptSubmit", + prompt: "What is on my calendar tomorrow?", + }), + dataDir, + }); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "private inbox" }, + }), + dataDir, + }); + assert.equal(readFileSync(path.join(dataDir, ARCADE_USED_FILE), "utf8"), "true"); + const second = await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "private inbox" }, + }), + dataDir, + }); + assert.equal(second.sent[0].properties.arcade_used_before, true); + assert.equal(isArcadeCall(second.sent[0]), true); +}); + +test("opt-out and missing data dir send nothing", async () => { + const input = copilotInput({ + hook_event_name: "PostToolUse", + tool_name: "arcade-Gmail_ListEmails", + tool_input: {}, + tool_result: { result_type: "success", text_result_for_llm: "[]" }, + }); + const cases = [ + ["ARCADE_PLUGIN_TELEMETRY=0", { ARCADE_PLUGIN_TELEMETRY: "0" }], + ["DO_NOT_TRACK=1", { DO_NOT_TRACK: "1" }], + ["COPILOT_OFFLINE=true", { COPILOT_OFFLINE: "true" }], + ["COPILOT_OFFLINE=1", { COPILOT_OFFLINE: "1" }], + ["no COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "" }], + ["relative COPILOT_PLUGIN_DATA", { COPILOT_PLUGIN_DATA: "relative/data" }], + ]; + for (const [label, env] of cases) { + const dataDir = tempDataDir(); + const { sent } = await captureTelemetry({ adapter: copilotAdapter, input, dataDir, env }); + assert.equal(sent.length, 0, label); + assert.deepEqual(readdirSync(dataDir), [], label); + } + for (const extra of [{ COPILOT_OFFLINE: "false" }, { COPILOT_OFFLINE: "0" }, { DISABLE_TELEMETRY: "1" }]) { + const dataDir = tempDataDir(); + await captureTelemetry({ + adapter: copilotAdapter, + input: copilotInput({ hook_event_name: "UserPromptSubmit", prompt: "What is on my calendar tomorrow?" }), + dataDir, + env: extra, + }); + const { sent } = await captureTelemetry({ adapter: copilotAdapter, input, dataDir, env: extra }); + assert.ok(sent.length > 0, JSON.stringify(extra)); + } + assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "true" }), true); + assert.equal(isOptedOut(copilotAdapter.optOutSwitches, { COPILOT_OFFLINE: "false" }), false); +}); + +test("malformed stdin sends nothing through runTelemetry", async () => { + const dataDir = tempDataDir(); + const sent = []; + await runTelemetry({ + input: { session_id: SESSION_ID, hook_event_name: "UserPromptSubmit" }, + adapter: copilotAdapter, + env: { ...process.env, COPILOT_PLUGIN_DATA: dataDir }, + enabled: true, + send: (event) => sent.push(event), + }); + assert.equal(sent.length, 0); +}); + +test("enabled Copilot manifest wiring matches hooks.enabled.json", async () => { + const rows = [ + ...HOOKS.filter((hook) => hook.script !== "telemetry.mjs"), + ...(await telemetryHookRows()).filter((row) => row.hosts.includes("copilot")), + ]; + const built = buildHookManifest("copilot", rows); + assert.deepEqual(built, enabledManifest); +}); + +test("checked-in Copilot manifest has no telemetry rows", () => { + assert.equal(readRepoFile(HOSTS.copilot.manifest).includes("telemetry.mjs"), false); +}); + +test("enabled telemetry commands exit quietly while telemetry is off", skipOnWindows, async () => { + const server = await startServer(); + const cwd = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-telemetry-off-")); + const dataDir = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-data-")); + const captureEnv = { COPILOT_PLUGIN_DATA: dataDir, ARCADE_PLUGIN_TELEMETRY: "1", ARCADE_PLUGIN_TELEMETRY_HOST: server.url }; + try { + for (const { command } of enabledTelemetryEntries()) { + for (const extra of [{}, { PLUGIN_ROOT: ROOT }]) { + const result = spawnSync(command, { + shell: true, + cwd, + env: hookEnv({ ...extra, ...captureEnv }), + input: JSON.stringify(SENDABLE_PROMPT), + encoding: "utf8", + }); + assert.equal(result.status, 0, command); + assert.equal(result.stdout, "", command); + assert.equal(result.stderr, "", command); + } + } + await sleep(200); + await assertCaptureObservesSend(server, hookEnv(captureEnv)); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + rmSync(cwd, { recursive: true, force: true }); + rmSync(dataDir, { recursive: true, force: true }); + await server.close(); + } +}); + +test( + "enabled telemetry powershell commands exit quietly while telemetry is off", + { skip: !POWERSHELL && "pwsh not found" }, + async () => { + const server = await startServer(); + const cwd = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-ps-")); + const dataDir = mkdtempSync(path.join(os.tmpdir(), "arcade-copilot-ps-data-")); + try { + const runPs = (script, env, input) => + spawnSync(String(POWERSHELL), ["-NoProfile", "-NonInteractive", "-Command", script], { + cwd, + env: hookEnv({ ARCADE_PLUGIN_TELEMETRY_HOST: server.url, ...env }), + input, + encoding: "utf8", + }); + for (const { powershell } of enabledTelemetryEntries()) { + assert.ok(powershell); + const input = JSON.stringify(SENDABLE_PROMPT); + const skipped = runPs(powershell, {}, input); + assert.equal(skipped.status, 0); + assert.equal(skipped.stdout, ""); + const ran = runPs(powershell, { PLUGIN_ROOT: ROOT, COPILOT_PLUGIN_DATA: dataDir }, input); + assert.equal(ran.status, 0); + assert.equal(ran.stdout, ""); + } + await sleep(200); + await assertCaptureObservesSend(server, hookEnv({ ARCADE_PLUGIN_TELEMETRY_HOST: server.url }), (env) => + exitCode( + String(POWERSHELL), + ["-NoProfile", "-NonInteractive", "-Command", `& '${process.execPath}' '${SEND_SCRIPT}'; exit $LASTEXITCODE`], + { cwd, env }, + ), + ); + assert.deepEqual(readdirSync(dataDir), []); + } finally { + rmSync(cwd, { recursive: true, force: true }); + rmSync(dataDir, { recursive: true, force: true }); + await server.close(); + } + }, +); + +test("the Copilot MCP tool matcher picks server tools, not built-ins", () => { + const matcher = new RegExp(`^(?:${MCP_TOOL_MATCHER})$`); + for (const name of ["arcade-Arcade_UseTool", "github-mcp-server-get_issue"]) assert.match(name, matcher); + for (const name of ["Bash", "Agent", "view"]) assert.doesNotMatch(name, matcher); +}); + +test("normalize reads tool_result text_result_for_llm and ignores top-level tool_response", () => { + const normalized = copilotAdapter.normalize({ + session_id: SESSION_ID, + agent_type: OPERATOR, + tool_name: "arcade-System_ManageAuthorization", + tool_response: '{"providers":[{"status":"authorization_required"}]}', + tool_result: { result_type: "success", text_result_for_llm: '{"providers":[]}' }, + transcript_path: "/SECRET/path", + cwd: "/SECRET/cwd", + }); + assert.equal(normalized.tool_response, '{"providers":[]}'); + assert.equal(normalized.prompt_id, undefined); + assert.equal(normalized.session_id, SESSION_ID); + const withPromptId = copilotAdapter.normalize({ session_id: SESSION_ID, prompt_id: "turn-1" }); + assert.equal(withPromptId.prompt_id, "turn-1"); + const built = buildEvent( + { ...normalized, hook_event_name: "PostToolUse" }, + { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }, + ); + assert.equal(built?.properties.auth_needed, false); + assertNoLeak(JSON.stringify(built), /SECRET/i); +}); + +test("normalize does not map SubagentStart camelCase fields into hook input", () => { + const normalized = copilotAdapter.normalize({ + sessionId: SESSION_ID, + agentName: OPERATOR, + agentDisplayName: "arcade-operator", + }); + assert.equal(normalized.session_id, undefined); + assert.equal(normalized.agent_type, undefined); +}); + +test("Copilot CLI sends no built-in tool events", () => { + const options = { adapter: copilotAdapter, os: "darwin", arcadeUsedBefore: false, appWork: true }; + assert.equal( + buildEvent(copilotInput({ hook_event_name: "PostToolUse", tool_name: "WebFetch" }), options), + null, + ); + assert.equal( + buildEvent( + copilotInput({ hook_event_name: "PostToolUse", tool_name: "Bash", tool_input: { command: "gh pr list" } }), + { ...options, cli: "gh" }, + ), + null, + ); +}); + +test("telemetry.mjs with --host copilot exits quietly while disabled", () => { + const dataDir = tempDataDir(); + const result = runTelemetryScript( + JSON.stringify(copilotInput({ hook_event_name: "PostToolUse", tool_name: "arcade-Gmail_ListEmails" })), + { COPILOT_PLUGIN_DATA: dataDir }, + "copilot", + ); + assert.equal(result.status, 0); + assert.equal(result.stdout, ""); + assert.deepEqual(readdirSync(dataDir), []); +}); + +test("arcade server prefix matches mcp.json", () => { + const [server] = Object.keys(JSON.parse(readRepoFile("mcp.json")).mcpServers); + assert.equal(COPILOT_ARCADE_SERVER, server); +});