Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions apps/vscode-e2e/fixtures/modes.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,20 @@
}
]
}
},
{
"match": {
"userMessage": "Use the `switch_mode` tool to switch to debug mode."
},
"response": {
"toolCalls": [
{
"name": "switch_mode",
"arguments": "{\"mode_slug\":\"debug\",\"reason\":\"User requested to switch to debug mode.\"}",
"id": "call_modes_switch_002"
}
]
}
}
]
}
95 changes: 95 additions & 0 deletions apps/vscode-e2e/src/fixtures/view-state.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import type { ChatCompletionRequest, ChatMessage, LLMock } from "@copilotkit/aimock"

const TASKS = ["A", "B", "C"] as const
const ROUNDS = 10

const MODE_SEQUENCES: Record<(typeof TASKS)[number], string[]> = {
A: ["ask", "debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code"],
B: ["debug", "architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask"],
C: ["architect", "orchestrator", "code", "ask", "debug", "architect", "orchestrator", "code", "ask", "debug"],
}

const markerFor = (taskName: (typeof TASKS)[number]) => `FOLLOWUP_MODE_ISOLATION_${taskName}`
const answerFor = (taskName: (typeof TASKS)[number], round: number) => `${taskName} follow-up round ${round}`
const callIdFor = (taskName: (typeof TASKS)[number], round: number) =>
`call_followup_mode_${taskName.toLowerCase()}_${String(round).padStart(2, "0")}`

const lastToolResultContains = (req: ChatCompletionRequest, toolCallId: string, expected: string[]) => {
const messages = Array.isArray(req?.messages) ? req.messages : []
const toolMessage = messages.filter((message: ChatMessage) => message?.role === "tool").at(-1)
const content = toolMessage?.content

return (
toolMessage?.tool_call_id === toolCallId &&
typeof content === "string" &&
expected.every((text) => content.includes(text))
)
}

const followupToolCall = (taskName: (typeof TASKS)[number], round: number) => ({
name: "ask_followup_question",
arguments: JSON.stringify({
question: `Task ${taskName}: choose mode for round ${round}`,
follow_up: [
{
text: answerFor(taskName, round),
mode: MODE_SEQUENCES[taskName][round - 1],
},
],
}),
id: callIdFor(taskName, round),
})

export const getFollowupModeIsolationPlan = () =>
TASKS.map((taskName) => ({
taskName,
marker: markerFor(taskName),
rounds: MODE_SEQUENCES[taskName].map((mode, index) => ({
round: index + 1,
answer: answerFor(taskName, index + 1),
mode,
})),
}))

export function addViewStateFixtures(mock: InstanceType<typeof LLMock>) {
for (const taskName of TASKS) {
mock.addFixture({
match: {
userMessage: markerFor(taskName),
},
response: {
toolCalls: [followupToolCall(taskName, 1)],
},
})

for (let round = 1; round < ROUNDS; round++) {
mock.addFixture({
match: {
predicate: (req) =>
lastToolResultContains(req, callIdFor(taskName, round), [answerFor(taskName, round)]),
},
response: {
toolCalls: [followupToolCall(taskName, round + 1)],
},
})
}

mock.addFixture({
match: {
predicate: (req) =>
lastToolResultContains(req, callIdFor(taskName, ROUNDS), [answerFor(taskName, ROUNDS)]),
},
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({
result: `Task ${taskName} completed ${ROUNDS} follow-up mode switches.`,
}),
id: `call_followup_mode_${taskName.toLowerCase()}_complete`,
},
],
},
})
}
}
37 changes: 37 additions & 0 deletions apps/vscode-e2e/src/runTest.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import { addUseMcpToolResultFixtures } from "./fixtures/use-mcp-tool"
import { addWriteToFileResultFixtures } from "./fixtures/write-to-file"
import { createScenarioWorkspace, removeScenarioWorkspace } from "./restart/scenarioWorkspace"
import { runRestartScenario } from "./restart/vscodeCoordinator"
import { toolResultContains } from "./fixtures/tool-result"
import { addViewStateFixtures } from "./fixtures/view-state"

function getCliFlagValue(flag: string) {
return process.argv.find((arg, index) => process.argv[index - 1] === flag)
Expand Down Expand Up @@ -143,6 +145,41 @@ async function main() {
addUseMcpToolResultFixtures(mock)
addWriteToFileResultFixtures(mock)
addDeepSeekV4Fixtures(mock)
addViewStateFixtures(mock)

// Model-agnostic predicate fixtures for the view-state suite's post-switch
// turns. They coexist with the legacy model-scoped regex fixture below
// (shared response id call_modes_post_switch_001) so the modes suite keeps
// its OpenRouter-scoped match while view-state runs under any default model.
mock.addFixture({
match: {
predicate: (req) => toolResultContains(req, "call_modes_switch_001", []),
},
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: "Switched to ❓ Ask mode as requested." }),
id: "call_modes_post_switch_001",
},
],
},
})

mock.addFixture({
match: {
predicate: (req) => toolResultContains(req, "call_modes_switch_002", []),
},
response: {
toolCalls: [
{
name: "attempt_completion",
arguments: JSON.stringify({ result: "Switched to 🪲 Debug mode as requested." }),
id: "call_modes_post_switch_002",
},
],
},
})

// The modes test (switch_mode → ask) triggers a second API call whose last
// user message starts with <environment_details> directly — no <user_message>
Expand Down
Loading
Loading