Skip to content

Commit 74f8b0d

Browse files
elkaixelkaix
committed
fix(agent-core-v2): restore turn-start model fallback
When the bound model no longer resolves at turn start, switch to the best-ranked ready model and emit ModelFallbackSwitched, a model-fallback warning, and model_fallback_triggered telemetry. #323 dropped this.
1 parent ecc2c3b commit 74f8b0d

9 files changed

Lines changed: 301 additions & 3 deletions

File tree

‎.changeset/restore-turn-model-fallback.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"@pymodel/pythinker-code": patch
3+
---
4+
5+
A session whose model was deleted or lost its provider switches to the best available model at the next turn, with a warning, instead of failing the turn.

‎packages/agent-core-v2/src/agent/llmRequester/llmRequesterService.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -709,7 +709,7 @@ export class AgentLLMRequesterService implements IAgentLLMRequesterService {
709709
let snapshot = this.turnConfigs.get(turnId);
710710
if (snapshot === undefined) {
711711
snapshot = {
712-
resolved: this.profile.resolveModelContext(),
712+
resolved: this.profile.resolveModelContext(turnId),
713713
params: this.profile.resolveRequestParams(),
714714
systemPrompt: this.profile.getSystemPrompt(),
715715
};

‎packages/agent-core-v2/src/agent/profile/profileOps.ts‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { nothing, original } from 'immer';
33
import { z } from 'zod';
44

55
import type { EnvironmentDisclosureSnapshot } from '#/app/agentProfileCatalog/agentProfileCatalog';
6-
import { AgentEvent2 } from '#/app/event/event2';
6+
import { AgentEvent2, Event2 } from '#/app/event/event2';
77
import type { ThinkingEffort } from '#human/llm/thinking';
88
import { defineState } from '#/state/state';
99

@@ -127,6 +127,18 @@ export class WarningIssued extends AgentEvent2<WarningIssuedPayload> {
127127
}
128128
export interface WarningIssued extends WarningIssuedPayload {}
129129

130+
export interface ModelFallbackSwitchedPayload {
131+
readonly turnId?: number;
132+
readonly fromModel: string;
133+
readonly toModel: string;
134+
}
135+
136+
export class ModelFallbackSwitched extends Event2<ModelFallbackSwitchedPayload> {
137+
static override readonly type = 'turn.model_fallback.switched';
138+
static override readonly observable = true;
139+
}
140+
export interface ModelFallbackSwitched extends ModelFallbackSwitchedPayload {}
141+
130142
export const profileKey = defineState(
131143
'profile',
132144
(): ProfileModelState => ({

‎packages/agent-core-v2/src/agent/profile/profileService.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,10 @@ import { IModelCatalog, type Model } from '#/llm-adapter/model/catalog';
88
import { type ModelOverrides } from '#/llm-adapter/model/model.types';
99
import { type ModelRequestParams, type SamplingOptions } from '#/llm-adapter/model/model-requester';
1010
import { IProtocolAdapterRegistry } from '#/llm-adapter/protocol/protocol';
11+
import { IModelService } from '#/llm-adapter/model/model';
12+
import { rankDefaultModelCandidates } from '#/llm-adapter/model/default-model-policy';
13+
import { resolveModelForReady } from '#/llm-adapter/model/model-auth';
14+
import { IProviderService } from '#/llm-adapter/provider/provider';
1115
import {
1216
drivesThinkingThroughTraits,
1317
modelSupportsThinkingEffort,
@@ -74,6 +78,7 @@ import {
7478
profileKey,
7579
ToolsResetActiveTools,
7680
ToolsSetActiveTools,
81+
ModelFallbackSwitched,
7782
WarningIssued,
7883
type ActiveToolsState,
7984
type ConfigUpdatePayload,
@@ -162,6 +167,8 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ
162167
@IPluginService private readonly plugins: IPluginService,
163168
@IAgentIdentity private readonly identity: IAgentIdentity,
164169
@IAgentAgentsMdReminderService private readonly agentsMdReminder: IAgentAgentsMdReminderService,
170+
@IModelService private readonly models: IModelService,
171+
@IProviderService private readonly providers: IProviderService,
165172
) {
166173
super();
167174
this.states.contributeState(profileKey);
@@ -442,7 +449,8 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ
442449
return this.resolveThinkingState(this.tryResolveRawModel()).effective;
443450
}
444451

445-
resolveModelContext(): ProfileModelContext {
452+
resolveModelContext(turnId?: number): ProfileModelContext {
453+
if (turnId !== undefined) this.ensureResolvableModel(turnId);
446454
const modelAlias = this.model;
447455
const model = this.modelCatalog.get(modelAlias);
448456
const loopControl = this.config.get<LoopControl>('loopControl');
@@ -458,6 +466,36 @@ export class AgentProfileService extends Disposable implements IAgentProfileServ
458466
};
459467
}
460468

469+
private ensureResolvableModel(turnId: number): void {
470+
const alias = this.modelAlias;
471+
if (alias === undefined) return;
472+
const models = this.models.list();
473+
const providers = this.providers.list();
474+
const defaultProvider = this.providers.getDefaultProvider();
475+
const isReady = (id: string): boolean =>
476+
resolveModelForReady(id, models, providers, defaultProvider).resolved;
477+
const resolution = resolveModelForReady(alias, models, providers, defaultProvider);
478+
if (resolution.resolved) return;
479+
const resolvedTo = rankDefaultModelCandidates(models).find(isReady);
480+
if (resolvedTo === undefined || resolvedTo === alias) return;
481+
this.update({ modelAlias: resolvedTo });
482+
this.telemetry.track2('model_fallback_triggered', {
483+
turn_id: turnId,
484+
from_model: alias,
485+
to_model: resolvedTo,
486+
});
487+
void this.dispatcher.dispatch(
488+
new ModelFallbackSwitched({ turnId, fromModel: alias, toModel: resolvedTo }),
489+
);
490+
void this.dispatcher.dispatch(
491+
new WarningIssued({
492+
agentId: this.scopeContext.agentId,
493+
code: 'model-fallback',
494+
message: `Model "${alias}" is no longer available (${resolution.reason}); switched to "${resolvedTo}".`,
495+
}),
496+
);
497+
}
498+
461499
resolveRequestParams(): ModelRequestParams {
462500
const model = this.tryResolveRawModel();
463501
const thinking = this.resolveThinkingState(model);

‎packages/agent-core-v2/src/app/telemetry/events.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,12 @@ export interface TurnEndedEvent {
8585
enabled_plugins?: string;
8686
}
8787

88+
export interface ModelFallbackEvent {
89+
turn_id: number;
90+
from_model: string;
91+
to_model: string;
92+
}
93+
8894
export interface PromptCacheProbeEvent {
8995
source: 'fork';
9096
turn_id: number;
@@ -652,6 +658,16 @@ export const telemetryEventDefinitions = {
652658
'Comma-separated sorted ids of enabled, loaded plugins when the turn ends; empty string for a known empty set, absent when no plugin snapshot is available',
653659
},
654660
}),
661+
model_fallback_triggered: defineAgentTelemetryEvent<ModelFallbackEvent>({
662+
owner: 'pythinker-code',
663+
comment:
664+
'At turn start the bound model no longer resolved, so the agent switched to the best-ranked ready model.',
665+
properties: {
666+
turn_id: 'Per-agent turn index (main or subagent); pair with agent_id to locate a turn within a session',
667+
from_model: 'Bound model alias that no longer resolved',
668+
to_model: 'Ready model alias the agent switched to',
669+
},
670+
}),
655671
prompt_cache_probe: defineAgentTelemetryEvent<PromptCacheProbeEvent>({
656672
owner: 'pythinker-code',
657673
comment:

‎packages/agent-core-v2/src/llm-adapter/model/default-model-policy.ts‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import type { ModelRecord, ModelsSection } from './model';
2+
3+
const TOOL_USE_CAPABILITY = 'tool_use';
4+
5+
export function rankDefaultModelCandidates(models: ModelsSection): string[] {
6+
return Object.entries(models)
7+
.filter(([, record]) => isEligibleDefaultModel(record))
8+
.map(([id, record]) => ({
9+
id,
10+
declaresToolUse: declaresToolUse(record),
11+
context: effectiveContextSize(record) ?? 0,
12+
}))
13+
.toSorted((a, b) => {
14+
if (a.declaresToolUse !== b.declaresToolUse) return a.declaresToolUse ? -1 : 1;
15+
if (a.context !== b.context) return b.context - a.context;
16+
const aParts = splitModelId(a.id);
17+
const bParts = splitModelId(b.id);
18+
if (aParts.providerId !== bParts.providerId) {
19+
return aParts.providerId.localeCompare(bParts.providerId);
20+
}
21+
const byModelPart = bParts.modelPart.localeCompare(aParts.modelPart, 'en', {
22+
numeric: true,
23+
});
24+
if (byModelPart !== 0) return byModelPart;
25+
return a.id.localeCompare(b.id);
26+
})
27+
.map((candidate) => candidate.id);
28+
}
29+
30+
function isEligibleDefaultModel(record: ModelRecord): boolean {
31+
const context = effectiveContextSize(record);
32+
if (context !== undefined && context <= 0) return false;
33+
const capabilities = effectiveCapabilities(record);
34+
if (capabilities === undefined || capabilities.length === 0) return true;
35+
return capabilities.some((entry) => entry.trim().toLowerCase() === TOOL_USE_CAPABILITY);
36+
}
37+
38+
function splitModelId(id: string): { readonly providerId: string; readonly modelPart: string } {
39+
const slash = id.indexOf('/');
40+
if (slash === -1) return { providerId: '', modelPart: id };
41+
return { providerId: id.slice(0, slash), modelPart: id.slice(slash + 1) };
42+
}
43+
44+
function effectiveCapabilities(record: ModelRecord): readonly string[] | undefined {
45+
return record.overrides?.capabilities ?? record.capabilities;
46+
}
47+
48+
function effectiveContextSize(record: ModelRecord): number | undefined {
49+
return record.overrides?.maxContextSize ?? record.maxContextSize;
50+
}
51+
52+
function declaresToolUse(record: ModelRecord): boolean {
53+
const capabilities = effectiveCapabilities(record);
54+
if (capabilities === undefined) return false;
55+
return capabilities.some((entry) => entry.trim().toLowerCase() === TOOL_USE_CAPABILITY);
56+
}

‎packages/agent-core-v2/test/agent/profile/modelFallback.test.ts‎

Lines changed: 157 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,157 @@
1+
import { emptyUsage } from '#human/llm/usage';
2+
import { afterEach, describe, expect, it } from 'vitest';
3+
4+
import { IEventBus } from '#/app/event/eventBus';
5+
import { IModelService } from '#/llm-adapter/model/model';
6+
import { IAgentLLMRequesterService } from '#/agent/llmRequester/llmRequester';
7+
import { IAgentProfileService } from '#/agent/profile/profile';
8+
import { ModelFallbackSwitched, WarningIssued } from '#/agent/profile/profileOps';
9+
10+
import { recordingTelemetry, type TelemetryRecord } from '../../app/telemetry/stubs';
11+
import {
12+
createTestAgent,
13+
llmGenerateServices,
14+
requesterFromGenerateFn,
15+
telemetryServices,
16+
type TestAgentContext,
17+
} from '../../harness';
18+
19+
function okGenerate() {
20+
let calls = 0;
21+
return {
22+
services: llmGenerateServices(
23+
requesterFromGenerateFn(async () => {
24+
calls += 1;
25+
return {
26+
id: `mf-${String(calls)}`,
27+
message: {
28+
role: 'assistant' as const,
29+
content: [{ type: 'text' as const, text: 'ok' }],
30+
toolCalls: [],
31+
},
32+
usage: emptyUsage(),
33+
finishReason: 'completed' as const,
34+
rawFinishReason: 'stop',
35+
};
36+
}),
37+
),
38+
calls: () => calls,
39+
};
40+
}
41+
42+
describe('turn-start model fallback', () => {
43+
let ctx: TestAgentContext;
44+
45+
afterEach(async () => {
46+
try {
47+
await ctx.expectResumeMatches();
48+
} finally {
49+
await ctx.dispose();
50+
}
51+
});
52+
53+
function requestTurn(turnId: number): Promise<unknown> {
54+
return ctx.get(IAgentLLMRequesterService).request({ source: { type: 'turn', turnId, step: 1 } });
55+
}
56+
57+
function captureSwitches(): ModelFallbackSwitched[] {
58+
const switched: ModelFallbackSwitched[] = [];
59+
ctx.get(IEventBus).subscribe(ModelFallbackSwitched, (event) => switched.push(event));
60+
return switched;
61+
}
62+
63+
it('switches a dead bound model to the best-ranked ready model', async () => {
64+
const generate = okGenerate();
65+
const records: TelemetryRecord[] = [];
66+
ctx = createTestAgent(generate.services, telemetryServices(recordingTelemetry(records)));
67+
const switched = captureSwitches();
68+
const notices: WarningIssued[] = [];
69+
ctx.get(IEventBus).subscribe(WarningIssued, (event) => notices.push(event));
70+
ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' });
71+
72+
await requestTurn(1);
73+
74+
expect(generate.calls()).toBe(1);
75+
expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model');
76+
expect(switched).toHaveLength(1);
77+
expect(switched[0]).toMatchObject({ turnId: 1, fromModel: 'dead-model', toModel: 'mock-model' });
78+
expect(notices).toHaveLength(1);
79+
expect(notices[0]?.code).toBe('model-fallback');
80+
expect(notices[0]?.message).toContain('dead-model');
81+
expect(notices[0]?.message).toContain('mock-model');
82+
const fallbackEvents = records.filter((record) => record.event === 'model_fallback_triggered');
83+
expect(fallbackEvents).toHaveLength(1);
84+
expect(fallbackEvents[0]?.properties).toMatchObject({
85+
turn_id: 1,
86+
from_model: 'dead-model',
87+
to_model: 'mock-model',
88+
});
89+
});
90+
91+
it('switches a model whose provider was deleted', async () => {
92+
ctx = createTestAgent(okGenerate().services, {
93+
initialConfig: {
94+
models: {
95+
'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 },
96+
},
97+
},
98+
});
99+
const switched = captureSwitches();
100+
ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' });
101+
102+
await requestTurn(1);
103+
104+
expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model');
105+
expect(switched).toHaveLength(1);
106+
expect(switched[0]).toMatchObject({ fromModel: 'orphan-model', toModel: 'mock-model' });
107+
});
108+
109+
it('skips an unready candidate that would win the ranking', async () => {
110+
ctx = createTestAgent(okGenerate().services, {
111+
initialConfig: {
112+
models: {
113+
'zombie-model': { provider: 'no-such-provider', model: 'zombie', maxContextSize: 100_000_000 },
114+
},
115+
},
116+
});
117+
const switched = captureSwitches();
118+
ctx.get(IAgentProfileService).update({ modelAlias: 'dead-model' });
119+
120+
await requestTurn(1);
121+
122+
expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model');
123+
expect(switched).toHaveLength(1);
124+
expect(switched[0]).toMatchObject({ fromModel: 'dead-model', toModel: 'mock-model' });
125+
});
126+
127+
it('keeps the dead alias when no candidate is ready', async () => {
128+
ctx = createTestAgent(okGenerate().services, {
129+
initialConfig: {
130+
models: {
131+
'aaa-model': { provider: 'no-such-provider', model: 'aaa', maxContextSize: 5_000 },
132+
'orphan-model': { provider: 'no-such-provider', model: 'orphan', maxContextSize: 1_000 },
133+
},
134+
},
135+
});
136+
const switched = captureSwitches();
137+
await ctx.get(IModelService).delete('mock-model');
138+
ctx.get(IAgentProfileService).update({ modelAlias: 'orphan-model' });
139+
140+
await requestTurn(1).catch(() => undefined);
141+
142+
expect(ctx.get(IAgentProfileService).getModel()).toBe('orphan-model');
143+
expect(switched).toHaveLength(0);
144+
});
145+
146+
it('leaves a healthy bound model untouched', async () => {
147+
const records: TelemetryRecord[] = [];
148+
ctx = createTestAgent(okGenerate().services, telemetryServices(recordingTelemetry(records)));
149+
const switched = captureSwitches();
150+
151+
await requestTurn(1);
152+
153+
expect(ctx.get(IAgentProfileService).getModel()).toBe('mock-model');
154+
expect(switched).toHaveLength(0);
155+
expect(records.filter((record) => record.event === 'model_fallback_triggered')).toHaveLength(0);
156+
});
157+
});

‎packages/agent-core-v2/test/session/agentLifecycle/agentLifecycle.test.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ import { agentContextOf, IAgentScopeContext } from '#/agent/scopeContext/scopeCo
3636
import { IAgentIdentity } from '#/app/agentIdentity/agentIdentity';
3737
import { IBuiltinAgentProfileLoader } from '#/app/agentProfileCatalog/builtinAgentProfileLoader';
3838
import { IModelCatalog } from '#/llm-adapter/model/catalog';
39+
import { IModelService } from '#/llm-adapter/model/model';
40+
import { IProviderService } from '#/llm-adapter/provider/provider';
3941
import type { ToolCall } from '#human/llm/message';
4042
import { IProtocolAdapterRegistry } from '#/llm-adapter/protocol/protocol';
4143
import { IHostClock } from '#/os/interface/hostClock';
@@ -325,6 +327,11 @@ describe('AgentLifecycleService', () => {
325327
cwd: '/tmp/pythinker-agentLifecycle-home',
326328
getEnv: () => undefined,
327329
} as unknown as IBootstrapService);
330+
ix.stub(IModelService, { list: () => ({}) } as unknown as IModelService);
331+
ix.stub(IProviderService, {
332+
list: () => ({}),
333+
getDefaultProvider: () => undefined,
334+
} as unknown as IProviderService);
328335
ix.stub(IFlagService, {
329336
_serviceBrand: undefined,
330337
enabled: () => false,

0 commit comments

Comments
 (0)