diff --git a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx
index 7c24a4d3c..80cc4ac82 100644
--- a/application/v2_ui/src/components/chat/OrchestrationRunView.tsx
+++ b/application/v2_ui/src/components/chat/OrchestrationRunView.tsx
@@ -1,4 +1,5 @@
// OrchestrationRunView.tsx
+import { ReasoningAdjustmentNotice } from './ReasoningAdjustmentNotice';
// The full step list for one run or one pending plan, with the narrowing edits and live status.
//
// This is the detail the inline card deliberately omits. It reads the RAW plan, not the edited
@@ -269,7 +270,10 @@ export function OrchestrationRunView({
const summary = stepRuntime[step.step_id]?.summary ?? '';
const removed = new Set(edits.removed_document_ids[step.step_id] ?? []);
const removable = new Set(stepRemovableDocumentIds(step));
- const documents = stepDocumentIds(step);
+ const explicitDocuments = stepDocumentIds(step);
+ const documents = step.capability_id === 'document_search' && explicitDocuments.length === 0
+ ? [...planInputDocuments].filter(([, document]) => document.selectedByUser).map(([id]) => id)
+ : explicitDocuments;
const args = readableArguments(step);
// A step can defer its documents to whatever an earlier step finds, so it may have
// none of its own to show.
@@ -477,6 +481,7 @@ export function OrchestrationRunView({
return (
+
{readOnly && !previewPlan ? (
diff --git a/application/v2_ui/src/components/chat/ReasoningAdjustmentNotice.tsx b/application/v2_ui/src/components/chat/ReasoningAdjustmentNotice.tsx
new file mode 100644
index 000000000..73b2559b9
--- /dev/null
+++ b/application/v2_ui/src/components/chat/ReasoningAdjustmentNotice.tsx
@@ -0,0 +1,16 @@
+// ReasoningAdjustmentNotice.tsx
+import { normalizeReasoningAdjustments, reasoningAdjustmentMessage } from '../../lib/reasoning';
+
+export function ReasoningAdjustmentNotice({ adjustments }: { adjustments: unknown }) {
+ const messages = [...new Set(normalizeReasoningAdjustments(adjustments).map((resolution) =>
+ reasoningAdjustmentMessage(
+ resolution, typeof resolution.model_name === 'string' ? resolution.model_name : undefined,
+ ),
+ ))];
+ if (!messages.length) return null;
+ return (
+
+ {messages.map((message) =>
{message}
)}
+
+ );
+}
diff --git a/application/v2_ui/src/components/workspaceAgents/AgentAdvancedFields.tsx b/application/v2_ui/src/components/workspaceAgents/AgentAdvancedFields.tsx
index b8104d3e8..3d38c531c 100644
--- a/application/v2_ui/src/components/workspaceAgents/AgentAdvancedFields.tsx
+++ b/application/v2_ui/src/components/workspaceAgents/AgentAdvancedFields.tsx
@@ -2,6 +2,8 @@
import type { Dispatch, SetStateAction } from 'react';
import { getModelSupportedLevels } from '../../lib/reasoning';
+import type { ModelCatalogEntry } from '../../lib/models';
+import { useBootstrapStore } from '../../stores/bootstrapStore';
import type { AgentConfiguration, AgentEditorOptions, AuthoringResource } from '../../lib/workspaceAuthoring';
import {
AGENT_INPUT_CLASS, agentModelChoices, agentStoredArrayEditError, agentText, clearAgentDraftFields, parseAgentSettings,
@@ -33,7 +35,12 @@ export function AgentAdvancedFields({
options: AgentEditorOptions;
}) {
const selectedModel = selectedAgentModel(draft, agentModelChoices(options));
- const levels = getModelSupportedLevels(selectedModel?.modelName || draft.model_id || draft.azure_openai_gpt_deployment || draft.azure_agent_apim_gpt_deployment);
+ const models = useBootstrapStore((state) => state.data?.catalogs?.models) as ModelCatalogEntry[] | undefined;
+ const catalogModel = models?.find((model) => draft.model_endpoint_id
+ ? model.endpoint_id === draft.model_endpoint_id && model.model_id === draft.model_id
+ : model.model_name === selectedModel?.modelName &&
+ model.deployment_name === (draft.azure_openai_gpt_deployment || draft.azure_agent_apim_gpt_deployment));
+ const levels = getModelSupportedLevels(catalogModel?.reasoning_capabilities);
const rawSettings = typeof draft._editor_settings_text === 'string' ? draft._editor_settings_text : JSON.stringify(draft.other_settings, null, 2);
const error = agentAdvancedError(draft, original);
return (
diff --git a/application/v2_ui/src/lib/chatRequestSelection.ts b/application/v2_ui/src/lib/chatRequestSelection.ts
index cc21d99a5..cafe55392 100644
--- a/application/v2_ui/src/lib/chatRequestSelection.ts
+++ b/application/v2_ui/src/lib/chatRequestSelection.ts
@@ -25,7 +25,7 @@
// being fixed here rather than the behaviour being matched.
import { agentInfoForSelection } from './agents';
-import { modelIdentityForSelection, type ModelCatalogEntry } from './models';
+import { findModel, modelIdentityForSelection, type ModelCatalogEntry } from './models';
import { requestReasoningEffort } from './reasoning';
import type { Json } from './types';
@@ -71,10 +71,10 @@ export function buildSelectionFields(input: SelectionInput): SelectionFields {
...modelIdentityForSelection(input.models, input.modelDeployment),
};
- // `none` is a real choice in the picker but not a value the endpoint takes, so it is
- // dropped here rather than at each caller: this is where a request's reasoning level is
- // decided, and the classic client's getCurrentReasoningEffort() returns null for it.
- const reasoningEffort = requestReasoningEffort(input.reasoningEffort);
+ const reasoningEffort = requestReasoningEffort(
+ input.reasoningEffort,
+ findModel(input.models, input.modelDeployment)?.reasoning_capabilities,
+ );
if (reasoningEffort) {
fields.reasoning_effort = reasoningEffort;
}
diff --git a/application/v2_ui/src/lib/composerGating.ts b/application/v2_ui/src/lib/composerGating.ts
index 02c6dc1b4..03f8764d4 100644
--- a/application/v2_ui/src/lib/composerGating.ts
+++ b/application/v2_ui/src/lib/composerGating.ts
@@ -34,6 +34,7 @@ export interface GatingInput {
imageGenerationActive: boolean;
/** True while an agent is selected in the composer. */
agentActive: boolean;
+ orchestrating?: boolean;
}
export interface ControlGating {
@@ -84,10 +85,10 @@ export function resolveGating(input: GatingInput): ControlGating {
// Read URLs needs both the capability and something to read.
const showUrlAccess = enabled(features, 'enable_url_access') && hasUrls;
- // Deep research needs a source to work from: the web, or URLs that have been provided.
+ // Orchestration research discovers its own sources. Ordinary chat keeps its source gate.
const showDeepResearch =
enabled(features, 'enable_source_review') &&
- (webSearchActive || (urlAccessActive && hasUrls) || hasUrls);
+ (input.orchestrating || webSearchActive || (urlAccessActive && hasUrls) || hasUrls);
return {
showDocuments: true,
@@ -96,9 +97,9 @@ export function resolveGating(input: GatingInput): ControlGating {
showUrlAccess,
showDeepResearch,
showFileUpload: enabled(features, 'enable_chat_file_uploads'),
- disabledByImageGeneration: imageGenerationActive,
- showModelPicker: !imageGenerationActive,
+ disabledByImageGeneration: imageGenerationActive && !input.orchestrating,
+ showModelPicker: !imageGenerationActive || Boolean(input.orchestrating),
modelPickerInactive: agentActive,
- showReasoning: !agentActive && !imageGenerationActive,
+ showReasoning: !agentActive && (!imageGenerationActive || Boolean(input.orchestrating)),
};
}
diff --git a/application/v2_ui/src/lib/messageDetails.ts b/application/v2_ui/src/lib/messageDetails.ts
index 3cd2e9e48..b155df7d8 100644
--- a/application/v2_ui/src/lib/messageDetails.ts
+++ b/application/v2_ui/src/lib/messageDetails.ts
@@ -209,7 +209,8 @@ export function buildDetailGroups(payload: Json | null | undefined): DetailGroup
pushRow(generation, 'Model', root.model_deployment_name);
pushRow(generation, 'Agent', root.agent_display_name || root.agent_name);
pushRow(generation, 'Augmented', formatBoolean(root.augmented));
- pushRow(generation, 'Reasoning effort', metadata.reasoning_effort);
+ pushRow(generation, 'Reasoning effort',
+ metadata.reasoning_mode === 'model_default' ? 'Model default' : metadata.reasoning_effort);
if (generation.length > 0) {
groups.push({ title: 'Generation', rows: generation });
}
diff --git a/application/v2_ui/src/lib/models.ts b/application/v2_ui/src/lib/models.ts
index f22d92ef2..57608f51f 100644
--- a/application/v2_ui/src/lib/models.ts
+++ b/application/v2_ui/src/lib/models.ts
@@ -17,10 +17,14 @@
// - `model_endpoint_id` requires `model_id` or `model_deployment`.
// So the fields are sent as a set or not at all.
+import type { ReasoningCapabilities } from './reasoning';
+
/** Catalog record fields, as produced by `_build_chat_model_catalog`. */
export interface ModelCatalogEntry {
selection_key?: string;
model_id?: string;
+ model_name?: string;
+ reasoning_capabilities?: ReasoningCapabilities;
deployment_name?: string;
endpoint_id?: string;
provider?: string;
diff --git a/application/v2_ui/src/lib/orchestration.ts b/application/v2_ui/src/lib/orchestration.ts
index 4e8321bfd..c69b3e70a 100644
--- a/application/v2_ui/src/lib/orchestration.ts
+++ b/application/v2_ui/src/lib/orchestration.ts
@@ -25,6 +25,7 @@ import { api, apiUrl, CREDENTIALS_MODE } from './apiClient';
import { readSsePost } from './sse';
import type { ComposerReference } from './composerDraft';
import type { ChatStreamEvent, Json } from './types';
+import { normalizeReasoningAdjustments, type ReasoningResolution } from './reasoning';
// `Json` is the shape of a step's `arguments` and the plan's opaque `inputs`/`outputs`, so it is
// part of this contract's surface. Re-exported here (rather than making consumers reach into
@@ -184,6 +185,8 @@ export interface OrchestrationPlanAction {
/** What the plan will act on, for the approval card. */
export interface OrchestrationPlanInputs {
+ /** Original positive selections resolved by the server, never inferred from planned usage. */
+ required_capabilities?: string[];
documents: OrchestrationPlanDocument[];
/** Older plans do not carry action metadata. Match steps by action_ref, not by name. */
actions?: OrchestrationPlanAction[];
@@ -202,6 +205,7 @@ export interface OrchestrationPlanInputs {
* the name a second time from the browser.
*/
export interface OrchestrationPlan {
+ reasoning_adjustments?: ReasoningResolution[];
plan_id: string;
run_id: string;
/** Conditional approval token for a manually held or revised plan. */
@@ -389,7 +393,12 @@ export const MAX_PLAN_INSTRUCTION_LENGTH = 2000;
* client's store does; a re-plan of the same turn sends the same id. The server honours it and
* echoes it back on the plan, rather than minting one of its own.
*/
-export interface OrchestrationPlanRequest {
+export interface OrchestrationSeeds {
+ required_capabilities?: string[];
+ [key: string]: unknown;
+}
+
+export interface OrchestrationPlanRequest extends OrchestrationSeeds {
message: string;
conversation_id?: string | null;
turn_id?: string;
@@ -433,6 +442,7 @@ export interface OrchestrationRunRequest {
* frame ends it too.
*/
export interface PlanStreamEvent {
+ reasoning_adjustments?: ReasoningResolution[];
type?: 'thought' | 'orchestration_plan' | 'orchestration_elicitation' | string;
plan?: OrchestrationPlan;
elicitation?: Elicitation;
@@ -861,6 +871,15 @@ async function readPlanStream(
handlers.onEditor?.(result.editor);
}
result.plan = event.plan ?? null;
+ if (result.plan && event.reasoning_adjustments?.length) {
+ result.plan = {
+ ...result.plan,
+ reasoning_adjustments: normalizeReasoningAdjustments([
+ ...(result.plan.reasoning_adjustments ?? []),
+ ...event.reasoning_adjustments,
+ ]),
+ };
+ }
result.completed = true;
if (result.plan) {
handlers.onPlan?.(result.plan);
diff --git a/application/v2_ui/src/lib/orchestrationController.ts b/application/v2_ui/src/lib/orchestrationController.ts
index 5758ba046..37c7c5d2a 100644
--- a/application/v2_ui/src/lib/orchestrationController.ts
+++ b/application/v2_ui/src/lib/orchestrationController.ts
@@ -31,6 +31,7 @@ import {
type ElicitationResponse,
type OrchestrationPlan,
type OrchestrationPlanRequest,
+ type OrchestrationSeeds,
type OrchestrationRunRequest,
type OrchestrationRequestError,
type PlanRevisionAction,
@@ -39,6 +40,7 @@ import {
} from './orchestration';
import { applyPlanEdits, isPlanApproved, isPlanAwaitingApproval, isPlanRunnable, normalizePlan } from './orchestrationPlan';
import type { Json } from './types';
+import { normalizeReasoningAdjustments, type ReasoningResolution } from './reasoning';
import { useChatStore } from '../stores/chatStore';
import {
selectEdits,
@@ -87,7 +89,7 @@ function makeTurnId(): string {
interface TurnContext {
message: string;
approvalMode: ApprovalMode;
- seeds: Record
;
+ seeds: OrchestrationSeeds;
revision: number;
pendingUserMessageId: string;
}
@@ -124,9 +126,9 @@ export interface StartPlanParams {
/**
* Manual-control selections that constrain the plan rather than being ignored:
* `selected_document_ids`, `agent_info`, the `model_*` quartet, `prompt_info`,
- * `web_search_enabled`. Assembled by the composer; passed through to the plan request as-is.
+ * `required_capabilities` and legacy `web_search_enabled`. Unchecked controls are neutral.
*/
- seeds?: Record;
+ seeds?: OrchestrationSeeds;
}
/**
@@ -333,6 +335,7 @@ async function dispatchPlan(
let produced = false;
let errored = false;
let failure = '';
+ let reasoningAdjustments: ReasoningResolution[] = [];
const isCurrentRequest = () =>
!controller.signal.aborted && activeControllers.get(currentConversationId) === controller;
await planOrchestration(
@@ -340,6 +343,12 @@ async function dispatchPlan(
{
onThought: (event) => {
if (isCurrentRequest()) {
+ reasoningAdjustments = normalizeReasoningAdjustments(
+ event.reasoning_adjustments, reasoningAdjustments,
+ );
+ useOrchestrationStore.getState().mergeReasoningAdjustments(
+ currentConversationId, currentTurnId, event.reasoning_adjustments,
+ );
useChatStore.getState()
.pushOrchestrationThought(currentConversationId, event as RunStreamEvent);
}
@@ -359,7 +368,12 @@ async function dispatchPlan(
}
adoptServerTurnId(plan.turn_id);
context.revision = plan.revision ?? context.revision;
- useOrchestrationStore.getState().setPlan(currentConversationId, currentTurnId, plan);
+ useOrchestrationStore.getState().setPlan(currentConversationId, currentTurnId, {
+ ...plan,
+ reasoning_adjustments: normalizeReasoningAdjustments([
+ ...reasoningAdjustments, ...(plan.reasoning_adjustments ?? []),
+ ]),
+ });
if (!selectPlan(useOrchestrationStore.getState(), currentConversationId, currentTurnId)) {
errored = true;
failure = 'The planner returned an invalid plan. Please try again.';
@@ -635,19 +649,29 @@ export async function approveAndRunPlan(params: {
const result = await runOrchestration(
runBody,
{
- onStep: (event) =>
- useOrchestrationStore.getState().applyStepEvent(conversationId, turnId, event),
+ onStep: (event) => {
+ const current = useOrchestrationStore.getState();
+ current.applyStepEvent(conversationId, turnId, event);
+ current.mergeReasoningAdjustments(conversationId, turnId, event.reasoning_adjustments);
+ },
// A run reports each step starting and finishing as a `thought`, the same event
// planning uses, so it lands in the same place a planning thought does — feeding the
// orchestration progress lane while the answer is still being assembled.
- onThought: (event) =>
+ onThought: (event) => {
+ useOrchestrationStore.getState().mergeReasoningAdjustments(
+ conversationId, turnId, event.reasoning_adjustments,
+ );
useChatStore
.getState()
- .pushOrchestrationThought(conversationId, event as RunStreamEvent),
+ .pushOrchestrationThought(conversationId, event as RunStreamEvent);
+ },
onContent: (_delta, accumulated) =>
useChatStore.getState().pushOrchestrationContent(conversationId, accumulated),
onDone: (event, accumulated) => {
settled = true;
+ useOrchestrationStore.getState().mergeReasoningAdjustments(
+ conversationId, turnId, event.reasoning_adjustments ?? event.metadata?.reasoning_adjustments,
+ );
useChatStore.getState().settleOrchestrationTurn(conversationId, {
status: 'completed',
event,
@@ -991,6 +1015,13 @@ export async function submitPlanRevision(
...editor, submission: { id: submissionId, fingerprint },
}));
const result = await reviseOrchestrationPlan(requestPlan.run_id, body, {
+ onThought: (event) => {
+ if (isEditorRequestCurrent(target, controller)) {
+ useOrchestrationStore.getState().mergeReasoningAdjustments(
+ conversationId, turnId, event.reasoning_adjustments,
+ );
+ }
+ },
onError: (message, error) => { failure = message; info = error; },
}, controller.signal);
if (!isEditorRequestCurrent(target, controller)) {
diff --git a/application/v2_ui/src/lib/orchestrationPlan.ts b/application/v2_ui/src/lib/orchestrationPlan.ts
index 136183dac..f96408c22 100644
--- a/application/v2_ui/src/lib/orchestrationPlan.ts
+++ b/application/v2_ui/src/lib/orchestrationPlan.ts
@@ -31,6 +31,7 @@ import type {
PlanStatus,
StepStatus,
} from './orchestration';
+import { normalizeReasoningAdjustments } from './reasoning';
/**
* The capability id of the answering step, from `TERMINAL_CAPABILITY_ID` in the registry.
@@ -213,6 +214,7 @@ export function normalizePlan(raw: unknown): OrchestrationPlan | null {
return {
plan_id: asString(source.plan_id),
+ reasoning_adjustments: normalizeReasoningAdjustments(source.reasoning_adjustments),
run_id: asString(source.run_id),
edit_version: typeof source.edit_version === 'string' ? source.edit_version : undefined,
turn_id: asString(source.turn_id),
@@ -277,6 +279,7 @@ function normalizeInputs(raw: unknown): OrchestrationPlanInputs {
}
return {
+ required_capabilities: asStringList(source.required_capabilities),
documents,
actions: source.actions !== undefined ? actions : undefined,
web: asBoolean(source.web, false),
diff --git a/application/v2_ui/src/lib/reasoning.ts b/application/v2_ui/src/lib/reasoning.ts
index 918f88423..d0eb2f378 100644
--- a/application/v2_ui/src/lib/reasoning.ts
+++ b/application/v2_ui/src/lib/reasoning.ts
@@ -1,95 +1,47 @@
// reasoning.ts
-// Which reasoning effort levels a model accepts, and which one is in effect.
-//
-// Mirrors getModelSupportedLevels and getCurrentModelReasoningEffort in
-// static/js/chat/chat-reasoning.js. Offering a level a model rejects produces a request the
-// endpoint has to strip, and hiding a level a model does support silently removes a
-// capability, so the mapping is kept in step with the existing client rather than guessed.
-//
-// The chosen level is stored per model in the `reasoningEffortSettings` user setting, which
-// the classic interface already owns. Sharing the setting means sharing how a model is keyed
-// in it, so both interfaces have to agree on the fallback order below.
+// Policy comes from the authorized server catalog; preference keys remain shared with classic.
-export type ReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high';
-
-/**
- * The stored per-model map, `{ 'gpt-5-mini': 'medium' }`.
- *
- * Values are read back as plain strings because the map is shared with another client and
- * with whatever an older release wrote; an unrecognised level is discarded on resolution
- * rather than trusted.
- */
+export type ReasoningEffort = 'none' | 'minimal' | 'low' | 'medium' | 'high' | 'xhigh';
export type ReasoningEffortSettings = Record;
-export const ALL_REASONING_LEVELS: ReasoningEffort[] = [
- 'none',
- 'minimal',
- 'low',
- 'medium',
- 'high',
-];
-
-export function getModelSupportedLevels(modelName?: string): ReasoningEffort[] {
- if (!modelName) {
- return ALL_REASONING_LEVELS;
- }
-
- const name = modelName.toLowerCase();
-
- // Models with no reasoning support at all.
- if (
- name.includes('gpt-4o') ||
- name.includes('gpt-4.1') ||
- name.includes('gpt-5-chat') ||
- name.includes('gpt-5-codex')
- ) {
- return ['none'];
- }
-
- if (name.includes('gpt-5-pro')) {
- return ['high'];
- }
-
- // The 5.1 series skips 'low'.
- if (name.includes('gpt-5.1')) {
- return ['none', 'minimal', 'medium', 'high'];
- }
-
- if (name.includes('gpt-5')) {
- return ['minimal', 'low', 'medium', 'high'];
- }
-
- // o-series reasoning models.
- if (/\bo[0-9]/.test(name)) {
- return ['low', 'medium', 'high'];
- }
-
- return ALL_REASONING_LEVELS;
+export interface ReasoningCapabilities {
+ status: 'supported' | 'unsupported' | 'unknown';
+ efforts: ReasoningEffort[];
+ default_effort: ReasoningEffort | null;
}
-/** True when the model offers a real choice worth surfacing a control for. */
-export function supportsReasoning(modelName?: string): boolean {
- const levels = getModelSupportedLevels(modelName);
- return levels.length > 1 || (levels.length === 1 && levels[0] !== 'none');
+export interface ReasoningResolution {
+ requested_effort: string | null;
+ effective_effort: string | null;
+ mode: 'explicit' | 'model_default';
+ adjustment_reason: string | null;
+ stage?: 'planner' | 'answer';
+ model_name?: string;
}
+export const ALL_REASONING_LEVELS: ReasoningEffort[] = [
+ 'none', 'minimal', 'low', 'medium', 'high', 'xhigh',
+];
+
export const REASONING_LABELS: Record = {
none: 'None',
minimal: 'Minimal',
low: 'Low',
medium: 'Medium',
high: 'High',
+ xhigh: 'XHigh',
};
-/**
- * How a model is identified in the stored map.
- *
- * `model_id` first, then the deployment name, matching `getCurrentModelName()` in
- * chat-reasoning.js. The order matters twice over. It decides the key a level is stored
- * under, so a level chosen in one interface is found again by the other. It also decides
- * which name the level set is derived from, and a deployment an administrator named
- * `chat-prod` says nothing about reasoning support where its model id `gpt-5-mini` does.
- */
+export function getModelSupportedLevels(policy?: ReasoningCapabilities): ReasoningEffort[] {
+ return policy?.status === 'supported' && Array.isArray(policy.efforts)
+ ? policy.efforts.filter((level) => ALL_REASONING_LEVELS.includes(level))
+ : [];
+}
+
+export function supportsReasoning(policy?: ReasoningCapabilities): boolean {
+ return getModelSupportedLevels(policy).length > 0;
+}
+
export function reasoningModelKey(
model: { model_id?: unknown; deployment_name?: unknown } | undefined,
fallback?: string,
@@ -100,38 +52,98 @@ export function reasoningModelKey(
return modelId || deployment || (fallback ?? '').trim();
}
-/**
- * The level in effect for a model, given what has been stored for it.
- *
- * Mirrors `getCurrentModelReasoningEffort()`: a model always has an effective level, so the
- * control shows a real value rather than an empty placeholder, and a stored level that the
- * current model does not accept is ignored instead of being sent and stripped.
- */
+export function resolveReasoningSelection(
+ modelKey: string | undefined,
+ saved?: ReasoningEffortSettings,
+ policy?: ReasoningCapabilities,
+): ReasoningResolution {
+ const requested = modelKey ? saved?.[modelKey] || null : null;
+ const levels = getModelSupportedLevels(policy);
+ const fallback = levels.includes('low')
+ ? 'low'
+ : levels.includes(policy?.default_effort as ReasoningEffort)
+ ? policy!.default_effort
+ : null;
+ const effective = levels.includes(requested as ReasoningEffort) ? requested : fallback;
+ return {
+ requested_effort: requested,
+ effective_effort: effective,
+ mode: effective === null ? 'model_default' : 'explicit',
+ adjustment_reason: requested && requested !== effective ? 'unsupported_effort' : null,
+ };
+}
+
export function resolveReasoningEffort(
- modelName: string | undefined,
+ modelKey: string | undefined,
saved?: ReasoningEffortSettings,
-): ReasoningEffort {
- const levels = getModelSupportedLevels(modelName);
+ policy?: ReasoningCapabilities,
+): ReasoningEffort | undefined {
+ const effective = resolveReasoningSelection(modelKey, saved, policy).effective_effort;
+ return effective === null ? undefined : effective as ReasoningEffort;
+}
- // gpt-5-pro takes `high` and nothing else, so a stored value cannot override it.
- if (modelName && modelName.toLowerCase().includes('gpt-5-pro')) {
- return 'high';
+/** Omitted and explicitly supported None are different provider requests. */
+export function requestReasoningEffort(
+ level: string | undefined,
+ policy?: ReasoningCapabilities,
+): string | undefined {
+ return getModelSupportedLevels(policy).includes(level as ReasoningEffort) ? level : undefined;
+}
+
+export function normalizeReasoningAdjustments(
+ value: unknown, previous: ReasoningResolution[] = [],
+): ReasoningResolution[] {
+ const entries = [...previous, ...(Array.isArray(value) ? value : [])];
+ const resolutions = entries.filter((item): item is ReasoningResolution =>
+ item !== null && typeof item === 'object' &&
+ (item.adjustment_reason === null || typeof item.adjustment_reason === 'string') &&
+ (item.mode === 'explicit' || item.mode === 'model_default') &&
+ (item.requested_effort === null || typeof item.requested_effort === 'string') &&
+ (item.effective_effort === null || typeof item.effective_effort === 'string'),
+ );
+ const latest = new Map();
+ for (const resolution of resolutions) {
+ const stage = resolution.stage === 'planner' || resolution.stage === 'answer' ? resolution.stage : '';
+ const modelName = typeof resolution.model_name === 'string' ? resolution.model_name : '';
+ latest.set(JSON.stringify([stage, modelName]), resolution);
}
+ return [...latest.values()].filter((resolution) => Boolean(resolution.adjustment_reason));
+}
- const stored = modelName ? saved?.[modelName] : undefined;
- if (stored && levels.includes(stored as ReasoningEffort)) {
- return stored as ReasoningEffort;
+/** Merge only the public reasoning projection, preserving other message metadata. */
+export function reasoningMetadataForEvent(event: {
+ metadata?: Record;
+ reasoning_effort?: string | null;
+ requested_reasoning_effort?: string | null;
+ reasoning_mode?: 'explicit' | 'model_default';
+ reasoning_adjustments?: ReasoningResolution[];
+}, previousAdjustments: ReasoningResolution[] = []): Record | undefined {
+ if (event.reasoning_effort === undefined && event.requested_reasoning_effort === undefined &&
+ event.reasoning_mode === undefined && event.reasoning_adjustments === undefined &&
+ previousAdjustments.length === 0) {
+ return event.metadata;
}
+ return {
+ ...event.metadata,
+ ...(event.reasoning_effort !== undefined ? { reasoning_effort: event.reasoning_effort } : {}),
+ ...(event.requested_reasoning_effort !== undefined
+ ? { requested_reasoning_effort: event.requested_reasoning_effort } : {}),
+ ...(event.reasoning_mode !== undefined ? { reasoning_mode: event.reasoning_mode } : {}),
+ ...(event.reasoning_adjustments !== undefined || previousAdjustments.length > 0
+ ? { reasoning_adjustments: normalizeReasoningAdjustments(
+ event.reasoning_adjustments ?? event.metadata?.reasoning_adjustments,
+ previousAdjustments,
+ ) } : {}),
+ };
+}
- return levels.includes('low') ? 'low' : levels[0];
+function effortLabel(effort: string | null): string {
+ return REASONING_LABELS[effort as ReasoningEffort] ?? (effort ? 'Saved effort' : 'Model default');
}
-/**
- * The value to send with a request, or undefined when nothing should be sent.
- *
- * Mirrors `getCurrentReasoningEffort()`, which returns null for `none`: the level is a real
- * choice in the picker but not a parameter the endpoint takes.
- */
-export function requestReasoningEffort(level: string | undefined): string | undefined {
- return !level || level === 'none' ? undefined : level;
+/** Never display provider errors or adjustment_reason text supplied in an event. */
+export function reasoningAdjustmentMessage(resolution: ReasoningResolution, modelName?: string): string {
+ const stage = resolution.stage === 'planner' ? 'Planner: ' : resolution.stage === 'answer' ? 'Answer: ' : '';
+ const effective = resolution.mode === 'model_default' ? 'Model default' : effortLabel(resolution.effective_effort);
+ return `${stage}${effortLabel(resolution.requested_effort)} could not be used${modelName ? ` for ${modelName}` : ''}; using ${effective}.`;
}
diff --git a/application/v2_ui/src/lib/types.ts b/application/v2_ui/src/lib/types.ts
index 464e4b373..85136ba40 100644
--- a/application/v2_ui/src/lib/types.ts
+++ b/application/v2_ui/src/lib/types.ts
@@ -7,6 +7,8 @@
// an index signature rather than being modelled exhaustively, so a backend addition never
// breaks the build.
+import type { ReasoningResolution } from './reasoning';
+
export type Json = Record;
export interface Conversation {
@@ -890,6 +892,10 @@ export interface WorkspaceAvailability {
* why almost everything here is optional.
*/
export interface ChatStreamEvent {
+ reasoning_adjustments?: ReasoningResolution[];
+ reasoning_effort?: string | null;
+ requested_reasoning_effort?: string | null;
+ reasoning_mode?: 'explicit' | 'model_default';
type?:
| 'thought'
| 'conversation_metadata'
diff --git a/application/v2_ui/src/stores/chatStore.ts b/application/v2_ui/src/stores/chatStore.ts
index e746246c7..696c7d39f 100644
--- a/application/v2_ui/src/stores/chatStore.ts
+++ b/application/v2_ui/src/stores/chatStore.ts
@@ -69,6 +69,11 @@ import {
resolveSendTarget,
} from '../lib/mentions';
import { buildSelectionFields } from '../lib/chatRequestSelection';
+import {
+ normalizeReasoningAdjustments,
+ reasoningMetadataForEvent,
+ type ReasoningResolution,
+} from '../lib/reasoning';
import { promptSelectionMetadata } from '../lib/promptRequest';
import type { RunStreamEvent } from '../lib/orchestration';
import {
@@ -106,6 +111,7 @@ import type { VisualStyle } from '../lib/visualPalettes';
import type {
AgentOption,
ChatMessage,
+ ChatStreamEvent,
ChatStreamRequest,
CollaborationConversation,
CollaborationMessage,
@@ -248,6 +254,7 @@ interface ChatState {
streaming: boolean;
streamingContent: string;
+ streamingReasoningAdjustments: ReasoningResolution[];
thoughts: ThoughtEntry[];
streamError: string | null;
streamAuthUrl: string | null;
@@ -812,6 +819,8 @@ function buildStreamHandlers(
*/
pendingUserMessageId?: string | null,
): ChatStreamHandlers {
+ const completionMetadata = (event: ChatStreamEvent) =>
+ reasoningMetadataForEvent(event, getState().streamingReasoningAdjustments);
return {
onUserMessagePersisted: (event) => {
const persistedId = String(event.user_message_id ?? event.message_id ?? '').trim();
@@ -837,11 +846,16 @@ function buildStreamHandlers(
typeof event.content === 'string'
? event.content
: String(event.thought ?? '');
- if (!content || !isCurrent()) {
+ const adjustmentUpdates = event.reasoning_adjustments ?? event.metadata?.reasoning_adjustments;
+ const hasAdjustmentUpdates = Array.isArray(adjustmentUpdates) && adjustmentUpdates.length > 0;
+ if (!isCurrent() || (!content && !hasAdjustmentUpdates)) {
return;
}
set((state) => ({
- thoughts: [
+ streamingReasoningAdjustments: normalizeReasoningAdjustments(
+ adjustmentUpdates, state.streamingReasoningAdjustments,
+ ),
+ thoughts: content ? [
...state.thoughts,
{
id: `${state.thoughts.length}`,
@@ -858,7 +872,7 @@ function buildStreamHandlers(
stepIndex:
typeof event.step_index === 'number' ? event.step_index : undefined,
},
- ],
+ ] : state.thoughts,
}));
},
onConversationMetadata: (event) => {
@@ -886,7 +900,7 @@ function buildStreamHandlers(
model_deployment_name: event.model_deployment_name,
agent_display_name: event.agent_display_name,
augmented: event.augmented,
- metadata: event.metadata,
+ metadata: completionMetadata(event),
// Carried onto the finished message so the reasoning steps stay
// available after the stream ends instead of disappearing with the
// streaming placeholder.
@@ -904,6 +918,7 @@ function buildStreamHandlers(
),
streaming: false,
streamingContent: '',
+ streamingReasoningAdjustments: [],
reconnectPhase: null,
}));
},
@@ -923,13 +938,14 @@ function buildStreamHandlers(
role: 'assistant',
content: accumulated,
timestamp: new Date().toISOString(),
+ metadata: completionMetadata(_event),
thoughts:
state.thoughts.length > 0 ? [...state.thoughts] : undefined,
},
],
}));
}
- set({ streaming: false, streamingContent: '', reconnectPhase: null });
+ set({ streaming: false, streamingContent: '', streamingReasoningAdjustments: [], reconnectPhase: null });
},
onError: (message, event) => {
if (!isCurrent()) {
@@ -938,6 +954,7 @@ function buildStreamHandlers(
set({
streaming: false,
streamingContent: '',
+ streamingReasoningAdjustments: [],
reconnectPhase: null,
streamError: message,
streamAuthUrl: foundryAuthUrl(event),
@@ -964,6 +981,7 @@ function buildStreamHandlers(
set({
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
reconnectPhase: 'reconnected',
streamError: null,
streamAuthUrl: null,
@@ -1099,6 +1117,7 @@ async function resumeChatStream(conversationId: string): Promise {
streaming: true,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: 'connecting',
@@ -1512,6 +1531,7 @@ export const useChatStore = create((set, get) => ({
streaming: false,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -1609,6 +1629,7 @@ export const useChatStore = create((set, get) => ({
messagesError: null,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -1787,6 +1808,7 @@ export const useChatStore = create((set, get) => ({
messagesError: null,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -2282,6 +2304,7 @@ export const useChatStore = create((set, get) => ({
streaming: willStream,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -2508,6 +2531,7 @@ export const useChatStore = create((set, get) => ({
streaming: true,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -2569,6 +2593,7 @@ export const useChatStore = create((set, get) => ({
streaming: false,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
reconnectPhase: null,
});
return;
@@ -2633,7 +2658,7 @@ export const useChatStore = create((set, get) => ({
web_search_citations:
event.web_search_citations as ChatMessage['web_search_citations'],
agent_citations: event.agent_citations as ChatMessage['agent_citations'],
- metadata: event.metadata,
+ metadata: reasoningMetadataForEvent(event),
thoughts: get().thoughts.length > 0 ? [...get().thoughts] : undefined,
};
set((state) => {
@@ -2828,6 +2853,7 @@ export const useChatStore = create((set, get) => ({
streaming: true,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
@@ -2878,6 +2904,7 @@ export const useChatStore = create((set, get) => ({
streaming: true,
streamingContent: '',
thoughts: [],
+ streamingReasoningAdjustments: [],
streamError: null,
streamAuthUrl: null,
reconnectPhase: null,
diff --git a/application/v2_ui/src/stores/orchestrationStore.ts b/application/v2_ui/src/stores/orchestrationStore.ts
index 30f084694..365c06bcc 100644
--- a/application/v2_ui/src/stores/orchestrationStore.ts
+++ b/application/v2_ui/src/stores/orchestrationStore.ts
@@ -22,6 +22,7 @@
// re-plans. What is persisted is the minimum needed to recognise a run that is already running.
import { create } from 'zustand';
+import { normalizeReasoningAdjustments } from '../lib/reasoning';
import { createElicitationDraft, type ElicitationDraft } from '../lib/elicitationAnswers';
import {
applyPlanEdits,
@@ -344,6 +345,7 @@ interface OrchestrationState {
/** Adopt a plan for a turn, replacing any pending question and re-seeding on a new revision. */
setPlan: (conversationId: string, turnId: string, plan: unknown) => void;
+ mergeReasoningAdjustments: (conversationId: string, turnId: string, adjustments: unknown) => void;
/** Forget a turn's plan. */
clearPlan: (conversationId: string, turnId: string) => void;
@@ -555,6 +557,20 @@ export const useOrchestrationStore = create((set, get) => ({
return true;
},
+ mergeReasoningAdjustments: (conversationId, turnId, adjustments) => {
+ if (!Array.isArray(adjustments) || !adjustments.length) return;
+ const key = scopeKey(conversationId, turnId);
+ set((state) => {
+ const plan = state.plans[key];
+ if (!plan) return {};
+ const merged = normalizeReasoningAdjustments([
+ ...(plan.reasoning_adjustments ?? []), ...adjustments,
+ ]);
+ if (JSON.stringify(merged) === JSON.stringify(plan.reasoning_adjustments ?? [])) return {};
+ return { plans: { ...state.plans, [key]: { ...plan, reasoning_adjustments: merged } } };
+ });
+ },
+
setPlan: (conversationId, turnId, rawPlan) => {
if (!conversationId || !turnId) {
return;
diff --git a/docs/explanation/features/CHAT_ORCHESTRATION.md b/docs/explanation/features/CHAT_ORCHESTRATION.md
index 8a0a39143..642e9af4b 100644
--- a/docs/explanation/features/CHAT_ORCHESTRATION.md
+++ b/docs/explanation/features/CHAT_ORCHESTRATION.md
@@ -1,6 +1,6 @@
# Chat Orchestration
-**Version: 0.261.103** (tracked in `application/single_app/config.py`)
+**Version: 0.261.104** (tracked in `application/single_app/config.py`)
**Implemented in version: 0.261.086**
**Knowledge phase added in version: 0.261.089**
@@ -12,6 +12,7 @@
**Selected/default model routing fixed in version: 0.261.103**
**Approval preference persistence fixed in version: 0.261.101**
**Conversational plan editing implemented in version: 0.261.102**
+**Capability-aware planning and reasoning compatibility fixed in version: 0.261.104**
## Overview
@@ -66,8 +67,10 @@ enablement lives in a nested capability record rather than a flag.
spend the planner's whole context on file names. A cheap search probe using the user's
contextualized request is aggregated to distinct documents instead. When the user has
already selected documents, no probe runs.
-- **Seeds as constraints.** Anything chosen in the composer narrows the plan rather than
- suggesting to it.
+- **Positive requirements and resource filters.** Supported selected tools, documents,
+ and agents must be used by the initial plan. Unchecked controls are neutral, not
+ permission denials. Other enabled, authorized capabilities remain available.
+ Workspace, tag, and document filters still bound source access.
- **Accessible actions by description.** Where action access is enabled, the planner
receives safe metadata for governed actions, not credentials, connection settings or
every action's function schemas. Scoped references distinguish actions with the same
@@ -78,6 +81,11 @@ enablement lives in a nested capability record rather than a flag.
- **The run ledger.** A compact, byte-bounded activity summary covering earlier searches,
produced artifacts, and answered questions. It helps avoid unnecessary repeated work,
but does not replace message history or prove that source evidence is available.
+- **Saved memory.** Since **0.261.104**, enabled Fact Memory supplies up to eight instruction
+ memories and four relevant embedded facts for planning, planner edits, and answering.
+ Current instructions take precedence. Private conversations use the caller's memory, or
+ the first active group's authorized memory in group/all source mode. Shared conversations,
+ including owner-held hidden source records, do not receive saved memory.
#### Conversational follow-ups
@@ -105,9 +113,20 @@ the same context. Changes to the referenced messages or their visibility require
plan; newly appended turns do not enter an older run.
These rules apply to Auto, countdown, and manual approval. They do not introduce rolling
-summaries or cross-conversation memory. First turns without history and simple
+summaries or cross-conversation transcript lookup. Existing scoped saved memories are
+separate from this history window. First turns without history and simple
acknowledgments do not require a resolution completion.
+Saved memory recall is read-only: it does not autosave facts or fill missing embeddings.
+Unavailable fact search is identified explicitly; query embeddings may still require a
+model call. Only audience and scope markers are retained with the plan or cached
+clarification, not the raw memory prompt. Current settings, membership, and audience are
+checked again before final synthesis; the recalled scope is reauthorized after the model
+call before publication. Questions and replays also enforce those boundaries.
+Answers preserve memory citations. See the
+[capability-context fix](../fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md#read-only-saved-memory)
+for scope and availability details.
+
Since **0.261.103**, an unused `clarification: null` in the resolver's JSON is
accepted as "no clarification needed", just like an empty string. It does not
discard the rest of a valid follow-up or require another model call. A request
@@ -124,8 +143,9 @@ distinct from an inaccessible or changed conversation.
Refused, filtered, absent, or incomplete completions are not retried as malformed
JSON. The resolver does not mistake provider failures for unsupported JSON formatting; only
an explicit unsupported-response-format error uses the existing no-format
-compatibility fallback. The separate plan generator retains its existing retry
-behavior. No new model setting or API version is required.
+compatibility fallback. The plan generator uses the same narrow format-error rule.
+Model failures are surfaced, not converted into an answer-only plan. No new
+model setting or API version is required.
#### Model selection
@@ -158,6 +178,14 @@ floor so reasoning does not consume the entire smaller visible-output allowance.
Anthropic completion flags are normalized at the protocol boundary, so successful
Claude follow-ups pass the same strict checks while truncation and refusal remain failures.
+Both chat interfaces consume a canonical, per-model reasoning policy. A configured
+model ID is a preference identity, not a model family. An unsupported stored effort
+uses the policy's supported application default with a visible notice; Luna Minimal
+becomes Low. Explicit supported None is sent unchanged. Unknown support or a narrowly
+classified provider rejection uses the model-managed default and reports that honestly,
+without switching deployments. See the
+[reasoning compatibility fix]({{ '/explanation/fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX/' | relative_url }}).
+
The saved assistant message and terminal stream identify the model that actually answered.
The existing V2 renderer displays that name. Empty, refused, filtered or failed answer
completions produce an error rather than a success-shaped empty turn.
@@ -232,17 +260,22 @@ documents directly whenever they are already known.
### Plan
-`functions_orchestration_planner.py` triages first. The point of triage is to stop a
-conversational question costing a planning round trip, so triage itself is heuristic rather
-than a model call — doing it with a model would spend exactly the round trip it saves. The
-heuristics are biased towards planning: a false positive costs one cheap call, while a
-false negative answers a document question without looking at the documents.
+Every Orchestrate request reaches `functions_orchestration_planner.py`, including short
+questions and acknowledgments. The planner returns a plan or an elicitation. There is
+no keyword/length shortcut that decides retrieval is unnecessary before the model sees
+the available capabilities. This deliberately adds a planning call to requests that
+previously bypassed it; ordinary chat is unchanged by that orchestration policy.
-Where a plan is needed, the planner returns either a plan or an elicitation.
+The context separates available capabilities, their actual unavailability reasons,
+positive user requirements, and authorized resources. Descriptors include outputs and
+per-plan limits, and the context carries the current UTC time. A model's claim that a
+feature is unavailable is not an authorization decision. Discovery failures surface as
+failures instead of silently replacing a catalog with an empty list.
-When eligible actions are available, short questions also reach planning: message length
-cannot distinguish a general question from a ticket-status lookup. The existing fast
-path remains when direct actions are disabled or unavailable.
+Initial plans cannot silently drop selected operations or documents. A subsequent
+reviewed edit can narrow them, with a visible warning. Planned Web use is kept separate
+from original Web selection during restoration. Current access and feature gates are
+checked again before execution.
`functions_orchestration_schema.py` holds both contracts and the validator. **Planner
output is treated as untrusted input.** A plan naming a capability that does not exist,
@@ -572,7 +605,7 @@ See [the Orchestration settings page](../../admin/orchestration.md) for the full
| `functions_orchestration_context.py` | Candidate documents, accessible agent/action metadata, seeds, bounded history snapshots, signals, run ledger |
| `functions_action_catalog.py` | Metadata-only action discovery, scoped references and fresh authorization |
| `functions_orchestration_actions.py` | Isolated, bounded execution of one selected action |
-| `functions_orchestration_planner.py` | Follow-up resolution, triage, plan synthesis, elicitation, re-planning |
+| `functions_orchestration_planner.py` | Follow-up resolution, capability-aware plan synthesis, elicitation, re-planning |
| `functions_orchestration_plan_editing.py` | Scoped plan changes, current source checks, and revised execution requests |
| `functions_orchestration_plan_revisions.py` | Durable edit holds, conditional revision publication, history, and execution claims |
| `functions_orchestration_adapters.py` | Capability adapters over existing functions |
@@ -667,8 +700,9 @@ research-selection rate is not itself a quality improvement.
- **A full page reload does not automatically restore the inline interview.** Drafts survive
paging and navigation within the current browser session; reload recovery is a separate
capability.
-- **Recent context only.** There is no orchestration rolling summary or cross-chat memory.
- A reference outside the retained window may need clarification.
+- **Recent transcript context only.** There is no orchestration rolling summary or
+ cross-chat transcript lookup. A reference outside the retained window may need
+ clarification. Enabled scoped fact memories are a separate, bounded source of context.
- **Automatic per-step model routing is not implemented.** Planning and research use the
selected/default answer model unless a dedicated planner override is configured.
Direct action execution receives the answer selection. Models are not selected
diff --git a/docs/explanation/features/V2_ORCHESTRATION_PLAN_EDITING.md b/docs/explanation/features/V2_ORCHESTRATION_PLAN_EDITING.md
index 32925d5c0..9227fa256 100644
--- a/docs/explanation/features/V2_ORCHESTRATION_PLAN_EDITING.md
+++ b/docs/explanation/features/V2_ORCHESTRATION_PLAN_EDITING.md
@@ -1,10 +1,13 @@
# V2 Orchestration Plan Editing
-**Version: 0.261.102**
+**Version: 0.261.104**
**Implemented in version: 0.261.102**, tracked in
`application/single_app/config.py`.
+**Reasoning compatibility and capability context fixed in version: 0.261.104**,
+using the same application version field.
+
## Overview
A proposed orchestration plan is not necessarily the plan a user wants to run.
@@ -66,6 +69,17 @@ changes, the current task, the latest instruction, and a bounded editor
conversation. The saved original request, selected sources, and conversation
snapshot retain their identities.
+Available capabilities come from server configuration, current access, and resource
+prerequisites. Selected controls and sources are positive requirements; an unchecked
+control does not veto a capability. A later edit can intentionally change an earlier
+selection, with a visible review warning when selected work is removed.
+
+Planner and answer models resolve reasoning effort independently from canonical
+model metadata. Unsupported saved levels are adjusted visibly rather than breaking
+Edit or Run. Runtime adjustments survive revision publication, editor projections,
+and run restoration without changing immutable historical plans. See
+[Reasoning compatibility]({{ '/explanation/fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX/' | relative_url }}).
+
The planner can return an updated plan, an explanation without changing the plan,
or a clarifying question. Questions remain inside the editor; answering one
continues that edit without removing the last valid preview.
@@ -134,7 +148,8 @@ conditional transitions, idempotency, history, and edit/run conflicts.
planner-to-persistence-to-execution flow, including clarification, error recovery,
preserved source selections, and the final revised request.
`functional_tests/test_orchestration_plan_revision_planner.py` covers the strict
-edit-output contract and the separation from initial planning's failure fallback.
+edit-output contract. Initial planning is strict too: a provider failure or an
+invalid plan no longer becomes a successful direct-answer fallback.
The orchestration UI harness covers the editor and existing narrowing-only Review
behavior. Model responses are deterministic in these tests; they establish the
diff --git a/docs/explanation/fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md b/docs/explanation/fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md
new file mode 100644
index 000000000..03727fce3
--- /dev/null
+++ b/docs/explanation/fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md
@@ -0,0 +1,151 @@
+# Capability-aware orchestration planning
+
+**Version: 0.261.104**
+
+**Fixed in version: 0.261.104**, tracked by `VERSION` in
+`application/single_app/config.py`.
+
+## Issue and root cause
+
+Orchestration could describe live retrieval as unauthorized even when the user
+expected the deployment's research capabilities to be available. Planner context
+combined the real server-resolved capability list with
+`user_selected.web_search: false`, conflating an unchecked default with a denial.
+Short requests could also bypass the planner entirely.
+
+Agent discovery had a separate shape mismatch: group-ID strings reached catalog
+code expecting group records. Its broad fallback then looked like a successfully
+resolved empty catalog. Initial model failures likewise looked like successful
+answer-only plans.
+
+The existing balanced research-depth guidance had not been removed. These fixes
+correct the surrounding context and failure contracts rather than introducing
+topic-specific routing.
+
+## Requirements versus availability
+
+The initial planner receives separate, authoritative information:
+
+| Context | Contract |
+| --- | --- |
+| Available capabilities | Server feature gates, orchestration allowlist, current caller access, and resource prerequisites. Descriptors include arguments, outputs, costs, and per-plan limits. |
+| Positive requirements | Selected supported controls, documents, and agents. Initial validation rejects silently dropped selections. |
+| Neutral controls | An unchecked control, including legacy false Web values, is not a veto. |
+| Authorized resources | Relevant documents, governed actions, current agent records, bounded conversation context, and enabled read-only saved memory. |
+| Time and prior activity | Current UTC time and bounded earlier-run summaries; neither grants permissions nor proves evidence was retrieved. |
+
+The planner can use another available capability without a manual opt-in. An
+explicit instruction such as "do not browse" remains an instruction. Selected
+workspace and document filters continue to bound retrieval; the fix does not
+widen authorization.
+
+Selected Deep Research no longer requires selecting Web first. Automatic web
+discovery still depends on the server's Web Search setting. Unsupported Image
+generation and ineligible URL controls are not silently converted into
+orchestration requirements.
+
+An existing Image selection blocks submission until the user chooses regular
+Chat with Image or explicitly excludes Image for this orchestration message.
+That exclusion does not erase the ordinary-chat Image preference. URL
+eligibility uses the resolved message, including attached prompts, rather than
+only the draft editor text.
+
+## Planning, editing, and execution
+
+Every Orchestrate request invokes the planner, including short questions.
+The model can choose a direct answer when context is sufficient. There is no
+keyword router, fixed research quota, compulsory Web step, or mandatory Deep
+Research step.
+
+Discovery and provider failures are explicit errors, not empty catalogs or
+successful answer-only plans. Missing, empty, or non-list model-authored steps
+are rejected before normalization; a missing final answering step is repaired
+only when real planned work remains. A failed edit preserves the prior plan. Later
+manual/editor narrowing remains possible and reports when original selected
+work is removed.
+
+The agent and action catalogs share fresh membership resolution. Client group
+IDs or records narrow current, role-checked group records; supplied names and
+roles are not trusted. Selected agents are resolved from the authorized catalog.
+
+Before execution, current feature and resource gates are rechecked. Stored plan
+usage and positive composer requirements are separate: restoring a model-chosen
+Web step does not turn the Web button into an original user selection. Completed
+usage reports come from actual executed capabilities, not merely proposed steps.
+
+## Read-only saved memory
+
+With **Fact Memory** enabled (`enable_fact_memory_plugin`), initial planning,
+planner edits, and final answers reuse the existing instruction/fact reader.
+Recall is limited to eight instructions and four relevant embedded facts, with
+each value bounded to 2,000 characters. Saved instructions are preferences
+subordinate to the current request; facts are background context, not permissions
+or proof of current external conditions.
+
+In a private conversation, personal/public source modes use the caller's memory.
+Group/all source modes with a selected group use the first active group's memory,
+after fresh membership authorization. Selecting an ID never grants access, and a
+denied group does not fall back to personal memory. Public document access does
+not create a public-memory scope.
+
+Shared conversations and their hidden source records receive no saved memory.
+Owning a source record does not establish a private audience; the owner-only
+orchestration API does not establish shared-memory authorization. This limitation
+is reported in planner context rather than treating personal memories as shared.
+
+Only audience and scope markers, not recalled prompt text, are saved with a plan
+or cached clarification outcome. These markers identify what was used; they do
+not grant access. Audience and current membership are checked before publishing
+plans, questions, and their replays. Each cached outcome keeps its own scope
+even if a later continuation changes sources.
+
+Recall and authorization are repeated before final synthesis so changed
+membership or disabled memory cannot reuse a stale payload. The actual recalled
+scope is checked again after the model call, before an answer or its memory
+citations can be published. Final answers retain the existing `fact_memory`
+citation format and provenance.
+
+Planning and orchestration recall do not autosave facts or backfill embeddings.
+Missing embeddings are reported as unavailable, while usable instruction
+memories can remain. Query embeddings may still require a model request.
+Disabled memory performs no memory-store or embedding access. Ordinary chat
+keeps its existing backfill behavior.
+
+## Files and regression coverage
+
+The contract is implemented in `functions_orchestration_context.py`,
+`functions_orchestration_registry.py`, `functions_orchestration_planner.py`,
+`functions_orchestration_schema.py`, the editing/revision modules, and
+`route_backend_orchestration.py`. Agent discovery uses
+`functions_action_catalog.py` and `functions_agent_catalog.py`. The V2 composer,
+request builder, plan normalization, and stores preserve positive selections.
+
+`functional_tests/test_orchestration_capability_context.py` exercises real group
+record resolution, authorization narrowing, requirements, and notice projection.
+Research-selection, conversation, clarification, revision, and hydration suites
+cover their related contracts. Browser tests cover the real composer and the
+combined editor/Flask workflow.
+
+`functional_tests/test_orchestration_memory_context.py` exercises the real reader
+through planning, editing, and final synthesis, including citations, scope
+revocation, audience changes during planning and synthesis, clarification replay,
+disabled memory, missing embeddings, and no writes.
+`functional_tests/test_fact_memory_read_only_context.py` covers the bounded
+reader and unchanged ordinary-chat backfill. Integration lives in
+`functions_orchestration_memory.py`, the executor/respond adapter, and the
+existing planning/revision routes.
+
+## Evaluation boundaries
+
+The research-planning evaluator captures actual serialized synthetic contexts,
+capability projections, guidance, and source fingerprints. Before/after variants
+retain their own contexts while keeping scenario permissions and model
+parameters paired. Captured source is data and is never executed.
+
+The suite includes an unchanged playlist request, synthetic coastal-planning
+paraphrases, a short question, explicit research requirements, neutral false Web
+selection, authorized documents and agents, saved preferences, and direct-answer
+controls. Controlled completions establish
+application behavior, not improved live model judgment. Live paired evaluation
+requires an explicitly selected deployment and call budget; it is not performed
+by ordinary regression tests.
diff --git a/docs/explanation/fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX.md b/docs/explanation/fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX.md
new file mode 100644
index 000000000..01af14d95
--- /dev/null
+++ b/docs/explanation/fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX.md
@@ -0,0 +1,95 @@
+# Orchestration reasoning-level compatibility
+
+**Version: 0.261.104**
+
+**Fixed in version: 0.261.104**, tracked by `VERSION` in
+`application/single_app/config.py`.
+
+## Issue and root cause
+
+Plan edits could fail while Auto or Review -> Run failed during answer generation.
+The selected GPT-5.6 Luna deployment rejected `reasoning_effort="minimal"` and
+reported the supported levels as None, Low, Medium, High, and XHigh.
+
+Both interfaces used outdated family-based reasoning choices. The V2 picker also
+inferred support from a preference key that could be an opaque model UUID.
+Orchestration passed the unsupported level to the provider. Initial planning
+concealed its failure with a direct-answer fallback, whereas editing correctly
+kept the previous plan. Ordinary chat appeared to work because it retried without
+the effort parameter; that did not mean Minimal had been honored.
+
+## Runtime policy
+
+`static/json/model_capabilities.json` now carries additive reasoning policies.
+`functions_model_capabilities.py` resolves the allowed levels from authorized
+canonical model metadata and declared aliases, independently of preference IDs.
+Existing non-reasoning and vision capability records are preserved.
+
+| Request | Effective behavior |
+| --- | --- |
+| Supported explicit level | Send it unchanged, including literal `none`. |
+| Unsupported level on a known model | Use its supported application default and show the correction. Luna Minimal becomes Low. |
+| No requested level | Omit the parameter; report model default. |
+| Unsupported or unknown reasoning support | Do not invent allowed levels. Omit the parameter and explain any discarded explicit choice. |
+| Provider rejects `reasoning_effort` | Retry once without that parameter only for the specific SDK HTTP 400 parameter/code rejection. Preserve the model and other valid arguments. |
+| Other provider failure | Propagate the failure; do not disguise it as compatibility recovery. |
+
+Low is the application's preferred supported fallback, not a claim about the
+provider's default. Model-default mode does not assert which effort the provider
+actually used.
+
+`model_endpoint_clients.py`, `functions_orchestration_models.py`, and
+`route_backend_chats.py` share this policy. A dedicated planner override keeps its
+own policy instead of inheriting the answer model's effort. JSON-format recovery
+is separate and only handles a rejected `response_format`.
+If both parameters are rejected, the binding remembers the reasoning omission
+before attempting recovery. A later JSON-format retry does not resend the
+rejected effort or reset its retry allowance. Model-default metadata survives
+even when the intermediate retry raises; a different explicit per-call effort
+retains its independent policy.
+
+## User-visible behavior and persistence
+
+V2 and classic selectors receive safe `reasoning_capabilities` metadata from
+`route_frontend_chats.py`. They retain the existing preference keys and merge
+corrected selections after preferences load rather than replacing unrelated
+model preferences.
+
+Live updates are merged by model and stage before filtering notices. A later
+resolution with no adjustment clears that stage's obsolete warning without
+removing a separate planner or answer correction.
+
+Corrections appear in the composer or relevant plan/answer surface. Saved message
+and run metadata distinguish:
+
+| Field | Meaning |
+| --- | --- |
+| `requested_reasoning_effort` | The original requested level. |
+| `reasoning_effort` | The effective explicit level, or null for model default. |
+| `reasoning_mode` | `explicit` or `model_default`. |
+| `reasoning_adjustments` | Safe requested/effective values, reason, canonical model name, and planner/answer stage. |
+
+The orchestration events, editor publication, and hydration projections preserve
+these notices. Display metadata does not rewrite immutable historical plans, add
+fake revisions, retarget models, or alter approval and concurrency rules. A
+failed edit still leaves the previous valid plan intact. Initial planning now
+also reports failures instead of manufacturing a successful direct-answer plan.
+
+## Validation and limitations
+
+Canonical policy and provider-error behavior are covered by
+`functional_tests/test_model_reasoning_capability_resolution.py` and
+`functional_tests/test_orchestration_model_selection.py`. Ordinary streaming and
+non-streaming paths are covered by `test_chat_reasoning_runtime.py`; empty-stream
+recovery has separate regression coverage. Combined reasoning/JSON rejection
+cases exercise both retry orders and require no more than three requests, with
+unchanged model identity, messages, and completion budget.
+
+`ui_tests/test_v2_reasoning_controls.py` exercises real composer behavior.
+`ui_tests/test_v2_orchestration_plan_editor_backend.py` forwards browser requests
+to the real Flask handlers, including stale Luna Minimal, a Web Search edit, and
+Run. Provider and storage boundaries are deterministic; these are not live
+model-quality evaluations.
+
+No API-version migration or Azure configuration change is required. The code
+must still be deployed before an existing hosted application gains this fix.
diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md
index 08453811f..d70e288ca 100644
--- a/docs/explanation/release_notes.md
+++ b/docs/explanation/release_notes.md
@@ -2,6 +2,22 @@
For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes).
+### **(v0.261.104)**
+
+#### Bug Fixes
+
+* **Model-Aware Reasoning Across Chat And Orchestration**
+ * Fixed plan editing and Auto/Review execution failures caused by unsupported reasoning levels. GPT-5.6 Luna's stale Minimal selection becomes Low with a visible adjustment, while supported None remains explicit.
+ * Both interfaces use canonical per-model capabilities. Narrow provider compatibility recovery reports Model default rather than claiming the rejected effort was honored; model identity and approval safeguards remain unchanged.
+ * (Ref: `functions_model_capabilities.py`, `model_endpoint_clients.py`, `functions_orchestration_models.py`, `route_backend_chats.py`, [Reasoning Compatibility Fix](fixes/ORCHESTRATION_REASONING_LEVEL_COMPATIBILITY_FIX.md))
+
+* **Capability-Aware Planning Without Hidden Shortcuts**
+ * Selected supported tools and sources are requirements, not a restriction to only those tools. Unchecked controls no longer imply that enabled Web Search or Deep Research is unauthorized.
+ * Every Orchestrate request reaches the planner, including short questions. Direct answers remain available; no topic rule forces research. Model and discovery failures are surfaced instead of becoming successful answer-only plans.
+ * Fixed authorized group-agent catalog discovery and preserved original selections separately from model-chosen plan usage. Current capabilities are rechecked before execution.
+ * Enabled saved memories now inform private-conversation planning, edits, and answers without autosave or embedding backfill. Scope and audience are rechecked before answering, and existing memory citations are preserved.
+ * (Ref: `functions_orchestration_context.py`, `functions_orchestration_registry.py`, `functions_orchestration_planner.py`, `functions_agent_catalog.py`, [Capability Context Fix](fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md))
+
### **(v0.261.100)**
#### New Features
diff --git a/docs/guides/review-and-edit-orchestration-plans.md b/docs/guides/review-and-edit-orchestration-plans.md
index 8987d0b23..d81df7a7f 100644
--- a/docs/guides/review-and-edit-orchestration-plans.md
+++ b/docs/guides/review-and-edit-orchestration-plans.md
@@ -4,7 +4,7 @@ title: "Review and edit orchestration plans"
description: "Refine proposed work with the planner before running it."
section: "Guides"
audience: user
-version: "0.261.102"
+version: "0.261.104"
---
## Decide what should run
@@ -18,6 +18,32 @@ Conversational plan editing was implemented in version **0.261.102**, recorded i
`application/single_app/config.py`. It is available in the V2 interface for plans
that have not started.
+## Choose requirements, not permissions
+
+Since **0.261.104**, every Orchestrate request reaches the planner, including
+short questions. Selected supported tools, documents, and agents tell it what
+the plan must use. Leaving Web Search or Deep Research unchecked does not
+forbid those capabilities: the planner can choose them when they are enabled,
+authorized, and useful for the task. Say "do not browse" when that is an actual
+requirement.
+
+Deep Research does not require selecting the Web button first. Its automatic
+source discovery still depends on the administrator enabling Web Search.
+Selected workspaces and document filters continue to bound document access.
+Image generation has no orchestration adapter; use ordinary chat for that work.
+If Image was already selected, Send and Enter pause for an explicit choice:
+**Use regular Chat with Image**, or **Use Orchestrate without Image for this
+message**. The second choice excludes Image only from that orchestration message
+and preserves your ordinary-chat Image preference.
+
+URL Access uses the full resolved message, including an attached prompt. Removing
+the URL from the draft clears that now-ineligible selection; it does not clear
+other selected requirements.
+
+Research is not compulsory. The planner can answer directly when the available
+context is enough. Capability lookup or model failures produce errors rather
+than a replacement answer-only plan.
+
## Review versus Edit
**Review** opens the existing drawer. You can inspect steps and their rationales,
@@ -53,6 +79,24 @@ Editor exchanges do not create duplicate messages in your main conversation.
The eventual answer follows your accepted changes while the original question
remains intact.
+If an accepted edit removes an originally selected operation or document, the
+preview reports that change for review. It does not rewrite your standing
+composer preferences.
+
+## Understand reasoning adjustments
+
+The reasoning picker uses the selected model's supported levels. For example,
+GPT-5.6 Luna supports **None**, **Low**, **Medium**, **High**, and **XHigh**, not
+Minimal. A previously saved Minimal choice becomes Low with a visible notice.
+This also applies to older saved plans when edited or run.
+
+**None** is an explicit level on models that support it. **Model default** means
+the request omitted the effort parameter; it does not claim the provider chose
+None or Low. If the provider rejects an otherwise supported effort, SimpleChat
+can retry once using the model default and reports the adjustment. Other model
+errors still stop the affected operation. Neither adjustment switches models
+or approves a plan.
+
## Understand the countdown pause
Opening Edit stops any countdown and establishes a manual-approval hold. The
@@ -87,6 +131,6 @@ cancelling it. Review it before explicitly choosing Cancel again.
## Related
-- [Chat orchestration]({{ '/explanation/features/CHAT_ORCHESTRATION/' | relative_url }})
-- [Plan editing architecture]({{ '/explanation/features/V2_ORCHESTRATION_PLAN_EDITING/' | relative_url }})
+- [Chat orchestration](https://github.com/microsoft/simplechat/blob/main/docs/explanation/features/CHAT_ORCHESTRATION.md)
+- [Plan editing architecture](https://github.com/microsoft/simplechat/blob/main/docs/explanation/features/V2_ORCHESTRATION_PLAN_EDITING.md)
- [Orchestration settings]({{ '/admin/orchestration/' | relative_url }})
diff --git a/docs/reference/actions/fact-memory.md b/docs/reference/actions/fact-memory.md
index 543eb15c4..caeb757dc 100644
--- a/docs/reference/actions/fact-memory.md
+++ b/docs/reference/actions/fact-memory.md
@@ -25,6 +25,20 @@ Use it for durable preferences or background facts. Do not store secrets or regu
- Assigning this action to an agent lets the agent read and write memories as part of its own tool calls.
- Users also need access to the action through workspace or governance policy where applicable.
+## Orchestration context
+
+Since **0.261.104**, private-conversation orchestration also recalls enabled saved
+instructions and relevant embedded facts when planning, editing a plan, and
+answering. This automatic context is read-only; it does not require assigning
+the action to an agent and does not autosave or backfill memory embeddings.
+Current requests override saved preferences. Scope and membership are rechecked
+before answering, and memory provenance remains available in citations.
+
+Shared conversations, including their hidden backing records, do not receive
+this automatic memory context. See the
+[orchestration memory boundaries](https://github.com/microsoft/simplechat/blob/main/docs/explanation/fixes/ORCHESTRATION_CAPABILITY_CONTEXT_FIX.md#read-only-saved-memory)
+for scope selection and missing-embedding behavior.
+
## Configuration overview
Assign/enable the built-in memory action; no external service fields are required.
diff --git a/docs/reference/chat-controls.md b/docs/reference/chat-controls.md
index e752b8ab1..a28830d23 100644
--- a/docs/reference/chat-controls.md
+++ b/docs/reference/chat-controls.md
@@ -4,6 +4,7 @@ title: "Chat interface controls"
description: "Reference for every documented control in the SimpleChat chat interface."
section: "Reference"
audience: user
+version: "0.261.104"
---
## How to use this reference
@@ -75,6 +76,14 @@ arrived during streaming. After granting access, send the message again.
| `reasoning-toggle-btn` | Opens reasoning-effort controls for models that support configurable reasoning. | Use it when a hard planning or analysis task needs more deliberate reasoning, or a simple task should be cheaper/faster. | Always available |
| `tts-autoplay-toggle-btn` | Toggles automatic spoken playback for AI responses. | Use it for hands-free review, accessibility, or listening while working in another window. | [`enable_text_to_speech`]({{ '/admin/knowledge/' | relative_url }}) |
+Since **0.261.104**, both interfaces use the selected model's declared reasoning
+levels rather than guessing from its configuration ID. Unsupported saved choices
+are adjusted visibly to a supported application default. For GPT-5.6 Luna,
+Minimal becomes Low; None and XHigh remain valid choices. When support is unknown
+or the parameter is unsupported, the request uses **Model default** instead of
+advertising invented options. Explicit **None** is distinct from omitting the
+parameter. Plans and answer metadata retain compatibility adjustments.
+
## Grounded search and document scope
{% include media.html src="reference/chat-controls-grounded-search.png" alt="Grounded Search panel with action, scope, document, tags, filters, and comparison controls visible." title="Grounded search and document scope" capture="Capture the Grounded Search panel with action, scope, document, tags, filters, and comparison controls visible." %}
@@ -162,6 +171,19 @@ for the complete workflow.
## Orchestration approval (V2 interface)
+In Orchestrate, selected Document Search, Web Search, Deep Research, and eligible
+URL Access controls are positive requirements, not the complete list of permitted
+tools. Unchecked controls are neutral. The planner may choose other enabled,
+authorized capabilities, while selected documents, agents, workspaces, and filters
+retain their intended constraints. Deep Research can be selected without also
+selecting Web Search. Image generation is unsupported in this mode and must be
+handled in ordinary chat or explicitly excluded for that message rather than
+silently discarded.
+
+Every Orchestrate request now invokes the planner, even a short question or
+acknowledgment. The planner may choose a direct answer; no topic rule forces
+research. See [Review and edit orchestration plans]({{ '/guides/review-and-edit-orchestration-plans/' | relative_url }}).
+
Account-level approval persistence was fixed in **0.261.101**. These controls appear
while Orchestrate is active and the administrator allows users to change approval
modes. The saved choice applies across chats and future visits; it does not alter
@@ -177,12 +199,27 @@ The composer reports an unsuccessful save rather than claiming the new mode was
remembered. Choose the mode again to retry. If no choice has been saved, the current
deployment default applies. See [Orchestration settings]({{ '/admin/orchestration/' | relative_url }}).
+## Orchestration input recovery (V2 interface)
+
+Since **0.261.104**, entering Orchestrate with Image already selected pauses
+submission behind an accessible alert. This applies to Send, Enter, and requests
+with an attached prompt; an unsupported selection is not silently ignored.
+
+| Control | What it does | Why you would use it | Enabled by |
+| --- | --- | --- | --- |
+| Use regular Chat with Image | Leaves Orchestrate and retains the Image selection. | Keep image generation as part of the request. | Orchestrate with Image already selected |
+| Use Orchestrate without Image for this message | Explicitly excludes Image from this orchestration message without changing the ordinary-chat Image preference. | Continue with supported orchestration work when an image is unnecessary for this turn. | Same input-recovery alert |
+
+The exclusion must be chosen again for a later message. URL Access eligibility
+uses the full resolved message, including attached prompts. Removing its URL
+clears only that selection; other requirements remain intact.
+
## Inline follow-up questions (V2 interface)
Implemented in **0.261.096**. These controls appear when chat orchestration needs more
information before it can plan the request. They use the composer's editing capabilities
without adding another model, agent, or execution toolbar. See
-[Chat Orchestration]({{ '/explanation/features/CHAT_ORCHESTRATION/' | relative_url }}).
+[Chat Orchestration](https://github.com/microsoft/simplechat/blob/main/docs/explanation/features/CHAT_ORCHESTRATION.md).
| Control | What it does | Why you would use it | Enabled by |
| --- | --- | --- | --- |
diff --git a/functional_tests/test_chat_reasoning_runtime.py b/functional_tests/test_chat_reasoning_runtime.py
new file mode 100644
index 000000000..0f49a8676
--- /dev/null
+++ b/functional_tests/test_chat_reasoning_runtime.py
@@ -0,0 +1,373 @@
+# test_chat_reasoning_runtime.py
+"""Functional tests for ordinary-chat reasoning integration.
+
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Executes the actual nonstreaming invocation and streaming branch through shared
+policy/retry functions. Azure seams and sockets are blocked; API errors use the
+real OpenAI SDK types. No Flask application or route graph is imported.
+"""
+
+import ast
+import copy
+from datetime import datetime
+import importlib
+import json
+import logging
+from pathlib import Path
+import socket
+import time
+from types import SimpleNamespace
+import unittest
+from unittest.mock import Mock, patch
+
+import httpx
+from openai import APIConnectionError, AuthenticationError, BadRequestError, RateLimitError
+
+from test_model_reasoning_capability_resolution import sdk_error
+from test_support.app_stubs import stubbed_config
+
+
+ROOT = Path(__file__).resolve().parents[1]
+ROUTE_FILE = ROOT / 'application' / 'single_app' / 'route_backend_chats.py'
+LUNA = 'gpt-5.6-luna'
+
+
+class ChatReasoningRuntimeTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ cls.source = ROUTE_FILE.read_text(encoding='utf-8')
+ cls.tree = ast.parse(cls.source)
+ with stubbed_config():
+ cls.clients = importlib.import_module('model_endpoint_clients')
+
+ def setUp(self):
+ network_guard = patch.object(socket, 'socket', side_effect=AssertionError('Network is blocked'))
+ network_guard.start()
+ self.addCleanup(network_guard.stop)
+ self.create = Mock()
+ self.usage = SimpleNamespace(prompt_tokens=8, completion_tokens=3, total_tokens=11)
+ self.completion = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content='Response'))], usage=self.usage,
+ )
+ self.chunk = SimpleNamespace(
+ choices=[SimpleNamespace(delta=SimpleNamespace(content='Response'))], usage=self.usage,
+ )
+ self.namespace = {
+ 'create_completion_with_reasoning': self.clients.create_completion_with_reasoning,
+ 'ModelEndpointBehavior': self.clients.ModelEndpointBehavior,
+ 'normalize_chat_completion_text': self.clients.normalize_chat_completion_text,
+ 'extract_chat_completion_response_text': self.clients.extract_chat_completion_response_text,
+ 'normalize_model_response_length': lambda value: value,
+ 'conversation_history_for_api': [{'role': 'user', 'content': 'Request'}],
+ 'reasoning_effort': 'minimal', 'reasoning_resolution': None,
+ 'gpt_reasoning_model_name': LUNA, 'gpt_model': 'custom-production-deployment',
+ 'gpt_provider': 'aoai', 'gpt_endpoint_id': 'authorized-endpoint',
+ 'gpt_response_length': 4096, 'gpt_response_length_parameter': 'max_completion_tokens',
+ 'gpt_client': SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=self.create))),
+ 'gpt_api_version': 'configured-version',
+ 'agent_citations_list': [], 'user_metadata': {}, 'enable_semantic_kernel': False,
+ 'user_enable_agents': False, 'active_group_id': None, 'document_scope': 'personal',
+ 'get_current_user_id': lambda: 'self', 'user_id': 'self', 'conversation_id': 'authorized-conversation',
+ '_prepare_conversation_context_for_invocation': lambda history, *args, **kwargs: (history, {}),
+ 'debug_print': Mock(), 'log_event': Mock(), 'datetime': datetime, 'logging': logging,
+ 'json': json, 'time': time, 'request_start_time': time.time(),
+ 'emit_thought': Mock(side_effect=lambda *args, **kwargs: {'type': 'thought', **kwargs}),
+ 'stream_cancel_requested': lambda: False, 'accumulated_content': '',
+ 'suppress_streamed_file_payload': False, 'token_usage_data': None,
+ }
+ helper_names = {
+ '_create_chat_completion_with_reasoning', '_build_chat_reasoning_metadata',
+ '_resolve_reasoning_effort_for_model', '_apply_response_length_for_model',
+ '_resolve_legacy_chat_reasoning_model_name',
+ }
+ helpers = [
+ copy.deepcopy(node) for node in self.tree.body
+ if isinstance(node, ast.FunctionDef) and node.name in helper_names
+ ]
+ invocation = copy.deepcopy(next(
+ node for node in ast.walk(self.tree)
+ if isinstance(node, ast.FunctionDef) and node.name == 'invoke_gpt_fallback'
+ ))
+ invocation.body = [
+ ast.Global(names=node.names) if isinstance(node, ast.Nonlocal) else node
+ for node in invocation.body
+ ]
+ stream_branch = next(
+ node for node in ast.walk(self.tree) if isinstance(node, ast.If)
+ and ast.unparse(node.test) == 'use_agent_streaming and selected_agent'
+ and any(
+ isinstance(child, ast.Assign)
+ and any(isinstance(target, ast.Name) and target.id == 'stream_params' for target in child.targets)
+ for child in node.orelse
+ )
+ )
+ stream_wrapper = ast.parse('def invoke_stream_branch():\n pass').body[0]
+ stream_wrapper.body = copy.deepcopy(stream_branch.orelse)
+ stream_wrapper.body.insert(0, ast.Global(names=[
+ 'reasoning_resolution', 'accumulated_content', 'token_usage_data',
+ ]))
+ module = ast.Module(body=[*helpers, invocation, stream_wrapper], type_ignores=[])
+ ast.fix_missing_locations(module)
+ exec(compile(module, str(ROUTE_FILE), 'exec'), self.namespace)
+
+ def metadata(self):
+ return self.namespace['_build_chat_reasoning_metadata'](
+ self.namespace['reasoning_resolution'], self.namespace['reasoning_effort'], LUNA,
+ )
+
+ def invoke(self, streaming):
+ if streaming:
+ list(self.namespace['invoke_stream_branch']())
+ else:
+ self.namespace['invoke_gpt_fallback']()
+
+ def test_supported_none_and_levels_sent_unchanged_in_both_routes(self):
+ for streaming in (False, True):
+ for effort in ('none', 'low', 'medium', 'high', 'xhigh', None):
+ with self.subTest(streaming=streaming, effort=effort):
+ self.create.reset_mock()
+ self.namespace['accumulated_content'] = ''
+ self.namespace['reasoning_effort'] = effort
+ self.create.side_effect = None
+ self.create.return_value = [self.chunk] if streaming else self.completion
+ self.invoke(streaming)
+ sent = self.create.call_args.kwargs
+ self.assertEqual(sent.get('reasoning_effort'), effort)
+ self.assertEqual('reasoning_effort' in sent, effort is not None)
+ self.assertEqual(sent['model'], 'custom-production-deployment')
+ self.assertEqual(sent['max_completion_tokens'], 4096)
+ self.assertEqual(sent['messages'], [{'role': 'user', 'content': 'Request'}])
+ self.assertEqual(self.metadata()['reasoning_effort'], effort)
+ self.assertEqual(self.metadata()['requested_reasoning_effort'], effort)
+ self.assertEqual(self.metadata()['reasoning_mode'], 'explicit' if effort else 'model_default')
+ self.assertEqual(self.metadata()['reasoning_adjustments'], [])
+ self.create.assert_called_once()
+
+ def test_stale_minimal_is_low_with_honest_metadata_in_both_routes(self):
+ for streaming in (False, True):
+ with self.subTest(streaming=streaming):
+ self.create.reset_mock()
+ self.create.return_value = [self.chunk] if streaming else self.completion
+ self.invoke(streaming)
+ self.create.assert_called_once()
+ self.assertEqual(self.create.call_args.kwargs['reasoning_effort'], 'low')
+ metadata = self.metadata()
+ self.assertEqual(metadata['reasoning_effort'], 'low')
+ self.assertEqual(metadata['reasoning_adjustments'][0], {
+ 'requested_effort': 'minimal', 'effective_effort': 'low', 'mode': 'explicit',
+ 'adjustment_reason': 'reasoning_effort_unsupported', 'model_name': LUNA, 'stage': 'answer',
+ })
+
+ def test_unknown_and_nonreasoning_models_do_not_receive_guessed_effort(self):
+ for streaming in (False, True):
+ for model in ('unknown-private-model', 'gpt-4o'):
+ with self.subTest(streaming=streaming, model=model):
+ self.create.reset_mock()
+ self.namespace['accumulated_content'] = ''
+ self.namespace['gpt_reasoning_model_name'] = model
+ self.create.return_value = [self.chunk] if streaming else self.completion
+ self.invoke(streaming)
+ self.assertNotIn('reasoning_effort', self.create.call_args.kwargs)
+ self.assertIsNone(self.metadata()['reasoning_effort'])
+ self.assertEqual(self.metadata()['reasoning_mode'], 'model_default')
+ self.assertTrue(self.metadata()['reasoning_adjustments'])
+
+ def test_provider_contradiction_retries_once_without_other_parameter_changes(self):
+ for streaming in (False, True):
+ with self.subTest(streaming=streaming):
+ self.create.reset_mock()
+ self.namespace['accumulated_content'] = ''
+ self.create.side_effect = [
+ sdk_error(), [self.chunk] if streaming else self.completion,
+ ]
+ self.invoke(streaming)
+ first, second = [call.kwargs for call in self.create.call_args_list]
+ self.assertEqual(first['reasoning_effort'], 'low')
+ self.assertEqual(second, {key: value for key, value in first.items() if key != 'reasoning_effort'})
+ metadata = self.metadata()
+ self.assertIsNone(metadata['reasoning_effort'])
+ self.assertEqual(metadata['reasoning_mode'], 'model_default')
+ self.assertEqual(metadata['reasoning_adjustments'][0]['requested_effort'], 'minimal')
+ self.assertEqual(metadata['reasoning_adjustments'][0]['adjustment_reason'], 'reasoning_parameter_rejected')
+ self.assertNotIn('private-provider-detail', json.dumps(metadata))
+
+ def test_unrelated_errors_do_not_trigger_reasoning_or_api_version_fallback(self):
+ errors = [
+ sdk_error(param='messages'),
+ sdk_error(code='invalid_request_error'),
+ sdk_error(AuthenticationError, status=401),
+ sdk_error(RateLimitError, status=429),
+ RuntimeError('invalid_request_error reasoning_effort api version not supported'),
+ APIConnectionError(request=httpx.Request('POST', 'https://provider.example.test')),
+ ]
+ for streaming in (False, True):
+ for error in errors:
+ with self.subTest(streaming=streaming, error=type(error).__name__):
+ self.create.reset_mock()
+ self.create.side_effect = error
+ with self.assertRaises(type(error)):
+ self.invoke(streaming)
+ self.create.assert_called_once()
+
+ def test_second_reasoning_failure_is_not_retried_again(self):
+ self.create.side_effect = [sdk_error(), sdk_error()]
+ with self.assertRaises(BadRequestError):
+ self.invoke(False)
+ self.assertEqual(self.create.call_count, 2)
+
+ def test_empty_stream_fallback_keeps_corrected_effort_and_original_adjustment(self):
+ self.create.side_effect = [[], self.completion]
+ self.invoke(True)
+ self.assertEqual(self.create.call_count, 2)
+ first, second = [call.kwargs for call in self.create.call_args_list]
+ self.assertEqual(second, {key: value for key, value in first.items() if key not in {'stream', 'stream_options'}})
+ self.assertEqual(second['reasoning_effort'], 'low')
+ self.assertEqual(self.metadata()['reasoning_adjustments'][0]['requested_effort'], 'minimal')
+ self.assertEqual(self.namespace['accumulated_content'], 'Response')
+
+ def test_empty_stream_after_compatibility_recovery_does_not_reintroduce_effort(self):
+ self.create.side_effect = [sdk_error(), [], self.completion]
+ self.invoke(True)
+ self.assertEqual(self.create.call_count, 3)
+ self.assertTrue(all('reasoning_effort' not in call.kwargs for call in self.create.call_args_list[1:]))
+ self.assertIsNone(self.metadata()['reasoning_effort'])
+ self.assertEqual(self.metadata()['reasoning_adjustments'][0]['requested_effort'], 'minimal')
+
+ def test_stream_adjustment_precedes_content_and_preserves_thought_envelope(self):
+ self.create.return_value = [self.chunk]
+ events = list(self.namespace['invoke_stream_branch']())
+ adjustment_index = next(
+ index for index, event in enumerate(events)
+ if isinstance(event, dict) and event.get('reasoning_adjustments')
+ )
+ content_index = next(index for index, event in enumerate(events) if isinstance(event, str) and '"content"' in event)
+ self.assertLess(adjustment_index, content_index)
+ adjustment = events[adjustment_index]['reasoning_adjustments'][0]
+ self.assertEqual(adjustment['effective_effort'], 'low')
+
+ serializer = copy.deepcopy(next(
+ node for node in ast.walk(self.tree)
+ if isinstance(node, ast.FunctionDef) and node.name == 'serialize_thought_event'
+ ))
+ module = ast.Module(body=[serializer], type_ignores=[])
+ ast.fix_missing_locations(module)
+ self.namespace['assistant_message_id'] = 'assistant-test'
+ exec(compile(module, str(ROUTE_FILE), 'exec'), self.namespace)
+ frame = self.namespace['serialize_thought_event'](
+ 'generation', 'Reasoning adjusted.', 2, reasoning_adjustments=[adjustment],
+ )
+ payload = json.loads(frame.removeprefix('data: '))
+ self.assertEqual(payload['type'], 'thought')
+ self.assertEqual(payload['step_type'], 'generation')
+ self.assertEqual(payload['reasoning_adjustments'], [adjustment])
+
+ def test_stream_iteration_failure_is_not_replayed(self):
+ def failing_stream():
+ yield self.chunk
+ raise sdk_error()
+ self.create.return_value = failing_stream()
+ with self.assertRaises(BadRequestError):
+ self.invoke(True)
+ self.create.assert_called_once()
+ self.assertEqual(self.namespace['accumulated_content'], 'Response')
+
+ def test_all_saved_assistant_paths_and_user_updates_use_effective_metadata(self):
+ metadata_dicts = []
+ for node in ast.walk(self.tree):
+ if not isinstance(node, ast.Dict):
+ continue
+ values = {
+ key.value: value for key, value in zip(node.keys, node.values)
+ if isinstance(key, ast.Constant) and isinstance(key.value, str)
+ }
+ metadata = values.get('metadata')
+ role = values.get('role')
+ if isinstance(role, ast.Constant) and role.value == 'assistant' and isinstance(metadata, ast.Dict):
+ if any(
+ isinstance(child, ast.Call) and isinstance(child.func, ast.Name)
+ and child.func.id == '_build_chat_reasoning_metadata'
+ for child in ast.walk(metadata)
+ ):
+ metadata_dicts.append(metadata)
+ self.assertEqual(len(metadata_dicts), 4, 'Normal, streamed, cancelled and interrupted messages must persist effective effort')
+ self.assertEqual(self.source.count("user_message_doc['metadata'].update(_build_chat_reasoning_metadata("), 2)
+ self.assertIn("'metadata': assistant_doc.get('metadata', {})", self.source)
+ self.assertNotIn("reasoning_effort != 'none'", self.source)
+
+ def test_no_completed_direct_call_does_not_report_requested_effort_as_applied(self):
+ self.assertIsNone(self.metadata()['reasoning_effort'])
+ self.assertEqual(self.metadata()['reasoning_adjustments'], [])
+
+ def test_authorized_endpoint_identity_uses_canonical_model_not_uuid_or_label(self):
+ model_id = 'f8c476df-c951-499c-b87d-98fd02597780'
+ endpoints = [{
+ 'id': endpoint_id, 'provider': 'aoai',
+ 'connection': {'endpoint': 'https://provider.example.test', 'api_version': 'configured-version'},
+ 'auth': {}, 'models': [{
+ 'id': model_id, 'modelName': canonical_name,
+ 'deploymentName': 'custom-production-deployment', 'displayName': 'GPT-5 Minimal',
+ }],
+ } for endpoint_id, canonical_name in (('first', LUNA), ('second', 'gpt-5-mini'))]
+ namespace = dict(self.namespace)
+ namespace.update({
+ 'get_streaming_model_endpoint_candidates': lambda *args, **kwargs: endpoints,
+ 'keyvault_model_endpoint_get_helper': lambda endpoint, *args, **kwargs: endpoint,
+ 'SecretReturnType': SimpleNamespace(VALUE='value'),
+ 'MODEL_ENDPOINT_PROVIDER_ALLOWLIST': {'aoai'},
+ 'infer_model_endpoint_protocol': lambda *args: 'azure_openai',
+ 'MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI': 'azure_openai',
+ '_normalize_model_icon_payload': lambda value: None,
+ 'normalize_model_response_length_from_model': lambda value: 4096,
+ 'build_streaming_multi_endpoint_client': lambda *args, **kwargs: self.namespace['gpt_client'],
+ })
+ names = {'resolve_streaming_multi_endpoint_gpt_config', '_build_model_endpoint_behavior_name'}
+ module = ast.Module(body=[
+ copy.deepcopy(node) for node in self.tree.body
+ if isinstance(node, ast.FunctionDef) and node.name in names
+ ], type_ignores=[])
+ ast.fix_missing_locations(module)
+ exec(compile(module, str(ROUTE_FILE), 'exec'), namespace)
+ resolver = namespace['resolve_streaming_multi_endpoint_gpt_config']
+ for endpoint_id, effective in (('first', 'low'), ('second', 'minimal')):
+ resolved = resolver(
+ {'enable_multi_model_endpoints': True},
+ {'model_id': model_id, 'model_endpoint_id': endpoint_id, 'model_provider': 'aoai'},
+ 'self',
+ )
+ self.assertEqual(resolved[1], 'custom-production-deployment')
+ self.assertEqual(resolved[6], endpoint_id)
+ self.assertEqual(resolved[7], model_id)
+ self.create.reset_mock()
+ self.create.return_value = self.completion
+ _, resolution = self.namespace['_create_chat_completion_with_reasoning'](
+ self.create, {'model': resolved[1], 'reasoning_effort': 'minimal'}, resolved[11],
+ )
+ self.assertEqual(resolution['effective_effort'], effective)
+ with self.assertRaises(LookupError):
+ resolver(
+ {'enable_multi_model_endpoints': True},
+ {'model_id': model_id, 'model_endpoint_id': 'unauthorized'},
+ 'self',
+ )
+
+ def test_legacy_apim_does_not_borrow_same_named_direct_endpoint_capabilities(self):
+ resolver = self.namespace['_resolve_legacy_chat_reasoning_model_name']
+ settings = {'gpt_model': {'selected': [{
+ 'deploymentName': 'production-answer', 'modelName': LUNA,
+ }]}}
+ self.assertEqual(resolver(settings, 'production-answer'), LUNA)
+ self.assertEqual(
+ resolver({**settings, 'enable_gpt_apim': True}, 'production-answer'),
+ 'production-answer',
+ )
+ self.assertEqual(resolver(settings, 'different-deployment'), 'different-deployment')
+ self.assertEqual(
+ resolver({'gpt_model': {'selected': [{'deploymentName': LUNA, 'modelName': ' '}]}}, LUNA),
+ LUNA,
+ )
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/functional_tests/test_chat_stream_empty_model_fallback.py b/functional_tests/test_chat_stream_empty_model_fallback.py
index 19c5bceca..219372cb8 100644
--- a/functional_tests/test_chat_stream_empty_model_fallback.py
+++ b/functional_tests/test_chat_stream_empty_model_fallback.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python3
# test_chat_stream_empty_model_fallback.py
+#!/usr/bin/env python3
"""
Functional test for empty model stream fallback.
-Version: 0.250.006
+Version: 0.261.104
Implemented in: 0.250.003; updated in 0.250.006
This test ensures non-agent model streams that complete without assistant text
@@ -12,6 +12,7 @@
import sys
from pathlib import Path
+from test_support.versioning import assert_app_version_at_least
ROOT = Path(__file__).resolve().parents[1]
@@ -37,7 +38,7 @@ def test_chat_stream_empty_model_fallback() -> None:
"Model stream returned no assistant content; retrying without streaming",
)
assert_contains(ROUTE_FILE, "fallback_params = {")
- assert_contains(ROUTE_FILE, "fallback_params.pop('reasoning_effort', None)")
+ assert_contains(ROUTE_FILE, "previous_resolution=reasoning_resolution")
assert_contains(ROUTE_FILE, "def _resolve_reasoning_effort_for_model")
assert_contains(ROUTE_FILE, "ModelEndpointBehavior(provider, model_name).resolve_reasoning_effort")
assert_contains(ROUTE_FILE, "ModelEndpointBehavior(provider, model_name).context_mode")
@@ -57,7 +58,7 @@ def test_chat_stream_empty_model_fallback() -> None:
"The selected model returned an empty response. Check the model endpoint API version and provider compatibility",
)
assert_contains(ROUTE_FILE, "payload.get('type') != 'thought'")
- assert_contains(CONFIG_FILE, 'VERSION = "0.250.006"')
+ assert_app_version_at_least("0.250.006")
print("✅ Empty model stream fallback markers verified.")
diff --git a/functional_tests/test_chat_stream_retry_multiendpoint_resolution_fix.py b/functional_tests/test_chat_stream_retry_multiendpoint_resolution_fix.py
index 2d7b6b009..c267e27a6 100644
--- a/functional_tests/test_chat_stream_retry_multiendpoint_resolution_fix.py
+++ b/functional_tests/test_chat_stream_retry_multiendpoint_resolution_fix.py
@@ -2,12 +2,12 @@
#!/usr/bin/env python3
"""
Functional test for chat stream retry multi-endpoint resolution.
-Version: 0.250.106
+Version: 0.261.104
Implemented in: 0.241.003
This test ensures the compatibility retry path reuses the in-app multi-endpoint
-resolver and Foundry fallback helpers instead of calling undefined script-only
-functions during GPT initialization.
+resolver instead of calling undefined script-only functions during GPT
+initialization. Provider compatibility recovery must retain the selected client.
"""
import os
@@ -90,8 +90,11 @@ def test_chat_api_uses_shared_multi_endpoint_resolution_for_retry_compatibility(
assert 'def get_foundry_api_version_candidates(' in route_source, (
'Expected route_backend_chats.py to define Foundry API-version fallback candidates in-app.'
)
- assert 'retry_client = build_streaming_multi_endpoint_client(' in route_source, (
- 'Expected Foundry fallback retries to reuse the in-app multi-endpoint client builder.'
+ assert '_create_chat_completion_with_reasoning(' in chat_api_source, (
+ 'Expected provider recovery to use the bounded shared reasoning helper.'
+ )
+ assert 'retry_client = build_streaming_multi_endpoint_client(' not in chat_api_source, (
+ 'A reasoning compatibility retry must not switch the selected client or API version.'
)
print('✅ Compatibility retry multi-endpoint resolution wiring passed')
diff --git a/functional_tests/test_fact_memory_history_context_leak_fix.py b/functional_tests/test_fact_memory_history_context_leak_fix.py
index bbc4c2c58..afa4132e3 100644
--- a/functional_tests/test_fact_memory_history_context_leak_fix.py
+++ b/functional_tests/test_fact_memory_history_context_leak_fix.py
@@ -2,7 +2,7 @@
#!/usr/bin/env python3
"""
Functional test for fact-memory history context leak fix.
-Version: 0.241.128
+Version: 0.261.104
Implemented in: 0.241.128
This test ensures saved instruction/fact memory citations stay available as
@@ -21,6 +21,7 @@
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
CONFIG_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'config.py')
ROUTE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'route_backend_chats.py')
+CONTEXT_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'functions_conversation_context.py')
FIX_DOC = os.path.join(
ROOT_DIR,
'docs',
@@ -64,6 +65,16 @@ def load_history_helpers():
f'Expected helpers {sorted(TARGET_FUNCTIONS)}, '
f'found {sorted(found_function_names)}'
)
+ context_tree = ast.parse(read_file_text(CONTEXT_FILE), filename=CONTEXT_FILE)
+ selected_nodes.extend(
+ copy.deepcopy(node) for node in context_tree.body
+ if isinstance(node, ast.Assign) and any(
+ isinstance(target, ast.Name) and target.id in {
+ 'CONVERSATION_CONTEXT_METADATA_TYPE', 'CONVERSATION_CONTEXT_FUNCTION_NAME',
+ }
+ for target in node.targets
+ )
+ )
module = ast.Module(body=selected_nodes, type_ignores=[])
ast.fix_missing_locations(module)
diff --git a/functional_tests/test_fact_memory_profile_and_mini_sk.py b/functional_tests/test_fact_memory_profile_and_mini_sk.py
index 4c8907686..8f14a4ca6 100644
--- a/functional_tests/test_fact_memory_profile_and_mini_sk.py
+++ b/functional_tests/test_fact_memory_profile_and_mini_sk.py
@@ -1,7 +1,7 @@
# test_fact_memory_profile_and_mini_sk.py
"""
Functional test for profile fact memory recall and mini-SK fact-memory support.
-Version: 0.240.085
+Version: 0.261.104
Implemented in: 0.240.077; 0.240.079; 0.240.081; 0.240.082; 0.240.083; 0.240.085
This test ensures fact memory supports instruction/fact memory types,
@@ -23,6 +23,7 @@
CONFIG_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'config.py')
STORE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'semantic_kernel_fact_memory_store.py')
ROUTE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'route_backend_chats.py')
+CONTEXT_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'functions_fact_memory_context.py')
PROFILE_ROUTE_FILE = os.path.join(ROOT_DIR, 'application', 'single_app', 'route_frontend_profile.py')
FEATURE_DOC = os.path.join(
ROOT_DIR,
@@ -82,7 +83,7 @@ class FakeCosmosResourceNotFoundError(Exception):
def load_tabular_fact_memory_helpers():
route_source = read_file_text(ROUTE_FILE)
- parsed = ast.parse(route_source, filename=ROUTE_FILE)
+ parsed = ast.parse(read_file_text(CONTEXT_FILE) + '\n' + route_source, filename=CONTEXT_FILE)
selected_nodes = []
selected_constant_names = {
'FACT_MEMORY_TYPE_FACT',
@@ -354,7 +355,7 @@ def test_route_sources_wire_chat_and_profile_fact_memory_paths():
assert route_source.count('enabled=fact_memory_enabled') >= 3, route_source
assert "settings.get('enable_fact_memory_plugin', False)" in route_source
assert "user_settings.get('enable_agents', True)" in route_source
- assert 'Fact Memory Recall' in route_source
+ assert 'Fact Memory Recall' in read_file_text(CONTEXT_FILE)
assert 'Instruction Memory' in route_source
assert "memory_type': 'instruction'" in route_source or 'FACT_MEMORY_TYPE_INSTRUCTION' in route_source
assert "'fact_memory'" in route_source
diff --git a/functional_tests/test_fact_memory_read_only_context.py b/functional_tests/test_fact_memory_read_only_context.py
new file mode 100644
index 000000000..de96ac601
--- /dev/null
+++ b/functional_tests/test_fact_memory_read_only_context.py
@@ -0,0 +1,199 @@
+# test_fact_memory_read_only_context.py
+"""Functional tests for shared read-only saved-memory context.
+
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Executes the real leaf module with storage, membership, embeddings and network
+stubbed. Planning must preserve scope, provenance and bounds without writes;
+normal chat must retain legacy embedding backfill.
+"""
+
+import importlib.util
+import json
+from pathlib import Path
+import socket
+import sys
+import types
+import unittest
+from unittest.mock import Mock, patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+CONTEXT_FILE = ROOT / 'application' / 'single_app' / 'functions_fact_memory_context.py'
+
+
+class MemoryContextTests(unittest.TestCase):
+ def setUp(self):
+ self.network_guard = patch.object(socket, 'socket', side_effect=AssertionError('Network is blocked'))
+ self.network_guard.start()
+ self.addCleanup(self.network_guard.stop)
+ self.facts = []
+ self.store = Mock()
+ self.store.list_facts.side_effect = self.list_facts
+ self.store.update_fact_embedding.return_value = None
+ self.store_factory = Mock(return_value=self.store)
+ self.embedding = Mock(return_value=([1.0, 0.0], {'model_deployment_name': 'embedding-test'}))
+ self.batch_embeddings = Mock(side_effect=lambda values: [self.embedding(value) for value in values])
+ self.membership = Mock()
+ self.logger = Mock()
+ stubs = {
+ 'functions_appinsights': types.SimpleNamespace(log_event=self.logger),
+ 'functions_content': types.SimpleNamespace(
+ generate_embedding=self.embedding, generate_embeddings_batch=self.batch_embeddings,
+ ),
+ 'functions_group': types.SimpleNamespace(assert_group_role=self.membership),
+ 'functions_message_artifacts': types.SimpleNamespace(make_json_serializable=lambda value: value),
+ 'semantic_kernel_fact_memory_store': types.SimpleNamespace(FactMemoryStore=self.store_factory),
+ }
+ self.module_guard = patch.dict(sys.modules, stubs)
+ self.module_guard.start()
+ self.addCleanup(self.module_guard.stop)
+ spec = importlib.util.spec_from_file_location('tested_fact_memory_context', CONTEXT_FILE)
+ self.context = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(self.context)
+
+ def list_facts(self, **kwargs):
+ return [
+ dict(fact) for fact in self.facts
+ if fact['scope_id'] == kwargs['scope_id']
+ and fact['scope_type'] == kwargs['scope_type']
+ and fact['memory_type'] == kwargs['memory_type']
+ ]
+
+ def add_fact(self, index, memory_type='fact', scope_type='user', scope_id='self', **kwargs):
+ self.facts.append({
+ 'id': f'memory-{index}', 'scope_type': scope_type, 'scope_id': scope_id,
+ 'memory_type': memory_type, 'value': f'Relevant saved memory {index}',
+ 'conversation_id': 'prior-authorized-conversation', 'agent_id': 'authorized-agent',
+ 'value_embedding': [1.0, 0.0], 'updated_at': '2026-09-07T00:00:00Z', **kwargs,
+ })
+
+ def payload(self, **kwargs):
+ args = {
+ 'scope_id': 'self', 'scope_type': 'user', 'authorized_user_id': 'self',
+ 'query_text': 'Relevant request', 'read_only': True, 'enabled': True,
+ }
+ args.update(kwargs)
+ return self.context.build_fact_memory_prompt_payload(**args)
+
+ def test_disabled_memory_has_no_accesses(self):
+ payload = self.payload(enabled=False, scope_type='group', scope_id='untrusted')
+ self.assertEqual(payload['context_messages'], [])
+ self.assertEqual(payload['thoughts'], [])
+ self.assertEqual(payload['citations'], [])
+ self.store_factory.assert_not_called()
+ self.membership.assert_not_called()
+ self.embedding.assert_not_called()
+ self.batch_embeddings.assert_not_called()
+
+ def test_unauthorized_scopes_fail_before_storage(self):
+ for kwargs in (
+ {'scope_id': 'someone-else'}, {'authorized_user_id': None},
+ {'scope_type': 'public', 'scope_id': 'public-space'},
+ {'scope_id': None}, {'scope_type': None},
+ ):
+ with self.subTest(kwargs=kwargs), self.assertRaises(PermissionError):
+ self.payload(**kwargs)
+ self.membership.side_effect = PermissionError('Membership revoked')
+ with self.assertRaises(PermissionError):
+ self.payload(scope_type='group', scope_id='revoked')
+ self.store_factory.assert_not_called()
+ self.embedding.assert_not_called()
+
+ def test_scoped_bounded_instruction_and_relevant_fact_provenance(self):
+ for index in range(15):
+ self.add_fact(index, value='value ' * 2000)
+ self.add_fact(
+ index + 20, memory_type='instruction', value='preference ' * 1000,
+ similarity={'unexpected': 'unbounded metadata' * 1000},
+ )
+ self.add_fact(100, scope_id='another-user', value='Never disclose this')
+ self.add_fact(101, value='Unrelated fact', value_embedding=[0.0, 1.0])
+ self.add_fact(102, value='No embedding', value_embedding=None)
+ payload = self.payload(
+ instruction_limit=1000, fact_limit=1000, query_text='query ' * 1000,
+ include_metadata=True, conversation_id='current-conversation', agent_id='current-agent',
+ )
+ self.assertEqual(len(payload['instruction_payload']['matched_facts']), 8)
+ self.assertEqual(len(payload['recall_payload']['matched_facts']), 4)
+ self.assertEqual(len(payload['citations']), 2)
+ self.assertEqual(len(payload['context_messages']), 3)
+ self.assertEqual(payload['recall_payload']['embedding_backfill_count'], 0)
+ for citation in payload['citations']:
+ for fact in citation['function_result']['facts']:
+ self.assertLessEqual(len(fact['value']), 2000)
+ self.assertEqual(fact['conversation_id'], 'prior-authorized-conversation')
+ self.assertEqual(fact['agent_id'], 'authorized-agent')
+ for fact in payload['recall_payload']['matched_facts']:
+ self.assertNotIn('value_embedding', fact)
+ serialized = json.dumps(payload)
+ self.assertNotIn('Never disclose this', serialized)
+ self.assertNotIn('Unrelated fact', serialized)
+ self.assertNotIn('No embedding', serialized)
+ self.assertIn('current user request takes precedence', serialized)
+ self.assertIn('not instructions or permissions', serialized)
+ self.assertLess(len(serialized), 150000)
+ self.assertLessEqual(len(self.embedding.call_args.args[0]), 2000)
+ self.store.update_fact_embedding.assert_not_called()
+ self.batch_embeddings.assert_not_called()
+ self.assertEqual(self.store.method_calls, [
+ unittest.mock.call.list_facts(scope_type='user', scope_id='self', memory_type='instruction'),
+ unittest.mock.call.list_facts(
+ scope_type='user', scope_id='self', memory_type='fact',
+ conversation_id='current-conversation', agent_id='current-agent',
+ ),
+ ])
+
+ def test_group_scope_uses_existing_fresh_membership_authorizer(self):
+ self.add_fact(1, scope_type='group', scope_id='selected-group')
+ self.add_fact(2, scope_type='group', scope_id='other-group')
+ payload = self.payload(scope_type='group', scope_id='selected-group')
+ self.membership.assert_called_with(
+ 'self', 'selected-group', allowed_roles=('Owner', 'Admin', 'DocumentManager', 'User'),
+ )
+ self.assertEqual([fact['id'] for fact in payload['recall_payload']['matched_facts']], ['memory-1'])
+ self.assertTrue(all(call.kwargs['scope_id'] == 'selected-group' for call in self.store.list_facts.call_args_list))
+ self.store.update_fact_embedding.assert_not_called()
+
+ def test_normal_chat_backfills_and_retains_existing_payload_shape(self):
+ self.add_fact(1, value_embedding=None)
+ payload = self.payload(read_only=False, authorized_user_id=None)
+ self.assertEqual(set(payload), {
+ 'context_messages', 'thoughts', 'citations', 'instruction_payload', 'recall_payload',
+ })
+ self.assertEqual(payload['recall_payload']['embedding_backfill_count'], 1)
+ self.assertEqual(len(payload['recall_payload']['matched_facts']), 1)
+ self.store.update_fact_embedding.assert_called_once()
+ self.batch_embeddings.assert_called_once()
+ self.membership.assert_not_called()
+
+ def test_query_embedding_failure_is_explicit_and_safely_logged(self):
+ self.add_fact(1)
+ self.embedding.side_effect = RuntimeError('secret provider detail')
+ payload = self.payload()
+ self.assertEqual(payload['recall_payload']['search_mode'], 'embedding_unavailable')
+ self.assertEqual(payload['recall_payload']['thought_content'], 'Fact memory search unavailable')
+ self.assertEqual(payload['recall_payload']['matched_facts'], [])
+ self.assertNotIn('secret provider detail', json.dumps(payload))
+ self.assertNotIn('secret provider detail', str(self.logger.call_args))
+ self.store.update_fact_embedding.assert_not_called()
+
+ def test_unembedded_read_only_facts_report_unavailable_without_backfill(self):
+ self.add_fact(1, value_embedding=None)
+ payload = self.payload()
+ self.assertEqual(payload['recall_payload']['search_mode'], 'embedding_unavailable')
+ self.assertEqual(payload['recall_payload']['embedding_backfill_count'], 0)
+ self.assertEqual(payload['recall_payload']['matched_facts'], [])
+ self.batch_embeddings.assert_not_called()
+ self.embedding.assert_not_called()
+ self.store.update_fact_embedding.assert_not_called()
+
+ def test_store_failure_propagates_instead_of_claiming_empty_memory(self):
+ self.store.list_facts.side_effect = RuntimeError('store unavailable')
+ with self.assertRaises(RuntimeError):
+ self.payload()
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/functional_tests/test_fact_memory_streaming_context_fix.py b/functional_tests/test_fact_memory_streaming_context_fix.py
index a14db289c..bb06a02fa 100644
--- a/functional_tests/test_fact_memory_streaming_context_fix.py
+++ b/functional_tests/test_fact_memory_streaming_context_fix.py
@@ -1,7 +1,7 @@
# test_fact_memory_streaming_context_fix.py
"""
Functional test for fact memory chat-context parity.
-Version: 0.240.051
+Version: 0.261.104
Implemented in: 0.240.050; 0.240.051
This test ensures both standard and streaming agent chat paths inject saved fact
@@ -13,6 +13,7 @@
import copy
import os
from test_support.versioning import assert_app_version_at_least
+from test_fact_memory_profile_and_mini_sk import CONTEXT_FILE, load_tabular_fact_memory_helpers
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -23,6 +24,7 @@
'docs',
'explanation',
'fixes',
+ 'v0.241.001',
'FACT_MEMORY_STREAMING_CONTEXT_FIX.md',
)
TARGET_FUNCTIONS = {
@@ -60,24 +62,10 @@ def visit_FunctionDef(self, node):
f'found {[node.name for node in selected_nodes]}'
)
- class FakeFactMemoryStore:
- created_instances = []
- next_facts = []
-
- def __init__(self):
- self.calls = []
- self.__class__.created_instances.append(self)
-
- def get_facts(self, **kwargs):
- self.calls.append(kwargs)
- return list(self.__class__.next_facts)
-
module = ast.Module(body=selected_nodes, type_ignores=[])
ast.fix_missing_locations(module)
- namespace = {
- 'FactMemoryStore': FakeFactMemoryStore,
- }
+ namespace, FakeFactMemoryStore, _ = load_tabular_fact_memory_helpers()
exec(compile(module, ROUTE_FILE, 'exec'), namespace)
return namespace, route_source, FakeFactMemoryStore
@@ -89,7 +77,7 @@ def test_get_facts_for_context_preserves_selected_agent_id():
namespace, _, fake_store_class = load_fact_memory_helpers()
fake_store_class.created_instances = []
fake_store_class.next_facts = [
- {'value': 'The user prefers hyphens instead of em dashes.'},
+ {'value': 'The user prefers hyphens instead of em dashes.', 'memory_type': 'fact', 'value_embedding': [1.0, 0.0]},
]
facts = namespace['get_facts_for_context'](
@@ -97,16 +85,17 @@ def test_get_facts_for_context_preserves_selected_agent_id():
scope_type='user',
conversation_id='conversation-456',
agent_id='agent-789',
+ query_text='who am i?',
)
assert '- The user prefers hyphens instead of em dashes.' in facts, facts
- assert '- agent_id: agent-789' in facts, facts
assert fake_store_class.created_instances, 'Expected FactMemoryStore to be instantiated.'
assert fake_store_class.created_instances[-1].calls == [{
'scope_type': 'user',
'scope_id': 'user-123',
'agent_id': 'agent-789',
'conversation_id': 'conversation-456',
+ 'memory_type': 'fact',
}], fake_store_class.created_instances[-1].calls
print('✅ Fact lookup preserves selected agent id')
@@ -120,7 +109,7 @@ def test_inject_fact_memory_context_adds_metadata_and_facts():
namespace, _, fake_store_class = load_fact_memory_helpers()
fake_store_class.created_instances = []
fake_store_class.next_facts = [
- {'value': 'The user prefers hyphens instead of em dashes.'},
+ {'value': 'The user prefers hyphens instead of em dashes.', 'memory_type': 'fact', 'value_embedding': [1.0, 0.0]},
]
conversation_history = [
@@ -132,6 +121,8 @@ def test_inject_fact_memory_context_adds_metadata_and_facts():
scope_type='user',
conversation_id='conversation-456',
agent_id='agent-789',
+ query_text='who am i?',
+ include_metadata=True,
)
assert conversation_history[0]['role'] == 'system', conversation_history
@@ -158,8 +149,9 @@ def test_route_wires_fact_memory_injection_for_standard_and_streaming_paths():
assert "agent_id=getattr(selected_agent, 'id', None)" in route_source, (
'Expected streaming injection to use the selected agent id.'
)
- assert '' in route_source, 'Expected fact memory system message markup.'
- assert '' in route_source, 'Expected conversation metadata system message markup.'
+ context_source = read_file_text(CONTEXT_FILE)
+ assert '' in context_source, 'Expected fact memory system message markup.'
+ assert '' in context_source, 'Expected conversation metadata system message markup.'
print('✅ Route wiring for standard and streaming fact injection passed')
return True
diff --git a/functional_tests/test_fact_memory_streaming_retrieval_fix.py b/functional_tests/test_fact_memory_streaming_retrieval_fix.py
index c2915cc02..4637c26cb 100644
--- a/functional_tests/test_fact_memory_streaming_retrieval_fix.py
+++ b/functional_tests/test_fact_memory_streaming_retrieval_fix.py
@@ -1,7 +1,7 @@
# test_fact_memory_streaming_retrieval_fix.py
"""
Functional test for fact memory streaming retrieval and visibility.
-Version: 0.240.081
+Version: 0.261.104
Implemented in: 0.240.081
This test ensures streaming chat uses backward-compatible agent defaults,
@@ -9,12 +9,9 @@
fact-memory usage through thoughts and a dedicated citation.
"""
-import ast
-import copy
import os
-import re
-from datetime import datetime
from test_support.versioning import assert_app_version_at_least
+from test_fact_memory_profile_and_mini_sk import CONTEXT_FILE, load_tabular_fact_memory_helpers
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
@@ -25,6 +22,7 @@
'docs',
'explanation',
'fixes',
+ 'v0.241.001',
'FACT_MEMORY_STREAMING_RETRIEVAL_FIX.md',
)
@@ -42,49 +40,8 @@ def read_config_version():
def load_fact_memory_retrieval_helpers():
- route_source = read_file_text(ROUTE_FILE)
- parsed = ast.parse(route_source, filename=ROUTE_FILE)
- target_functions = {
- '_tokenize_fact_memory_text',
- '_is_identity_fact_memory_query',
- '_looks_like_profile_fact',
- 'retrieve_relevant_fact_memory_entries',
- 'build_fact_memory_citation',
- 'build_fact_memory_recall_payload',
- }
- selected_nodes = []
-
- for node in parsed.body:
- if isinstance(node, ast.FunctionDef) and node.name in target_functions:
- selected_nodes.append(copy.deepcopy(node))
-
- assert len(selected_nodes) == len(target_functions), (
- f'Expected helpers {sorted(target_functions)}, '
- f'found {[node.name for node in selected_nodes]}'
- )
-
- class FakeFactMemoryStore:
- next_facts = []
- created_instances = []
-
- def __init__(self):
- self.calls = []
- self.__class__.created_instances.append(self)
-
- def list_facts(self, **kwargs):
- self.calls.append(kwargs)
- return list(self.__class__.next_facts)
-
- namespace = {
- 'FactMemoryStore': FakeFactMemoryStore,
- 'datetime': datetime,
- 'make_json_serializable': lambda value: value,
- 're': re,
- }
- module = ast.Module(body=selected_nodes, type_ignores=[])
- ast.fix_missing_locations(module)
- exec(compile(module, ROUTE_FILE, 'exec'), namespace)
- return namespace, FakeFactMemoryStore
+ namespace, fake_store_class, _ = load_tabular_fact_memory_helpers()
+ return namespace, fake_store_class
def test_fact_memory_retrieval_uses_request_relevance():
@@ -94,9 +51,9 @@ def test_fact_memory_retrieval_uses_request_relevance():
namespace, fake_store_class = load_fact_memory_retrieval_helpers()
fake_store_class.created_instances = []
fake_store_class.next_facts = [
- {'id': '1', 'value': "User's name is Paul.", 'updated_at': '2026-04-07T00:00:00Z'},
- {'id': '2', 'value': 'User lives in Alexandria.', 'updated_at': '2026-04-06T00:00:00Z'},
- {'id': '3', 'value': 'Server timeout is 30 seconds.', 'updated_at': '2026-04-05T00:00:00Z'},
+ {'id': '1', 'memory_type': 'fact', 'value_embedding': [1.0, 0.0], 'value': "User's name is Paul.", 'updated_at': '2026-04-07T00:00:00Z'},
+ {'id': '2', 'memory_type': 'fact', 'value_embedding': [0.92, 0.08], 'value': 'User lives in Alexandria.', 'updated_at': '2026-04-06T00:00:00Z'},
+ {'id': '3', 'memory_type': 'fact', 'value_embedding': [0.0, 1.0], 'value': 'Server timeout is 30 seconds.', 'updated_at': '2026-04-05T00:00:00Z'},
]
recall_payload = namespace['build_fact_memory_recall_payload'](
@@ -108,7 +65,7 @@ def test_fact_memory_retrieval_uses_request_relevance():
include_metadata=True,
)
- assert recall_payload['thought_content'] == 'Fact memory search found 2 relevant memories'
+ assert recall_payload['thought_content'] == 'Fact memory search found 2 relevant facts'
assert len(recall_payload['context_messages']) == 2, recall_payload
assert "User's name is Paul." in recall_payload['context_messages'][1]['content']
assert 'User lives in Alexandria.' in recall_payload['context_messages'][1]['content']
@@ -118,6 +75,7 @@ def test_fact_memory_retrieval_uses_request_relevance():
'scope_type': 'user',
'scope_id': 'user-123',
'conversation_id': 'conversation-456',
+ 'memory_type': 'fact',
}]
print('✅ Fact memory retrieval relevance passed')
@@ -131,10 +89,11 @@ def test_streaming_route_wires_fact_memory_visibility_and_agent_default():
route_source = read_file_text(ROUTE_FILE)
assert "user_settings.get('enable_agents', True)" in route_source
- assert 'force_enable_agents = bool(request_agent_info)' in route_source
- assert 'Fact Memory Recall' in route_source
+ assert 'force_enable_agents = _has_chat_agent_selection(request_agent_info)' in route_source
+ context_source = read_file_text(CONTEXT_FILE)
+ assert 'Fact Memory Recall' in context_source
assert "yield emit_thought(" in route_source and "'fact_memory'" in route_source
- assert 'Retrieved saved fact memories relevant to the current request.' in route_source
+ assert 'Retrieved saved facts relevant to the current request.' in context_source
print('✅ Streaming route fact-memory wiring passed')
return True
diff --git a/functional_tests/test_model_reasoning_capability_resolution.py b/functional_tests/test_model_reasoning_capability_resolution.py
new file mode 100644
index 000000000..44ddfeab1
--- /dev/null
+++ b/functional_tests/test_model_reasoning_capability_resolution.py
@@ -0,0 +1,335 @@
+# test_model_reasoning_capability_resolution.py
+"""
+Functional tests for canonical reasoning policies and bounded provider recovery.
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Exercises real catalog matching and SDK errors without Azure clients or network
+calls. Covers stale preferences, explicit None, unknown support, safe metadata,
+unchanged vision decisions, immutable request arguments and streaming boundaries.
+"""
+
+import importlib
+import json
+import unittest
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+import httpx
+from openai import APIConnectionError, AuthenticationError, BadRequestError, RateLimitError
+
+from test_support.app_stubs import stubbed_config
+
+
+LUNA = "gpt-5.6-luna"
+LUNA_EFFORTS = ["none", "low", "medium", "high", "xhigh"]
+MODEL_UUID = "f8c476df-c951-499c-b87d-98fd02597780"
+
+
+def sdk_error(error_type=BadRequestError, *, status=400, param="reasoning_effort",
+ code="unsupported_value", nested=False):
+ """Construct the actual SDK exception shape without making a request."""
+ detail = {
+ "message": "Unsupported value: reasoning_effort minimal; private-provider-detail",
+ "type": "invalid_request_error", "param": param, "code": code,
+ }
+ return error_type(
+ detail["message"],
+ response=httpx.Response(
+ status, request=httpx.Request("POST", "https://provider.example.test/chat/completions")
+ ),
+ body={"error": detail} if nested else detail,
+ )
+
+
+class ReasoningPolicyTests(unittest.TestCase):
+ @classmethod
+ def setUpClass(cls):
+ with stubbed_config():
+ cls.capabilities = importlib.import_module("functions_model_capabilities")
+ cls.clients = importlib.import_module("model_endpoint_clients")
+ cls.capabilities.load_model_capability_catalog(force_refresh=True)
+
+ def policy(self, model=LUNA):
+ return self.capabilities.resolve_model_reasoning_policy(model)
+
+ def resolve(self, effort, model=LUNA):
+ return self.capabilities.resolve_model_reasoning_effort(model, effort)
+
+ def test_luna_contract_and_normalized_aliases(self):
+ for name in (LUNA, "GPT 5.6 LUNA", "gpt_5_6_luna", "gpt-5.6-luna-2026-06-25",
+ "gpt-5.6-luna-eastus", "gpt-5.6"):
+ with self.subTest(name=name):
+ self.assertEqual(self.policy(name), {
+ "status": "supported", "efforts": LUNA_EFFORTS, "default_effort": "low",
+ })
+
+ def test_authorized_canonical_identity_not_configuration_id_or_display_label(self):
+ record = {
+ "id": MODEL_UUID, "model_id": MODEL_UUID, "modelName": LUNA,
+ "deploymentName": "production-answer", "displayName": "GPT-5 Minimal",
+ }
+ self.assertEqual(self.policy(record)["efforts"], LUNA_EFFORTS)
+ self.assertEqual(self.policy({"behavior_name": LUNA, "deployment": "custom"})["efforts"],
+ LUNA_EFFORTS)
+ for value in (
+ MODEL_UUID, {"id": MODEL_UUID}, {"displayName": LUNA}, {"name": LUNA},
+ {"modelName": "acme-private-model", "deploymentName": LUNA, "displayName": LUNA},
+ ):
+ with self.subTest(value=value):
+ self.assertEqual(self.policy(value)["status"], "unknown")
+ second_endpoint = {**record, "modelName": "gpt-5-mini"}
+ self.assertIn("minimal", self.policy(second_endpoint)["efforts"])
+ self.assertNotIn("minimal", self.policy(record)["efforts"])
+
+ def test_longest_prefix_does_not_invent_new_version_or_chat_variant_support(self):
+ for name in ("gpt-5.99", "gpt-5.99-custom", "gpt-5.6.1", "gpt-5.3-chat-eastus",
+ "gpt-5-chat", "gpt-5.1-chat", "acme-model", "gpt-6", None, {}):
+ with self.subTest(name=name):
+ self.assertEqual(self.policy(name), {
+ "status": "unknown", "efforts": [], "default_effort": None,
+ })
+ self.assertEqual(self.policy("gpt-5-pro-prod")["efforts"], ["high"])
+ self.assertEqual(self.policy("o1-mini-2024-09-12")["status"], "unsupported")
+
+ def test_supported_values_preserved_and_luna_minimal_corrected_to_low(self):
+ for value in LUNA_EFFORTS:
+ with self.subTest(value=value):
+ self.assertEqual(self.resolve(value), {
+ "requested_effort": value, "effective_effort": value,
+ "mode": "explicit", "adjustment_reason": None,
+ })
+ self.assertEqual(self.resolve("minimal"), {
+ "requested_effort": "minimal", "effective_effort": "low",
+ "mode": "explicit", "adjustment_reason": "reasoning_effort_unsupported",
+ })
+ self.assertEqual(self.resolve(" HIGH ")["effective_effort"], "high")
+ self.assertEqual(self.resolve("max")["effective_effort"], "low")
+
+ def test_legitimate_minimal_and_model_specific_fallbacks(self):
+ for model in ("gpt-5", "gpt-5-mini", "gpt-5-nano"):
+ self.assertEqual(self.resolve("minimal", model)["effective_effort"], "minimal")
+ self.assertIsNone(self.resolve("minimal", model)["adjustment_reason"])
+ self.assertEqual(self.resolve("minimal", "gpt-5-pro")["effective_effort"], "high")
+ self.assertEqual(self.resolve("low", "gpt-5.4-pro")["effective_effort"], "medium")
+ self.assertEqual(self.policy("gpt-5.1")["efforts"], ["none", "low", "medium", "high"])
+ self.assertEqual(self.policy("gpt-5.2")["efforts"], LUNA_EFFORTS)
+ self.assertEqual(self.policy("gpt-5.3-codex")["efforts"], ["low", "medium", "high", "xhigh"])
+ for model in ("o1", "o3", "o3-mini", "o4-mini"):
+ self.assertEqual(self.policy(model)["efforts"], ["low", "medium", "high"])
+
+ def test_absent_effort_never_injects_application_fallback(self):
+ for value in (None, "", " "):
+ for model in (LUNA, "gpt-5-pro", "gpt-4o", "unknown"):
+ with self.subTest(value=value, model=model):
+ self.assertEqual(self.resolve(value, model), {
+ "requested_effort": None, "effective_effort": None,
+ "mode": "model_default", "adjustment_reason": None,
+ })
+ self.assertEqual(self.resolve("none")["mode"], "explicit")
+ self.assertEqual(self.resolve("none", "gpt-5")["effective_effort"], "low")
+
+ def test_unknown_and_unsupported_efforts_use_honest_default_metadata(self):
+ for model, reason in (
+ ("gpt-4o", "reasoning_parameter_unsupported"),
+ ("o1-mini", "reasoning_parameter_unsupported"),
+ ("unknown", "reasoning_capability_unknown"),
+ ):
+ for effort in ("none", "high"):
+ self.assertEqual(self.resolve(effort, model), {
+ "requested_effort": effort, "effective_effort": None,
+ "mode": "model_default", "adjustment_reason": reason,
+ })
+
+ def test_policy_sources_and_original_boolean_catalog_fields(self):
+ catalog_path = Path(self.capabilities.__file__).parent / self.capabilities.CATALOG_FILENAME
+ document = json.loads(catalog_path.read_text(encoding="utf-8"))
+ sources = {source["id"] for source in document["sources"]}
+ for model in document["models"]:
+ for value in model.get("capabilities", {}).values():
+ self.assertIsInstance(value, bool)
+ if policy := model.get("reasoningPolicy"):
+ self.assertTrue(policy["sourceIds"])
+ self.assertTrue(set(policy["sourceIds"]) <= sources)
+ self.assertEqual(self.policy(model["id"])["status"], policy["status"])
+ public = self.policy()
+ public["efforts"].clear()
+ self.assertEqual(self.policy()["efforts"], LUNA_EFFORTS)
+
+ def test_reasoning_only_legacy_records_do_not_change_vision(self):
+ resolve_vision = self.capabilities.resolve_model_vision_support
+ self.assertEqual(resolve_vision("gpt-4o"), (True, "inferred"))
+ self.assertEqual(resolve_vision("o3-mini"), (True, "inferred"))
+ self.assertEqual(resolve_vision("gpt-5.3-chat"), (False, "catalog"))
+ self.assertEqual(resolve_vision(LUNA), (True, "catalog"))
+ self.assertEqual(resolve_vision({"modelName": LUNA, "supportsVision": False}),
+ (False, "declared"))
+
+ def test_missing_or_malformed_reasoning_metadata_is_unknown(self):
+ for policy in (None, {}, [], {"status": "supported", "efforts": ["imaginary"]},
+ {"status": "supported", "efforts": ["low"], "default_effort": "high"}):
+ with self.subTest(policy=policy):
+ with patch.object(self.capabilities, "_CATALOG_CACHE", {
+ "test-model": {"reasoningPolicy": policy}
+ }):
+ self.assertEqual(self.policy("test-model")["status"], "unknown")
+ with patch.object(self.capabilities, "_CATALOG_CACHE", {}):
+ self.assertEqual(self.policy()["status"], "unknown")
+ self.assertEqual(self.resolve("minimal")["mode"], "model_default")
+
+ def test_endpoint_behavior_uses_the_shared_policy(self):
+ behavior = self.clients.ModelEndpointBehavior("aoai", LUNA)
+ self.assertEqual(behavior.resolve_reasoning_effort("minimal"), "low")
+ self.assertEqual(behavior.resolve_reasoning_effort("none"), "none")
+ self.assertEqual(behavior.resolve_reasoning_effort(None), "")
+ self.assertEqual(
+ self.clients.ModelEndpointBehavior("aoai", "gpt-5").resolve_reasoning_effort("minimal"),
+ "minimal",
+ )
+ self.assertEqual(
+ self.clients.ModelEndpointBehavior("aoai", "gpt-5.99").resolve_reasoning_effort("high"),
+ "",
+ )
+
+ def test_completion_applies_policy_without_mutating_parameters(self):
+ for effort, effective in (("minimal", "low"), ("none", "none"), ("xhigh", "xhigh"),
+ (None, None)):
+ with self.subTest(effort=effort):
+ params = {
+ "model": "production-answer", "messages": [{"role": "user", "content": "hello"}],
+ "reasoning_effort": effort, "max_completion_tokens": 8192, "stream": True,
+ }
+ create = Mock(return_value=iter(["token"]))
+ result, resolution = self.clients.create_completion_with_reasoning(create, params, LUNA)
+ self.assertIs(result, create.return_value)
+ self.assertEqual(params["reasoning_effort"], effort)
+ self.assertEqual(create.call_args.kwargs.get("reasoning_effort"), effective)
+ self.assertEqual(resolution["effective_effort"], effective)
+ self.assertIs(create.call_args.kwargs["messages"], params["messages"])
+ create.assert_called_once()
+
+ def test_exact_parameter_rejection_retries_once_and_only_omits_effort(self):
+ for nested in (False, True):
+ for code in ("unsupported_value", "unsupported_parameter"):
+ with self.subTest(nested=nested, code=code):
+ params = {
+ "model": "production-answer", "messages": [{"role": "user", "content": "hello"}],
+ "reasoning_effort": "high", "max_completion_tokens": 8192,
+ "response_format": {"type": "json_object"}, "stream": True,
+ }
+ create = Mock(side_effect=[sdk_error(code=code, nested=nested), "completion"])
+ with patch.object(self.clients, "log_event") as log:
+ result, resolution = self.clients.create_completion_with_reasoning(
+ create, params, LUNA
+ )
+ self.assertEqual(result, "completion")
+ self.assertEqual(create.call_count, 2)
+ self.assertEqual(create.call_args_list[0].kwargs, params)
+ self.assertEqual(create.call_args_list[1].kwargs, {
+ key: value for key, value in params.items() if key != "reasoning_effort"
+ })
+ self.assertEqual(resolution, {
+ "requested_effort": "high", "effective_effort": None,
+ "mode": "model_default", "adjustment_reason": "reasoning_parameter_rejected",
+ })
+ self.assertNotIn("private-provider-detail", str(log.call_args_list))
+ self.assertNotIn("private-provider-detail", json.dumps(resolution))
+
+ def test_second_rejection_propagates_and_no_effort_means_no_retry(self):
+ error = sdk_error()
+ for model, effort, calls in ((LUNA, "high", 2), (LUNA, None, 1), ("unknown", "high", 1)):
+ with self.subTest(model=model, effort=effort):
+ create = Mock(side_effect=error)
+ with self.assertRaises(BadRequestError) as raised:
+ self.clients.create_completion_with_reasoning(
+ create, {"model": model, "messages": [], "reasoning_effort": effort}, model
+ )
+ self.assertIs(raised.exception, error)
+ self.assertEqual(create.call_count, calls)
+
+ def test_resolution_callback_records_recovery_before_a_failing_retry(self):
+ observations = []
+ retry_error = sdk_error(param="response_format", code="unsupported_parameter")
+
+ def create(**parameters):
+ self.assertEqual(observations[-1]["effective_effort"],
+ parameters.get("reasoning_effort"))
+ if "reasoning_effort" in parameters:
+ raise sdk_error()
+ raise retry_error
+
+ with self.assertRaises(BadRequestError) as raised:
+ self.clients.create_completion_with_reasoning(
+ create,
+ {"model": LUNA, "messages": [], "reasoning_effort": "minimal",
+ "response_format": {"type": "json_object"}},
+ LUNA, on_resolution=observations.append,
+ )
+ self.assertIs(raised.exception, retry_error)
+ self.assertEqual(observations, [
+ {"requested_effort": "minimal", "effective_effort": "low", "mode": "explicit",
+ "adjustment_reason": "reasoning_effort_unsupported"},
+ {"requested_effort": "minimal", "effective_effort": None, "mode": "model_default",
+ "adjustment_reason": "reasoning_parameter_rejected"},
+ ])
+
+ def test_resolution_callback_cannot_mutate_request_or_returned_metadata(self):
+ observations = []
+
+ def observe(resolution):
+ observations.append(dict(resolution))
+ resolution.update(requested_effort="changed", effective_effort="high")
+
+ create = Mock(return_value="completion")
+ result, resolution = self.clients.create_completion_with_reasoning(
+ create, {"model": LUNA, "messages": [], "reasoning_effort": "none"},
+ LUNA, on_resolution=observe,
+ )
+ self.assertEqual(result, "completion")
+ self.assertEqual(create.call_args.kwargs["reasoning_effort"], "none")
+ self.assertEqual(resolution, {
+ "requested_effort": "none", "effective_effort": "none", "mode": "explicit",
+ "adjustment_reason": None,
+ })
+ self.assertEqual(observations, [resolution])
+
+ def test_unrelated_errors_never_trigger_compatibility_recovery(self):
+ errors = [
+ sdk_error(param="response_format"), sdk_error(param="messages"),
+ sdk_error(code="context_length_exceeded"), sdk_error(code="content_filter"),
+ sdk_error(param=None), sdk_error(code="invalid_request_error"),
+ sdk_error(AuthenticationError, status=401), sdk_error(RateLimitError, status=429),
+ APIConnectionError(request=httpx.Request("POST", "https://provider.example.test")),
+ ValueError("reasoning_effort is unsupported"),
+ ]
+ for error in errors:
+ with self.subTest(error=type(error).__name__, code=getattr(error, "code", None)):
+ create = Mock(side_effect=error)
+ self.assertFalse(self.clients.is_reasoning_parameter_rejection(error))
+ with self.assertRaises(type(error)) as raised:
+ self.clients.create_completion_with_reasoning(
+ create, {"model": LUNA, "messages": [], "reasoning_effort": "high"}, LUNA
+ )
+ self.assertIs(raised.exception, error)
+ create.assert_called_once()
+
+ def test_stream_iteration_errors_are_not_replayed(self):
+ error = sdk_error()
+
+ def stream():
+ yield "already delivered"
+ raise error
+
+ create = Mock(return_value=stream())
+ result, resolution = self.clients.create_completion_with_reasoning(
+ create, {"model": LUNA, "messages": [], "reasoning_effort": "low", "stream": True}, LUNA
+ )
+ self.assertEqual(next(result), "already delivered")
+ with self.assertRaises(BadRequestError):
+ next(result)
+ self.assertEqual(resolution["effective_effort"], "low")
+ create.assert_called_once()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/functional_tests/test_model_vision_capability_resolution.py b/functional_tests/test_model_vision_capability_resolution.py
index a792caa1d..f5576b515 100644
--- a/functional_tests/test_model_vision_capability_resolution.py
+++ b/functional_tests/test_model_vision_capability_resolution.py
@@ -2,7 +2,7 @@
# test_model_vision_capability_resolution.py
"""
Functional test for how the application decides a model can accept images.
-Version: 0.261.084
+Version: 0.261.104
Implemented in: 0.261.084
Multi-Modal Vision Analysis sends page images to a model, so it can only offer
@@ -45,13 +45,13 @@
def test_the_catalog_declares_vision_support_for_every_model():
- """A model missing the field falls through to a guess it should not need."""
+ """Boolean capability records stay complete; reasoning-only records stay separate."""
print("Testing catalog completeness...")
assert_app_version_at_least("0.261.084")
document = json.loads(CATALOG.read_text(encoding="utf-8"))
- models = document.get("models") or []
+ models = [model for model in document.get("models") or [] if "capabilities" in model]
assert models, "The capability catalog lists no models."
missing = [
@@ -151,12 +151,12 @@ def test_an_unknown_model_still_falls_back_to_the_heuristic():
"""Refusing to guess would hide working models from existing deployments."""
print("\nTesting the heuristic fallback...")
- # The catalog covers current models; gpt-4o predates it and is not listed.
+ # gpt-4o has a reasoning-only record, not a vision declaration.
# A great many deployments still run it, so the heuristic still has to
# recognise it rather than the model disappearing from the picker.
supports, source = resolve("gpt-4o")
assert supports is True, (
- "gpt-4o resolved as not vision-capable. It is absent from the catalog, "
+ "gpt-4o resolved as not vision-capable. It has no vision declaration, "
"so the heuristic has to carry it, or existing deployments would lose "
"the model they are using."
)
diff --git a/functional_tests/test_orchestration_action_planning.py b/functional_tests/test_orchestration_action_planning.py
index 36b3a8570..d59dfba64 100644
--- a/functional_tests/test_orchestration_action_planning.py
+++ b/functional_tests/test_orchestration_action_planning.py
@@ -1,7 +1,7 @@
# test_orchestration_action_planning.py
"""Functional coverage for knowledge-phase action planning and opt-in.
-Version: 0.261.098
+Version: 0.261.104
Implemented in: 0.261.098
Exercises the real registry, planner and validator with model/storage seams mocked.
@@ -12,10 +12,12 @@
import json
import sys
from types import SimpleNamespace
+from unittest.mock import patch
import pytest
from test_support.app_stubs import APP_ROOT, stubbed_config
+from test_support.orchestration_research import document_action_policy_module
SETTINGS = {
@@ -36,9 +38,11 @@
}
-@pytest.fixture
+@pytest.fixture(scope='module')
def modules():
- with stubbed_config(cognitive_services_scope='https://cognitiveservices.azure.com/.default'):
+ with stubbed_config(cognitive_services_scope='https://cognitiveservices.azure.com/.default'), patch.dict(
+ sys.modules, {'functions_document_actions': document_action_policy_module()},
+ ):
yield SimpleNamespace(**{
name: importlib.import_module(f'functions_orchestration_{name}')
for name in ('registry', 'context', 'schema', 'planner')
@@ -105,11 +109,11 @@ def unexpected(*args, **kwargs):
) == []
-def test_short_action_requests_reach_planning_without_changing_disabled_fast_path(modules):
+def test_short_requests_reach_planning_with_or_without_action_access(modules):
question = 'Ticket 42 status?'
context = modules.context.build_planner_context(question, actions=[ACTION])
assert modules.planner.triage_request(question, context) != 'trivial'
- assert modules.planner.triage_request(question, {}) == 'trivial'
+ assert modules.planner.triage_request(question, {}) != 'trivial'
def test_normalized_plan_identifies_action_safely_and_keeps_phase_order(modules):
@@ -141,7 +145,7 @@ def test_planner_passes_both_action_and_agent_catalogs_to_validation(modules, mo
'arguments': {'agent_name': 'specialist', 'task': 'Separate specialist task.'},
})
monkeypatch.setattr(modules.planner, 'resolve_planner_client', lambda settings: (None, 'planner'))
- monkeypatch.setattr(modules.planner, '_call_planner', lambda *args: (json.dumps(plan), None))
+ monkeypatch.setattr(modules.planner, '_call_planner', lambda *args, **kwargs: (json.dumps(plan), None))
context = modules.context.build_planner_context(
'Gather findings.', agents=[{'name': 'specialist'}], actions=[ACTION],
)
@@ -158,19 +162,28 @@ def test_planner_passes_both_action_and_agent_catalogs_to_validation(modules, mo
def test_elicitation_retry_keeps_request_gates(modules, monkeypatch):
replies = iter([json.dumps({'kind': 'elicitation'}), json.dumps(raw_plan())])
+ supplied = []
+
+ def complete(_client, _deployment, messages, *args, **kwargs):
+ assert kwargs['require_complete_response'] is True
+ supplied.append(json.loads(messages[1]['content']))
+ return next(replies), None
+
monkeypatch.setattr(modules.planner, 'resolve_planner_client', lambda settings: (None, 'planner'))
- monkeypatch.setattr(modules.planner, '_call_planner', lambda *args: (next(replies), None))
+ monkeypatch.setattr(modules.planner, '_call_planner', complete)
def reject(*args, **kwargs):
raise modules.schema.PlanValidationError('Unrenderable question')
monkeypatch.setattr(modules.planner, 'normalize_elicitation', reject)
context = modules.context.build_planner_context('Look up ticket 42.', actions=[ACTION])
- _, result = modules.planner.plan_request(
- 'Look up ticket 42.', context, 'conversation', 'actor', settings=SETTINGS,
- request_context={'action_catalog': []},
- )
- assert all(step['capability_id'] != 'action_invoke' for step in result['steps'])
+ with pytest.raises(modules.planner.PlannerError):
+ modules.planner.plan_request(
+ 'Look up ticket 42.', context, 'conversation', 'actor', settings=SETTINGS,
+ request_context={'action_catalog': []},
+ )
+ assert len(supplied) == 2
+ assert all('action_invoke' not in item['capability_availability']['available'] for item in supplied)
def test_route_combines_answer_and_action_usage_without_dropping_either():
diff --git a/functional_tests/test_orchestration_agent_selection.py b/functional_tests/test_orchestration_agent_selection.py
index b54e907a9..537d9faaf 100644
--- a/functional_tests/test_orchestration_agent_selection.py
+++ b/functional_tests/test_orchestration_agent_selection.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_agent_selection.py
"""
Functional test for orchestration agent selection.
-Version: 0.261.089
+Version: 0.261.104
Implemented in: 0.261.087
An agent's configuration is not all equally safe to show a planner. Its naming fields are
@@ -18,11 +18,14 @@
import ast
import os
import sys
+import types
+from unittest.mock import Mock, patch
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
from test_support.app_stubs import APP_ROOT, stubbed_app_imports # noqa: E402
from test_support.versioning import assert_app_version_at_least # noqa: E402
+from test_support.orchestration_research import _definitions # noqa: E402
CONTEXT = 'functions_orchestration_context.py'
ROUTE = 'route_backend_orchestration.py'
@@ -58,6 +61,22 @@ def _tree(module):
return ast.parse(handle.read())
+def _catalog_stub(builder):
+ module = types.ModuleType('functions_agent_catalog')
+ module.build_accessible_agent_catalog = builder
+ module.build_agent_catalog_key = _definitions(
+ 'functions_agent_catalog.py', names={'build_agent_catalog_key'},
+ )['build_agent_catalog_key']
+ return module
+
+
+def _agent_settings():
+ from functions_orchestration_registry import get_capability
+
+ descriptor = get_capability('agent_invoke')
+ return {key: True for key in (*descriptor['settings_gates'], *descriptor['settings_gates_any'])}
+
+
def test_projection_withholds_agent_internals():
"""The planner sees an agent's naming fields and nothing else."""
print("Testing the agent planner projection...")
@@ -175,15 +194,23 @@ def test_a_seeded_agent_is_a_hard_constraint():
# A seeded agent must narrow the catalog. The user has already made the choice
# this catalog exists to inform, so offering alternatives invites the planner to
# overrule them -- the same reason a seeded document turns off the candidate probe.
- catalog = resolve_agent_catalog(
- 'user-1',
- seeds={'agent': {'name': 'chosen_one', 'display_name': 'Chosen One'}},
- )
+ actual = {
+ 'id': 'agent-1', 'name': 'chosen_one', 'display_name': 'Current name',
+ 'scope_type': 'personal', 'scope_id': 'user-1',
+ }
+ builder = Mock(return_value=[actual, {**actual, 'id': 'agent-2', 'name': 'another'}])
+ with patch.dict(sys.modules, {'functions_agent_catalog': _catalog_stub(builder)}):
+ catalog = resolve_agent_catalog(
+ 'user-1', settings=_agent_settings(),
+ seeds={'agent': {'name': 'chosen_one', 'display_name': 'Untrusted client name'}},
+ )
names = [a.get('name') for a in (catalog or [])]
assert names == ['chosen_one'], (
f"a seeded agent must be the only one offered, got {names}. A user who "
f"picked an agent has stated a constraint, not a preference."
)
+ assert catalog[0]['display_name'] == 'Current name'
+ builder.assert_called_once()
print(" ok a user-selected agent is the only one offered")
return True
@@ -194,30 +221,22 @@ def test_a_seeded_agent_is_a_hard_constraint():
return False
-def test_catalog_resolution_fails_soft():
- """A catalog lookup that raises degrades to 'no agents', never breaks planning."""
- print("Testing that catalog resolution fails soft...")
+def test_catalog_resolution_failure_is_explicit():
+ """A failed lookup cannot impersonate a successful empty authorized catalog."""
+ print("Testing that catalog failure is explicit...")
try:
- tree = _tree(CONTEXT)
- target = None
- for node in ast.walk(tree):
- if isinstance(node, ast.FunctionDef) and node.name == 'resolve_agent_catalog':
- target = node
- assert target is not None, 'resolve_agent_catalog not found'
-
- handlers = [n for n in ast.walk(target) if isinstance(n, ast.ExceptHandler)]
- assert handlers, (
- 'resolve_agent_catalog must handle a failing lookup. It is a multi-query Cosmos '
- 'traversal; a transient failure there must cost the plan its agents, not the '
- 'user their answer.'
- )
- for handler in handlers:
- raises = [n for n in ast.walk(handler) if isinstance(n, ast.Raise)]
- assert not raises, (
- 'the catalog handler re-raises; planning must continue without agents'
- )
-
- print(" ok a failed lookup degrades to no agents")
+ with stubbed_app_imports():
+ from functions_orchestration_context import CatalogResolutionError, resolve_agent_catalog
+
+ builder = Mock(side_effect=RuntimeError('PRIVATE_STORAGE_DETAIL'))
+ with patch.dict(sys.modules, {'functions_agent_catalog': _catalog_stub(builder)}):
+ try:
+ resolve_agent_catalog('user-1', settings=_agent_settings())
+ except CatalogResolutionError as exc:
+ assert 'PRIVATE_STORAGE_DETAIL' not in exc.message
+ else:
+ raise AssertionError('A failed lookup was treated as an empty catalog.')
+ print(" ok a failed lookup is not an authorization decision")
return True
except Exception as e:
print(f"Test failed: {e}")
@@ -274,7 +293,7 @@ def test_route_resolves_the_catalog_once_per_plan():
test_nameless_agents_are_dropped,
test_projection_is_applied_where_the_context_is_built,
test_a_seeded_agent_is_a_hard_constraint,
- test_catalog_resolution_fails_soft,
+ test_catalog_resolution_failure_is_explicit,
test_route_resolves_the_catalog_once_per_plan,
]
results = []
diff --git a/functional_tests/test_orchestration_capability_context.py b/functional_tests/test_orchestration_capability_context.py
new file mode 100644
index 000000000..80b4e844c
--- /dev/null
+++ b/functional_tests/test_orchestration_capability_context.py
@@ -0,0 +1,220 @@
+# test_orchestration_capability_context.py
+"""
+Regression tests for truthful orchestration resources, requirements and reasoning notices.
+
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Executes production definitions with explicit offline storage boundaries. In particular,
+the real agent label-map functions receive records resolved by the real membership helper.
+"""
+
+import json
+import sys
+import types
+import unittest
+from contextlib import ExitStack
+from copy import deepcopy
+from pathlib import Path
+from unittest.mock import Mock, patch
+
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+ sys.path.insert(0, str(ROOT))
+
+# Standalone execution needs the repository path before importing local test helpers.
+from functional_tests.test_support.orchestration_research import _definitions, planner_runtime # noqa: E402
+
+
+class AgentDiscoveryTests(unittest.TestCase):
+ def setUp(self):
+ self.stack = ExitStack()
+ self.addCleanup(self.stack.close)
+ for target in ("socket.create_connection", "socket.socket.connect", "socket.socket.connect_ex"):
+ self.stack.enter_context(patch(target, side_effect=AssertionError("Unexpected network access.")))
+ self.current_groups = [{"id": "allowed-group", "name": "Current group name"}]
+ self.groups = types.SimpleNamespace(
+ get_user_groups=Mock(side_effect=lambda _user: deepcopy(self.current_groups)),
+ assert_group_role=Mock(),
+ )
+ identifier = _definitions("functions_agent_delegation.py", names={"_identifier"})["_identifier"]
+ self.group_api = _definitions("functions_action_catalog.py", seed={
+ "_identifier": identifier,
+ "import_module": lambda name: {"functions_group": self.groups}[name],
+ }, names={
+ "_GROUP_ROLES", "_INVALID_REFERENCE", "_stored_identifier", "_require_actor",
+ "_selected_group_ids", "_current_groups", "_assert_group_access", "resolve_current_user_groups",
+ })
+ self.model_reads = Mock(return_value=[])
+ self.action_reads = Mock(return_value=[])
+ self.agent_reads = Mock(return_value=[{"id": "agent-1", "name": "Group helper"}])
+ self.catalog_api = _definitions("functions_agent_catalog.py", seed={
+ "resolve_current_user_groups": self.group_api["resolve_current_user_groups"],
+ "normalize_model_endpoints": lambda values: (values, False),
+ "get_group_model_endpoints": self.model_reads,
+ "get_global_actions": lambda **_kwargs: [],
+ "filter_governed_global_actions_for_user": lambda _user, actions: actions,
+ "SecretReturnType": types.SimpleNamespace(NAME="name"),
+ "get_group_actions": self.action_reads,
+ "filter_actions_by_action_type_access": lambda _user, actions, *_args: actions,
+ "_should_include_global_agents": lambda _settings: False,
+ "get_group_agents": self.agent_reads,
+ "_serialize_catalog_agent": lambda agent, **scope: {**agent, **scope},
+ }, names={
+ "build_accessible_agent_catalog", "_build_model_label_map",
+ "_add_model_labels_from_endpoints", "_build_action_label_map", "_add_action_labels",
+ "build_agent_catalog_key",
+ })
+ self.settings = {
+ "enable_group_workspaces": True, "allow_group_agents": True,
+ "allow_group_custom_endpoints": True, "allow_group_plugins": True,
+ }
+
+ def catalog(self, selectors):
+ return self.catalog_api["build_accessible_agent_catalog"](
+ "user-1", settings=self.settings, user_groups=selectors,
+ )
+
+ def test_id_strings_are_resolved_before_real_model_and_action_label_maps(self):
+ catalog = self.catalog(["allowed-group"])
+ self.assertEqual(catalog[0]["scope_id"], "allowed-group")
+ self.assertEqual(catalog[0]["scope_name"], "Current group name")
+ self.model_reads.assert_called_once_with("allowed-group")
+ self.action_reads.assert_called_once_with("allowed-group", return_type="name")
+ self.groups.assert_group_role.assert_called_once_with(
+ "user-1", "allowed-group", allowed_roles=("Owner", "Admin", "DocumentManager", "User"),
+ )
+
+ def test_client_group_records_only_narrow_current_membership(self):
+ catalog = self.catalog([
+ {"id": "allowed-group", "name": "FORGED LABEL", "role": "Owner"},
+ {"id": "foreign-group", "name": "FORGED OTHER GROUP"},
+ ])
+ self.assertEqual([row["scope_id"] for row in catalog], ["allowed-group"])
+ self.assertNotIn("FORGED", repr(catalog))
+ self.agent_reads.assert_called_once_with("allowed-group")
+
+ def test_nonmember_and_revoked_group_scopes_do_not_reach_resource_reads(self):
+ self.assertEqual(self.catalog(["foreign-group"]), [])
+ self.groups.assert_group_role.side_effect = PermissionError("Membership was revoked.")
+ self.assertEqual(self.catalog(["allowed-group"]), [])
+ self.model_reads.assert_not_called()
+ self.action_reads.assert_not_called()
+ self.agent_reads.assert_not_called()
+
+ def context_resolver(self, builder):
+ runtime = self.stack.enter_context(planner_runtime())
+ module = types.ModuleType("functions_agent_catalog")
+ module.build_accessible_agent_catalog = builder
+ module.build_agent_catalog_key = self.catalog_api["build_agent_catalog_key"]
+ self.stack.enter_context(patch.dict(sys.modules, {"functions_agent_catalog": module}))
+ context = _definitions("functions_orchestration_context.py", seed=runtime.registry, names={
+ "_text", "CatalogResolutionError", "resolve_agent_catalog",
+ })
+ descriptor = runtime.registry["get_capability"]("agent_invoke")
+ settings = {
+ key: True for key in (*descriptor["settings_gates"], *descriptor["settings_gates_any"])
+ }
+ return context, settings
+
+ def test_selected_agent_is_resolved_from_current_authorized_records(self):
+ agent = {
+ "id": "agent-1", "name": "chosen", "display_name": "Current name",
+ "scope_type": "personal", "scope_id": "user-1",
+ }
+ builder = Mock(return_value=[agent, {**agent, "id": "agent-2", "name": "alternative"}])
+ context, settings = self.context_resolver(builder)
+ result = context["resolve_agent_catalog"](
+ "user-1", seeds={"agent": {"name": "chosen", "display_name": "FORGED"}},
+ settings=settings,
+ )
+ self.assertEqual(result, [agent])
+ builder.assert_called_once()
+ builder.return_value = []
+ with self.assertRaises(context["CatalogResolutionError"]):
+ context["resolve_agent_catalog"](
+ "user-1", seeds={"agent": {"name": "chosen"}}, settings=settings,
+ )
+
+ def test_catalog_failure_is_not_reported_as_an_empty_authorized_catalog(self):
+ context, settings = self.context_resolver(Mock(side_effect=RuntimeError("Private storage error.")))
+ with self.assertRaises(context["CatalogResolutionError"]) as raised:
+ context["resolve_agent_catalog"]("user-1", settings=settings)
+ self.assertNotIn("Private storage error", raised.exception.message)
+
+
+class RequirementAndNoticeTests(unittest.TestCase):
+ def setUp(self):
+ self.runtime_scope = planner_runtime()
+ self.runtime = self.runtime_scope.__enter__()
+ self.addCleanup(self.runtime_scope.__exit__, None, None, None)
+
+ def test_normalization_cannot_silently_lose_selected_work(self):
+ plan = {"steps": [{"capability_id": "respond", "arguments": {}}]}
+ check = self.runtime.schema["validate_plan_requirements"]
+ with self.assertRaises(self.runtime.schema["PlanValidationError"]):
+ check(deepcopy(plan), {"required_capabilities": ["deep_research"]})
+ with self.assertRaises(self.runtime.schema["PlanValidationError"]):
+ check(deepcopy(plan), {"agent": {"name": "chosen"}})
+ self.assertEqual(check(deepcopy(plan), {"web_search": False}), plan)
+
+ def test_explicit_later_narrowing_stays_possible_but_visible(self):
+ plan = {"steps": [{"capability_id": "respond", "arguments": {}}]}
+ checked = self.runtime.schema["validate_plan_requirements"](
+ plan, {"web_search": True}, allow_changes=True,
+ )
+ self.assertTrue(checked["validation"]["repairs"])
+ self.assertIn("Review this change", checked["validation"]["repairs"][0])
+ self.runtime.schema["validate_plan_requirements"](
+ checked, {"web_search": True}, allow_changes=True,
+ )
+ self.assertEqual(len(checked["validation"]["repairs"]), 1)
+
+ def test_selected_documents_must_be_in_the_effective_search_or_read_scope(self):
+ check = self.runtime.schema["validate_plan_requirements"]
+ seeds = {"document_ids": ["document-a", "document-b"]}
+ plan = {"steps": [{
+ "capability_id": "document_search", "arguments": {"document_ids": ["document-a"]},
+ }]}
+ with self.assertRaises(self.runtime.schema["PlanValidationError"]):
+ check(deepcopy(plan), seeds)
+ plan["steps"][0]["arguments"] = {"query": "Search the selected documents."}
+ check(plan, seeds)
+
+ def test_runtime_notices_report_effective_default_and_keep_model_roles_separate(self):
+ events = _definitions("functions_orchestration_events.py")
+ binding = types.SimpleNamespace(
+ behavior_name="gpt-5.6-luna", deployment="custom-deployment",
+ reasoning_resolution={
+ "requested_effort": "minimal", "effective_effort": "low",
+ "mode": "explicit", "adjustment_reason": "unsupported_value",
+ },
+ )
+ planner = events["build_model_reasoning_metadata"](binding, "planner")
+ binding.reasoning_resolution.update(
+ effective_effort=None, mode="model_default", adjustment_reason="provider_rejected",
+ )
+ answer = events["build_model_reasoning_metadata"](binding, "answer")
+ self.assertIsNone(answer["reasoning_effort"])
+ self.assertEqual(answer["reasoning_mode"], "model_default")
+ adjustments = events["merge_reasoning_adjustments"](
+ planner["reasoning_adjustments"], answer["reasoning_adjustments"],
+ )
+ self.assertEqual(len(adjustments), 2)
+ frame = events["build_reasoning_adjustment_event"](adjustments)
+ payload = json.loads(frame.removeprefix("data:").strip())
+ self.assertEqual(payload["type"], "thought")
+ self.assertIn("Model default", payload["content"])
+ done = json.loads(events["build_run_done_event"](
+ "conversation", **answer,
+ ).removeprefix("data:").strip())
+ self.assertEqual(done["reasoning_mode"], "model_default")
+ self.assertEqual(done["reasoning_adjustments"], answer["reasoning_adjustments"])
+ self.assertEqual(events["merge_reasoning_adjustments"](
+ {"malformed": "not an array"}, "not an array", [None, {"stage": []}],
+ ), [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/functional_tests/test_orchestration_conversation_context.py b/functional_tests/test_orchestration_conversation_context.py
index 966c50f93..dac19576f 100644
--- a/functional_tests/test_orchestration_conversation_context.py
+++ b/functional_tests/test_orchestration_conversation_context.py
@@ -1,7 +1,7 @@
# test_orchestration_conversation_context.py
"""
Functional regressions for bounded, conversation-aware orchestration.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.096
Resolver response compatibility and bounded recovery: 0.261.103
@@ -23,6 +23,7 @@
from test_support.app_stubs import stubbed_app_imports, stubbed_config
from test_support.versioning import assert_app_version_at_least
+from test_support.orchestration_research import document_action_policy_module
LATEST = 'Which are open on Wednesdays?'
@@ -60,6 +61,10 @@ def fake_module(name, **values):
def load_modules():
+ # These tests need actual policy behavior, not the document engine's Azure bootstrap.
+ policy = patch.dict(sys.modules, {'functions_document_actions': document_action_policy_module()})
+ policy.start()
+ unittest.addModuleCleanup(policy.stop)
with stubbed_config(cognitive_services_scope='https://cognitiveservices.azure.com/.default'):
return SimpleNamespace(**{
name: importlib.import_module(f'functions_orchestration_{name}')
@@ -252,14 +257,14 @@ def test_new_topic_uses_unchanged_request_and_no_old_constraints(self):
self.assertEqual(result['resolved_message'], 'Explain Python generators.')
self.assertEqual(result['message_ids'], [])
- def test_factual_follow_up_is_not_trivial_but_transformation_can_be(self):
+ def test_factual_follow_up_and_transformation_both_reach_planning(self):
planner = self.modules.planner
result, _ = self.resolve()
self.assertNotEqual(planner.triage_request(LATEST, {
'request_resolution': result
}), 'trivial')
result['requires_retrieval'] = False
- self.assertEqual(planner.triage_request('Put those in a table', {
+ self.assertNotEqual(planner.triage_request('Put those in a table', {
'request_resolution': result
}), 'trivial')
self.assertNotEqual(planner.triage_request('Put those in a table', {
diff --git a/functional_tests/test_orchestration_conversation_context_routes.py b/functional_tests/test_orchestration_conversation_context_routes.py
index 9b1e859fd..a60825537 100644
--- a/functional_tests/test_orchestration_conversation_context_routes.py
+++ b/functional_tests/test_orchestration_conversation_context_routes.py
@@ -1,7 +1,7 @@
# test_orchestration_conversation_context_routes.py
"""
Functional tests for conversation context across real orchestration HTTP/SSE routes.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.096
Prompt attachment integration: 0.261.097
Direct action integration: 0.261.098
@@ -266,6 +266,46 @@ def test_manual_model_is_used_for_resolution_planning_and_answer_in_all_approval
for client in self.model_clients:
client.close.assert_called_once_with()
+ def test_stale_luna_minimal_is_corrected_through_every_approval_mode(self):
+ selection = self.use_modern_models()
+ selection.update(model_id='luna-model', model_deployment='gpt-5.6-luna')
+ normal_completion = self.model.chat.completions.create
+
+ def reject_unsupported_minimal(**kwargs):
+ if kwargs.get('reasoning_effort') == 'minimal':
+ raise BadRequestError(
+ "Unsupported reasoning_effort: minimal. Supported values: none, low, medium, high, xhigh.",
+ response=HttpResponse(400, request=HttpRequest('POST', 'https://model.example.test')),
+ body={'error': {'code': 'unsupported_value', 'param': 'reasoning_effort'}},
+ )
+ return normal_completion(**kwargs)
+
+ self.model.chat.completions.create = reject_unsupported_minimal
+ for mode in ('auto', 'timed', 'manual'):
+ with self.subTest(mode=mode):
+ self.messages.items.clear()
+ self.runs.items.clear()
+ self.model.calls.clear()
+ for row in winery_history():
+ self.messages.upsert_item(row)
+ plan = self.planned(**selection, reasoning_effort='minimal', approval_mode=mode)
+ self.assertEqual(plan['reasoning_adjustments'][0]['effective_effort'], 'low')
+ events = frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ self.assertEqual(len(self.model.calls), 3)
+ for call in self.model.calls:
+ self.assertEqual(call['model'], 'gpt-5.6-luna')
+ self.assertEqual(call['reasoning_effort'], 'low')
+ terminal = next(event for event in events if event.get('type') == 'orchestration_done')
+ self.assertEqual(terminal['requested_reasoning_effort'], 'minimal')
+ self.assertEqual(terminal['reasoning_effort'], 'low')
+ self.assertEqual(terminal['reasoning_mode'], 'explicit')
+ self.assertEqual({item['stage'] for item in terminal['reasoning_adjustments']}, {'planner', 'answer'})
+ stored = self.runs.read_item(plan['run_id'], 'conv1')
+ answer = self.messages.read_item(stored['assistant_message_id'], 'conv1')
+ self.assertEqual(answer['reasoning_effort'], 'low')
+ self.assertEqual(answer['requested_reasoning_effort'], 'minimal')
+
def test_admin_default_is_pinned_to_the_plan_and_cannot_be_retargeted_at_run_time(self):
self.use_modern_models()
plan = self.planned()
@@ -278,6 +318,18 @@ def test_admin_default_is_pinned_to_the_plan_and_cannot_be_retargeted_at_run_tim
self.assertEqual({call['model'] for call in self.model.calls}, {'gpt-5.6-terra'})
self.assertEqual(self.runs.read_item(plan['run_id'], 'conv1')['seeds']['model'], TERRA_SELECTION)
+ def test_malformed_model_step_lists_never_publish_an_executable_plan(self):
+ for proposal in (
+ {'kind': 'plan'}, {'kind': 'plan', 'steps': []},
+ {'kind': 'plan', 'steps': 'not-a-list'},
+ ):
+ with self.subTest(proposal=proposal):
+ self.model.plan_override = proposal
+ _response, events = self.plan()
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertFalse(any(event.get('type') == 'orchestration_plan' for event in events))
+ self.assertFalse(any(row.get('plan') for row in self.runs.items.values()))
+
def test_replanning_uses_the_original_model_not_replacement_answer_controls(self):
self.use_modern_models()
self.planned()
@@ -834,6 +886,9 @@ def test_clarification_answer_survives_replan_and_execution(self):
def test_declined_clarification_is_not_asked_again(self):
elicitation = self.clarification()
+ self.model.plan_override = {
+ 'kind': 'plan', 'steps': [{'capability_id': 'respond', 'arguments': {}}],
+ }
plan = self.planned(
revision=1, elicitation=elicitation,
elicitation_response={'action': 'decline', 'content': {}},
@@ -841,6 +896,10 @@ def test_declined_clarification_is_not_asked_again(self):
self.assertEqual([step['capability_id'] for step in plan['steps']], ['respond'])
self.assertEqual(self.search_queries, [])
self.assertEqual(self.runs.read_item(plan['run_id'], 'conv1')['answered_questions'][0]['action'], 'decline')
+ self.assertTrue(any(
+ call['messages'][0]['content'] == self.modules.planner.PLANNER_SYSTEM_PROMPT
+ for call in self.model.calls
+ ))
def test_successive_clarifications_preserve_answers_without_creating_phantom_runs(self):
first = self.clarification()
@@ -958,19 +1017,38 @@ def test_incomplete_or_mismatched_clarification_is_explicitly_rejected(self):
)
self.assertEqual(response.status_code, 400)
- def test_history_transformation_bypasses_retrieval_but_keeps_answer_context(self):
+ def test_planner_can_reuse_history_without_retrieval_and_keeps_answer_context(self):
self.model.resolution_override = {
'relationship': 'follow_up',
'resolved_message': 'Put the previously listed Grants Pass wineries in a table.',
'message_ids': ['u2', 'a2'], 'requires_retrieval': False, 'clarification': '',
}
+ self.model.plan_override = {
+ 'kind': 'plan', 'steps': [{'capability_id': 'respond', 'arguments': {}}],
+ }
plan = self.planned(message='Put those in a table.')
self.assertEqual([step['capability_id'] for step in plan['steps']], ['respond'])
self.run_plan(plan)
self.assertEqual(self.search_queries, [])
- self.assertEqual(len(self.model.calls), 2)
+ self.assertEqual(len(self.model.calls), 3)
+ self.assertEqual(self.model.calls[1]['messages'][0]['content'], self.modules.planner.PLANNER_SYSTEM_PROMPT)
self.assertIn('Schmidt', json.dumps(self.model.calls[-1]))
+ def test_short_requests_reach_the_planner_with_available_capabilities(self):
+ self.messages.items.clear()
+ self.settings['enable_web_search'] = True
+ self.model.plan_override = {
+ 'kind': 'plan', 'steps': [{'capability_id': 'respond', 'arguments': {}}],
+ }
+ plan = self.planned(message='Hi!', turn_id='short-turn')
+ self.assertEqual([step['capability_id'] for step in plan['steps']], ['respond'])
+ self.assertEqual(len(self.model.calls), 1)
+ call = self.model.calls[0]
+ self.assertEqual(call['messages'][0]['content'], self.modules.planner.PLANNER_SYSTEM_PROMPT)
+ context = json.loads(call['messages'][1]['content'])
+ self.assertIn('web_search', context['capability_availability']['available'])
+ self.assertNotIn('web_search', context['user_selected'])
+
def test_legacy_pending_run_uses_its_saved_user_message_cutoff(self):
plan = self.planned()
record = self.runs.items[('conv1', plan['run_id'])]
diff --git a/functional_tests/test_orchestration_elicitation_context.py b/functional_tests/test_orchestration_elicitation_context.py
index 6b9af4447..9d1d9aa6f 100644
--- a/functional_tests/test_orchestration_elicitation_context.py
+++ b/functional_tests/test_orchestration_elicitation_context.py
@@ -1,7 +1,7 @@
# test_orchestration_elicitation_context.py
"""
Behavioral coverage for persisted inline clarification and execution context.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.096
Conversation-context, prompt-snapshot, and action integration: 0.261.099
Atomic revision-store fixture isolation: 0.261.103
@@ -32,6 +32,7 @@
import test_orchestration_conversation_context_routes as server_context_tests # noqa: E402
from test_support.app_stubs import APP_ROOT, stubbed_config # noqa: E402
+from test_support.orchestration_research import _definitions # noqa: E402
from test_support.versioning import assert_app_version_at_least # noqa: E402
@@ -344,6 +345,9 @@ def setUp(self):
),
'functions_agent_catalog': module(
'functions_agent_catalog', build_accessible_agent_catalog=lambda *args, **kwargs: [],
+ build_agent_catalog_key=_definitions(
+ 'functions_agent_catalog.py', names={'build_agent_catalog_key'},
+ )['build_agent_catalog_key'],
),
}
self.stack.enter_context(installed_modules(stubs))
@@ -753,6 +757,30 @@ def filtered_search(query, user_id, document_ids=None, **kwargs):
self.assertEqual(expected, found)
def test_real_plan_answer_run_preserves_original_and_rich_context(self):
+ self.settings.update(enable_semantic_kernel=True, allow_user_agents=True)
+ agent = {'id': 'main-agent', 'name': 'main-agent', 'scope_type': 'personal', 'scope_id': 'owner'}
+ self.stack.enter_context(patch.object(
+ sys.modules['functions_agent_catalog'], 'build_accessible_agent_catalog', return_value=[agent],
+ ))
+ agent_tasks = []
+
+ async def invoke_agent(selected, task, **kwargs):
+ self.assertEqual(selected['name'], 'main-agent')
+ agent_tasks.append(task)
+ return {'response': 'Agent considered the selected context.', 'citations': []}
+
+ self.stack.enter_context(patch.object(
+ self.route, 'capture_execution_identity', return_value=types.SimpleNamespace(user_id='owner'),
+ ))
+ self.stack.enter_context(patch.dict(sys.modules, {
+ 'agent_delegation_runtime': module(
+ 'agent_delegation_runtime', invoke_scoped_agent=invoke_agent,
+ delegation_citations=lambda _budget: [],
+ ),
+ 'semantic_kernel_plugins.plugin_invocation_logger': module(
+ 'semantic_kernel_plugins.plugin_invocation_logger', get_plugin_logger=lambda: None,
+ ),
+ }))
original_prompt = {'id': 'main', 'name': 'Main prompt', 'content': 'Original prompt wording'}
elicitation = self.begin(
question(generic=True), prompt_info=original_prompt,
@@ -767,7 +795,16 @@ def test_real_plan_answer_run_preserves_original_and_rich_context(self):
self.assertEqual(0, self.store.next_turn_index('conv', 'owner'))
self.assertEqual(0, len(self.messages.items))
- self.model_outputs.append(planned(['own']))
+ model_plan = planned(['own'])
+ model_plan['steps'].insert(-1, {
+ 'step_id': 'selected-agent', 'capability_id': 'agent_invoke',
+ 'arguments': {
+ 'agent_name': 'main-agent',
+ 'task': 'Review the selected document and prioritize accessibility.',
+ },
+ })
+ model_plan['steps'][-1].setdefault('depends_on', []).append('selected-agent')
+ self.model_outputs.append(model_plan)
payload = self.reply_payload(
elicitation,
{'files': [], 'style': 'brief', 'sections': ['risks', 'actions'], 'approved': False, 'count': 0},
@@ -800,9 +837,9 @@ def test_real_plan_answer_run_preserves_original_and_rich_context(self):
self.assertEqual(['own'], record['answered_questions'][0]['answer']['files'])
self.assertIs(False, record['answered_questions'][0]['answer']['approved'])
self.assertEqual(0, record['answered_questions'][0]['answer']['count'])
- for context in self.builder_contexts[-2:]:
- self.assertIn('prioritize accessibility', context['user_request'])
- self.assertEqual(1, len(context['clarifications']))
+ self.assertEqual(2, len(self.builder_contexts))
+ self.assertIn('prioritize accessibility', self.builder_contexts[-1]['user_request'])
+ self.assertEqual(1, len(self.builder_contexts[-1]['clarifications']))
self.assertEqual(1, len(self.messages.items))
duplicate = event_document(self.post_reply(payload), 'orchestration_plan')
@@ -810,7 +847,9 @@ def test_real_plan_answer_run_preserves_original_and_rich_context(self):
self.assertEqual(1, len(self.messages.items))
self.assertEqual(2, len(self.planner_contexts))
self.assertEqual(1, len(self.store.list_conversation_runs('conv', 'owner')))
- frames(self.run_plan(plan))
+ self.assert_run_completed(plan)
+ self.assertEqual(1, len(agent_tasks))
+ self.assertIn('prioritize accessibility', agent_tasks[0])
self.assertEqual(['own'], self.document_reads)
self.assertEqual('personal', self.analysis_calls[0]['doc_scope'])
self.assertIn('prioritize accessibility', self.analysis_calls[0]['prompt'])
@@ -846,7 +885,7 @@ def test_chat_attachment_file_and_image_are_real_sources(self):
self.assertIn(f'Content of the {role} upload.', json.dumps(self.answer_prompts))
self.assertEqual('chat', self.executed_contexts[-1].elicitation_references[0]['scope']['kind'])
- def test_primitive_only_legacy_reply_and_trivial_continuation(self):
+ def test_primitive_only_legacy_reply_reaches_planner_and_preserves_false_values(self):
self.settings['chat_orchestration_ledger_max_runs'] = 0
raw = {
'kind': 'elicitation', 'message': 'Choose the response preferences.',
@@ -866,9 +905,11 @@ def test_primitive_only_legacy_reply_and_trivial_continuation(self):
for key in ('elicitation_id', 'elicitation_revision', 'elicitation_submission_id'):
payload.pop(key)
payload['elicitation'] = {'requested_schema': {'properties': {}}}
- with patch.object(self.route, 'triage_request', return_value='trivial'):
- plan = event_document(self.post_reply(payload), 'orchestration_plan')
- self.assertEqual(1, len(self.planner_contexts), 'A trivial continuation must not spend another planner call')
+ self.model_outputs.append({
+ 'kind': 'plan', 'steps': [{'capability_id': 'respond', 'arguments': {}}],
+ })
+ plan = event_document(self.post_reply(payload), 'orchestration_plan')
+ self.assertEqual(2, len(self.planner_contexts), 'The planner must consider the accepted clarification')
self.assertEqual(1, plan['revision'])
record = self.store.get_orchestration_run(plan['run_id'], 'owner', 'conv')
self.assertIs(False, record['answered_questions'][0]['answer']['approved'])
@@ -906,9 +947,9 @@ def test_answers_accumulate_across_questions_and_retries(self):
self.assertEqual(2, plan['revision'])
record = self.store.get_orchestration_run(plan['run_id'], 'owner', 'conv')
self.assertEqual(2, len(record['answered_questions']))
- for context in self.builder_contexts[-2:]:
- self.assertIn('First clarification wording.', context['user_request'])
- self.assertIn('Second clarification wording.', context['user_request'])
+ self.assertEqual(3, len(self.builder_contexts))
+ self.assertIn('First clarification wording.', self.builder_contexts[-1]['user_request'])
+ self.assertIn('Second clarification wording.', self.builder_contexts[-1]['user_request'])
stale = {**first_payload, 'elicitation_submission_id': 'late-answer'}
self.assertEqual(409, self.post_reply(stale).status_code)
changed_retry = deepcopy(second_payload)
@@ -966,7 +1007,12 @@ def test_group_and_multiple_public_sources_keep_original_selections(self):
elicitation = self.begin(
selected_document_ids=['group-doc'], doc_scope='group', active_group_ids=['group-a'],
)
- self.model_outputs.append(planned(['public-doc', 'another-public-doc']))
+ model_plan = planned(['group-doc', 'public-doc'])
+ more = planned(['another-public-doc'])['steps'][0]
+ more['step_id'] = 'gather-more'
+ model_plan['steps'].insert(-1, more)
+ model_plan['steps'][-1]['depends_on'].append('gather-more')
+ self.model_outputs.append(model_plan)
payload = self.reply_payload(elicitation, context={'files': {'references': [
reference('public-doc', scope_kind='public', scope_id='public-a'),
reference('another-public-doc', scope_kind='public', scope_id='public-b'),
@@ -980,7 +1026,7 @@ def test_group_and_multiple_public_sources_keep_original_selections(self):
frames(self.run_plan(plan))
self.assertEqual(['public-a', 'public-b'], self.analysis_calls[0]['active_public_workspace_id'])
self.assertEqual('public-a', self.executed_contexts[0].active_public_workspace_id)
- self.assertEqual(['public-doc', 'another-public-doc'], self.document_reads)
+ self.assertEqual(['group-doc', 'public-doc', 'another-public-doc'], self.document_reads)
def test_unapproved_shares_failed_processing_and_non_file_messages_are_rejected(self):
elicitation = self.begin()
diff --git a/functional_tests/test_orchestration_memory_context.py b/functional_tests/test_orchestration_memory_context.py
new file mode 100644
index 000000000..a6dcaf6f3
--- /dev/null
+++ b/functional_tests/test_orchestration_memory_context.py
@@ -0,0 +1,336 @@
+# test_orchestration_memory_context.py
+"""Functional regressions for audience-bound orchestration memory.
+
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Uses real Flask routes, revisions, executor, adapters and the shared memory reader.
+Only storage, membership, embedding and model boundaries are replaced. All network
+access is blocked; planning and answering may read memory but must never write it.
+"""
+
+import json
+import sys
+import unittest
+from unittest.mock import patch
+
+from azure.core.exceptions import AzureError
+
+import test_fact_memory_read_only_context as fact_tests
+import test_orchestration_conversation_context_routes as context_tests
+import test_orchestration_plan_revision_routes as revision_tests
+
+
+class OrchestrationMemoryTests(unittest.TestCase):
+ plan = context_tests.ConversationRouteTests.plan
+ planned = context_tests.ConversationRouteTests.planned
+ run_plan = context_tests.ConversationRouteTests.run_plan
+ open_editor = revision_tests.PlanRevisionRouteTests.open_editor
+ request_revision = revision_tests.PlanRevisionRouteTests.request_revision
+ revise = revision_tests.PlanRevisionRouteTests.revise
+ run_editor_plan = revision_tests.PlanRevisionRouteTests.run_editor_plan
+ list_facts = fact_tests.MemoryContextTests.list_facts
+ add_fact = fact_tests.MemoryContextTests.add_fact
+
+ def setUp(self):
+ revision_tests.PlanRevisionRouteTests.setUp(self)
+ fact_tests.MemoryContextTests.setUp(self)
+ self.settings['enable_fact_memory_plugin'] = True
+ leaf_patch = patch.dict(sys.modules, {'functions_fact_memory_context': self.context})
+ leaf_patch.start()
+ self.addCleanup(leaf_patch.stop)
+ self.add_fact(1, scope_id='user1', memory_type='instruction', value='Prefer an accessible itinerary.')
+ self.add_fact(2, scope_id='user1', value='Saved destination: Crescent City.')
+ self.add_fact(3, scope_id='other-user', value='OTHER USER PRIVATE MEMORY')
+ self.addCleanup(self.assert_no_memory_writes)
+
+ def assert_no_memory_writes(self):
+ self.assertTrue(all(call[0] == 'list_facts' for call in self.store.method_calls), self.store.method_calls)
+ self.batch_embeddings.assert_not_called()
+
+ def planner_memory(self, calls=None):
+ return json.loads((calls or self.model.calls)[-1]['messages'][1]['content'])['memory']
+
+ def answer_calls(self):
+ return [
+ call for call in self.model.calls
+ if call['messages'][0]['content'] == self.modules.adapters.RESPONSE_CONTEXT_POLICY
+ ]
+
+ def shared_source(self):
+ conversation = self.conversations.read_item('conv1', 'conv1')
+ conversation.update(
+ conversation_kind='collaboration_source',
+ collaboration_conversation_id='shared-conversation',
+ chat_type='personal_single_user', is_hidden=True,
+ )
+ self.conversations.upsert_item(conversation)
+
+ def group_plan(self):
+ self.settings['enable_group_workspaces'] = True
+ self.add_fact(4, scope_type='group', scope_id='group1', value='GROUP MEMORY')
+ return self.planned(doc_scope='group', active_group_ids=['group1'])
+
+ def test_private_planning_and_answering_use_scoped_memory_and_preserve_citations(self):
+ plan = self.planned()
+ memory = self.planner_memory()
+ self.assertEqual(memory['status'], 'available')
+ self.assertEqual(memory['scope_type'], 'user')
+ self.assertIn('Saved destination: Crescent City.', json.dumps(memory))
+ self.assertNotIn('OTHER USER PRIVATE MEMORY', json.dumps(memory))
+ self.assertNotIn('Saved destination:', json.dumps(list(self.runs.items.values())))
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ answer = self.answer_calls()[-1]['messages']
+ self.assertIn('Saved destination: Crescent City.', json.dumps(answer))
+ self.assertIn('subordinate to the latest request', answer[0]['content'])
+ self.assertIn('Which are open on Wednesdays?', answer[-1]['content'])
+ terminal = next(event for event in events if event.get('type') == 'orchestration_done')
+ self.assertEqual(len(terminal['agent_citations']), 2)
+ self.assertTrue(all(citation['plugin_name'] == 'fact_memory' for citation in terminal['agent_citations']))
+ self.assertIn('prior-authorized-conversation', json.dumps(terminal['agent_citations']))
+ saved_answers = [item for item in self.messages.items.values() if item.get('role') == 'assistant']
+ self.assertTrue(any('fact_memory' in json.dumps(item.get('agent_citations')) for item in saved_answers))
+
+ def test_editing_refreshes_memory_without_persisting_raw_prompt_context(self):
+ editor = self.open_editor(self.planned())
+ self.add_fact(5, scope_id='user1', value='NEWLY SAVED MEMORY')
+ revised, _body = self.revise(editor, revision_tests.revised_plan())
+ self.assertIn('NEWLY SAVED MEMORY', json.dumps(self.planner_memory(self.edit_calls)))
+ record = self.runs.read_item(revised['plan']['run_id'], 'conv1')
+ self.assertEqual(record['memory_audience']['kind'], 'personal')
+ self.assertNotIn('NEWLY SAVED MEMORY', json.dumps(record))
+
+ def test_disabled_memory_performs_no_reads_or_embeddings_through_plan_and_run(self):
+ self.settings['enable_fact_memory_plugin'] = False
+ plan = self.planned()
+ self.assertEqual(self.planner_memory()['status'], 'disabled')
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ self.store_factory.assert_not_called()
+ self.membership.assert_not_called()
+ self.embedding.assert_not_called()
+
+ def test_shared_source_owner_does_not_load_personal_or_seeded_group_memory(self):
+ self.shared_source()
+ self.settings['enable_group_workspaces'] = True
+ plan = self.planned(doc_scope='group', active_group_ids=['group1'])
+ memory = self.planner_memory()
+ self.assertEqual(memory['status'], 'unavailable')
+ self.assertEqual(memory['messages'], [])
+ self.assertIn('shared conversations', memory['notices'][0])
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ self.store_factory.assert_not_called()
+ self.membership.assert_not_called()
+
+ def test_private_group_workspace_reads_only_authorized_group_memory(self):
+ plan = self.group_plan()
+ memory = self.planner_memory()
+ self.assertEqual(memory['scope_type'], 'group')
+ self.assertIn('GROUP MEMORY', json.dumps(memory))
+ self.assertNotIn('Saved destination:', json.dumps(memory))
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ self.membership.assert_called_with(
+ 'user1', 'group1', allowed_roles=('Owner', 'Admin', 'DocumentManager', 'User'),
+ )
+ self.assertTrue(all(call.kwargs['scope_id'] == 'group1' for call in self.store.list_facts.call_args_list))
+
+ def test_revoked_group_scope_blocks_run_before_any_execution(self):
+ plan = self.group_plan()
+ planned_queries = list(self.search_queries)
+ self.membership.side_effect = PermissionError('revoked')
+ response = self.run_plan(plan)
+ self.assertEqual(response.status_code, 409)
+ self.assertEqual(response.get_json()['code'], 'memory_scope_unavailable')
+ self.assertEqual(self.search_queries, planned_queries)
+ self.assertEqual(self.answer_calls(), [])
+
+ def test_revoked_group_scope_is_rechecked_after_retrieval_before_answer(self):
+ plan = self.group_plan()
+ self.after_search = lambda: setattr(self.membership, 'side_effect', PermissionError('revoked'))
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertEqual(self.answer_calls(), [])
+
+ def test_disabling_memory_during_retrieval_removes_final_context_and_citations(self):
+ plan = self.planned()
+ self.after_search = lambda: self.settings.update(enable_fact_memory_plugin=False)
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertFalse(any(event.get('error') for event in events), events)
+ self.assertNotIn('Saved destination:', json.dumps(self.answer_calls()))
+ terminal = next(event for event in events if event.get('type') == 'orchestration_done')
+ self.assertEqual(terminal.get('agent_citations'), [])
+
+ def test_changing_audience_after_planning_blocks_the_saved_plan(self):
+ plan = self.planned()
+ self.shared_source()
+ response = self.run_plan(plan)
+ self.assertEqual(response.status_code, 409)
+ self.assertEqual(response.get_json()['code'], 'memory_audience_changed')
+ self.assertEqual(self.answer_calls(), [])
+
+ def test_changing_audience_during_retrieval_blocks_answer_synthesis(self):
+ plan = self.planned()
+ self.after_search = self.shared_source
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertEqual(self.answer_calls(), [])
+
+ def test_changing_audience_during_planning_prevents_publication(self):
+ def change_after_memory_read():
+ if self.store.list_facts.called:
+ self.shared_source()
+
+ self.before_plan_reply = change_after_memory_read
+ _response, events = self.plan()
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertFalse(any(event.get('type') == 'orchestration_plan' for event in events), events)
+
+ def test_changing_audience_during_synthesis_prevents_answer_publication(self):
+ plan = self.planned()
+ completion = self.model.chat.completions.create
+
+ def change_during_answer(**kwargs):
+ response = completion(**kwargs)
+ if kwargs['messages'][0]['content'] == self.modules.adapters.RESPONSE_CONTEXT_POLICY:
+ self.shared_source()
+ return response
+
+ self.model.chat.completions.create = change_during_answer
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertFalse(any(event.get('type') == 'orchestration_done' for event in events), events)
+ self.assertFalse(self.runs.read_item(plan['run_id'], 'conv1').get('assistant_message_id'))
+
+ def test_revoked_group_membership_during_synthesis_blocks_answer_and_citations(self):
+ plan = self.group_plan()
+ completion = self.model.chat.completions.create
+
+ def revoke_during_answer(**kwargs):
+ response = completion(**kwargs)
+ if kwargs['messages'][0]['content'] == self.modules.adapters.RESPONSE_CONTEXT_POLICY:
+ self.membership.side_effect = PermissionError('revoked')
+ return response
+
+ self.model.chat.completions.create = revoke_during_answer
+ events = context_tests.frames(self.run_plan(plan))
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertFalse(any(event.get('type') == 'orchestration_done' for event in events), events)
+ self.assertNotIn('GROUP MEMORY', json.dumps(events))
+ stored = self.runs.read_item(plan['run_id'], 'conv1')
+ self.assertEqual(stored['status'], 'failed')
+ self.assertFalse(stored.get('assistant_message_id'))
+
+ def private_memory_question(self):
+ question = revision_tests.question()
+ question['message'] = 'Should the visit include your saved destination: Crescent City?'
+ return question
+
+ def test_audience_change_during_planner_clarification_prevents_save_and_emit(self):
+ self.model.plan_override = self.private_memory_question()
+
+ def change_after_memory_read():
+ if self.store.list_facts.called:
+ self.shared_source()
+
+ self.before_plan_reply = change_after_memory_read
+ _response, events = self.plan()
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertNotIn('saved destination: Crescent City', json.dumps(events))
+ self.assertFalse(any(event.get('type') == 'orchestration_elicitation' for event in events))
+ self.assertFalse(any(row.get('question') for row in self.runs.items.values()))
+
+ def test_group_revocation_during_planner_clarification_prevents_save_and_emit(self):
+ self.settings['enable_group_workspaces'] = True
+ self.add_fact(4, scope_type='group', scope_id='group1', value='GROUP MEMORY')
+ self.model.plan_override = self.private_memory_question()
+
+ def revoke_after_memory_read():
+ if self.store.list_facts.called:
+ self.membership.side_effect = PermissionError('revoked')
+
+ self.before_plan_reply = revoke_after_memory_read
+ _response, events = self.plan(doc_scope='group', active_group_ids=['group1'])
+ self.assertTrue(any(event.get('error') for event in events), events)
+ self.assertFalse(any(event.get('type') == 'orchestration_elicitation' for event in events))
+ self.assertFalse(any(row.get('question') for row in self.runs.items.values()))
+
+ def test_pending_question_replay_rechecks_memory_audience_without_new_model_call(self):
+ self.model.plan_override = self.private_memory_question()
+ _response, events = self.plan()
+ self.assertTrue(any(event.get('type') == 'orchestration_elicitation' for event in events))
+ calls_before = len(self.model.calls)
+ self.shared_source()
+ _response, replay = self.plan()
+ self.assertTrue(any(event.get('error') for event in replay), replay)
+ self.assertNotIn('saved destination: Crescent City', json.dumps(replay))
+ self.assertEqual(len(self.model.calls), calls_before)
+
+ def test_completed_submission_replay_rechecks_memory_audience(self):
+ self.model.plan_override = self.private_memory_question()
+ _response, events = self.plan()
+ first_question = next(event['elicitation'] for event in events if event.get('type') == 'orchestration_elicitation')
+ answer = {
+ 'revision': 1, 'elicitation': first_question,
+ 'elicitation_response': {'action': 'accept', 'content': {'day': 'Friday'}},
+ }
+ _response, answered = self.plan(**answer)
+ self.assertTrue(any(event.get('type') == 'orchestration_elicitation' for event in answered), answered)
+ calls_before = len(self.model.calls)
+ self.shared_source()
+ _response, replay = self.plan(**answer)
+ self.assertTrue(any(event.get('error') for event in replay), replay)
+ self.assertNotIn('saved destination: Crescent City', json.dumps(replay))
+ self.assertEqual(len(self.model.calls), calls_before)
+
+ def test_completed_submission_replay_uses_its_original_memory_scope(self):
+ self.settings['enable_group_workspaces'] = True
+ self.add_fact(4, scope_type='group', scope_id='group1', value='GROUP MEMORY')
+ self.model.plan_override = self.private_memory_question()
+ _response, events = self.plan(doc_scope='group', active_group_ids=['group1'])
+ first_question = next(event['elicitation'] for event in events if event.get('type') == 'orchestration_elicitation')
+ answer = {
+ 'revision': 1, 'elicitation': first_question,
+ 'elicitation_response': {'action': 'accept', 'content': {'day': 'Friday'}},
+ }
+ _response, answered = self.plan(**answer)
+ self.assertTrue(any(event.get('type') == 'orchestration_elicitation' for event in answered), answered)
+ pending = next(row for row in self.runs.items.values() if row.get('question'))
+ self.assertEqual(
+ pending['submissions'][-1]['outcome']['memory_scope'], {'type': 'group', 'id': 'group1'},
+ )
+ # A later continuation may have a different scope; it must not authorize an older outcome.
+ pending['turn_context']['memory_scope'] = None
+ self.membership.side_effect = PermissionError('revoked')
+ calls_before = len(self.model.calls)
+ _response, replay = self.plan(**answer)
+ self.assertTrue(any(event.get('error') for event in replay), replay)
+ self.assertFalse(any(event.get('type') == 'orchestration_elicitation' for event in replay))
+ self.assertEqual(len(self.model.calls), calls_before)
+
+ def test_missing_fact_embeddings_are_reported_without_backfill(self):
+ self.facts[1]['value_embedding'] = None
+ self.planned()
+ memory = self.planner_memory()
+ self.assertEqual(memory['status'], 'partial')
+ self.assertIn('could not be searched', memory['notices'][0])
+ self.assertIn('accessible itinerary', json.dumps(memory))
+ self.assertNotIn('Saved destination:', json.dumps(memory))
+ self.embedding.assert_not_called()
+
+ def test_memory_storage_failure_does_not_replace_the_previous_plan(self):
+ editor = self.open_editor(self.planned())
+ before = self.runs.read_item(editor['plan']['run_id'], 'conv1')['plan']
+ self.store.list_facts.side_effect = AzureError('private connection details')
+ response, events, _body = self.request_revision(editor)
+ self.assertEqual(response.status_code, 200)
+ self.assertTrue(any(event.get('code') == 'memory_context_unavailable' for event in events), events)
+ self.assertNotIn('private connection details', json.dumps(events))
+ self.assertEqual(self.runs.read_item(editor['plan']['run_id'], 'conv1')['plan'], before)
+ self.assertEqual(self.edit_calls, [])
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/functional_tests/test_orchestration_model_selection.py b/functional_tests/test_orchestration_model_selection.py
index e55681147..0d80a2c5d 100644
--- a/functional_tests/test_orchestration_model_selection.py
+++ b/functional_tests/test_orchestration_model_selection.py
@@ -1,8 +1,9 @@
# test_orchestration_model_selection.py
"""
Functional regressions for authorized orchestration model selection and SDK parameters.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.103
+Canonical reasoning resolution and recovery: 0.261.104
Exercises the real selection/binding code with endpoint authorization and client creation
replaced at their existing boundaries. No Azure resources or credentials are used.
@@ -15,11 +16,14 @@
from types import SimpleNamespace
from unittest.mock import Mock, patch
+from openai import AuthenticationError, RateLimitError
+
from test_orchestration_conversation_context import (
LATEST, RESOLVED, fake_module, load_modules, winery_history,
)
from test_support.app_stubs import stubbed_config
from test_support.versioning import assert_app_version_at_least
+from test_model_reasoning_capability_resolution import sdk_error
TERRA_SELECTION = {
@@ -415,6 +419,305 @@ def test_non_reasoning_models_keep_temperature_and_legacy_token_parameter(self):
'model': 'gpt-4o', 'messages': [], 'max_tokens': 1200, 'temperature': 0.3,
})
+ def test_luna_uuid_and_custom_deployment_resolve_before_the_first_request(self):
+ model = self.endpoint['models'][0]
+ model.update(id='f8c476df-c951-499c-b87d-98fd02597780', modelName='gpt-5.6-luna',
+ deploymentName='production-answer', displayName='GPT-5 Minimal')
+ selection = {**TERRA_SELECTION, 'model_id': model['id'], 'model_deployment': 'production-answer'}
+ binding = self.models.resolve_orchestration_model(
+ self.settings, user_id='user1', seeds={'model': selection, 'reasoning_effort': 'minimal'},
+ )
+ self.addCleanup(binding.close)
+ self.assertEqual(binding.reasoning_effort, 'minimal')
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': 'minimal', 'effective_effort': 'low', 'mode': 'explicit',
+ 'adjustment_reason': 'reasoning_effort_unsupported',
+ })
+ self.client.chat.completions.create.assert_not_called()
+ binding.create_completion(messages=[], max_tokens=1200, temperature=0)
+ parameters = self.client.chat.completions.create.call_args.kwargs
+ self.assertEqual(parameters['reasoning_effort'], 'low')
+ self.assertEqual(parameters['max_completion_tokens'], 8192)
+ self.assertEqual(parameters['model'], 'production-answer')
+ self.assertEqual(binding.reasoning_effort, 'minimal')
+
+ def test_none_and_explicit_omission_do_not_inherit_the_binding_effort(self):
+ binding = self.models.OrchestrationModel(self.client, 'gpt-5.6-luna', reasoning_effort='high')
+ for effort, expected in (('none', 'none'), (None, None), ('', None)):
+ binding.create_completion(messages=[], max_tokens=1200, reasoning_effort=effort)
+ self.assertEqual(
+ self.client.chat.completions.create.call_args.kwargs.get('reasoning_effort'), expected,
+ )
+ self.assertEqual(binding.reasoning_resolution['effective_effort'], expected)
+ self.assertEqual(binding.reasoning_effort, 'high')
+ binding.create_completion(messages=[], max_tokens=1200)
+ self.assertEqual(binding.reasoning_resolution['effective_effort'], 'high')
+
+ def test_provider_recovery_updates_resolution_without_changing_budget_or_selection(self):
+ binding = self.models.OrchestrationModel(
+ self.client, 'production-answer', behavior_name='gpt-5.6-luna',
+ reasoning_effort='minimal', response_length=2048,
+ )
+ self.client.chat.completions.create.side_effect = [sdk_error(), 'completion']
+ result = binding.create_completion(
+ messages=[{'role': 'user', 'content': 'answer'}], max_tokens=1200,
+ use_model_response_length=True, response_format={'type': 'json_object'},
+ )
+ self.assertEqual(result, 'completion')
+ first, retry = self.client.chat.completions.create.call_args_list
+ self.assertEqual(first.kwargs['reasoning_effort'], 'low')
+ self.assertEqual(retry.kwargs, {
+ key: value for key, value in first.kwargs.items() if key != 'reasoning_effort'
+ })
+ self.assertEqual(retry.kwargs['max_completion_tokens'], 2048)
+ self.assertEqual(binding.reasoning_effort, 'minimal')
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': 'minimal', 'effective_effort': None, 'mode': 'model_default',
+ 'adjustment_reason': 'reasoning_parameter_rejected',
+ })
+
+ def test_combined_reasoning_and_json_recovery_never_reintroduces_rejected_effort(self):
+ messages = [{'role': 'user', 'content': 'Return one JSON object.'}]
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(
+ message=SimpleNamespace(content='{"steps": []}', refusal=None), finish_reason='stop',
+ )],
+ usage=SimpleNamespace(total_tokens=12),
+ )
+ for requested in ('minimal', 'low', 'none'):
+ with self.subTest(requested=requested):
+ binding = self.models.OrchestrationModel(
+ self.client, 'custom-planner', behavior_name='gpt-5.6-luna',
+ reasoning_effort=requested,
+ )
+ self.client.chat.completions.create.reset_mock()
+
+ def create(**parameters):
+ if 'reasoning_effort' in parameters:
+ raise sdk_error()
+ if 'response_format' in parameters:
+ raise sdk_error(param='response_format', code='unsupported_parameter')
+ return response
+
+ self.client.chat.completions.create.side_effect = create
+ result, usage = self.modules.planner._call_planner(
+ binding.as_planner_client(), binding.deployment, messages,
+ max_tokens=1200, require_complete_response=True,
+ )
+ calls = self.client.chat.completions.create.call_args_list
+ self.assertEqual(len(calls), 3)
+ self.assertEqual(calls[0].kwargs['reasoning_effort'], (
+ 'low' if requested == 'minimal' else requested
+ ))
+ self.assertEqual(calls[1].kwargs, {
+ key: value for key, value in calls[0].kwargs.items() if key != 'reasoning_effort'
+ })
+ self.assertEqual(calls[2].kwargs, {
+ key: value for key, value in calls[1].kwargs.items() if key != 'response_format'
+ })
+ for call in calls:
+ self.assertIs(call.kwargs['messages'], messages)
+ self.assertEqual(call.kwargs['model'], 'custom-planner')
+ self.assertEqual(call.kwargs['max_completion_tokens'], 8192)
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': requested, 'effective_effort': None, 'mode': 'model_default',
+ 'adjustment_reason': 'reasoning_parameter_rejected',
+ })
+ self.assertEqual(binding.reasoning_effort, requested)
+ self.assertEqual(result, '{"steps": []}')
+ self.assertIs(usage, response.usage)
+
+ def test_recovery_state_survives_failed_retry_without_swallowing_unrelated_errors(self):
+ for error in (
+ sdk_error(param='messages', code='invalid_request_error'),
+ sdk_error(param='response_format', code='unsupported_parameter'),
+ ):
+ with self.subTest(parameter=error.param):
+ binding = self.models.OrchestrationModel(
+ self.client, 'gpt-5.6-luna', reasoning_effort='minimal',
+ )
+ self.client.chat.completions.create.reset_mock()
+ self.client.chat.completions.create.side_effect = [sdk_error(), error]
+ with self.assertRaises(type(error)) as raised:
+ binding.create_completion(messages=[], max_tokens=1200)
+ self.assertIs(raised.exception, error)
+ self.assertEqual(self.client.chat.completions.create.call_count, 2)
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': 'minimal', 'effective_effort': None, 'mode': 'model_default',
+ 'adjustment_reason': 'reasoning_parameter_rejected',
+ })
+ self.client.chat.completions.create.side_effect = None
+ for override, effective, reason in (
+ ('none', 'none', None), ('high', 'high', None), (None, None, None),
+ ('minimal', None, 'reasoning_parameter_rejected'),
+ ):
+ binding.create_completion(
+ messages=[], max_tokens=1200, reasoning_effort=override,
+ )
+ parameters = self.client.chat.completions.create.call_args.kwargs
+ self.assertEqual(parameters.get('reasoning_effort'), effective)
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': override, 'effective_effort': effective,
+ 'mode': 'model_default' if effective is None else 'explicit',
+ 'adjustment_reason': reason,
+ })
+ self.assertEqual(binding.reasoning_effort, 'minimal')
+ fresh_binding = self.models.OrchestrationModel(
+ self.client, 'gpt-5.6-luna', reasoning_effort='minimal',
+ )
+ fresh_binding.create_completion(messages=[], max_tokens=1200)
+ self.assertEqual(
+ self.client.chat.completions.create.call_args.kwargs['reasoning_effort'], 'low',
+ )
+
+ def test_combined_recovery_does_not_loop_on_a_second_format_rejection(self):
+ binding = self.models.OrchestrationModel(
+ self.client, 'gpt-5.6-luna', reasoning_effort='low',
+ )
+ format_error = sdk_error(param='response_format', code='unsupported_parameter')
+ self.client.chat.completions.create.side_effect = [
+ sdk_error(), format_error, format_error,
+ ]
+ with self.assertRaises(type(format_error)) as raised:
+ self.modules.planner._call_planner(
+ binding.as_planner_client(), binding.deployment, [], max_tokens=1200,
+ )
+ self.assertIs(raised.exception, format_error)
+ calls = self.client.chat.completions.create.call_args_list
+ self.assertEqual(len(calls), 3)
+ self.assertNotIn('reasoning_effort', calls[1].kwargs)
+ self.assertNotIn('reasoning_effort', calls[2].kwargs)
+ self.assertNotIn('response_format', calls[2].kwargs)
+ self.assertEqual(binding.reasoning_resolution['mode'], 'model_default')
+
+ def test_json_then_reasoning_recovery_preserves_the_same_three_attempt_bound(self):
+ binding = self.models.OrchestrationModel(
+ self.client, 'gpt-5.6-luna', reasoning_effort='low',
+ )
+ messages = [{'role': 'user', 'content': 'Return a JSON object.'}]
+ response = SimpleNamespace(
+ choices=[SimpleNamespace(message=SimpleNamespace(content='{}'))], usage=None,
+ )
+ self.client.chat.completions.create.side_effect = [
+ sdk_error(param='response_format', code='unsupported_parameter'), sdk_error(), response,
+ ]
+ result, _usage = self.modules.planner._call_planner(
+ binding.as_planner_client(), binding.deployment, messages, max_tokens=1200,
+ )
+ calls = self.client.chat.completions.create.call_args_list
+ self.assertEqual(len(calls), 3)
+ self.assertEqual(calls[1].kwargs, {
+ key: value for key, value in calls[0].kwargs.items() if key != 'response_format'
+ })
+ self.assertEqual(calls[2].kwargs, {
+ key: value for key, value in calls[1].kwargs.items() if key != 'reasoning_effort'
+ })
+ self.assertEqual(result, '{}')
+ self.assertEqual(binding.reasoning_resolution, {
+ 'requested_effort': 'low', 'effective_effort': None, 'mode': 'model_default',
+ 'adjustment_reason': 'reasoning_parameter_rejected',
+ })
+
+ def test_combined_recovery_propagates_auth_and_rate_errors_without_more_attempts(self):
+ for error_type, status in ((AuthenticationError, 401), (RateLimitError, 429)):
+ with self.subTest(status=status):
+ binding = self.models.OrchestrationModel(
+ self.client, 'gpt-5.6-luna', reasoning_effort='low',
+ )
+ final_error = sdk_error(error_type, status=status)
+ self.client.chat.completions.create.reset_mock()
+ self.client.chat.completions.create.side_effect = [
+ sdk_error(), sdk_error(param='response_format', code='unsupported_parameter'),
+ final_error,
+ ]
+ with self.assertRaises(error_type) as raised:
+ self.modules.planner._call_planner(
+ binding.as_planner_client(), binding.deployment, [], max_tokens=1200,
+ )
+ self.assertIs(raised.exception, final_error)
+ calls = self.client.chat.completions.create.call_args_list
+ self.assertEqual(len(calls), 3)
+ self.assertNotIn('reasoning_effort', calls[2].kwargs)
+ self.assertEqual(binding.reasoning_resolution['effective_effort'], None)
+ self.assertEqual(binding.reasoning_resolution['mode'], 'model_default')
+
+ def test_planner_override_resolves_its_own_policy_without_inheriting_answer_effort(self):
+ planner_endpoint = model_endpoint()
+ planner_endpoint['id'] = 'planner-endpoint'
+ planner_endpoint['models'] = [{
+ 'id': 'planner-model', 'deploymentName': 'custom-planner', 'modelName': 'gpt-5-mini',
+ }]
+ self.settings.update({
+ 'chat_orchestration_planner_model_endpoint_id': 'planner-endpoint',
+ 'chat_orchestration_planner_model_id': 'planner-model',
+ })
+ self.runtime.resolve_model_endpoint_from_context.side_effect = [self.endpoint, planner_endpoint]
+ planner = self.resolve(TERRA_SELECTION, planner=True)
+ self.assertEqual(planner.reasoning_resolution['mode'], 'model_default')
+ planner.create_completion(messages=[], max_tokens=1200)
+ self.assertNotIn('reasoning_effort', self.client.chat.completions.create.call_args.kwargs)
+ planner.create_completion(messages=[], max_tokens=1200, reasoning_effort='minimal')
+ self.assertEqual(self.client.chat.completions.create.call_args.kwargs['reasoning_effort'], 'minimal')
+ self.assertEqual(planner.answer_model_selection(), TERRA_SELECTION)
+
+ def test_legacy_custom_deployment_uses_the_configured_canonical_name(self):
+ self.settings.update(enable_multi_model_endpoints=False, gpt_model={
+ 'selected': [{'deploymentName': 'legacy-answer', 'modelName': 'gpt-5.6-luna'}]
+ })
+ binding = self.resolve({'model_deployment': 'legacy-answer'})
+ self.assertEqual(binding.reasoning_resolution['effective_effort'], 'high')
+ binding.create_completion(messages=[], max_tokens=1200)
+ self.assertEqual(
+ self.legacy_client.chat.completions.create.call_args.kwargs['max_completion_tokens'], 8192,
+ )
+
+ def test_legacy_planner_override_uses_its_own_canonical_record(self):
+ self.settings['gpt_model']['selected'].append({
+ 'deploymentName': 'custom-planner', 'modelName': 'gpt-5-pro',
+ })
+ self.settings['chat_orchestration_planner_deployment'] = 'custom-planner'
+ binding = self.resolve(TERRA_SELECTION, planner=True)
+ self.assertEqual(binding.behavior_name, 'gpt-5-pro')
+ self.assertIsNone(binding.reasoning_resolution['effective_effort'])
+ binding.create_completion(messages=[], max_tokens=1200, reasoning_effort='low')
+ self.assertEqual(
+ self.legacy_client.chat.completions.create.call_args.kwargs['reasoning_effort'], 'high',
+ )
+ self.assertEqual(binding.answer_model_selection(), TERRA_SELECTION)
+
+ def test_apim_never_borrows_same_named_direct_aoai_model_metadata(self):
+ for deployment, direct_model, expected in (
+ ('custom-answer', 'gpt-5.6-luna', None),
+ ('gpt-5.6-luna', 'gpt-4o', 'high'),
+ ):
+ for planner in (False, True):
+ with self.subTest(deployment=deployment, planner=planner):
+ self.settings.update(
+ enable_multi_model_endpoints=False,
+ enable_gpt_apim=True,
+ azure_apim_gpt_deployment=deployment,
+ chat_orchestration_planner_deployment=deployment if planner else '',
+ gpt_model={'selected': [{
+ 'deploymentName': deployment, 'modelName': direct_model,
+ }]},
+ )
+ binding = self.resolve({'model_deployment': deployment}, planner=planner)
+ self.assertEqual(binding.behavior_name, '')
+ binding.create_completion(
+ messages=[], max_tokens=1200, reasoning_effort='high', temperature=0.3,
+ )
+ parameters = self.legacy_client.chat.completions.create.call_args.kwargs
+ self.assertEqual(parameters['model'], deployment)
+ self.assertEqual(parameters.get('reasoning_effort'), expected)
+ self.assertEqual(binding.reasoning_resolution['effective_effort'], expected)
+ if expected is None:
+ self.assertEqual(parameters['max_tokens'], 1200)
+ self.assertEqual(parameters['temperature'], 0.3)
+ self.assertEqual(
+ binding.reasoning_resolution['adjustment_reason'],
+ 'reasoning_capability_unknown',
+ )
+
def test_binding_cannot_be_retargeted_and_closes_its_sdk_client_once(self):
binding = self.resolve(TERRA_SELECTION)
with self.assertRaises(self.models.OrchestrationModelError):
diff --git a/functional_tests/test_orchestration_phase_ordering.py b/functional_tests/test_orchestration_phase_ordering.py
index 46ca32a80..22348a86e 100644
--- a/functional_tests/test_orchestration_phase_ordering.py
+++ b/functional_tests/test_orchestration_phase_ordering.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_phase_ordering.py
"""
Functional test for chat orchestration phase ordering.
-Version: 0.261.089
+Version: 0.261.104
Implemented in: 0.261.087
A plan runs in three phases: collect knowledge, reason on it and answer, then create
@@ -20,7 +20,7 @@
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-from test_support.app_stubs import stubbed_app_imports # noqa: E402
+from test_support.orchestration_research import stubbed_orchestration_imports # noqa: E402
from test_support.versioning import assert_app_version_at_least # noqa: E402
SETTINGS = {
@@ -43,7 +43,7 @@ def test_phases_are_ordered_and_indexed():
"""The phase tuple is ordered, and every capability lands in a real one."""
print("Testing the phase ordering...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_registry as registry
assert registry.CAPABILITY_PHASES == ('knowledge', 'reasoning', 'output'), (
@@ -80,7 +80,7 @@ def test_gathering_after_answering_is_reordered():
"""A plan that answers before it gathers is repaired, not run as written."""
print("Testing that gathering is moved ahead of answering...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_schema as schema
plan = schema.normalize_plan(
@@ -119,7 +119,7 @@ def test_backwards_dependency_is_dropped_and_reported():
"""A gathering step may not wait on the answer."""
print("Testing that a backwards dependency is dropped...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_schema as schema
plan = schema.normalize_plan(
@@ -164,7 +164,7 @@ def test_ordering_within_a_phase_is_preserved():
"""Sorting by phase must not shuffle steps that share one."""
print("Testing that the planner's own order survives within a phase...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_schema as schema
plan = schema.normalize_plan(
diff --git a/functional_tests/test_orchestration_plan_revision_planner.py b/functional_tests/test_orchestration_plan_revision_planner.py
index 22aea085f..e8859f85d 100644
--- a/functional_tests/test_orchestration_plan_revision_planner.py
+++ b/functional_tests/test_orchestration_plan_revision_planner.py
@@ -1,7 +1,7 @@
# test_orchestration_plan_revision_planner.py
"""
Functional tests for the plan editor's strict planner contract.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.102
Authorized model routing through editor replanning: 0.261.103
@@ -14,6 +14,9 @@
from types import SimpleNamespace
from unittest.mock import Mock, patch
+from httpx import Request
+from openai import APIError
+
from test_orchestration_conversation_context import load_modules
@@ -145,7 +148,9 @@ def test_declined_clarification_cannot_fall_back_to_an_answer_plan(self):
def test_provider_failure_is_safe_and_does_not_fall_back(self):
with (
patch.object(self.planner, 'resolve_planner_client', return_value=(object(), 'planner')),
- patch.object(self.planner, '_call_planner', side_effect=RuntimeError('PRIVATE_PROVIDER_DETAIL')),
+ patch.object(self.planner, '_call_planner', side_effect=APIError(
+ 'PRIVATE_PROVIDER_DETAIL', request=Request('POST', 'https://model.example'), body=None,
+ )),
):
with self.assertRaises(self.planner.PlannerError) as raised:
self.planner.plan_request(
diff --git a/functional_tests/test_orchestration_plan_revision_routes.py b/functional_tests/test_orchestration_plan_revision_routes.py
index 73c8a841b..3be879280 100644
--- a/functional_tests/test_orchestration_plan_revision_routes.py
+++ b/functional_tests/test_orchestration_plan_revision_routes.py
@@ -1,7 +1,7 @@
# test_orchestration_plan_revision_routes.py
"""
Functional tests for conversational, pre-execution plan revisions.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.102
Authorized model routing through revisions and clarification: 0.261.103
@@ -17,6 +17,8 @@
from unittest.mock import patch
from azure.core.exceptions import AzureError
+from httpx import Request
+from openai import APIError
import test_orchestration_conversation_context_routes as context_routes
from test_orchestration_model_selection import TERRA_SELECTION
@@ -333,7 +335,15 @@ def test_unavailable_capability_and_invalid_model_output_keep_previous_plan(self
editor = self.open_editor()
invalid = revised_plan()
invalid['steps'][0]['capability_id'] = 'not_enabled_or_registered'
- for reply in (invalid, {'kind': 'plan', 'steps': []}, RuntimeError('provider-secret')):
+ provider_error = APIError(
+ 'provider-secret', request=Request('POST', 'https://model.example'), body=None,
+ )
+ for reply in (
+ invalid, {'kind': 'plan', 'steps': []},
+ {'kind': 'plan', 'revised_request': 'Keep the current task.', 'steps': []},
+ {'kind': 'plan', 'revised_request': 'Keep the current task.', 'steps': 'not-a-list'},
+ provider_error,
+ ):
with self.subTest(reply=type(reply).__name__):
self.edit_responses.append(reply)
_response, events, _body = self.request_revision(editor)
diff --git a/functional_tests/test_orchestration_plan_revision_store.py b/functional_tests/test_orchestration_plan_revision_store.py
index 0c4802526..1be6ef3aa 100644
--- a/functional_tests/test_orchestration_plan_revision_store.py
+++ b/functional_tests/test_orchestration_plan_revision_store.py
@@ -1,7 +1,7 @@
# test_orchestration_plan_revision_store.py
"""
Functional tests for the pre-execution plan revision persistence boundary.
-Version: 0.261.102
+Version: 0.261.104
Implemented in: 0.261.102
Uses real storage helpers and SDK batch formatting with an atomic in-memory container.
@@ -811,11 +811,25 @@ def test_safe_projection_excludes_private_record_and_provider_metadata(self):
})
held['plan'].update(raw_provider_response='PRIVATE_PLAN', seeds={'key': 'PRIVATE_PLAN_SEED'})
held['plan']['steps'][0]['result'] = {'key': 'PRIVATE_STEP'}
+ held['seeds']['web_search'] = True
+ held['plan']['reasoning_adjustments'] = [{
+ 'requested_effort': 'minimal', 'effective_effort': 'low', 'mode': 'explicit',
+ 'adjustment_reason': 'reasoning_effort_unsupported', 'stage': 'planner',
+ 'model_name': 'gpt-5.6-luna', 'raw': 'PRIVATE_PROVIDER_RESPONSE',
+ }, {
+ 'adjustment_reason': 'reasoning_effort_unsupported',
+ 'effective_effort': {'key': 'PRIVATE_MALFORMED_METADATA'},
+ }]
+ original_plan = deepcopy(held['plan'])
state = self.revisions.plan_editor_state(held, 'user1')
self.assertNotIn('PRIVATE_', json.dumps(state))
self.assertNotIn('_etag', json.dumps(state))
self.assertEqual(state['chat'][0]['content'], 'Safe reply')
self.assertEqual(state['pending'], editor_question())
+ self.assertEqual(state['plan']['inputs']['required_capabilities'], ['web_search'])
+ self.assertEqual(len(state['plan']['reasoning_adjustments']), 1)
+ self.assertEqual(state['plan']['reasoning_adjustments'][0]['effective_effort'], 'low')
+ self.assertEqual(held['plan'], original_plan)
self.assert_error('not_found', self.revisions.plan_editor_state, held, 'other-user', status=404)
def test_release_does_not_hide_operational_failure_or_clear_pending_outcome(self):
diff --git a/functional_tests/test_orchestration_plan_schema.py b/functional_tests/test_orchestration_plan_schema.py
index f333e966c..d9c60261b 100644
--- a/functional_tests/test_orchestration_plan_schema.py
+++ b/functional_tests/test_orchestration_plan_schema.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_plan_schema.py
"""
Functional test for the chat orchestration plan contract and validator.
-Version: 0.261.085
+Version: 0.261.104
Implemented in: 0.261.085
Planner output is untrusted input. A plan arrives as JSON written by a language model, and
@@ -19,13 +19,16 @@
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-from test_support.app_stubs import stubbed_app_imports # noqa: E402
+from test_support.orchestration_research import stubbed_orchestration_imports as stubbed_app_imports # noqa: E402
from test_support.versioning import assert_app_version_at_least # noqa: E402
SETTINGS = {
'enable_user_workspace': True,
'enable_web_search': True,
'chat_orchestration_max_steps': 6,
+ 'document_action_capabilities': {
+ 'analyze': {'enabled': False}, 'comparison': {'enabled': False},
+ },
}
diff --git a/functional_tests/test_orchestration_prompt_instruction.py b/functional_tests/test_orchestration_prompt_instruction.py
index 3d6600f83..c3fb95085 100644
--- a/functional_tests/test_orchestration_prompt_instruction.py
+++ b/functional_tests/test_orchestration_prompt_instruction.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_prompt_instruction.py
"""
Functional test for orchestration treating a selected prompt as an instruction.
-Version: 0.261.092
+Version: 0.261.104
Implemented in: 0.261.092
A saved prompt is a standing instruction: it says what kind of work this is. Orchestration used
@@ -15,7 +15,7 @@
1. The planner is shown the prompt's wording, capped so an unbounded saved prompt cannot
consume the planner's budget.
- 2. A selected prompt makes a request non-trivial, alongside a chosen document or agent.
+ 2. Every request reaches planning, with or without a selected prompt, document or agent.
3. The stored plan names the prompt rather than quoting it. The plan document is kept and
shown, and the wording is already in the message the plan was built from.
@@ -34,6 +34,7 @@
sys.path.insert(0, str(REPO_ROOT / "functional_tests"))
from test_support.versioning import assert_app_version_at_least # noqa: E402
+from test_support.orchestration_research import _definitions # noqa: E402
CONTEXT_PY = APP_DIR / "functions_orchestration_context.py"
PLANNER_PY = APP_DIR / "functions_orchestration_planner.py"
@@ -102,7 +103,7 @@ def test_the_planner_is_shown_the_prompts_wording():
def test_no_prompt_reads_as_no_prompt():
- """`triage_request` tests this the same way it tests the other selections."""
+ """Absent selections are neutral, not made-up instructions."""
print("Testing absent prompts...")
module = _extract(CONTEXT_PY, {"_text", "_selected_prompt"})
@@ -121,9 +122,9 @@ def test_no_prompt_reads_as_no_prompt():
return True
-def test_a_selected_prompt_makes_the_request_non_trivial():
- """Reaching for stored instructions is a statement that this work has a shape."""
- print("Testing triage...")
+def test_requests_reach_planning_with_or_without_selected_instructions():
+ """Neither message length nor an unchecked control may bypass planning."""
+ print("Testing the all-request planning contract...")
module = _extract(
PLANNER_PY,
@@ -137,26 +138,19 @@ def test_a_selected_prompt_makes_the_request_non_trivial():
triage = module["triage_request"]
bare = {"user_selected": {}}
- assert triage("hi", bare) == "trivial", (
- "a remark with nothing selected must still be trivial, or every message plans"
- )
+ assert triage("hi", bare) == "simple"
with_prompt = {"user_selected": {"prompt": {"name": "Quarterly review", "content": "..."}}}
- assert triage("hi", with_prompt) == "complex", (
- "a selected prompt must count as pointing at something, like a document or an agent"
- )
+ assert triage("hi", with_prompt) == "simple"
- # The signals it already honoured must not have been displaced by the new one.
for signal, value in (
("documents", ["doc-1"]),
("agent", "Researcher"),
("web_search", True),
):
- assert triage("hi", {"user_selected": {signal: value}}) == "complex", (
- f"the existing {signal} signal must still make a request complex"
- )
+ assert triage("hi", {"user_selected": {signal: value}}) == "simple"
- print(" ok a selected prompt makes the request complex")
+ print(" ok selected instructions do not alter the all-request planning contract")
return True
@@ -164,7 +158,8 @@ def test_the_stored_plan_names_the_prompt_rather_than_quoting_it():
"""The plan document is kept and shown; the wording is already in the message."""
print("Testing plan inputs...")
- module = _extract(SCHEMA_PY, {"build_plan_inputs"})
+ registry = _definitions("functions_orchestration_registry.py")
+ module = _definitions("functions_orchestration_schema.py", seed=registry)
build_plan_inputs = module["build_plan_inputs"]
seeds = {
@@ -194,7 +189,7 @@ def test_the_stored_plan_names_the_prompt_rather_than_quoting_it():
test_version_is_at_least_the_implementing_release,
test_the_planner_is_shown_the_prompts_wording,
test_no_prompt_reads_as_no_prompt,
- test_a_selected_prompt_makes_the_request_non_trivial,
+ test_requests_reach_planning_with_or_without_selected_instructions,
test_the_stored_plan_names_the_prompt_rather_than_quoting_it,
]
diff --git a/functional_tests/test_orchestration_registry_contract.py b/functional_tests/test_orchestration_registry_contract.py
index 6cbe735df..ad4b9f48d 100644
--- a/functional_tests/test_orchestration_registry_contract.py
+++ b/functional_tests/test_orchestration_registry_contract.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_registry_contract.py
"""
Functional test for the chat orchestration capability registry.
-Version: 0.261.085
+Version: 0.261.104
Implemented in: 0.261.085
The registry is the only capability information the planner model ever sees, and it is
@@ -20,15 +20,24 @@
sys.path.append(os.path.dirname(os.path.abspath(__file__)))
-from test_support.app_stubs import stubbed_app_imports # noqa: E402
+from test_support.orchestration_research import stubbed_orchestration_imports # noqa: E402
from test_support.versioning import assert_app_version_at_least # noqa: E402
+def _settings(**values):
+ return {
+ 'document_action_capabilities': {
+ 'analyze': {'enabled': False}, 'comparison': {'enabled': False},
+ },
+ **values,
+ }
+
+
def test_descriptors_are_well_formed():
"""Every descriptor carries the fields the planner and validator both rely on."""
print("Testing orchestration capability descriptors...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_registry as registry
required_fields = (
@@ -86,17 +95,17 @@ def test_gates_withhold_capabilities():
"""A capability whose settings gate is off must not be offered."""
print("Testing orchestration capability gating...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_registry as registry
# Nothing enabled: only the terminal capability survives, because a plan has
# to be able to end even in a deployment with everything switched off.
- bare = registry.resolve_available_capability_ids({})
+ bare = registry.resolve_available_capability_ids(_settings())
assert bare == [registry.TERMINAL_CAPABILITY_ID], (
f"An empty deployment offered {bare}"
)
- with_web = registry.resolve_available_capability_ids({'enable_web_search': True})
+ with_web = registry.resolve_available_capability_ids(_settings(enable_web_search=True))
assert registry.CAPABILITY_WEB_SEARCH in with_web
assert registry.CAPABILITY_DOCUMENT_SEARCH not in with_web, (
"Document search must need a workspace to search"
@@ -106,7 +115,7 @@ def test_gates_withhold_capabilities():
for workspace_key in (
'enable_user_workspace', 'enable_group_workspaces', 'enable_public_workspaces'
):
- ids = registry.resolve_available_capability_ids({workspace_key: True})
+ ids = registry.resolve_available_capability_ids(_settings(**{workspace_key: True}))
assert registry.CAPABILITY_DOCUMENT_SEARCH in ids, (
f"{workspace_key} alone should permit document search"
)
@@ -124,10 +133,10 @@ def test_administrator_narrowing():
"""The enabled-capability list narrows the registry without breaking plans."""
print("Testing orchestration capability narrowing...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_registry as registry
- settings = {'enable_user_workspace': True, 'enable_web_search': True}
+ settings = _settings(enable_user_workspace=True, enable_web_search=True)
full = registry.resolve_available_capability_ids(settings)
# No opinion means everything, not nothing. An administrator who has never
@@ -154,23 +163,24 @@ def test_administrator_narrowing():
def test_planner_projection_hides_internals():
- """Gates, adapters and caps are the application's business, not the model's."""
+ """Gate internals stay private; outputs and limits help the model choose feasible work."""
print("Testing orchestration planner projection...")
try:
- with stubbed_app_imports():
+ with stubbed_orchestration_imports():
import functions_orchestration_registry as registry
- settings = {'enable_user_workspace': True, 'enable_web_search': True}
+ settings = _settings(enable_user_workspace=True, enable_web_search=True)
available = registry.resolve_available_capabilities(settings)
projection = registry.build_planner_capability_projection(available)
assert projection, "The projection was empty"
leaked = {'gate', 'settings_gates', 'settings_gates_any', 'adapter',
- 'max_per_plan', 'document_action_type', 'requires_scope'}
+ 'document_action_type', 'requires_scope'}
for entry in projection:
overlap = leaked & set(entry.keys())
assert not overlap, f"Planner projection leaked {sorted(overlap)}"
assert entry['when_to_use'], "Guidance is what the planner chooses on"
+ assert 'produces' in entry and 'max_per_plan' in entry
client = registry.build_capability_client_projection(available)
for entry in client:
diff --git a/functional_tests/test_orchestration_research_selection.py b/functional_tests/test_orchestration_research_selection.py
index fbcf4a96f..23bde9308 100644
--- a/functional_tests/test_orchestration_research_selection.py
+++ b/functional_tests/test_orchestration_research_selection.py
@@ -2,7 +2,7 @@
"""
Functional contracts for balanced orchestration research selection and its opt-in evaluator.
-Version: 0.261.099
+Version: 0.261.104
Implemented in: 0.261.099
Runs actual planner, capability projection, request gates and plan normalization with
@@ -37,6 +37,7 @@
capture_baseline,
case_inputs,
load_case_suite,
+ OfflineBadRequestError,
planner_runtime,
)
from functional_tests.test_support.versioning import assert_app_version_at_least # noqa: E402
@@ -69,6 +70,13 @@ def model_plan(capability=None, rationale="Additional discovery and checked deta
}
+def unsupported_json_error():
+ return OfflineBadRequestError(
+ "SYNTHETIC_PRIVATE_PROVIDER_DETAIL",
+ body={"param": "response_format", "code": "unsupported_parameter"},
+ )
+
+
class ScriptedClient:
"""An SDK-shaped completion seam, not an alternative planner."""
@@ -131,19 +139,27 @@ def plan(self, case_id, replies, settings_overrides=None, request_overrides=None
settings.update(settings_overrides or {})
request_context.update(request_overrides or {})
client = ScriptedClient(*replies)
+ self.last_client = client
+ kwargs.setdefault("seeds", self.runtime.context["resolve_seeds"](case.get("request") or {}))
with patch.dict(self.runtime.planner, {
"resolve_planner_client": lambda settings: (client, "synthetic-deployment"),
}):
kind, document = self.runtime.planner["plan_request"](
case["message"], context, "synthetic-conversation", request_context["user_id"],
- settings=settings, request_context=request_context, authorized_document_ids=[],
+ settings=settings, request_context=request_context,
+ authorized_document_ids=[
+ document["document_id"] for document in context.get("candidate_documents", [])
+ ],
**kwargs,
)
return kind, document, client
def test_balanced_fixture_has_evidence_based_review_not_a_research_quota(self):
self.assertEqual(len(self.cases), len(self.suite["cases"]))
- self.assertEqual(len(self.cases), 11)
+ self.assertTrue({
+ "coastal-tide-planning", "short-tide-question", "coastal-planning-paraphrase",
+ "explicit-web-selection", "explicit-research-selection", "neutral-web-selection",
+ } <= set(self.cases))
self.assertEqual(
set(self.cases["original-playlist"]["acceptable_choices"]),
{"web_search", "deep_research"},
@@ -157,6 +173,22 @@ def test_balanced_fixture_has_evidence_based_review_not_a_research_quota(self):
self.assertIn("Human semantic review", self.suite["rubric"]["review_method"])
self.assertNotIn("target_research_rate", self.suite)
+ def test_authorized_resource_and_memory_scenarios_reach_real_context_projection(self):
+ for case_id in (
+ "authorized-document-context", "authorized-agent-context", "saved-preference-context",
+ ):
+ with self.subTest(case_id=case_id):
+ _settings, caller, context = case_inputs(self.runtime, self.suite, self.cases[case_id])
+ if case_id == "authorized-document-context":
+ self.assertEqual(context["candidate_documents"][0]["document_id"], "synthetic-guide")
+ elif case_id == "authorized-agent-context":
+ self.assertEqual(context["agents"][0]["name"], "writing-coach")
+ self.assertEqual(caller["agent_catalog"][0]["name"], "writing-coach")
+ else:
+ self.assertEqual(context["memory"]["status"], "available")
+ self.assertIn("metric units", json.dumps(context["memory"]["messages"]))
+ self.assertNotIn("evidence_objectives", context)
+
def test_source_capture_is_the_actual_prompt_and_full_projection(self):
snapshot = capture_baseline()
tree = ast.parse((APP_ROOT / "functions_orchestration_planner.py").read_text(encoding="utf-8"))
@@ -198,12 +230,13 @@ def test_original_playlist_is_exact_and_nontrivial_without_selected_resources(se
"I want a nastalgic vibe we were born in early 80s with bluegrass, country and more modern "
"stuff from the 2000s to fun contemporary stuff playing now."
))
- self.assertGreater(len(message), self.runtime.planner["TRIVIAL_MAX_CHARACTERS"])
self.assertEqual(self.runtime.planner["triage_request"](message, {}), "simple")
self.assertEqual(
self.runtime.planner["triage_request"](self.cases["stable-direct"]["message"], {}),
- "trivial",
+ "simple",
)
+ self.assertEqual(self.runtime.planner["triage_request"]("Thanks!", {}), "simple")
+ self.assertNotIn("PLANNING_SIGNAL_PATTERN", self.runtime.planner)
def test_initial_and_replan_share_the_actual_prompt_and_projection(self):
for hint in (None, "The earlier lookup covered one perspective; reconsider remaining evidence needs."):
@@ -250,6 +283,11 @@ def test_valid_model_depth_is_preserved_not_routed_by_topic_or_length(self):
def test_direct_choice_never_gets_automatic_research_inserted(self):
for case_id in self.cases:
with self.subTest(case=case_id):
+ if case_id in ("explicit-web-selection", "explicit-research-selection"):
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan(case_id, [model_plan()])
+ self.assertEqual(len(self.last_client.calls), 1)
+ continue
_, document, client = self.plan(case_id, [model_plan()])
self.assertEqual([step["capability_id"] for step in document["steps"]], ["respond"])
self.assertEqual(len(client.calls), 1)
@@ -265,14 +303,14 @@ def test_feature_role_and_allowlist_gates_apply_to_initial_plans_and_replans(sel
for case_id in ("research-disabled", "research-role-missing", "research-allowlist-excluded"):
for hint in (None, "Check whether additional discovery is justified."):
with self.subTest(case=case_id, replan_hint=hint):
- _, document, client = self.plan(case_id, [model_plan("deep_research")], replan_hint=hint)
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan(case_id, [model_plan("deep_research")], replan_hint=hint)
+ client = self.last_client
payload = json.JSONDecoder().raw_decode(client.calls[0]["messages"][1]["content"])[0]
self.assertEqual(
[item["id"] for item in payload["capabilities"]],
["web_search", "respond"],
)
- self.assertEqual([step["capability_id"] for step in document["steps"]], ["respond"])
- self.assertFalse(document["validation"]["ok"])
def test_role_policy_is_actual_fail_closed_claim_normalization(self):
for roles, allowed in (
@@ -280,6 +318,13 @@ def test_role_policy_is_actual_fail_closed_claim_normalization(self):
(["DeepResearchUser"], True), (["deepresearchuser"], True), ("DeepResearchUser", True),
):
with self.subTest(roles=roles):
+ if not allowed:
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan(
+ "original-playlist", [model_plan("deep_research")],
+ request_overrides={"user_roles": roles},
+ )
+ continue
_, document, _ = self.plan(
"original-playlist", [model_plan("deep_research")],
request_overrides={"user_roles": roles},
@@ -297,14 +342,13 @@ def test_invalid_elicitation_retry_retains_request_role_gate(self):
"type": "object", "properties": {"nested": {"type": "object"}},
},
}
- _, document, client = self.plan(
- "research-role-missing", [invalid_question, model_plan("deep_research")],
- )
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan("research-role-missing", [invalid_question, model_plan("deep_research")])
+ client = self.last_client
self.assertEqual(len(client.calls), 2)
for call in client.calls:
payload = json.loads(call["messages"][1]["content"])
self.assertNotIn("deep_research", [item["id"] for item in payload["capabilities"]])
- self.assertEqual([step["capability_id"] for step in document["steps"]], ["respond"])
def test_research_cap_is_one_and_does_not_require_a_preceding_web_step(self):
raw = model_plan("deep_research")
@@ -315,11 +359,78 @@ def test_research_cap_is_one_and_does_not_require_a_preceding_web_step(self):
self.assertEqual([step["capability_id"] for step in document["steps"]], ["deep_research", "respond"])
self.assertTrue(document["validation"]["repairs"])
- def test_unparseable_completion_falls_back_without_inserting_research(self):
- _, document, client = self.plan("broad-discovery-checking", ["not a plan"])
- self.assertEqual([step["capability_id"] for step in document["steps"]], ["respond"])
- self.assertIn("planner_fallback_reason", document)
- self.assertEqual(len(client.calls), 1)
+ def test_unparseable_completion_cannot_masquerade_as_a_direct_choice(self):
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan("broad-discovery-checking", ["not a plan"])
+ self.assertEqual(len(self.last_client.calls), 1)
+
+ def test_missing_or_malformed_work_never_becomes_an_inserted_answer_only_plan(self):
+ malformed = [
+ {'kind': 'plan'},
+ {'kind': 'plan', 'steps': []},
+ {'kind': 'plan', 'steps': 'not-a-list'},
+ {'kind': 'plan', 'steps': [None]},
+ {'kind': 'plan', 'steps': [{}]},
+ ]
+ for proposal in malformed:
+ with self.subTest(proposal=proposal), self.assertRaises(self.runtime.planner['PlannerError']):
+ self.plan('stable-direct', [proposal])
+ self.assertEqual(len(self.last_client.calls), 1)
+
+ def test_real_model_authored_work_can_still_receive_a_missing_terminal_step(self):
+ proposal = model_plan('web_search')
+ proposal['steps'] = proposal['steps'][:-1]
+ kind, plan, _client = self.plan('focused-current-lookup', [proposal])
+ self.assertEqual(kind, 'plan')
+ self.assertEqual([step['capability_id'] for step in plan['steps']], ['web_search', 'respond'])
+ self.assertTrue(plan['validation']['repairs'])
+
+ def test_unselected_web_is_neutral_and_actual_availability_is_authoritative(self):
+ _, _, client = self.plan("neutral-web-selection", [model_plan("web_search")])
+ payload = json.loads(client.calls[0]["messages"][1]["content"])
+ self.assertNotIn("web_search", payload["user_selected"])
+ self.assertEqual(payload["required_capabilities"], [])
+ self.assertIn("web_search", payload["capability_availability"]["available"])
+ self.assertIn("deep_research", payload["capability_availability"]["available"])
+ self.assertNotIn("web_search", payload["capability_availability"]["unavailable"])
+ self.assertTrue(payload["capability_availability"]["web_discovery_enabled"])
+
+ def test_research_discovery_reports_the_server_web_setting_not_the_manual_control(self):
+ _, _, client = self.plan(
+ "neutral-web-selection", [model_plan()], settings_overrides={"enable_web_search": False},
+ )
+ payload = json.loads(client.calls[0]["messages"][1]["content"])
+ self.assertFalse(payload["capability_availability"]["web_discovery_enabled"])
+ self.assertIn("deep_research", payload["capability_availability"]["available"])
+
+ def test_selected_controls_are_required_not_an_available_capability_allowlist(self):
+ for case_id, selected in (
+ ("explicit-web-selection", "web_search"),
+ ("explicit-research-selection", "deep_research"),
+ ):
+ with self.subTest(selected=selected):
+ _, document, client = self.plan(case_id, [model_plan(selected)])
+ payload = json.loads(client.calls[0]["messages"][1]["content"])
+ self.assertEqual(payload["required_capabilities"], [selected])
+ self.assertEqual(payload["capability_availability"]["available"], [
+ "web_search", "deep_research", "respond",
+ ])
+ self.assertEqual(document["steps"][0]["capability_id"], selected)
+
+ def test_unavailable_selected_operation_fails_before_any_model_call(self):
+ with self.assertRaises(self.runtime.planner["PlannerError"]):
+ self.plan(
+ "explicit-research-selection", [], settings_overrides={"enable_source_review": False},
+ )
+ self.assertEqual(self.last_client.calls, [])
+
+ def test_json_recovery_does_not_reclassify_a_different_rejected_parameter(self):
+ error = OfflineBadRequestError("Unsupported reasoning.", body={
+ "param": "reasoning_effort", "code": "unsupported_value",
+ "message": "The requested reasoning_effort is unsupported with response_format.",
+ })
+ self.assertFalse(self.runtime.planner["_unsupported_json_format"](error))
+ self.assertTrue(self.runtime.planner["_unsupported_json_format"](unsupported_json_error()))
class EvaluationContracts(OfflineTestCase):
@@ -368,15 +479,23 @@ def test_paired_calls_share_context_capabilities_parameters_and_client(self):
self.assertEqual(request["observed_model"], "synthetic-model")
def test_all_synthetic_cases_share_gates_without_exposing_review_annotations(self):
- client = ScriptedClient(*[model_plan() for _ in range(2 * len(self.cases))])
+ replies = [
+ model_plan(
+ "web_search" if case_id == "explicit-web-selection" else
+ "deep_research" if case_id == "explicit-research-selection" else None
+ )
+ for case_id in self.cases for _ in range(2)
+ ]
+ client = ScriptedClient(*replies)
report = evaluation.run_comparison(
self.baseline, client=client, deployment="synthetic-deployment",
call_cap=2 * len(self.cases),
)
self.assertEqual(report["requests_made"], 2 * len(self.cases))
for result in report["results"]:
- expected = ["web_search", "respond"] if result["case_id"].startswith("research-") else [
- "web_search", "deep_research", "respond",
+ expected = [
+ capability["id"]
+ for capability in self.baseline["contexts"][result["case_id"]]["capabilities"]
]
self.assertEqual(result["available_capabilities"], expected)
for call in client.calls:
@@ -384,6 +503,38 @@ def test_all_synthetic_cases_share_gates_without_exposing_review_annotations(sel
self.assertNotIn("evidence_objectives", payload)
self.assertNotIn("overuse_risk", payload)
+ def test_available_document_can_be_used_in_both_paired_variants(self):
+ proposal = {
+ "kind": "plan", "intent": {"summary": "Read the supplied visitor guide."},
+ "steps": [
+ {
+ "step_id": "read", "capability_id": "document_analyze", "title": "Read guide",
+ "arguments": {
+ "document_ids": ["synthetic-guide"],
+ "analysis_prompt": "Summarize visitor access and accessibility restrictions.",
+ },
+ },
+ {
+ "step_id": "answer", "capability_id": "respond", "title": "Answer",
+ "arguments": {}, "depends_on": ["read"],
+ },
+ ],
+ }
+ report = evaluation.run_comparison(
+ self.baseline, client=ScriptedClient(proposal, proposal),
+ deployment="synthetic-deployment", call_cap=2,
+ case_ids=["authorized-document-context"],
+ )
+ self.assertEqual(report["status"], "completed")
+ for result in report["results"]:
+ self.assertIn("document_analyze", result["available_capabilities"])
+ self.assertEqual(result["outcome"], "plan")
+ self.assertTrue(result["validation"]["ok"])
+ self.assertEqual(
+ [step["capability_id"] for step in result["selected_steps"]],
+ ["document_analyze", "respond"],
+ )
+
def test_sdk_automatic_retries_and_invalid_budgets_are_rejected_before_calls(self):
client = ScriptedClient(model_plan(), model_plan())
for cap in (None, 0, 1, True):
@@ -412,7 +563,7 @@ def test_changed_capability_contract_or_parameters_fail_preflight(self):
self.assertEqual(client.calls, [])
def test_successful_response_format_retry_is_counted_and_classified(self):
- client = ScriptedClient(RuntimeError("SYNTHETIC_PRIVATE_PROVIDER_DETAIL"), model_plan(), model_plan())
+ client = ScriptedClient(unsupported_json_error(), model_plan(), model_plan())
report = self.compare(client, call_cap=3)
self.assertEqual(report["status"], "completed_with_recoveries")
self.assertEqual(report["requests_made"], len(client.calls))
@@ -424,7 +575,7 @@ def test_successful_response_format_retry_is_counted_and_classified(self):
self.assertNotIn("SYNTHETIC_PRIVATE_PROVIDER_DETAIL", json.dumps(report))
def test_cap_counts_fallback_retry_and_stops_before_any_extra_provider_request(self):
- client = ScriptedClient(RuntimeError("SYNTHETIC_PRIVATE_PROVIDER_DETAIL"), model_plan())
+ client = ScriptedClient(unsupported_json_error(), model_plan())
report = self.compare(client, call_cap=2)
self.assertEqual(report["status"], "budget_exhausted")
self.assertEqual(len(client.calls), 2)
@@ -454,11 +605,11 @@ def test_provider_failure_is_not_scored_as_a_successful_direct_answer(self):
)
report = self.compare(client, call_cap=4)
self.assertEqual(report["status"], "provider_failure")
- self.assertEqual(report["requests_made"], 2)
+ self.assertEqual(report["requests_made"], 1)
self.assertEqual(len(report["results"]), 1)
result = report["results"][0]
self.assertFalse(result["semantic_review_eligible"])
- self.assertEqual([step["capability_id"] for step in result["selected_steps"]], ["respond"])
+ self.assertEqual(result["selected_steps"], [])
self.assertNotIn("planner_fallback_reason", result)
self.assertNotIn("SYNTHETIC_PRIVATE_PROVIDER_DETAIL", json.dumps(report))
@@ -474,12 +625,12 @@ def test_loaded_sdk_error_base_is_handled_without_importing_sdk_offline(self):
report = self.compare(client, call_cap=4)
self.assertEqual(report["status"], "provider_failure")
- self.assertEqual(report["requests_made"], 2)
+ self.assertEqual(report["requests_made"], 1)
self.assertNotIn("SYNTHETIC_PRIVATE_PROVIDER_DETAIL", json.dumps(report))
def test_unparseable_reply_and_normalization_repairs_are_reported_honestly(self):
report = self.compare(ScriptedClient("not parseable", model_plan()))
- self.assertEqual(report["status"], "completed_with_planner_fallbacks")
+ self.assertEqual(report["status"], "completed_with_planner_failures")
self.assertEqual(report["results"][0]["fallback_classification"], "unparseable_reply")
self.assertFalse(report["results"][0]["semantic_review_eligible"])
raw = model_plan("deep_research")
@@ -490,6 +641,18 @@ def test_unparseable_reply_and_normalization_repairs_are_reported_honestly(self)
self.assertEqual(len(repaired["results"][0]["selected_steps"]), 2)
self.assertTrue(repaired["results"][0]["validation"]["repairs"])
+ def test_comparison_uses_each_actual_context_contract_without_executing_snapshot_code(self):
+ self.baseline["contexts"]["original-playlist"]["user_selected"]["web_search"] = False
+ self.baseline["source_definitions"]["build_planner_context"] = "raise RuntimeError('must not execute')"
+ client = ScriptedClient(model_plan("web_search"), model_plan("deep_research"))
+ report = self.compare(client)
+ self.assertTrue(report["context_changed"])
+ before = json.loads(client.calls[0]["messages"][1]["content"])
+ after = json.loads(client.calls[1]["messages"][1]["content"])
+ self.assertIs(before["user_selected"]["web_search"], False)
+ self.assertNotIn("web_search", after["user_selected"])
+ self.assertEqual(report["status"], "completed")
+
def test_normalization_exception_keeps_request_accounting_without_raw_errors(self):
invalid = model_plan()
invalid["revision"] = "SYNTHETIC_PRIVATE_INVALID_VALUE"
diff --git a/functional_tests/test_orchestration_run_hydration_routes.py b/functional_tests/test_orchestration_run_hydration_routes.py
index cf458ea9e..f11e4557d 100644
--- a/functional_tests/test_orchestration_run_hydration_routes.py
+++ b/functional_tests/test_orchestration_run_hydration_routes.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_orchestration_run_hydration_routes.py
"""
Functional test for the orchestration run hydration endpoints and their projections.
-Version: 0.261.099
+Version: 0.261.104
Implemented in: 0.261.099
Orchestration runs have always been persisted, but nothing in the browser read them back, so a
@@ -23,10 +23,12 @@
import re
import sys
from pathlib import Path
+from copy import deepcopy
sys.path.append(str(Path(__file__).resolve().parent))
from test_support.versioning import assert_app_version_at_least # noqa: E402
+from test_support.orchestration_research import _definitions # noqa: E402
IMPLEMENTED_IN = "0.261.099"
@@ -90,7 +92,13 @@ def _load_projections():
missing = wanted - {node.name for node in picked}
if missing:
raise AssertionError(f"missing helpers in the route module: {sorted(missing)}")
- namespace = {}
+ registry = _definitions("functions_orchestration_registry.py")
+ events = _definitions("functions_orchestration_events.py")
+ namespace = {
+ "deepcopy": deepcopy,
+ "required_capability_ids": registry["required_capability_ids"],
+ "merge_reasoning_adjustments": events["merge_reasoning_adjustments"],
+ }
exec(compile(ast.Module(body=picked, type_ignores=[]), str(ROUTE_FILE), "exec"), namespace)
return namespace
@@ -175,6 +183,7 @@ def test_detail_projection_adds_only_the_plan():
print("Testing the run detail projection adds only the plan...")
try:
helpers = _load_projections()
+ original = deepcopy(STORED_RUN)
summary = helpers["_run_summary_row"](STORED_RUN)
detail = helpers["_run_detail_row"](STORED_RUN)
@@ -185,6 +194,15 @@ def test_detail_projection_adds_only_the_plan():
assert detail[key] == value, f"{key!r} disagrees between the listing and the detail"
assert detail["plan"]["steps"][0]["step_id"] == "s1", "the plan must be returned in full"
assert "seeds" not in detail, "the seeds stay server-side even in the detail"
+ assert STORED_RUN == original, "Projection must not rewrite immutable saved plans"
+ assert detail["plan"]["inputs"]["required_capabilities"] == []
+ automatic = deepcopy(STORED_RUN)
+ automatic["plan"]["inputs"]["web"] = True
+ automatic["seeds"] = {"web_search": False}
+ projected = helpers["_run_detail_row"](automatic)
+ assert projected["plan"]["inputs"]["required_capabilities"] == []
+ automatic["seeds"]["web_search"] = True
+ assert helpers["_run_detail_row"](automatic)["plan"]["inputs"]["required_capabilities"] == ["web_search"]
for forbidden in (
"conversation_context", "request_resolution", "user_message_fingerprint",
):
diff --git a/functional_tests/test_support/orchestration_research.py b/functional_tests/test_support/orchestration_research.py
index e5a8075ea..01b823f8d 100644
--- a/functional_tests/test_support/orchestration_research.py
+++ b/functional_tests/test_support/orchestration_research.py
@@ -2,7 +2,7 @@
"""
Offline source loading and synthetic inputs for research-planner evaluation.
-Version: 0.261.100
+Version: 0.261.104
Implemented in: 0.261.099
Only production definitions are executed, never their application imports. In particular,
@@ -22,7 +22,7 @@
from contextlib import contextmanager
from datetime import datetime, timezone
from pathlib import Path
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Iterable, List, Optional
from unittest.mock import patch
@@ -33,6 +33,21 @@
REGISTRY_FILE = "functions_orchestration_registry.py"
+class OfflineAPIError(RuntimeError):
+ """SDK-shaped error seam; importing the real SDK is unnecessary offline."""
+
+
+class OfflineBadRequestError(OfflineAPIError):
+ def __init__(self, message, *, body=None):
+ super().__init__(message)
+ self.body = body or {}
+ self.status_code = 400
+
+
+class OfflineAzureError(RuntimeError):
+ pass
+
+
def _assignment(tree, name):
return next(
node for node in tree.body
@@ -56,7 +71,7 @@ def _definitions(filename, seed=None, names=None):
body.append(node)
namespace = {
"json": json, "logging": logging, "re": re, "uuid": uuid, "hashlib": hashlib,
- "Any": Any, "Dict": Dict, "List": List, "Optional": Optional,
+ "Any": Any, "Dict": Dict, "Iterable": Iterable, "List": List, "Optional": Optional,
# Production telemetry is intentionally disabled for this isolated evaluation.
"log_event": lambda *args, **kwargs: None,
**(seed or {}),
@@ -68,6 +83,30 @@ def _definitions(filename, seed=None, names=None):
return namespace
+def document_action_policy_module():
+ """Load the actual pure settings policy without importing its execution engines."""
+ limits = _definitions("functions_document_analysis.py", names={
+ "CHAT_DOCUMENT_ANALYSIS_MAX_DOCUMENTS", "WORKFLOW_DOCUMENT_ANALYSIS_MAX_DOCUMENTS",
+ })
+ module = types.ModuleType("functions_document_actions")
+ module.__dict__.update(_definitions(
+ "functions_document_actions.py", seed={**limits, "copy": copy},
+ ))
+ return module
+
+
+@contextmanager
+def stubbed_orchestration_imports():
+ """Use real document capability defaults, not an import failure as a disabled gate."""
+ # Keep this opt-in so tests of the document action engine can import their real subject.
+ from .app_stubs import stubbed_app_imports
+
+ with stubbed_app_imports(), patch.dict(sys.modules, {
+ "functions_document_actions": document_action_policy_module(),
+ }):
+ yield
+
+
@contextmanager
def planner_runtime():
"""Expose actual planner, context, registry and schema functions without Azure imports."""
@@ -79,38 +118,48 @@ def planner_runtime():
}))
registry = _definitions(REGISTRY_FILE)
schema = _definitions("functions_orchestration_schema.py", seed=registry)
+ events = _definitions("functions_orchestration_events.py")
delegation = _definitions("functions_agent_delegation.py", names={"AGENT_PLUGIN_TYPE"})
catalog = _definitions("functions_action_catalog.py", seed=delegation)
context = _definitions("functions_orchestration_context.py", seed={
**registry, "build_action_planner_projection": catalog["build_action_planner_projection"],
- "deepcopy": copy.deepcopy,
+ "deepcopy": copy.deepcopy, "datetime": datetime, "timezone": timezone,
}, names={
"SELECTED_PROMPT_LENGTH", "_text", "_string_list", "_history_text",
- "_extract_urls", "_selected_prompt", "build_conversation_signals",
+ "_extract_urls", "_selected_prompt", "build_conversation_signals", "resolve_seeds",
"build_planner_context", "conversation_reference_messages",
"_elicitation_answer_text", "build_elicitation_user_request",
})
planner = _definitions(PLANNER_FILE, seed={
**registry, **schema,
+ "build_model_reasoning_metadata": events["build_model_reasoning_metadata"],
"conversation_reference_messages": context["conversation_reference_messages"],
+ "APIError": getattr(sys.modules.get("openai"), "APIError", OfflineAPIError),
+ "BadRequestError": getattr(sys.modules.get("openai"), "BadRequestError", OfflineBadRequestError),
+ "AzureError": OfflineAzureError,
})
def no_configured_client(settings):
raise planner["PlannerError"]("No explicit evaluation client was supplied.")
planner["resolve_planner_client"] = no_configured_client
- with patch.dict(sys.modules, {"functions_source_review": review}):
+ with patch.dict(sys.modules, {
+ "functions_source_review": review,
+ "functions_document_actions": document_action_policy_module(),
+ }):
yield types.SimpleNamespace(
planner=planner, registry=registry, schema=schema, context=context,
)
def capture_baseline():
- """Capture the actual current prompt/projection, without invoking capability gates."""
+ """Capture current guidance and real synthetic context, without resource access."""
planner_source = (APP_ROOT / PLANNER_FILE).read_text(encoding="utf-8")
registry_source = (APP_ROOT / REGISTRY_FILE).read_text(encoding="utf-8")
+ context_source = (APP_ROOT / "functions_orchestration_context.py").read_text(encoding="utf-8")
planner_tree = ast.parse(planner_source)
registry_tree = ast.parse(registry_source)
+ context_tree = ast.parse(context_source)
config_tree = ast.parse((APP_ROOT / "config.py").read_text(encoding="utf-8"))
registry = _definitions(REGISTRY_FILE)
definitions = {}
@@ -119,23 +168,60 @@ def capture_baseline():
("build_planner_messages", planner_tree, planner_source),
("CAPABILITY_REGISTRY", registry_tree, registry_source),
("build_planner_capability_projection", registry_tree, registry_source),
+ ("build_planner_context", context_tree, context_source),
+ ("resolve_seeds", context_tree, context_source),
):
node = next(
(item for item in tree.body if isinstance(item, ast.FunctionDef) and item.name == name),
None,
)
definitions[name] = ast.get_source_segment(source, node or _assignment(tree, name))
+ suite = load_case_suite()
+ with planner_runtime() as runtime:
+ contexts = {}
+ for case in suite["cases"]:
+ settings, caller, context = case_inputs(runtime, suite, case)
+
+ def capture_call(_client, _deployment, messages, **_kwargs):
+ contexts[case["id"]] = json.loads(messages[1]["content"])
+ # A renderable question ends planning without invoking any execution path.
+ return json.dumps({
+ "kind": "elicitation", "message": "Synthetic capture only.",
+ "requested_schema": {
+ "type": "object", "properties": {"detail": {"type": "string"}},
+ },
+ }), None
+
+ with patch.dict(runtime.planner, {
+ "resolve_planner_client": lambda _settings: (None, "synthetic-capture"),
+ "_call_planner": capture_call,
+ }):
+ runtime.planner["plan_request"](
+ case["message"], context, "synthetic-capture", caller["user_id"],
+ settings=settings, request_context=caller,
+ seeds=runtime.context["resolve_seeds"](case.get("request") or {}),
+ )
return {
- "schema_version": 1,
+ "schema_version": 2,
"captured_at": datetime.now(timezone.utc).isoformat(),
"app_version": ast.literal_eval(_assignment(config_tree, "VERSION").value),
"capture_method": (
- "AST literal extraction and execution of registry definitions only; "
- "no app imports or gates invoked"
+ "Offline production context/planner/registry definitions with synthetic inputs "
+ "and controlled completions; no application bootstrap or service access"
),
"planner_system_prompt": ast.literal_eval(
_assignment(planner_tree, "PLANNER_SYSTEM_PROMPT").value
),
+ "contexts": contexts,
+ "case_inputs_sha256": {
+ case["id"]: hashlib.sha256(
+ json.dumps(
+ {"settings": suite["settings"], "roles": suite["user_roles"], "case": case},
+ sort_keys=True, separators=(",", ":"),
+ ).encode("utf-8")
+ ).hexdigest()
+ for case in suite["cases"]
+ },
"capabilities": registry["build_planner_capability_projection"](
registry["CAPABILITY_REGISTRY"]
),
@@ -148,6 +234,7 @@ def capture_baseline():
"source_sha256": {
PLANNER_FILE: hashlib.sha256(planner_source.encode("utf-8")).hexdigest(),
REGISTRY_FILE: hashlib.sha256(registry_source.encode("utf-8")).hexdigest(),
+ "functions_orchestration_context.py": hashlib.sha256(context_source.encode("utf-8")).hexdigest(),
},
"source_definitions": definitions,
}
@@ -166,12 +253,18 @@ def case_inputs(runtime, suite, case):
"user_id": "synthetic-evaluation-user",
"user_roles": copy.deepcopy(case.get("user_roles", suite["user_roles"])),
"message_urls": [],
- "agent_catalog": [],
+ "agent_catalog": copy.deepcopy(case.get("agents", [])),
}
signals = runtime.context["build_conversation_signals"](
case.get("prior_messages", []), case["message"],
)
context = runtime.context["build_planner_context"](
case["message"], ledger=copy.deepcopy(case.get("earlier_runs")), signals=signals,
+ seeds=runtime.context["resolve_seeds"](case.get("request") or {}),
+ candidates=copy.deepcopy(case.get("candidate_documents", [])),
+ agents=request_context["agent_catalog"],
+ memory_context=copy.deepcopy(case.get("memory_context")),
)
+ if "request_time_utc" in context:
+ context["request_time_utc"] = "2026-09-07T12:00:00+00:00"
return settings, request_context, context
diff --git a/functional_tests/test_support/orchestration_research_cases.json b/functional_tests/test_support/orchestration_research_cases.json
index 533d588bb..25550db5a 100644
--- a/functional_tests/test_support/orchestration_research_cases.json
+++ b/functional_tests/test_support/orchestration_research_cases.json
@@ -1,5 +1,5 @@
{
- "version": "0.261.099",
+ "version": "0.261.104",
"implemented_in": "0.261.099",
"purpose": "Public synthetic planner-selection evaluation; no real conversations or credentials.",
"rubric": {
@@ -41,7 +41,7 @@
"evidence_objectives": ["Explain a stable, well-established concept accurately and simply."],
"overuse_risk": "External research adds cost without addressing a missing evidence need.",
"underuse_risk": "A direct answer still needs to be correct; no novel evidence gathering is requested.",
- "review_notes": "The production triage shortcut may answer directly without calling the planner."
+ "review_notes": "The planner should recognize that existing knowledge is sufficient; a route shortcut must not make this decision."
},
{
"id": "long-simple-drafting",
@@ -138,6 +138,62 @@
"overuse_risk": "Ignoring prior context or repeating completed work adds cost and violates the requested scope.",
"underuse_risk": "The summary must stay within the facts actually supplied, not assert newly checked details."
},
+ {
+ "id": "coastal-tide-planning",
+ "message": "We will be near Crescent City, California, September 9-14, 2026. Suggest places for tide pooling, with coordinates and suitable low-tide times for each day. Use reliable local access information and tide predictions, and explain uncertainty rather than inventing times.",
+ "acceptable_choices": ["web_search", "deep_research"],
+ "evidence_objectives": [
+ "Find relevant coastal locations and verify their coordinates and access constraints.",
+ "Use appropriate tide predictions for the requested September 9-14, 2026 dates, including station and time-zone context.",
+ "Choose sufficient discovery and source reading without inventing authorization restrictions."
+ ],
+ "overuse_risk": "Repeated searches that do not improve location, access, or prediction coverage add unnecessary work.",
+ "underuse_risk": "An answer from memory cannot establish these requested tide times or current access conditions.",
+ "review_notes": "Synthetic equivalent of the reported tide-pooling failure, not a transcript. Either retrieval capability can be appropriate if its planned coverage is credible."
+ },
+ {
+ "id": "short-tide-question",
+ "message": "When is low tide at Crescent City on September 10, 2026?",
+ "acceptable_choices": ["web_search"],
+ "evidence_objectives": ["Retrieve a relevant tide prediction with station, date, and time-zone context."],
+ "overuse_risk": "A broad coastal research project is unnecessary for a focused prediction lookup.",
+ "underuse_risk": "The short wording must not cause the route to bypass capability-aware planning."
+ },
+ {
+ "id": "coastal-planning-paraphrase",
+ "message": "Help us choose where and when to visit intertidal areas around Crescent City during September 9 through 14, 2026. I need map-ready locations and daily timing grounded in the relevant tide tables, with local access limitations noted.",
+ "acceptable_choices": ["web_search", "deep_research"],
+ "evidence_objectives": ["Ground locations, local access, and dated tide predictions in appropriate sources."],
+ "overuse_risk": "A paraphrase does not independently justify a larger research budget.",
+ "underuse_risk": "Lack of a manual Web selection does not mean source retrieval is forbidden."
+ },
+ {
+ "id": "explicit-web-selection",
+ "message": "Explain how a lighthouse lens directs light, with a reliable source.",
+ "request": {"web_search_enabled": true, "required_capabilities": ["web_search"]},
+ "acceptable_choices": ["web_search"],
+ "evidence_objectives": ["Honor the explicitly selected Web operation and ground the explanation."],
+ "overuse_risk": "The selected operation does not require unrelated research.",
+ "underuse_risk": "Do not silently drop an explicit selection merely because a memory-based explanation is possible."
+ },
+ {
+ "id": "explicit-research-selection",
+ "message": "Compare different approaches to measuring coastal ecosystem recovery, including conflicting evidence and limitations.",
+ "request": {"required_capabilities": ["deep_research"]},
+ "acceptable_choices": ["deep_research"],
+ "evidence_objectives": ["Honor explicitly selected Deep Research with relevant source discovery and comparison."],
+ "overuse_risk": "Deep Research should not trigger a redundant separate Web step just for seeding.",
+ "underuse_risk": "The selected Deep Research control must reach the model and survive normalization."
+ },
+ {
+ "id": "neutral-web-selection",
+ "message": "What are the published opening hours for the Battery Point Lighthouse museum this September?",
+ "request": {"web_search_enabled": false},
+ "acceptable_choices": ["web_search"],
+ "evidence_objectives": ["Find an applicable published schedule and distinguish it from access conditions."],
+ "overuse_risk": "A focused schedule question does not require broad unrelated coastal research.",
+ "underuse_risk": "False means the manual control was not selected, not that an available capability is unauthorized."
+ },
{
"id": "research-disabled",
"message": "Explore several independent accounts of recent coastal restoration projects, compare their reported ecological outcomes, and explain important evidence gaps with sources.",
@@ -164,6 +220,54 @@
"evidence_objectives": ["Honor administrator narrowing while keeping the terminal answer available."],
"overuse_risk": "Feature enablement and the required role do not override the orchestration capability allowlist.",
"underuse_risk": "A narrowed plan should not pretend its limited coverage is comprehensive research."
+ },
+ {
+ "id": "authorized-document-context",
+ "message": "Summarize the visitor-access guidance in my Visitor guide PDF, noting accessibility restrictions and any missing details. External research is unnecessary unless the guide leaves a material gap.",
+ "settings": {
+ "enable_user_workspace": true,
+ "chat_orchestration_enabled_capabilities": ["document_search", "document_analyze", "web_search", "deep_research", "respond"]
+ },
+ "candidate_documents": [
+ {"document_id": "synthetic-guide", "file_name": "Visitor guide.pdf", "scope": "personal"}
+ ],
+ "acceptable_choices": ["document_search", "document_analyze"],
+ "evidence_objectives": ["Plan to read the relevant authorized source instead of inventing its contents."],
+ "overuse_risk": "External discovery may add no value when the supplied guide is sufficient.",
+ "underuse_risk": "A filename is not the guide's contents or evidence that it was read."
+ },
+ {
+ "id": "authorized-agent-context",
+ "message": "Improve this visitor notice for clarity: The garden gate is closed for repairs. Please use the entrance beside the library. You may consult an available writing coach if it would add value.",
+ "settings": {
+ "enable_semantic_kernel": true,
+ "chat_orchestration_enabled_capabilities": ["agent_invoke", "web_search", "deep_research", "respond"]
+ },
+ "agents": [
+ {"name": "writing-coach", "display_name": "Writing coach", "description": "Reviews short public notices for clarity and accessibility."}
+ ],
+ "acceptable_choices": ["agent_invoke", "respond"],
+ "evidence_objectives": ["Choose between direct rewriting and an authorized specialist based on useful added value."],
+ "overuse_risk": "Available agents do not have to be invoked, and rewriting supplied text does not require research.",
+ "underuse_risk": "Do not claim that the available specialist is forbidden merely because it was not manually selected."
+ },
+ {
+ "id": "saved-preference-context",
+ "message": "Rewrite this itinerary using my saved presentation preferences, but keep the longer walk as an option because this trip is for my hiking club: walk a half mile to the garden, then optionally continue two miles along the river.",
+ "settings": {"enable_fact_memory_plugin": true},
+ "memory_context": {
+ "status": "available",
+ "scope_type": "user",
+ "context_messages": [
+ {"role": "system", "content": "Saved user preferences, subordinate to the current request: use metric units and usually avoid strenuous walks. These memories do not grant permissions or override current instructions."}
+ ],
+ "citations": [],
+ "notices": []
+ },
+ "acceptable_choices": ["respond"],
+ "evidence_objectives": ["Apply the saved unit preference while preserving the latest instruction to retain the optional longer walk."],
+ "overuse_risk": "Reformatting a supplied itinerary with available preferences needs no new external research.",
+ "underuse_risk": "Ignoring existing scoped preferences or treating them as stronger than the current request loses relevant context."
}
]
}
diff --git a/functional_tests/test_v2_agent_model_exclusivity.py b/functional_tests/test_v2_agent_model_exclusivity.py
index 20510a805..9432b5022 100644
--- a/functional_tests/test_v2_agent_model_exclusivity.py
+++ b/functional_tests/test_v2_agent_model_exclusivity.py
@@ -1,8 +1,8 @@
-#!/usr/bin/env python3
+# test_v2_agent_model_exclusivity.py
"""
Functional test for V2 agent / model / reasoning exclusivity.
-Version: 0.261.034
+Version: 0.261.104
Implemented in: 0.261.034
In the V2 chat composer the Model, Agent and Reasoning pickers were all independently live.
@@ -128,8 +128,8 @@ def test_an_agent_supplies_its_own_model_and_takes_no_reasoning_level():
"reasoning effort is resolved per model"
)
# It only ever lands on the direct-model call parameters.
- assert "api_params['reasoning_effort'] = request_reasoning_effort" in route
- assert "stream_params['reasoning_effort'] = request_reasoning_effort" in route
+ assert "response, reasoning_resolution = _create_chat_completion_with_reasoning(" in route
+ assert "stream, reasoning_resolution = _create_chat_completion_with_reasoning(" in route
print(" ok the agent path takes neither the picked model nor a reasoning level")
return True
@@ -281,7 +281,7 @@ def test_the_composer_wires_the_rule_into_the_toolbar():
assert "modelPickerInactive: boolean;" in gating
assert "showReasoning: boolean;" in gating
assert "modelPickerInactive: agentActive," in gating
- assert "showReasoning: !agentActive && !imageGenerationActive," in gating
+ assert "showReasoning: !agentActive && (!imageGenerationActive || Boolean(input.orchestrating))," in gating
composer = read(V2_SRC, "components", "chat", "Composer.tsx")
diff --git a/functional_tests/test_v2_agent_model_exclusivity_logic.ts b/functional_tests/test_v2_agent_model_exclusivity_logic.ts
index 20d71036c..c462a9905 100644
--- a/functional_tests/test_v2_agent_model_exclusivity_logic.ts
+++ b/functional_tests/test_v2_agent_model_exclusivity_logic.ts
@@ -1,7 +1,7 @@
// test_v2_agent_model_exclusivity_logic.ts
// Behavioural checks for the V2 agent / model / reasoning exclusivity.
//
-// Version: 0.261.034
+// Version: 0.261.104
// Implemented in: 0.261.034
//
// The V2 interface has no unit test runner, and adding one would pull in a test framework for
@@ -27,6 +27,8 @@ import {
} from '../application/v2_ui/src/lib/chatRequestSelection';
import { resolveGating } from '../application/v2_ui/src/lib/composerGating';
import type { ModelCatalogEntry } from '../application/v2_ui/src/lib/models';
+import type { ReasoningCapabilities } from '../application/v2_ui/src/lib/reasoning';
+import modelCatalog from '../application/single_app/static/json/model_capabilities.json';
let failures = 0;
function check(name: string, condition: boolean, detail?: unknown) {
@@ -39,6 +41,8 @@ function check(name: string, condition: boolean, detail?: unknown) {
}
/* ---- fixtures ---- */
+const reasoningPolicy = modelCatalog.models.find((model) => model.id === 'gpt-5')!
+ .reasoningPolicy as ReasoningCapabilities;
/** Shaped like `_build_chat_model_catalog` output, including the per-endpoint selection key. */
const MODELS: ModelCatalogEntry[] = [
@@ -49,6 +53,7 @@ const MODELS: ModelCatalogEntry[] = [
endpoint_id: 'endpoint-a',
provider: 'azure_openai',
display_name: 'GPT-5 (East)',
+ reasoning_capabilities: reasoningPolicy,
},
{
// The same deployment name on a second endpoint: why the key is not the name.
@@ -58,6 +63,7 @@ const MODELS: ModelCatalogEntry[] = [
endpoint_id: 'endpoint-b',
provider: 'azure_openai',
display_name: 'GPT-5 (West)',
+ reasoning_capabilities: reasoningPolicy,
},
];
diff --git a/functional_tests/test_v2_reasoning_effort_logic.mjs b/functional_tests/test_v2_reasoning_effort_logic.mjs
index ddec4b8d0..e6537ac0e 100644
--- a/functional_tests/test_v2_reasoning_effort_logic.mjs
+++ b/functional_tests/test_v2_reasoning_effort_logic.mjs
@@ -1,150 +1,140 @@
// test_v2_reasoning_effort_logic.mjs
-//
-// Runtime test for the V2 per-model reasoning effort resolution.
-// Version: 0.261.036
-// Implemented in: 0.261.036
-//
-// The companion test, test_v2_reasoning_effort_persistence.py, asserts that the composer is
-// wired to the shared user setting and that the keys it writes are ones the route accepts.
-// Those are source assertions: they prove the pieces are connected, not that the right level
-// comes out.
-//
-// This file executes the resolution itself, because its failure modes are all silent. A level
-// stored under the wrong key is simply never found again. A stored level that the newly
-// selected model does not accept is sent and then stripped by the endpoint, so the user sees a
-// control claiming an effort that was never applied. And `none` is a real choice in the picker
-// but not a value the endpoint takes, so sending it looks like a working request.
-//
-// Run directly with `node functional_tests/test_v2_reasoning_effort_logic.mjs`. Requires Node
-// 22.6 or newer, which strips the TypeScript types so the real module can be imported rather
-// than a copy of it.
+// Version: 0.261.104
+// Implemented in: 0.261.104
+// Execute real frontend resolution against the canonical Python policy, not a second family table.
import assert from 'node:assert/strict';
+import { execFileSync } from 'node:child_process';
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import vm from 'node:vm';
import {
- getModelSupportedLevels,
- reasoningModelKey,
- requestReasoningEffort,
- resolveReasoningEffort,
- supportsReasoning,
+ getModelSupportedLevels, reasoningModelKey, requestReasoningEffort,
+ resolveReasoningEffort, resolveReasoningSelection, supportsReasoning,
+ normalizeReasoningAdjustments, reasoningAdjustmentMessage,
+ reasoningMetadataForEvent,
} from '../application/v2_ui/src/lib/reasoning.ts';
+const root = fileURLToPath(new URL('../', import.meta.url));
+const policies = JSON.parse(execFileSync('python', ['-c', `
+import json, sys
+sys.path.insert(0, r'application\\single_app')
+from functions_model_capabilities import resolve_model_reasoning_policy
+print(json.dumps({name: resolve_model_reasoning_policy(name) for name in
+ ['gpt-5.6-luna', 'gpt-5', 'gpt-5.1', 'gpt-5-pro', 'o3', 'gpt-4o', 'unknown-private-model']}))
+`], { cwd: root, encoding: 'utf8' }));
+const luna = policies['gpt-5.6-luna'];
const checks = [];
-function check(name, fn) {
- checks.push([name, fn]);
-}
-
-/* --------------------------------- the key ---------------------------------- */
-
-check('a model is keyed by its model id, not its deployment name', () => {
- // getCurrentModelName() in chat-reasoning.js reads dataset.modelId first, so a level
- // stored by either interface has to land on the same entry.
- assert.equal(
- reasoningModelKey({ model_id: 'gpt-5-mini', deployment_name: 'chat-prod' }),
- 'gpt-5-mini',
- );
-});
+const check = (name, run) => checks.push([name, run]);
-check('the deployment name is used when there is no model id', () => {
- assert.equal(reasoningModelKey({ deployment_name: 'gpt-5-mini' }), 'gpt-5-mini');
- assert.equal(reasoningModelKey({ model_id: ' ', deployment_name: 'gpt-5' }), 'gpt-5');
+check('storage keys stay id-first, independently of canonical identity', () => {
+ assert.equal(reasoningModelKey({ model_id: 'opaque-uuid', deployment_name: 'chat-prod' }), 'opaque-uuid');
+ assert.equal(reasoningModelKey({ deployment_name: 'chat-prod' }), 'chat-prod');
+ assert.equal(reasoningModelKey(undefined, 'old-key'), 'old-key');
+ assert.equal(reasoningModelKey(undefined), '');
});
-
-check('a missing catalog record falls back to what the picker shows', () => {
- assert.equal(reasoningModelKey(undefined, 'gpt-5-mini'), 'gpt-5-mini');
- assert.equal(reasoningModelKey(undefined, undefined), '');
+check('Luna accepts exactly the observed endpoint levels', () => {
+ assert.deepEqual(getModelSupportedLevels(luna), ['none', 'low', 'medium', 'high', 'xhigh']);
+ assert.equal(getModelSupportedLevels(policies['gpt-5']).includes('minimal'), true);
+ assert.equal(getModelSupportedLevels(policies['gpt-5.1']).includes('low'), true);
});
-
-/* ------------------------------ stored levels -------------------------------- */
-
-check('a stored level is restored for its own model', () => {
- const saved = { 'gpt-5-mini': 'high' };
- assert.equal(resolveReasoningEffort('gpt-5-mini', saved), 'high');
+check('stale Minimal resolves to Low without mutating unrelated preferences', () => {
+ const saved = { 'opaque-uuid': 'minimal', other: 'high' };
+ assert.deepEqual(resolveReasoningSelection('opaque-uuid', saved, luna), {
+ requested_effort: 'minimal', effective_effort: 'low',
+ mode: 'explicit', adjustment_reason: 'unsupported_effort',
+ });
+ assert.deepEqual(saved, { 'opaque-uuid': 'minimal', other: 'high' });
+ assert.equal(resolveReasoningEffort('other', saved, luna), 'high');
+ assert.equal(resolveReasoningEffort('new-model', saved, luna), 'low');
});
-
-check('a level stored for one model does not follow the user to another', () => {
- const saved = { 'gpt-5-mini': 'high' };
- // o3 has its own entry, or it has not been chosen for and takes the default.
- assert.equal(resolveReasoningEffort('o3', saved), 'low');
-});
-
-check('a stored level the model does not accept is ignored', () => {
- // The 5.1 series skips `low`, so a level carried over from an o-series model cannot be
- // honoured and must not be sent for the endpoint to strip. It falls back the way
- // getCurrentModelReasoningEffort() does: `low` when offered, otherwise the first level,
- // which for this family is `none`.
- assert.equal(resolveReasoningEffort('gpt-5.1', { 'gpt-5.1': 'low' }), 'none');
- // gpt-5 has no `none`, so a stored `none` from a 5.1 model is discarded for `low`.
- assert.equal(resolveReasoningEffort('gpt-5', { 'gpt-5': 'none' }), 'low');
-});
-
-/* --------------------------------- defaults ---------------------------------- */
-
-check('an unset model defaults to low, as the classic client does', () => {
- assert.equal(resolveReasoningEffort('gpt-5-mini', {}), 'low');
- assert.equal(resolveReasoningEffort('gpt-5-mini', undefined), 'low');
- assert.equal(resolveReasoningEffort('o3', undefined), 'low');
-});
-
-check('a model without low takes its first supported level', () => {
- // The 5.1 series offers none, minimal, medium and high.
- assert.equal(resolveReasoningEffort('gpt-5.1', {}), 'none');
+check('every catalog-supported choice survives unchanged, including None', () => {
+ for (const policy of Object.values(policies)) {
+ for (const level of getModelSupportedLevels(policy)) {
+ assert.equal(resolveReasoningEffort('id', { id: level }, policy), level);
+ assert.equal(requestReasoningEffort(level, policy), level);
+ }
+ }
+ assert.equal(requestReasoningEffort('none', luna), 'none');
+ assert.equal(requestReasoningEffort(undefined, luna), undefined);
+ assert.equal(requestReasoningEffort('minimal', luna), undefined);
});
-
-check('gpt-5-pro is always high, whatever was stored', () => {
- assert.equal(resolveReasoningEffort('gpt-5-pro', {}), 'high');
- assert.equal(resolveReasoningEffort('gpt-5-pro', { 'gpt-5-pro': 'minimal' }), 'high');
+check('unknown and unsupported policies never invent supported levels', () => {
+ for (const policy of [undefined, policies['gpt-4o'], policies['unknown-private-model']]) {
+ assert.deepEqual(getModelSupportedLevels(policy), []);
+ assert.equal(supportsReasoning(policy), false);
+ assert.equal(resolveReasoningEffort('id', { id: 'high' }, policy), undefined);
+ assert.equal(requestReasoningEffort('none', policy), undefined);
+ }
});
-
-check('no model selected still resolves to a level', () => {
- assert.equal(resolveReasoningEffort(undefined, undefined), 'low');
- assert.equal(resolveReasoningEffort('', {}), 'low');
+check('a single-level policy uses its supported fallback', () => {
+ assert.equal(resolveReasoningEffort('pro', { pro: 'low' }, policies['gpt-5-pro']), 'high');
});
-
-/* ------------------------------- what is sent -------------------------------- */
-
-check('none is never sent to the endpoint', () => {
- // getCurrentReasoningEffort() returns null for none; the endpoint takes no such value.
- assert.equal(requestReasoningEffort('none'), undefined);
- assert.equal(requestReasoningEffort(''), undefined);
- assert.equal(requestReasoningEffort(undefined), undefined);
+check('safe notices describe omission as Model default and ignore provider error prose', () => {
+ const adjustment = {
+ requested_effort: 'minimal', effective_effort: null, mode: 'model_default',
+ adjustment_reason: '', stage: 'answer',
+ };
+ assert.equal(normalizeReasoningAdjustments([null, {}, adjustment]).length, 1);
+ assert.equal(reasoningAdjustmentMessage(adjustment), 'Answer: Minimal could not be used; using Model default.');
});
-
-check('a real level is passed through unchanged', () => {
- assert.equal(requestReasoningEffort('minimal'), 'minimal');
- assert.equal(requestReasoningEffort('high'), 'high');
+check('latest stage/model correction wins without merging planner and answer notices', () => {
+ const first = {
+ requested_effort: 'minimal', effective_effort: 'low', mode: 'explicit',
+ adjustment_reason: 'unsupported_effort', stage: 'answer', model_name: 'gpt-5.6-luna',
+ };
+ const planner = { ...first, stage: 'planner' };
+ const latest = { ...first, effective_effort: null, mode: 'model_default', adjustment_reason: 'provider_rejected' };
+ assert.deepEqual(normalizeReasoningAdjustments([first, planner, latest]), [latest, planner]);
+ assert.deepEqual(normalizeReasoningAdjustments([first, { ...latest, adjustment_reason: null }]), []);
+ const cleared = { ...first, requested_effort: 'low', adjustment_reason: null };
+ assert.deepEqual(normalizeReasoningAdjustments([cleared], [latest, planner]), [planner]);
+ assert.deepEqual(normalizeReasoningAdjustments(undefined, [latest, planner]), [latest, planner]);
+ assert.deepEqual(reasoningMetadataForEvent({
+ reasoning_adjustments: [cleared],
+ }, [latest, planner]), { reasoning_adjustments: [planner] });
+ assert.deepEqual(reasoningMetadataForEvent({
+ metadata: { unrelated: 'keep', reasoning_adjustments: [cleared] },
+ }, [latest, planner]), { unrelated: 'keep', reasoning_adjustments: [planner] });
});
-
-/* ------------------------- models with no choice ----------------------------- */
-
-check('a model with no reasoning offers nothing to choose', () => {
- for (const model of ['gpt-4o', 'gpt-4.1-mini', 'gpt-5-chat', 'gpt-5-codex']) {
- assert.deepEqual(getModelSupportedLevels(model), ['none'], model);
- assert.equal(supportsReasoning(model), false, model);
- }
+check('terminal public reasoning fields override stale metadata without losing other fields', () => {
+ assert.deepEqual(reasoningMetadataForEvent({
+ metadata: { reasoning_effort: 'minimal', unrelated: 'keep' },
+ reasoning_effort: null,
+ requested_reasoning_effort: 'minimal',
+ reasoning_mode: 'model_default',
+ reasoning_adjustments: [],
+ }), {
+ unrelated: 'keep', reasoning_effort: null, requested_reasoning_effort: 'minimal',
+ reasoning_mode: 'model_default', reasoning_adjustments: [],
+ });
});
-
-check('a reasoning model does offer a choice', () => {
- for (const model of ['gpt-5', 'gpt-5.1', 'gpt-5-pro', 'o3']) {
- assert.equal(supportsReasoning(model), true, model);
+check('classic and V2 agree for the same projected policy and stored preference', () => {
+ const option = { dataset: {
+ modelId: 'opaque-uuid', modelName: 'gpt-5.6-luna', deploymentName: 'prod',
+ reasoningCapabilities: JSON.stringify(luna),
+ } };
+ const modelSelect = { value: 'prod', selectedIndex: 0, options: [option] };
+ const source = readFileSync(new URL('../application/single_app/static/js/chat/chat-reasoning.js', import.meta.url), 'utf8')
+ .replace(/^import .*;$/gm, '').replace(/^export /gm, '');
+ const context = vm.createContext({
+ document: { getElementById: (id) => id === 'model-select' ? modelSelect : null },
+ console,
+ });
+ vm.runInContext(source, context);
+ for (const level of ['minimal', ...luna.efforts]) {
+ vm.runInContext(`reasoningEffortSettings = { 'opaque-uuid': '${level}' };`, context);
+ assert.equal(
+ vm.runInContext('getCurrentReasoningEffort()', context),
+ resolveReasoningEffort('opaque-uuid', { 'opaque-uuid': level }, luna),
+ );
}
+ option.dataset.reasoningCapabilities = JSON.stringify(policies['unknown-private-model']);
+ assert.equal(vm.runInContext('getCurrentReasoningEffort()', context), null);
});
-/* ----------------------------------- runner ---------------------------------- */
-
-let passed = 0;
-let failed = 0;
-
-for (const [name, fn] of checks) {
- try {
- await fn();
- console.log(`ok ${name}`);
- passed += 1;
- } catch (error) {
- console.log(`FAIL ${name}`);
- console.log(` ${error.message}`);
- failed += 1;
- }
+for (const [name, run] of checks) {
+ await run();
+ console.log(`ok ${name}`);
}
-
-console.log(`\n${passed}/${passed + failed} runtime checks passed`);
-process.exit(failed > 0 ? 1 : 0);
+console.log(`${checks.length}/${checks.length} reasoning behavior checks passed`);
diff --git a/functional_tests/test_v2_reasoning_effort_persistence.py b/functional_tests/test_v2_reasoning_effort_persistence.py
index c564f0a8f..852dbfeac 100644
--- a/functional_tests/test_v2_reasoning_effort_persistence.py
+++ b/functional_tests/test_v2_reasoning_effort_persistence.py
@@ -1,349 +1,147 @@
-#!/usr/bin/env python3
+# test_v2_reasoning_effort_persistence.py
"""
-Functional test for V2 reasoning effort persistence.
+Functional regressions for canonical reasoning projection and preference contracts.
+Version: 0.261.104
+Implemented in: 0.261.104
-Version: 0.261.036
-Implemented in: 0.261.036
-
-The V2 reasoning level used to live only in the composer's local state. It was never read
-from or written to /api/user/settings, so it was lost on every remount -- navigating away
-and back, or reloading -- and it was never cleared when the model changed, which meant a
-level chosen for gpt-5 was still sent after switching to gpt-4o, a model that has no
-reasoning at all.
-
-The fix reuses the contract the classic interface already has rather than inventing a second
-one: the level is stored per model in the `reasoningEffortSettings` user setting, and the
-chosen model is stored in `preferredModelId` / `preferredModelDeployment`, which is what
-`_build_initial_chat_model_selection` restores the picker from.
-
-Three things are pinned here.
-
-**The keys have to be whitelisted.** /api/user/settings validates against `allowed_keys` in
-route_backend_users.py and drops anything outside it **without complaining** -- the POST
-still returns success and the value never arrives, so the preference appears to save and is
-gone on the next load.
-
-**The key a level is stored under has to match the classic interface.** Both write the same
-map, so `getCurrentModelName()` and `reasoningModelKey()` must agree on model id before
-deployment name. If they disagree, a level set in one interface is invisible in the other.
-
-**The level has to be derived, not remembered.** The composer must clear the effort for a
-model that offers no choice, or the stale value is sent to a model that rejects it.
-
-The resolution itself is exercised by the companion Node test, which is run from here.
+Executes actual catalog/initial-selection functions without Flask/Azure startup, then the
+Node behavioral tests. Real late-load, migration and remount behavior is in the UI suite.
"""
-import re
-import shutil
+import ast
+import copy
+import json
import subprocess
import sys
from pathlib import Path
-REPO_ROOT = Path(__file__).resolve().parents[1]
-APP_DIR = REPO_ROOT / "application" / "single_app"
-V2_SRC = REPO_ROOT / "application" / "v2_ui" / "src"
-LEGACY_CHAT_JS = APP_DIR / "static" / "js" / "chat"
-LOGIC_TEST = REPO_ROOT / "functional_tests" / "test_v2_reasoning_effort_logic.mjs"
-
-IMPLEMENTED_IN = "0.261.036"
-
-# The settings this fix depends on, all of which are shared with the classic interface.
-SHARED_SETTING_KEYS = (
- "reasoningEffortSettings",
- "preferredModelId",
- "preferredModelDeployment",
-)
-
-sys.path.insert(0, str(REPO_ROOT / "functional_tests"))
-
-from test_support.versioning import assert_app_version_at_least # noqa: E402
-
-
-def _read(path):
- return path.read_text(encoding="utf-8")
-
-
-def _allowed_keys():
- """The whitelist the settings route validates against."""
- users = _read(APP_DIR / "route_backend_users.py")
- block = re.search(r"allowed_keys = \{(.*?)\}", users, re.DOTALL)
- assert block, "Could not find allowed_keys in route_backend_users.py"
- return set(re.findall(r"['\"]([A-Za-z_][A-Za-z0-9_]*)['\"]", block.group(1)))
-
-
-def _writable_keys():
- """The keys the V2 client declares it may write."""
- settings = _read(V2_SRC / "lib" / "userSettings.ts")
- block = re.search(
- r"export const WRITABLE_USER_SETTING_KEYS = \[(.*?)\] as const;", settings, re.DOTALL
- )
- assert block, "Could not find WRITABLE_USER_SETTING_KEYS in userSettings.ts"
- return set(re.findall(r"'([^']+)'", block.group(1)))
-
-
-def test_the_shared_keys_are_declared_and_accepted():
- """A key outside the route's whitelist is discarded silently, so both sides must list it."""
- print("Testing the shared settings keys...")
-
- writable = _writable_keys()
- allowed = _allowed_keys()
-
- for key in SHARED_SETTING_KEYS:
- assert key in writable, (
- f"{key!r} is written by the V2 composer but is not declared in "
- "WRITABLE_USER_SETTING_KEYS, so the whitelist test cannot cover it"
+ROOT = Path(__file__).resolve().parents[1]
+APP = ROOT / "application" / "single_app"
+sys.path.insert(0, str(APP))
+
+from functions_model_capabilities import REASONING_IDENTIFIER_FIELDS, resolve_model_reasoning_policy # noqa: E402
+
+
+def _catalog_functions():
+ source = ast.parse((APP / "route_frontend_chats.py").read_text(encoding="utf-8"))
+ names = {
+ "_normalize_chat_model_value", "_build_chat_model_catalog",
+ "_build_initial_chat_model_selection", "_chat_model_reasoning_metadata",
+ }
+ namespace = {
+ "resolve_model_reasoning_policy": resolve_model_reasoning_policy,
+ "REASONING_IDENTIFIER_FIELDS": REASONING_IDENTIFIER_FIELDS,
+ "sanitize_model_endpoints_for_frontend": copy.deepcopy,
+ "normalize_model_endpoints": lambda endpoints: (endpoints, False),
+ "_filter_chat_model_endpoints_by_governance": lambda user, endpoints, feature: endpoints,
+ }
+ module = ast.Module(body=[node for node in source.body if isinstance(node, ast.FunctionDef) and node.name in names], type_ignores=[])
+ exec(compile(module, "route_frontend_chats.py", "exec"), namespace)
+ return namespace
+
+
+def test_authorized_model_policy_is_identical_on_initial_and_refreshed_catalogs():
+ namespace = _catalog_functions()
+ model = {
+ "id": "opaque-uuid", "modelName": "gpt-5.6-luna",
+ "deploymentName": "chat-prod", "displayName": "Friendly display",
+ "api_key": "never-project-this",
+ }
+ settings = {"enable_multi_model_endpoints": True, "model_endpoints": [
+ {"id": endpoint_id, "models": [model], "endpoint": "https://internal.invalid", "key": "secret"}
+ for endpoint_id in ("first", "second")
+ ]}
+ catalog = namespace["_build_chat_model_catalog"](
+ user_id="caller", settings=settings, user_settings_dict={}, user_groups_raw=[],
+ )
+ assert len(catalog) == 2
+ assert catalog[0]["selection_key"] != catalog[1]["selection_key"]
+ for item in catalog:
+ initial = namespace["_build_initial_chat_model_selection"](
+ chat_model_options=catalog, preferred_model_id=item["selection_key"],
)
- assert key in allowed, (
- f"{key!r} is not in allowed_keys, so /api/user/settings will return success and "
- "then discard it"
+ assert initial["model_name"] == item["model_name"] == "gpt-5.6-luna"
+ assert initial["reasoning_capabilities"] == item["reasoning_capabilities"] == resolve_model_reasoning_policy("gpt-5.6-luna")
+ serialized = json.dumps(catalog)
+ assert "never-project-this" not in serialized and "internal.invalid" not in serialized
+ assert "secret" not in serialized
+
+
+def test_legacy_and_apim_models_have_safe_policies_without_new_identity_keys():
+ namespace = _catalog_functions()
+ for settings in (
+ {"gpt_model": {"selected": [{"deploymentName": "custom", "modelName": "gpt-5.6-luna"}]}},
+ {"enable_gpt_apim": True, "azure_apim_gpt_deployment": "gpt-5.6-luna"},
+ ):
+ catalog = namespace["_build_chat_model_catalog"](
+ user_id="caller", settings=settings, user_settings_dict={}, user_groups_raw=[],
)
-
- print(f"All {len(SHARED_SETTING_KEYS)} shared keys are declared and accepted!")
- return True
-
-
-def test_the_storage_key_matches_the_classic_client():
- """Both interfaces write the same map, so they must key a model the same way."""
- print("Testing the per-model storage key...")
-
- legacy = _read(LEGACY_CHAT_JS / "chat-reasoning.js")
- # The classic client keys on dataset.modelId first, falling back to the deployment name.
- assert "selectedOption?.dataset?.modelId || selectedOption?.dataset?.deploymentName" in legacy, (
- "chat-reasoning.js no longer resolves the model name id-first; the V2 key must "
- "follow whatever it does now or the shared map splits in two"
- )
- assert "reasoningEffortSettings[modelName]" in legacy, (
- "chat-reasoning.js no longer stores the level per model"
- )
-
- reasoning = _read(V2_SRC / "lib" / "reasoning.ts")
- assert "export function reasoningModelKey" in reasoning, (
- "V2 needs a single place that decides how a model is keyed in the shared map"
- )
- assert "return modelId || deployment ||" in reasoning, (
- "reasoningModelKey must prefer the model id, matching getCurrentModelName()"
- )
-
- print("Storage key test passed!")
- return True
-
-
-def test_the_composer_reads_and_writes_the_shared_map():
- """The level has to survive a remount, which means reading and writing the setting."""
- print("Testing composer persistence...")
-
- composer = _read(V2_SRC / "components" / "chat" / "Composer.tsx")
-
- assert "state.settings.reasoningEffortSettings" in composer, (
- "The composer must read the stored level, or it starts empty on every mount"
- )
- assert "reasoningEffortSettings: { ...saved, ...levels }" in composer, (
- "A chosen level must be written back into the shared map under the model's key"
- )
- assert "resolveReasoningEffort(reasoningKey, reasoningEffortSettings)" in composer, (
- "The level in effect must be resolved from the model and the stored map"
- )
-
- # Derived, not remembered: a model that offers no choice must carry no level at all, and
- # a deployment with no model catalog must not have one guessed for it.
- derived = re.search(
- r"reasoningKey && reasoningLevels\.length > 0\s*\?\s*resolveReasoningEffort\(", composer
- )
- assert derived, (
- "The effort must be cleared for a model with no reasoning and left alone when there "
- "is no model identity, or a stale or invented level is sent"
- )
- assert re.search(r"if \(!reasoningKey\) \{\s*return;", composer), (
- "The sync effect must leave the session's own choice alone when there is no model "
- "to derive a level from"
- )
-
- # The route stores this setting whole, and the app renders before the settings load
- # necessarily finishes, so an early write would replace the map with a single entry.
- assert "pendingLevels.current = { ...pendingLevels.current, [reasoningKey]: level }" in composer, (
- "A level chosen before the stored map arrives must be held, not written into an "
- "empty map, which would discard every other model's level"
- )
- assert "if (!settingsLoaded || Object.keys(pendingLevels.current).length === 0)" in composer, (
- "The held levels must be written once the map has been read, or the choice is lost"
- )
- assert "useUserSettingsStore.getState().settings" in composer, (
- "The write must merge into the map as it stands at write time, not as it was when "
- "the choice was made"
- )
- assert "settingsFailed" in composer, (
- "A settings load that failed leaves no map to merge into; the user has to be told "
- "the level is not being saved rather than left to discover it"
- )
-
- # Derived, not remembered: a model that offers no choice must carry no level at all.
- derived = re.search(
- r"reasoningLevels\.length > 0\s*\?\s*resolveReasoningEffort\(", composer
- )
- assert derived, (
- "The effort must be cleared for a model with no reasoning, or a stale level is sent "
- "to a model that rejects it"
- )
-
- # There is always an effective level once a model is known, so the control is clearable
- # only where none is derived -- a deployment with no model catalog.
- reasoning_control = composer.split(
- "{gating.showReasoning && reasoningLevels.length > 0 && ("
- )[1].split(")}")[0]
- assert "clearable={!reasoningKey}" in reasoning_control, (
- "The reasoning picker should be clearable only where no level is in effect; a model "
- "with a level already has `None` as an explicit option where it is supported"
- )
-
- print("Composer persistence test passed!")
- return True
-
-
-def test_the_model_selection_is_remembered():
- """Per-model memory is meaningless if the model itself is not restored."""
- print("Testing model selection persistence...")
-
- composer = _read(V2_SRC / "components" / "chat" / "Composer.tsx")
- assert "preferredModelId: modelSelectionKey(model)" in composer, (
- "The chosen model must be saved as its selection key, which is what "
- "_build_initial_chat_model_selection matches on"
- )
- assert "preferredModelDeployment: deployment" in composer, (
- "The deployment name is the server's fallback when the selection key no longer "
- "resolves, so it is saved too"
- )
- assert "rememberModelSelection(value)" in composer, (
- "The save must be wired to the model picker's change handler"
- )
-
- # The server side of the contract, which is what makes the saved keys matter.
- bootstrap = _read(APP_DIR / "route_backend_v2.py")
- assert 'user_settings_dict.get("preferredModelId")' in bootstrap, (
- "The bootstrap must still resolve the initial model from preferredModelId"
- )
-
- print("Model selection test passed!")
- return True
-
-
-def test_none_is_not_sent_to_the_endpoint():
- """`none` is a choice in the picker but not a value the endpoint takes."""
- print("Testing the none level...")
-
- legacy = _read(LEGACY_CHAT_JS / "chat-reasoning.js")
- assert "return effort === 'none' ? null : effort;" in legacy, (
- "chat-reasoning.js no longer suppresses none; V2 should follow whatever it does now"
- )
-
- reasoning = _read(V2_SRC / "lib" / "reasoning.ts")
- assert "export function requestReasoningEffort" in reasoning, (
- "V2 needs one place that decides what is safe to send"
- )
-
- # Every request's routing fields are built here, for both the send and the retry path,
- # so this is the one place the level has to be filtered.
- selection = _read(V2_SRC / "lib" / "chatRequestSelection.ts")
- assert "requestReasoningEffort(input.reasoningEffort)" in selection, (
- "The reasoning level must be filtered where a request's routing fields are built, "
- "or `none` reaches the endpoint"
- )
- assert "if (input.reasoningEffort)" not in selection, (
- "The raw level must not be assigned directly; `none` would pass straight through"
- )
-
- store = _read(V2_SRC / "stores" / "chatStore.ts")
- assert "requestBody.reasoning_effort =" not in store, (
- "The level must not be attached outside buildSelectionFields, which is also what "
- "keeps it off the agent path"
- )
- assert "reasoning_effort: options?.reasoningEffort," not in store, (
- "The retry path must not send the raw level, or `none` reaches the endpoint"
- )
-
- print("None-level test passed!")
- return True
-
-
-def test_reasoning_logic_behaves():
- """Run the companion runtime test, which executes the resolution itself.
-
- The assertions above prove the composer is wired to the setting. They cannot prove that
- the right level comes out of it, because that is behaviour rather than shape. The Node
- test does that, and is run from here so it cannot quietly rot next to a suite that never
- invokes it.
-
- Node is not otherwise required to work on this repository, so its absence is reported
- rather than failed. A Node that is present and reports a failure is a failure.
- """
- print("Testing reasoning resolution behaviour...")
- if not LOGIC_TEST.exists():
- raise AssertionError(f"The runtime logic test is missing: {LOGIC_TEST}")
-
- node = shutil.which("node")
- if not node:
- print(" Node is not installed; skipping the runtime logic test.")
- print(f" Run it with: node {LOGIC_TEST.relative_to(REPO_ROOT)}")
- return True
-
- completed = subprocess.run(
- [node, str(LOGIC_TEST)],
- capture_output=True,
- text=True,
- cwd=str(REPO_ROOT),
- )
- output = (completed.stdout or "") + (completed.stderr or "")
-
- # Node below 22.6 cannot import TypeScript directly. That is a limitation of the
- # environment, not a defect in the code under test.
- if completed.returncode != 0 and "Unknown file extension" in output:
- print(" This Node cannot import TypeScript directly (needs 22.6 or newer); skipping.")
- return True
-
- for line in output.splitlines():
- if line.strip():
- print(f" {line}")
-
- if completed.returncode != 0:
- raise AssertionError("The runtime logic test failed; see the output above.")
-
- print("Reasoning resolution test passed!")
- return True
+ assert catalog[0]["reasoning_capabilities"] == resolve_model_reasoning_policy("gpt-5.6-luna")
+ assert "model_id" not in catalog[0] and "endpoint_id" not in catalog[0]
+ legacy = namespace["_build_chat_model_catalog"](
+ user_id="caller",
+ settings={"enable_gpt_apim": True, "azure_apim_gpt_deployment": "z-first,a-second"},
+ user_settings_dict={}, user_groups_raw=[],
+ )
+ initial = namespace["_build_initial_chat_model_selection"](chat_model_options=legacy)
+ assert initial["deployment_name"] == "z-first"
+ apim = namespace["_build_chat_model_catalog"](
+ user_id="caller",
+ settings={
+ "enable_gpt_apim": True, "azure_apim_gpt_deployment": "custom",
+ "gpt_model": {"selected": [{"deploymentName": "custom", "modelName": "gpt-5.6-luna"}]},
+ },
+ user_settings_dict={}, user_groups_raw=[],
+ )
+ assert apim[0]["model_name"] == apim[0]["selection_key"] == "custom"
+ assert apim[0]["reasoning_capabilities"]["status"] == "unknown"
+
+
+def test_shared_preference_keys_remain_writable_and_request_path_is_centralized():
+ users = (APP / "route_backend_users.py").read_text(encoding="utf-8")
+ writable = (ROOT / "application" / "v2_ui" / "src" / "lib" / "userSettings.ts").read_text(encoding="utf-8")
+ for key in ("reasoningEffortSettings", "preferredModelId", "preferredModelDeployment"):
+ assert key in users and key in writable
+ composer = (ROOT / "application" / "v2_ui" / "src" / "components" / "chat" / "Composer.tsx").read_text(encoding="utf-8")
+ assert "reasoningEffortSettings: { ...saved, ...levels }" in composer
+ assert "if (!settingsLoaded || Object.keys(pendingLevels.current).length === 0)" in composer
+
+
+def test_catalog_identity_matches_authorized_record_policy_priority():
+ namespace = _catalog_functions()
+ fixtures = [
+ ({"modelName": " ", "behavior_name": "gpt-5.6-luna", "deploymentName": "custom"}, "gpt-5.6-luna"),
+ ({"modelName": " ", "deploymentName": "gpt-5.6-luna"}, "gpt-5.6-luna"),
+ ({"modelName": "unknown-private", "behavior_name": "gpt-5.6-luna", "deploymentName": "gpt-5.6-luna"}, "unknown-private"),
+ ({"modelName": 17, "behavior_name": "gpt-5.6-luna", "deploymentName": "custom"}, "gpt-5.6-luna"),
+ ({"deploymentName": "custom", "displayName": "gpt-5.6-luna", "id": "gpt-5.6-luna"}, "custom"),
+ ]
+ for model, expected_name in fixtures:
+ for settings in (
+ {"gpt_model": {"selected": [model]}},
+ {"enable_multi_model_endpoints": True, "model_endpoints": [{"id": "endpoint", "models": [model]}]},
+ ):
+ catalog = namespace["_build_chat_model_catalog"](
+ user_id="caller", settings=settings, user_settings_dict={}, user_groups_raw=[],
+ )
+ initial = namespace["_build_initial_chat_model_selection"](chat_model_options=catalog)
+ assert initial["model_name"] == catalog[0]["model_name"] == expected_name
+ assert initial["reasoning_capabilities"] == catalog[0]["reasoning_capabilities"] == resolve_model_reasoning_policy(model)
+ assert catalog[0]["deployment_name"] == model["deploymentName"]
-def test_version_was_incremented():
- """The application version records when this shipped."""
- print("Testing version...")
- version = assert_app_version_at_least(
- IMPLEMENTED_IN,
- reason="V2 per-model reasoning effort persistence.",
- )
- print(f" config.py VERSION is {version}.")
- print("Version test passed!")
- return True
+def test_frontend_reasoning_behavior():
+ subprocess.run(["node", str(ROOT / "functional_tests" / "test_v2_reasoning_effort_logic.mjs")], cwd=ROOT, check=True)
if __name__ == "__main__":
tests = [
- test_the_shared_keys_are_declared_and_accepted,
- test_the_storage_key_matches_the_classic_client,
- test_the_composer_reads_and_writes_the_shared_map,
- test_the_model_selection_is_remembered,
- test_none_is_not_sent_to_the_endpoint,
- test_reasoning_logic_behaves,
- test_version_was_incremented,
+ test_authorized_model_policy_is_identical_on_initial_and_refreshed_catalogs,
+ test_legacy_and_apim_models_have_safe_policies_without_new_identity_keys,
+ test_shared_preference_keys_remain_writable_and_request_path_is_centralized,
+ test_catalog_identity_matches_authorized_record_policy_priority,
+ test_frontend_reasoning_behavior,
]
-
- results = []
for test in tests:
- print(f"\nRunning {test.__name__}...")
- try:
- results.append(bool(test()))
- except Exception as exc: # noqa: BLE001 - surface any failure with a traceback
- print(f"Test failed: {exc}")
- import traceback
-
- traceback.print_exc()
- results.append(False)
-
- print(f"\nResults: {sum(results)}/{len(results)} tests passed")
- sys.exit(0 if all(results) else 1)
+ test()
+ print(f"{len(tests)}/{len(tests)} reasoning projection checks passed")
diff --git a/scripts/evaluate_orchestration_research_planning.py b/scripts/evaluate_orchestration_research_planning.py
index 4e7f29ec9..95525a11e 100644
--- a/scripts/evaluate_orchestration_research_planning.py
+++ b/scripts/evaluate_orchestration_research_planning.py
@@ -1,8 +1,8 @@
# evaluate_orchestration_research_planning.py
"""
-Small, opt-in paired evaluation of research-selection guidance.
+Small, opt-in paired evaluation of research-selection guidance and context.
-Version: 0.261.099
+Version: 0.261.104
Implemented in: 0.261.099
Default invocation lists synthetic cases without network access. Capture BEFORE changing
@@ -13,10 +13,10 @@
Live example (only against an explicitly approved evaluation deployment):
- python scripts\\evaluate_orchestration_research_planning.py --mode live --baseline .\\research-baseline.json --output .\\research-comparison.json --endpoint https://YOUR-EVAL.openai.azure.com --deployment YOUR-PLANNER --api-version 2024-10-21 --api-key-env SIMPLECHAT_EVAL_KEY --call-cap 30 --repeat 1
+ python scripts\\evaluate_orchestration_research_planning.py --mode live --baseline .\\research-baseline.json --output .\\research-comparison.json --endpoint https://YOUR-EVAL.openai.azure.com --deployment YOUR-PLANNER --api-version 2024-10-21 --api-key-env SIMPLECHAT_EVAL_KEY --case original-playlist --call-cap 4 --repeat 1
-Use --case original-playlist to select a case (repeat --case for several). All 11 cases
-need 22 primary requests per repetition; retries also consume the explicit call cap.
+Use --case original-playlist to select a case (repeat --case for several). Each case
+needs two primary requests per repetition; retries also consume the explicit call cap.
Alternatively use --entra-token-env with an explicitly obtained evaluation bearer token.
No default credential chain, application settings, real conversations, web searches,
answer generation, or automatic model grading are used.
@@ -26,12 +26,12 @@
would otherwise defeat request accounting. The production planner's response-format
fallback is retained and counted. Do not inject a client with hidden transport retries.
-Both variants use the current production planner/context/normalizer and identical
-synthetic availability, deployment and parameters. Only the captured system prompt and
-capability guidance differ. Triage is recorded, but every case exercises the planner,
-including cases the production route might answer without a planning call. This measures
-planner selection, not end-to-end answer quality. Review the rubric for overuse AND
-underuse, not an arbitrary research rate. Mock tests establish contracts only.
+Both variants use the current production planner/normalizer and identical synthetic
+availability, deployment and parameters. Each uses its captured real model-facing context,
+system prompt and capability guidance. Snapshots are data, never executable source.
+Every case exercises the planner, as production now does. This measures planner selection,
+not end-to-end answer quality. Review the rubric for overuse AND underuse, not an arbitrary
+research rate. Mock tests establish contracts only.
Outputs never overwrite an existing file. Provider failures and exhausted budgets are
explicit unsuccessful outcomes with nonzero CLI exit status, not successful direct-answer
@@ -60,6 +60,7 @@
# Direct script execution needs the repository path before this offline support import.
from functional_tests.test_support.orchestration_research import ( # noqa: E402
capture_baseline,
+ OfflineBadRequestError,
case_inputs,
load_case_suite,
planner_runtime,
@@ -150,6 +151,10 @@ def create(self, **kwargs):
status = getattr(exc, "status_code", None)
if isinstance(status, int) and 100 <= status <= 599:
event["http_status"] = status
+ bad_request_type = getattr(sys.modules.get("openai"), "BadRequestError", OfflineBadRequestError)
+ if isinstance(bad_request_type, type) and isinstance(exc, bad_request_type):
+ # Only the production classifier decides whether JSON-format recovery is valid.
+ raise
raise EvaluationProviderFailure("The evaluation planner request failed.") from None
finally:
event["duration_ms"] = round((time.perf_counter() - started) * 1000, 3)
@@ -182,8 +187,8 @@ def _validate_comparison(baseline, candidate, suite, case_ids, call_cap, repetit
raise EvaluationConfigurationError("Supply an explicit positive integer call cap.")
if type(repetitions) is not int or repetitions < 1:
raise EvaluationConfigurationError("Repetitions must be a positive integer.")
- if not isinstance(baseline, dict) or baseline.get("schema_version") != 1:
- raise EvaluationConfigurationError("A captured schema-version-1 baseline is required.")
+ if not isinstance(baseline, dict) or baseline.get("schema_version") != 2:
+ raise EvaluationConfigurationError("A schema-version-2 baseline with captured contexts is required.")
if not _text(baseline.get("planner_system_prompt")):
raise EvaluationConfigurationError("The baseline has no captured planner prompt.")
if baseline.get("parameters") != candidate["parameters"]:
@@ -193,9 +198,10 @@ def _validate_comparison(baseline, candidate, suite, case_ids, call_cap, repetit
raise EvaluationConfigurationError("The baseline has no valid capability projection.")
if any(not _text(item.get(name)) for item in original for name in ("summary", "when_to_use")):
raise EvaluationConfigurationError("The baseline capability guidance is incomplete.")
- # Only descriptive guidance may differ in this paired selection comparison.
+ # Planner-facing descriptions, outputs and newly exposed limits may change, not
+ # the executable capability identity, arguments, phase or cost.
without_guidance = lambda entries: [
- {key: value for key, value in item.items() if key not in ("summary", "when_to_use")}
+ {key: item.get(key) for key in ("id", "label", "phase", "inputs", "cost")}
for item in entries
]
if without_guidance(original) != without_guidance(candidate["capabilities"]):
@@ -205,6 +211,15 @@ def _validate_comparison(baseline, candidate, suite, case_ids, call_cap, repetit
if not selected or len(set(selected)) != len(selected) or not set(selected) <= all_ids:
raise EvaluationConfigurationError("Select distinct case IDs from the committed synthetic suite.")
cases = [case for case in suite["cases"] if case["id"] in selected]
+ for case in cases:
+ captured = (baseline.get("contexts") or {}).get(case["id"])
+ if not isinstance(captured, dict) or captured.get("message") != case["message"]:
+ raise EvaluationConfigurationError("The baseline is missing a selected case's captured context.")
+ if (
+ (baseline.get("case_inputs_sha256") or {}).get(case["id"])
+ != candidate["case_inputs_sha256"][case["id"]]
+ ):
+ raise EvaluationConfigurationError("Baseline and candidate synthetic case inputs must match.")
if call_cap < 2 * len(cases) * repetitions:
raise EvaluationConfigurationError("The call cap cannot cover the requested paired primary calls.")
return cases
@@ -223,16 +238,23 @@ def _run_variant(runtime, suite, case, snapshot, client, deployment, variant, re
raw_proposals = []
message_digests = []
original_call = planner["_call_planner"]
+ original_messages = planner["build_planner_messages"]
client.labels = {"case_id": case["id"], "variant": variant, "repetition": repetition}
- def observed_call(configured_client, configured_deployment, messages):
+ def captured_messages(current_context, replan_hint=None, edit_context=None):
+ captured = copy.deepcopy(snapshot["contexts"][case["id"]])
+ captured["capabilities"] = copy.deepcopy(current_context["capabilities"])
+ return original_messages(captured, replan_hint=replan_hint, edit_context=edit_context)
+
+ def observed_call(configured_client, configured_deployment, messages, **kwargs):
message_digests.append(_digest(messages))
- reply, usage = original_call(configured_client, configured_deployment, messages)
+ reply, usage = original_call(configured_client, configured_deployment, messages, **kwargs)
raw_proposals.append(copy.deepcopy(planner["extract_planner_json"](reply)))
return reply, usage
started = time.perf_counter()
processing_failed = False
+ planner_failure = None
with patch.dict(planner, {
"PLANNER_SYSTEM_PROMPT": snapshot["planner_system_prompt"],
"PLANNER_TEMPERATURE": snapshot["parameters"]["temperature"],
@@ -241,34 +263,42 @@ def observed_call(configured_client, configured_deployment, messages):
copy.deepcopy(projection_by_id[item["id"]]) for item in capabilities
],
"resolve_planner_client": lambda settings: (client, deployment),
+ "build_planner_messages": captured_messages,
"_call_planner": observed_call,
}):
try:
kind, document = planner["plan_request"](
case["message"], context, "synthetic-evaluation-conversation",
request_context["user_id"], settings=settings, request_context=request_context,
- authorized_document_ids=[],
+ authorized_document_ids=[
+ document["document_id"] for document in context.get("candidate_documents", [])
+ ],
+ seeds=runtime.context["resolve_seeds"](case.get("request") or {}),
)
+ except EvaluationBudgetExceeded:
+ kind, document = "error", {}
+ planner_failure = "budget_exhausted"
+ except EvaluationProviderFailure:
+ kind, document = "error", {}
+ planner_failure = "provider_failure"
+ except planner["PlannerError"] as exc:
+ kind, document = "error", {}
+ planner_failure = {
+ "unparseable_plan": "unparseable_reply",
+ "repeated_elicitation": "elicitation_failure",
+ "model_request_failed": "provider_failure",
+ }.get(exc.reason, "validation_failure")
except (ValueError, TypeError, AttributeError, KeyError, OverflowError):
# Malformed model fields can fail beyond the normalizer's repair contract.
# Keep request accounting and never persist a raw exception or claim success.
kind, document = "error", {}
processing_failed = True
requests = client.requests[before:]
- fallback = None
+ fallback = planner_failure
if client.blocked_requests > blocked_before:
fallback = "budget_exhausted"
elif processing_failed:
fallback = "planner_processing_error"
- elif "planner_fallback_reason" in document:
- if not raw_proposals:
- fallback = "provider_failure"
- elif not raw_proposals[-1]:
- fallback = "unparseable_reply"
- elif raw_proposals[-1].get("kind") == "elicitation":
- fallback = "elicitation_fallback"
- else:
- fallback = "validation_fallback"
recoveries = []
if not fallback and any(item["status"] == "provider_error" for item in requests):
recoveries.append("retry_without_response_format")
@@ -282,6 +312,7 @@ def observed_call(configured_client, configured_deployment, messages):
"kind": kind,
"outcome": fallback or kind,
"fallback_classification": fallback,
+ "failure_classification": fallback,
"recoveries": recoveries,
"semantic_review_eligible": fallback is None,
"proposals": [
@@ -300,6 +331,7 @@ def observed_call(configured_client, configured_deployment, messages):
},
"request_numbers": [item["request_number"] for item in requests],
"message_sha256": message_digests,
+ "context_sha256": _digest(snapshot["contexts"][case["id"]]),
"duration_ms": round((time.perf_counter() - started) * 1000, 3),
}
@@ -313,7 +345,7 @@ def run_comparison(baseline, *, client, deployment, call_cap, repetitions=1, cas
cases = _validate_comparison(baseline, candidate, suite, case_ids, call_cap, repetitions)
counted = CountedPlannerClient(client, call_cap)
report = {
- "schema_version": 1,
+ "schema_version": 2,
"status": "running",
"started_at": datetime.now(timezone.utc).isoformat(),
"review_status": "not_reviewed",
@@ -328,6 +360,10 @@ def run_comparison(baseline, *, client, deployment, call_cap, repetitions=1, cas
baseline["planner_system_prompt"] != candidate["planner_system_prompt"]
or baseline["capabilities"] != candidate["capabilities"]
),
+ "context_changed": any(
+ baseline["contexts"][case["id"]] != candidate["contexts"][case["id"]]
+ for case in cases
+ ),
"sdk_max_retries": 0,
"request_accounting": "SDK completion-create attempts, including failures; automatic SDK retries disabled.",
"call_cap": call_cap,
@@ -339,6 +375,7 @@ def run_comparison(baseline, *, client, deployment, call_cap, repetitions=1, cas
"source_sha256": snapshot.get("source_sha256"),
"prompt_sha256": _digest(snapshot["planner_system_prompt"]),
"projection_sha256": _digest(snapshot["capabilities"]),
+ "context_sha256": _digest(snapshot["contexts"]),
}
for name, snapshot in (("baseline", baseline), ("candidate", candidate))
},
@@ -367,7 +404,7 @@ def run_comparison(baseline, *, client, deployment, call_cap, repetitions=1, cas
break
if report["status"] == "running":
if any(item["fallback_classification"] for item in report["results"]):
- report["status"] = "completed_with_planner_fallbacks"
+ report["status"] = "completed_with_planner_failures"
elif any(item["recoveries"] for item in report["results"]):
report["status"] = "completed_with_recoveries"
elif any(
diff --git a/ui_tests/test_chat_reasoning_runtime_notices.py b/ui_tests/test_chat_reasoning_runtime_notices.py
new file mode 100644
index 000000000..8cb1f460a
--- /dev/null
+++ b/ui_tests/test_chat_reasoning_runtime_notices.py
@@ -0,0 +1,177 @@
+# test_chat_reasoning_runtime_notices.py
+"""
+Classic chat reasoning-recovery notices through real stream and history rendering.
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Loads local application modules and vendored Markdown/sanitizer assets. Only API
+responses are mocked; the existing browser fixture supports local/Azure Playwright.
+"""
+
+import pytest
+from playwright.sync_api import expect
+
+from test_chat_streaming_thinking_placeholder import HARNESS_PATH, _start_static_test_server
+from test_v2_orchestration_approval_persistence import approval_browser, connect_options # noqa: F401
+
+pytestmark = pytest.mark.ui
+
+
+@pytest.fixture
+def classic_reasoning_page(approval_browser):
+ with _start_static_test_server() as origin:
+ context = approval_browser.new_context(viewport={"width": 1280, "height": 900})
+ page = context.new_page()
+ errors = []
+ page.on("pageerror", lambda error: errors.append(str(error)))
+ try:
+ page.goto(f"{origin}/{HARNESS_PATH}")
+ for asset in ("marked.min.js", "purify.min.js"):
+ page.add_script_tag(url=f"{origin}/application/single_app/static/js/chat/{asset}")
+ page.evaluate("""async () => {
+ window.appSettings = {enable_thoughts: true, enable_text_to_speech: false, documentActionCapabilities: {}};
+ window.enable_document_classification = false;
+ window.currentConversationId = 'classic-reasoning';
+ window.scrollChatToBottom = () => {};
+ window.reasoningInjected = false;
+ window.savedReasoningMessages = [];
+ document.getElementById('test-root').innerHTML = `
+
+
+
+ gpt-4o `;
+ window.fetch = (url) => {
+ const path = String(url);
+ if (path === '/api/chat/stream') {
+ const body = new ReadableStream({
+ start(controller) {
+ window.emitClassicReasoningEvent = (event) => {
+ controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\\n\\n`));
+ if (event.done) controller.close();
+ };
+ },
+ });
+ return Promise.resolve(new Response(body, {headers: {'Content-Type': 'text/event-stream'}}));
+ }
+ const result = path.startsWith('/conversation/classic-reasoning/messages')
+ ? {messages: window.savedReasoningMessages}
+ : {success: true, messages: [], documents: [], thoughts: []};
+ return Promise.resolve(new Response(JSON.stringify(result), {headers: {'Content-Type': 'application/json'}}));
+ };
+ window.classicMessages = await import('/application/single_app/static/js/chat/chat-messages.js');
+ window.classicStreaming = await import('/application/single_app/static/js/chat/chat-streaming.js');
+ window.currentConversationId = 'classic-reasoning';
+ }""")
+ yield page
+ assert errors == []
+ finally:
+ context.close()
+
+
+@pytest.mark.parametrize("terminal_source", ["metadata", "top_level", "thought_only", "cancelled"])
+def test_classic_runtime_adjustment_is_live_nonrepeating_safe_and_reloaded(classic_reasoning_page, terminal_source):
+ page = classic_reasoning_page
+ page.evaluate("""() => {
+ window.classicStreaming.sendMessageWithStreaming(
+ {message: 'Answer this request.', conversation_id: 'classic-reasoning', reasoning_effort: 'minimal'},
+ 'pending-classic-user', 'classic-reasoning', {allowRecovery: false},
+ );
+ }""")
+ page.wait_for_function("() => Boolean(window.emitClassicReasoningEvent)")
+ model_name = (
+ ' '
+ if terminal_source == "top_level" else "gpt-5.6-luna"
+ )
+ first = {
+ "requested_effort": "minimal", "effective_effort": "low", "mode": "explicit",
+ "adjustment_reason": "reasoning_effort_unsupported", "stage": "answer", "model_name": model_name,
+ }
+ thought = {
+ "type": "thought", "step_type": "generation", "content": "Adjusting reasoning.",
+ "reasoning_adjustments": [first],
+ }
+ page.evaluate("(event) => window.emitClassicReasoningEvent(event)", thought)
+ notice = page.locator("#chatbox .reasoning-adjustment-notices")
+ expect(notice).to_have_count(1)
+ expect(notice).to_contain_text("using Low.")
+ expect(notice).to_have_attribute("role", "status")
+ expect(notice).to_have_attribute("aria-live", "polite")
+ page.evaluate("() => { window.initialReasoningNotice = document.querySelector('.reasoning-adjustment-notices'); }")
+ page.evaluate("(event) => window.emitClassicReasoningEvent(event)", thought)
+ expect(notice).to_have_count(1)
+ assert page.evaluate("() => window.initialReasoningNotice === document.querySelector('.reasoning-adjustment-notices')")
+ latest = {
+ **first, "effective_effort": None, "mode": "model_default",
+ "adjustment_reason": (
+ ""
+ if terminal_source == "top_level" else "reasoning_parameter_rejected"
+ ),
+ }
+ page.evaluate("(event) => window.emitClassicReasoningEvent(event)", {
+ "type": "thought", "step_type": "generation", "content": "Using the model default.",
+ "reasoning_adjustments": [latest],
+ })
+ expected = f"Answer: Minimal could not be used for {model_name}; using Model default."
+ expect(notice).to_have_text(expected)
+ expect(notice.locator("img, script")).to_have_count(0)
+ page.evaluate("(event) => window.emitClassicReasoningEvent(event)", {"content": "A useful answer."})
+ expect(notice).to_have_text(expected)
+ terminal = {
+ "done": True, "message_id": "classic-answer", "conversation_id": "classic-reasoning",
+ "full_content": "A useful answer.", "metadata": {"fixture_marker": "preserved"},
+ }
+ if terminal_source in ("metadata", "cancelled"):
+ terminal["metadata"]["reasoning_adjustments"] = [latest]
+ elif terminal_source == "top_level":
+ terminal["reasoning_adjustments"] = [latest]
+ if terminal_source == "cancelled":
+ terminal.update(cancelled=True, message_persisted=True)
+ page.evaluate("(event) => window.emitClassicReasoningEvent(event)", terminal)
+ expect(page.locator('[data-message-id="classic-answer"] .reasoning-adjustment-notices')).to_have_text(expected)
+ expect(notice).to_have_count(1)
+ page.evaluate("""async (adjustment) => {
+ window.savedReasoningMessages = [
+ {id: 'classic-user', role: 'user', content: 'Answer this request.',
+ metadata: {reasoning_adjustments: [adjustment]}},
+ {id: 'classic-answer', conversation_id: 'classic-reasoning', role: 'assistant',
+ content: 'A useful answer.', metadata: {reasoning_adjustments: [adjustment]}},
+ ];
+ await window.classicMessages.loadMessages('classic-reasoning');
+ }""", latest)
+ expect(notice).to_have_count(1)
+ expect(notice).to_have_text(expected)
+ expect(notice.locator("img, script")).to_have_count(0)
+ assert page.evaluate("() => window.reasoningInjected") is False
+
+
+def test_classic_notice_distinguishes_explicit_none_and_removes_a_superseded_correction(classic_reasoning_page):
+ page = classic_reasoning_page
+ page.evaluate("""async () => {
+ window.classicReasoning = await import('/application/single_app/static/js/chat/chat-reasoning.js');
+ window.reasoningPayload = {reasoning_adjustments: [
+ {requested_effort: 'high', effective_effort: 'none', mode: 'explicit',
+ adjustment_reason: 'reasoning_effort_unsupported', stage: 'answer', model_name: 'Model'},
+ {requested_effort: 'minimal', effective_effort: 'low', mode: 'explicit',
+ adjustment_reason: 'reasoning_effort_unsupported', stage: 'planner', model_name: 'Model'},
+ null, {mode: 'invalid'},
+ ]};
+ window.originalReasoningPayload = JSON.stringify(window.reasoningPayload);
+ window.classicAdjustments = window.classicReasoning.getMessageReasoningAdjustments(window.reasoningPayload);
+ window.classicMessages.appendMessage('AI', 'A saved answer.', null, 'saved-reasoning',
+ false, [], [], [], null, null, {metadata: {reasoning_adjustments: window.classicAdjustments}});
+ }""")
+ notice = page.locator("#chatbox .reasoning-adjustment-notices")
+ expect(notice.locator("p")).to_have_count(2)
+ expect(notice).to_contain_text("Answer: High could not be used for Model; using None.")
+ expect(notice).to_contain_text("Planner: Minimal could not be used for Model; using Low.")
+ page.evaluate("""() => {
+ const latest = window.classicReasoning.getMessageReasoningAdjustments({reasoning_adjustments: [
+ {requested_effort: 'high', effective_effort: 'high', mode: 'explicit',
+ adjustment_reason: null, stage: 'answer', model_name: 'Model'},
+ ]}, window.classicAdjustments);
+ window.classicReasoning.renderMessageReasoningAdjustments(
+ document.querySelector('[data-message-id="saved-reasoning"]'), latest);
+ }""")
+ expect(notice.locator("p")).to_have_count(1)
+ expect(notice).to_have_text("Planner: Minimal could not be used for Model; using Low.")
+ assert page.evaluate("() => JSON.stringify(window.reasoningPayload) === window.originalReasoningPayload")
diff --git a/ui_tests/test_v2_orchestration_composer.py b/ui_tests/test_v2_orchestration_composer.py
index 5cd299ac3..9cca7c97b 100644
--- a/ui_tests/test_v2_orchestration_composer.py
+++ b/ui_tests/test_v2_orchestration_composer.py
@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+# test_v2_orchestration_composer.py
"""
UI test for the V2 Composer's orchestration mode: the toggle, the manual-controls disclosure.
-Version: 0.261.085
+Version: 0.261.104
Implemented in: 0.261.085
Orchestration inverts the composer. The Orchestrate toggle appears only where the deployment ships
@@ -14,7 +14,7 @@
This test drives the REAL Composer over a seeded bootstrap (no server, no credentials) and asserts:
* The Orchestrate toggle is absent unless both the feature flag and the bootstrap switch are on.
- * When available it is off by default and the classic manual controls are shown.
+ * When available it is on by default and manual controls are collapsed.
* Turning it on collapses the capability toggles and the model/agent/reasoning pickers behind the
disclosure, while the attach-a-file and voice-input controls stay visible; opening the
disclosure brings the manual controls back.
@@ -223,11 +223,33 @@ def test_disclosure_restores_the_manual_controls():
return False
+def test_orchestration_controls_do_not_advertise_image_generation():
+ """Unsupported image generation is disabled without blocking available Deep Research."""
+ page = _PAGE
+ page.evaluate(_SEED_COMPOSER, {
+ "features": _features(
+ enable_source_review=True, enable_deep_source_review=True,
+ enable_image_generation=True, enable_web_search=True,
+ ),
+ "orchestration": _orchestration(),
+ })
+ page.click('#mount-a [title="Manual controls"]')
+ image = page.get_by_title("Image unavailable in Orchestrate", exact=True)
+ assert image.is_disabled()
+ assert page.get_by_title("Deep research", exact=True).is_enabled()
+ page.get_by_title("Deep research", exact=True).click()
+ assert page.get_by_title("Web", exact=True).get_attribute("aria-pressed") == "false"
+ page.get_by_title("Orchestrate", exact=True).click()
+ assert page.get_by_title("Image", exact=True).is_enabled()
+ return True
+
+
PAGE_TESTS = [
test_toggle_hidden_unless_feature_and_switch_are_on,
test_toggle_is_on_by_default_with_controls_collapsed,
test_turning_off_restores_the_classic_composer,
test_disclosure_restores_the_manual_controls,
+ test_orchestration_controls_do_not_advertise_image_generation,
]
diff --git a/ui_tests/test_v2_orchestration_plan_editor_backend.py b/ui_tests/test_v2_orchestration_plan_editor_backend.py
index 0be545f59..88b63d0fe 100644
--- a/ui_tests/test_v2_orchestration_plan_editor_backend.py
+++ b/ui_tests/test_v2_orchestration_plan_editor_backend.py
@@ -1,7 +1,7 @@
# test_v2_orchestration_plan_editor_backend.py
"""
Browser-to-Flask regression for editing and running an orchestration plan.
-Version: 0.261.103
+Version: 0.261.104
Implemented in: 0.261.102
Selected model continuity through editing and execution: 0.261.103
@@ -12,10 +12,12 @@
"""
import json
+import re
import sys
from pathlib import Path
from types import SimpleNamespace
from urllib.parse import urlsplit
+from unittest.mock import patch
import pytest
from playwright.sync_api import expect
@@ -26,6 +28,7 @@
# Reuse the versioned HTTP fixture and the real-component browser harness.
import test_orchestration_plan_revision_routes as backend_tests # noqa: E402
import test_v2_orchestration_plan_editor as editor_tests # noqa: E402
+from test_model_reasoning_capability_resolution import sdk_error # noqa: E402
from test_v2_orchestration_plan_editor import ( # noqa: E402, F401
connect_options,
editor_assets,
@@ -40,7 +43,34 @@
def integrated_editor(request, editor_browser, editor_assets):
backend = backend_tests.PlanRevisionRouteTests()
backend.setUp()
- selection = backend.use_modern_models() if request.param == 'terra' else {}
+ selection = backend.use_modern_models() if request.param != 'legacy' else {}
+ if request.param == 'luna-stale':
+ selection.update(model_id='luna-model', model_deployment='gpt-5.6-luna')
+ backend.settings['enable_web_search'] = True
+ backend.provider_attempts = []
+ backend.web_queries = []
+ complete = backend.model.chat.completions.create
+
+ def reject_unsupported_effort(**kwargs):
+ backend.provider_attempts.append(kwargs)
+ if kwargs.get('reasoning_effort') == 'minimal':
+ raise sdk_error()
+ return complete(**kwargs)
+
+ def web_search(**kwargs):
+ backend.web_queries.append(kwargs['web_search_query_text'])
+ kwargs['system_messages_for_augmentation'].append({
+ 'role': 'system', 'content': 'Web evidence: opening hours from the winery website.',
+ })
+ return True
+
+ backend.model.chat.completions.create = reject_unsupported_effort
+ boundary = backend_tests.context_routes.fake_module(
+ 'route_backend_chats', perform_web_search=web_search,
+ )
+ patcher = patch.dict(sys.modules, {'route_backend_chats': boundary})
+ patcher.start()
+ backend.addCleanup(patcher.stop)
context = editor_browser.new_context(viewport={'width': 1440, 'height': 900})
page = context.new_page()
errors = []
@@ -82,7 +112,11 @@ def forward(route):
page.route('**/*', forward)
page.on('pageerror', lambda error: errors.append(str(error)))
try:
- plan = backend.planned(approval_mode='timed', **selection)
+ plan = backend.planned(
+ approval_mode='manual' if request.param == 'luna-stale' else 'timed',
+ reasoning_effort='minimal' if request.param == 'luna-stale' else '',
+ **selection,
+ )
seeded = SimpleNamespace(assets=editor_assets, editors={'conv1': {'plan': plan}})
editor_tests.mount(page, seeded, 'conv1', 'turn1')
record = backend.runs.read_item(plan['run_id'], 'conv1')
@@ -219,3 +253,45 @@ def test_stale_cancel_does_not_discard_another_tabs_new_question(integrated_edit
assert discard['elicitation_id'] == newest['pending']['elicitation_id']
assert discard['elicitation_id'] != old_question['elicitation_id']
assert backend.editor(newest['plan']['run_id'])['pending'] is None
+
+
+@pytest.mark.parametrize('integrated_editor', ['luna-stale'], indirect=True)
+def test_stale_minimal_is_visibly_adjusted_before_editing_and_running_web_search(integrated_editor):
+ page, backend, requests, original = integrated_editor
+ notice = re.compile(r'Minimal.*Low', re.IGNORECASE | re.DOTALL)
+ expect(page.get_by_text(notice).first).to_be_visible()
+ assert original['reasoning_adjustments'][0]['effective_effort'] == 'low'
+ assert backend.runs.read_item(original['run_id'], 'conv1')['seeds']['reasoning_effort'] == 'minimal'
+
+ dialog = editor_tests.open_editor(page)
+ task = 'Add a web search for the latest winery opening hours.'
+ revised = backend_tests.revised_plan(task, searches=0)
+ revised['steps'].insert(0, {
+ 'step_id': 'web', 'capability_id': 'web_search', 'title': 'Search current opening hours',
+ 'arguments': {'query': task},
+ })
+ revised['steps'][-1]['depends_on'] = ['web']
+ backend.edit_responses.append(revised)
+ editor_tests.ask(page, task)
+ editor_tests.wait_revision(page, 1, 'conv1', 'turn1')
+ current = editor_tests.state(page, 'conv1', 'turn1')
+ assert current['plan']['inputs']['required_capabilities'] == []
+ assert current['plan']['inputs']['web'] is True
+ expect(dialog.get_by_text(notice).first).to_be_visible()
+
+ dialog.get_by_role('button', name='Run saved revision').click()
+ expect(dialog).to_have_count(0)
+ page.wait_for_function(
+ "() => window.OrchHarness.stores.chat.useChatStore.getState().messages.length === 2",
+ )
+ saved = backend.runs.read_item(current['plan']['run_id'], 'conv1')
+ assert saved['status'] == 'completed'
+ assert backend.web_queries == [task]
+ assert all(call['model'] == 'gpt-5.6-luna' for call in backend.provider_attempts)
+ assert all(call['reasoning_effort'] == 'low' for call in backend.provider_attempts)
+ assert 'Web evidence: opening hours' in json.dumps(backend.model.calls[-1]['messages'])
+ assistant = next(row for row in backend.messages.items.values() if row['id'] == saved['assistant_message_id'])
+ assert assistant['metadata']['reasoning_effort'] == 'low'
+ assert assistant['metadata']['requested_reasoning_effort'] == 'minimal'
+ assert {item['stage'] for item in assistant['metadata']['reasoning_adjustments']} == {'planner', 'answer'}
+ assert len([entry for entry in requests if entry['path'] == '/api/v2/orchestration/run']) == 1
diff --git a/ui_tests/test_v2_reasoning_controls.py b/ui_tests/test_v2_reasoning_controls.py
new file mode 100644
index 000000000..b5d1ba2f0
--- /dev/null
+++ b/ui_tests/test_v2_reasoning_controls.py
@@ -0,0 +1,697 @@
+# test_v2_reasoning_controls.py
+"""
+Real-Composer reasoning, capability selections, and saved-plan notice regressions.
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Reuse the local/Azure Playwright fixtures without live model, Azure or retrieval calls.
+"""
+
+import json
+import sys
+from pathlib import Path
+from urllib.parse import urlsplit
+
+import pytest
+from playwright.sync_api import expect
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "application" / "single_app"))
+
+from functions_model_capabilities import resolve_model_reasoning_policy # noqa: E402
+from test_v2_orchestration_approval_persistence import ( # noqa: E402, F401
+ approval_assets, approval_browser, approval_ui, connect_options,
+ CHAT_PATH, PLAN_PATH, mount, message_box, send_button, settings_response,
+)
+
+pytestmark = pytest.mark.ui
+
+
+def configure(api, *, orchestrating=True):
+ models = [
+ {
+ "selection_key": "global::east:luna-uuid", "model_id": "luna-uuid",
+ "deployment_name": "production", "endpoint_id": "east", "provider": "aoai",
+ "model_name": "gpt-5.6-luna", "display_name": "Luna",
+ "reasoning_capabilities": resolve_model_reasoning_policy("gpt-5.6-luna"),
+ },
+ {
+ "selection_key": "global::west:other-uuid", "model_id": "other-uuid",
+ "deployment_name": "production", "endpoint_id": "west", "provider": "aoai",
+ "model_name": "gpt-5", "display_name": "Other model",
+ "reasoning_capabilities": resolve_model_reasoning_policy("gpt-5"),
+ },
+ ]
+ api.bootstrap["catalogs"].update(models=models, initial_model_selection=models[0])
+ api.bootstrap["features"].update({
+ "enable_chat_orchestration": orchestrating, "enable_source_review": True,
+ "enable_deep_source_review": True, "enable_web_search": True,
+ "enable_url_access": True, "enable_image_generation": True,
+ })
+ api.settings["reasoningEffortSettings"] = {"luna-uuid": "minimal", "other-uuid": "high"}
+
+
+def open_manual(page):
+ page.get_by_title("Manual controls", exact=True).click()
+
+
+def choose_effort(page, current, desired):
+ page.get_by_role("button", name=current, exact=True).click()
+ page.get_by_role("listbox", name="Reasoning options").get_by_role("option", name=desired, exact=True).click()
+
+
+@pytest.mark.parametrize("width", [1440, 390])
+@pytest.mark.parametrize("mode", ["manual", "auto"])
+def test_stale_minimal_is_corrected_once_even_with_manual_controls_collapsed(approval_ui, width, mode):
+ open_page, api = approval_ui
+ configure(api)
+ api.settings["orchestrationApprovalMode"] = mode
+ page = open_page(width)
+ mount(page, api)
+ notice = page.get_by_role("status").filter(has_text="Minimal could not be used")
+ expect(notice).to_have_count(1)
+ expect(notice).to_contain_text("using Low")
+ expect(page.get_by_role("button", name="Low", exact=True)).to_have_count(0)
+ page.wait_for_function("() => window.OrchHarness.stores.userSettings.useUserSettingsStore.getState().settings.reasoningEffortSettings['luna-uuid'] === 'low'")
+ open_manual(page)
+ expect(page.get_by_role("button", name="Low", exact=True)).to_be_visible()
+ message_box(page).fill("A short request")
+ expect(notice).to_have_count(1)
+ with page.expect_response(settings_response):
+ page.evaluate("() => window.OrchHarness.stores.userSettings.useUserSettingsStore.getState().flush()")
+ assert api.settings["reasoningEffortSettings"] == {"luna-uuid": "low", "other-uuid": "high"}
+ assert api.settings["darkModeEnabled"] is True
+ mount(page, api)
+ expect(page.get_by_role("status").filter(has_text="Minimal could not be used")).to_have_count(0)
+ open_manual(page)
+ expect(page.get_by_role("button", name="Low", exact=True)).to_be_visible()
+
+
+def test_none_and_xhigh_are_real_choices_and_none_is_sent_explicitly(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.settings["reasoningEffortSettings"]["luna-uuid"] = "high"
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ page.get_by_role("button", name="High", exact=True).click()
+ options = page.get_by_role("listbox", name="Reasoning options").get_by_role("option")
+ expect(options).to_have_text(["None", "Low", "Medium", "High", "XHigh"])
+ options.filter(has_text="XHigh").click()
+ choose_effort(page, "XHigh", "None")
+ message_box(page).fill("Hello")
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ assert api.plans[-1]["reasoning_effort"] == "none"
+ assert api.plans[-1]["model_id"] == "luna-uuid"
+ assert api.plans[-1]["model_endpoint_id"] == "east"
+ assert api.plans[-1]["required_capabilities"] == []
+ assert api.plans[-1]["web_search_enabled"] is False
+
+
+def test_late_settings_merge_preserves_choices_for_two_models(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.hold_reads = True
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ choose_effort(page, "Low", "XHigh")
+ page.get_by_role("button", name="Luna", exact=True).click()
+ page.get_by_role("option", name="Other model", exact=True).click()
+ choose_effort(page, "Low", "Medium")
+ assert not any("reasoningEffortSettings" in write["settings"] for write in api.writes)
+ api.release_reads()
+ expect(page.get_by_role("button", name="Medium", exact=True)).to_be_visible()
+ with page.expect_response(settings_response):
+ page.evaluate("() => window.OrchHarness.stores.userSettings.useUserSettingsStore.getState().flush()")
+ assert api.settings["reasoningEffortSettings"] == {"luna-uuid": "xhigh", "other-uuid": "medium"}
+ assert api.settings["darkModeEnabled"] is True
+ page.get_by_role("button", name="Other model", exact=True).click()
+ page.get_by_role("option", name="Luna", exact=True).click()
+ expect(page.get_by_role("button", name="XHigh", exact=True)).to_be_visible()
+
+
+@pytest.mark.parametrize("policy_name", ["gpt-4o", "unknown-private-model"])
+def test_unsupported_or_unknown_policy_omits_reasoning_without_erasing_preferences(approval_ui, policy_name):
+ open_page, api = approval_ui
+ configure(api, orchestrating=False)
+ api.bootstrap["catalogs"]["models"][0]["reasoning_capabilities"] = resolve_model_reasoning_policy(policy_name)
+ page = open_page()
+ mount(page, api)
+ expect(page.get_by_role("status").filter(has_text="using Model default")).to_be_visible()
+ expect(page.get_by_role("button", name="Low", exact=True)).to_have_count(0)
+ message_box(page).fill("Hello")
+ with page.expect_response(lambda response: urlsplit(response.url).path == CHAT_PATH):
+ send_button(page).click()
+ assert "reasoning_effort" not in api.chats[-1]
+ assert api.settings["reasoningEffortSettings"]["luna-uuid"] == "minimal"
+
+
+def test_selected_supported_controls_are_positive_requirements_without_web_opt_in(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ expect(page.get_by_title("Deep research", exact=True)).to_be_enabled()
+ page.get_by_title("Deep research", exact=True).click()
+ expect(page.get_by_title("Web", exact=True)).to_have_attribute("aria-pressed", "false")
+ expect(page.get_by_title("Image unavailable in Orchestrate", exact=True)).to_be_disabled()
+ message_box(page).fill("Read https://example.test/report")
+ page.get_by_title("Read URLs", exact=True).click()
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ assert api.plans[-1]["required_capabilities"] == ["deep_research", "url_fetch"]
+ assert api.plans[-1]["web_search_enabled"] is False
+ assert not any("image" in key for key in api.plans[-1])
+
+
+@pytest.mark.parametrize("width,manual_controls", [(1440, True), (390, False)])
+@pytest.mark.parametrize("choice", ["regular_chat", "orchestrate_without_image"])
+def test_preselected_image_requires_an_explicit_compatible_choice_before_click_or_enter(
+ approval_ui, width, manual_controls, choice,
+):
+ open_page, api = approval_ui
+ configure(api)
+ api.bootstrap["orchestration"]["show_manual_controls"] = manual_controls
+ page = open_page(width)
+ mount(page, api)
+ page.get_by_title("Orchestrate", exact=True).click()
+ page.get_by_title("Image", exact=True).click()
+ message_box(page).fill("Draw an illustration.")
+ page.get_by_title("Orchestrate", exact=True).click()
+ notice = page.get_by_role("alert").filter(has_text="Orchestrate cannot generate images")
+ expect(notice).to_be_visible()
+ expect(send_button(page)).to_be_disabled()
+ send_button(page).dispatch_event("click")
+ message_box(page).press("Enter")
+ expect(notice).to_be_focused()
+ expect(message_box(page)).to_have_value("Draw an illustration.")
+ assert api.plans == [] and api.chats == []
+
+ if choice == "regular_chat":
+ page.get_by_role("button", name="Use regular Chat with Image", exact=True).click()
+ expect(page.get_by_title("Image", exact=True)).to_have_attribute("aria-pressed", "true")
+ with page.expect_response(lambda response: urlsplit(response.url).path == CHAT_PATH):
+ message_box(page).press("Enter")
+ assert api.chats[-1]["image_generation"] is True
+ assert api.plans == []
+ return
+
+ page.get_by_role("button", name="Use Orchestrate without Image for this message", exact=True).click()
+ expect(message_box(page)).to_be_focused()
+ expect(page.get_by_role("status").filter(has_text="This orchestration message will not generate images")).to_be_visible()
+ expect(send_button(page)).to_be_enabled()
+ assert api.plans == [] and api.chats == []
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ message_box(page).press("Enter")
+ assert api.plans[-1]["required_capabilities"] == []
+ assert not any("image" in key for key in api.plans[-1])
+ page.wait_for_function("() => !window.OrchHarness.stores.chat.useChatStore.getState().streaming")
+ message_box(page).fill("A second illustration.")
+ expect(send_button(page)).to_be_disabled()
+ message_box(page).press("Enter")
+ expect(notice).to_be_focused()
+ assert len(api.plans) == 1
+ page.get_by_role("button", name="Use regular Chat with Image", exact=True).click()
+ expect(page.get_by_title("Image", exact=True)).to_have_attribute("aria-pressed", "true")
+
+
+@pytest.mark.parametrize("previous_selection", ["edited_out", "previous_message", "capability_disabled"])
+def test_no_url_means_no_hidden_url_requirement_on_later_submission(approval_ui, previous_selection):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ page.get_by_title("Deep research", exact=True).click()
+ message_box(page).fill("Read https://example.test/report")
+ page.get_by_title("Read URLs", exact=True).click()
+ if previous_selection == "previous_message":
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ assert api.plans[-1]["required_capabilities"] == ["deep_research", "url_fetch"]
+ page.wait_for_function("() => !window.OrchHarness.stores.chat.useChatStore.getState().streaming")
+ elif previous_selection == "capability_disabled":
+ api.bootstrap["features"]["enable_url_access"] = False
+ page.evaluate("() => window.OrchHarness.stores.bootstrap.useBootstrapStore.getState().refresh()")
+ expect(page.get_by_role("status").filter(has_text="still be sent for server validation")).to_be_visible()
+ message_box(page).fill("Research another topic without an explicit link.")
+ expect(page.get_by_title("Read URLs", exact=True)).to_have_count(0)
+ expect(page.get_by_role("status").filter(has_text="selected retrieval requirement is no longer available")).to_have_count(0)
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ message_box(page).press("Enter")
+ assert api.plans[-1]["required_capabilities"] == ["deep_research"]
+ assert api.plans[-1]["web_search_enabled"] is False
+
+
+def test_url_selection_uses_the_resolved_attached_prompt_not_only_typed_text(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.bootstrap["catalogs"]["prompts"] = [{
+ "id": "url-prompt", "name": "URL prompt", "content": "Read https://example.test/prompt",
+ "scope_type": "personal",
+ }]
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ page.get_by_role("button", name="Prompt", exact=True).click()
+ page.get_by_role("option", name="URL prompt", exact=True).click()
+ page.get_by_title("Read URLs", exact=True).click()
+ message_box(page).fill("Summarize that source.")
+ expect(page.get_by_title("Read URLs", exact=True)).to_have_attribute("aria-pressed", "true")
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ assert api.plans[-1]["required_capabilities"] == ["url_fetch"]
+ assert "https://example.test/prompt" in api.plans[-1]["message"]
+
+
+def test_deep_research_respects_availability_and_role_projection(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.bootstrap["features"]["enable_source_review"] = False
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ expect(page.get_by_title("Deep research", exact=True)).to_have_count(0)
+
+
+def test_crawl_depth_setting_does_not_disable_the_authorized_research_operation(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.bootstrap["features"]["enable_deep_source_review"] = False
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ expect(page.get_by_title("Deep research", exact=True)).to_be_enabled()
+
+
+def test_availability_refresh_does_not_silently_drop_a_selected_requirement(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ open_manual(page)
+ page.get_by_title("Deep research", exact=True).click()
+ api.bootstrap["features"]["enable_source_review"] = False
+ page.evaluate("() => window.OrchHarness.stores.bootstrap.useBootstrapStore.getState().refresh()")
+ expect(page.get_by_role("status").filter(has_text="still be sent for server validation")).to_be_visible()
+ message_box(page).fill("Research this.")
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ assert api.plans[-1]["required_capabilities"] == ["deep_research"]
+
+
+def test_saved_plan_and_backend_default_notices_are_safe_and_do_not_make_revisions(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ const store = H.stores.orchestration.useOrchestrationStore.getState();
+ store.setPlan('approval-chat', 'saved-turn', {
+ plan_id: 'saved-plan', run_id: 'saved-run', turn_id: 'saved-turn', revision: 4,
+ conversation_id: 'approval-chat',
+ intent: { summary: 'Saved plan', complexity: 'simple' },
+ approval: { mode: 'manual', state: 'pending', timeout_seconds: 0 },
+ status: 'awaiting_approval', steps: [{step_id: 'answer', capability_id: 'respond'}],
+ reasoning_adjustments: [{
+ requested_effort: 'minimal', effective_effort: null, mode: 'model_default',
+ adjustment_reason: ' ', stage: 'planner',
+ model_name: 'Luna ',
+ }],
+ });
+ H.mount('mount-b', 'OrchestrationPlanCard', {conversationId: 'approval-chat', turnId: 'saved-turn'});
+ }""")
+ notice = page.locator("#mount-b").get_by_role("status")
+ expect(notice).to_contain_text("Planner: Minimal could not be used for Luna ; using Model default.")
+ expect(notice.locator("b, img, script")).to_have_count(0)
+ assert page.evaluate("() => window.OrchHarness.stores.orchestration.useOrchestrationStore.getState().plans['approval-chat\\u0000saved-turn'].revision") == 4
+ assert api.plans == []
+
+
+@pytest.mark.parametrize("terminal_kind,terminal_adjustment,clear_source", [
+ ("done", False, None), ("done", True, None), ("cancelled", False, None),
+ ("done", False, "thought"), ("done", False, "top_level"),
+ ("done", False, "metadata"), ("cancelled", False, "metadata"),
+])
+def test_ordinary_thought_corrections_are_live_latest_and_preserved_at_completion(
+ approval_ui, terminal_kind, terminal_adjustment, clear_source,
+):
+ open_page, api = approval_ui
+ configure(api, orchestrating=False)
+ page = open_page()
+ mount(page, api)
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ H.mount('mount-b', 'MessageList');
+ const originalFetch = window.fetch;
+ window.fetch = (url, options) => {
+ if (!String(url).endsWith('/api/chat/stream')) return originalFetch(url, options);
+ const stream = new ReadableStream({
+ start(controller) {
+ window.emitOrdinaryReasoningEvent = (event) => {
+ controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\\n\\n`));
+ if (event.done) controller.close();
+ };
+ },
+ });
+ return Promise.resolve(new Response(stream, {headers: {'Content-Type': 'text/event-stream'}}));
+ };
+ }""")
+ message_box(page).fill("Answer this request.")
+ send_button(page).click()
+ page.wait_for_function("() => Boolean(window.emitOrdinaryReasoningEvent)")
+ first = {
+ "requested_effort": "minimal", "effective_effort": "low", "mode": "explicit",
+ "adjustment_reason": "reasoning_effort_unsupported",
+ "model_name": "gpt-5.6-luna", "stage": "answer",
+ }
+ thought = {
+ "type": "thought", "step_type": "generation", "content": "Adjusting reasoning.",
+ "reasoning_adjustments": [first],
+ }
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", thought)
+ notice = page.locator("#mount-b").get_by_role("status").filter(has_text="Minimal could not be used")
+ expect(notice).to_have_count(1)
+ expect(notice).to_contain_text("using Low")
+ assert page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().streamingContent") == ""
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", thought)
+ latest = {
+ **first, "effective_effort": None, "mode": "model_default",
+ "adjustment_reason": " ",
+ }
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", {
+ "type": "thought", "step_type": "generation", "reasoning_adjustments": [latest],
+ })
+ expect(notice).to_have_count(1)
+ expect(notice).to_contain_text("using Model default")
+ expect(notice.locator("img, script")).to_have_count(0)
+ assert page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().thoughts.length") == 2
+ cleared = {
+ **first, "requested_effort": "low", "adjustment_reason": None,
+ }
+ if clear_source == "thought":
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", {
+ "type": "thought", "step_type": "generation", "reasoning_adjustments": [cleared],
+ })
+ expect(notice).to_have_count(0)
+ assert page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().thoughts.length") == 2
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", {"content": "A useful answer."})
+ expect(page.locator("#mount-b").get_by_text("A useful answer.", exact=True)).to_be_visible()
+ terminal = {
+ "done": True, "cancelled": terminal_kind == "cancelled", "message_id": "ordinary-answer",
+ "reasoning_effort": None, "requested_reasoning_effort": "minimal", "reasoning_mode": "model_default",
+ "metadata": {"fixture_marker": "preserved"},
+ }
+ if terminal_adjustment:
+ terminal["reasoning_adjustments"] = [latest]
+ if clear_source:
+ terminal.update(reasoning_effort="low", requested_reasoning_effort="low", reasoning_mode="explicit")
+ if clear_source == "top_level":
+ terminal["reasoning_adjustments"] = [cleared]
+ elif clear_source == "metadata":
+ terminal["metadata"]["reasoning_adjustments"] = [cleared]
+ page.evaluate("(event) => window.emitOrdinaryReasoningEvent(event)", terminal)
+ page.wait_for_function("() => !window.OrchHarness.stores.chat.useChatStore.getState().streaming")
+ expect(notice).to_have_count(0 if clear_source else 1)
+ if not clear_source:
+ expect(notice).to_contain_text("using Model default")
+ result = page.evaluate("""() => {
+ const state = window.OrchHarness.stores.chat.useChatStore.getState();
+ return {metadata: state.messages.find((message) => message.role === 'assistant').metadata,
+ live: state.streamingReasoningAdjustments};
+ }""")
+ assert result["metadata"].get("reasoning_adjustments", []) == ([] if clear_source else [latest])
+ assert result["metadata"]["reasoning_effort"] == ("low" if clear_source else None)
+ assert result["metadata"]["requested_reasoning_effort"] == ("low" if clear_source else "minimal")
+ assert result["metadata"]["reasoning_mode"] == ("explicit" if clear_source else "model_default")
+ assert result["metadata"]["fixture_marker"] == "preserved"
+ assert result["live"] == []
+
+
+def test_documents_prompt_and_agent_survive_positive_seeds_without_model_override(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.bootstrap["catalogs"]["agents"] = [{
+ "id": "selected-agent", "name": "Helper", "display_name": "Helper",
+ "scope_type": "global",
+ }]
+ api.bootstrap["catalogs"]["prompts"] = [{
+ "id": "selected-prompt", "name": "Brief answer", "content": "Keep the answer brief.",
+ "scope_type": "personal",
+ }]
+ page = open_page()
+ mount(page, api)
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ H.unmount('mount-a');
+ H.mount('mount-a', 'Composer', {}, {initialEntries: [{
+ pathname: '/chat',
+ search: '?document_ids=selected-doc&doc_scope=personal',
+ state: {contextDocuments: [{
+ document: {id: 'selected-doc', file_name: 'Report.pdf'},
+ scope: {kind: 'personal', id: null, name: 'My workspace'},
+ }]},
+ }]});
+ }""")
+ open_manual(page)
+ page.get_by_role("button", name="Agent", exact=True).click()
+ page.get_by_role("option", name="Helper", exact=True).click()
+ page.get_by_role("button", name="Prompt", exact=True).click()
+ page.get_by_role("option", name="Brief answer", exact=True).click()
+ page.get_by_title("Web", exact=True).click()
+ message_box(page).fill("Read this document.")
+ with page.expect_response(lambda response: urlsplit(response.url).path == PLAN_PATH):
+ send_button(page).click()
+ request = api.plans[-1]
+ assert request["required_capabilities"] == ["document_search", "web_search"]
+ assert request["selected_document_ids"] == ["selected-doc"]
+ assert request["agent_info"]["id"] == "selected-agent"
+ assert request["prompt_info"]["id"] == "selected-prompt"
+ assert not any(key in request for key in ("model_id", "model_endpoint_id", "reasoning_effort"))
+
+
+def test_real_chat_completion_displays_backend_omission_instead_of_claiming_low(approval_ui):
+ open_page, api = approval_ui
+ configure(api, orchestrating=False)
+ api.settings["reasoningEffortSettings"]["luna-uuid"] = "low"
+ page = open_page()
+
+ def complete(route):
+ api.chats.append(route.request.post_data_json)
+ events = [
+ {"content": "Completed answer."},
+ {
+ "done": True, "message_id": "answer-with-adjustment",
+ "reasoning_adjustments": [{
+ "requested_effort": "low", "effective_effort": None,
+ "mode": "model_default", "adjustment_reason": "provider_rejected",
+ "stage": "answer", "model_name": "gpt-5.6-luna",
+ }],
+ "metadata": {"reasoning_effort": None, "reasoning_mode": "model_default"},
+ },
+ ]
+ route.fulfill(content_type="text/event-stream", body="".join(f"data: {json.dumps(event)}\n\n" for event in events))
+
+ page.route("**/api/chat/stream", complete)
+ mount(page, api)
+ message_box(page).fill("Hello")
+ with page.expect_response(lambda response: urlsplit(response.url).path == CHAT_PATH):
+ send_button(page).click()
+ page.evaluate("() => window.OrchHarness.mount('mount-b', 'MessageList', {})")
+ expect(page.locator("#mount-b").get_by_role("status").filter(has_text="using Model default")).to_be_visible()
+ assert api.chats[-1]["reasoning_effort"] == "low"
+
+
+def test_run_thought_corrections_appear_before_completion_and_latest_stage_wins(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ api.settings["reasoningEffortSettings"]["luna-uuid"] = "low"
+ page = open_page()
+ mount(page, api)
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ H.stores.orchestration.useOrchestrationStore.getState().setPlan('approval-chat', 'live-turn', {
+ plan_id: 'live-plan', run_id: 'live-run', turn_id: 'live-turn',
+ conversation_id: 'approval-chat', revision: 4, edit_version: 'unchanged-v4',
+ intent: { summary: 'Saved plan', complexity: 'simple' },
+ approval: { mode: 'manual', state: 'pending', timeout_seconds: 0 },
+ status: 'awaiting_approval',
+ steps: [{step_id: 'answer', capability_id: 'respond', title: 'Answer'}],
+ reasoning_adjustments: [{
+ requested_effort: 'minimal', effective_effort: 'low', mode: 'explicit',
+ adjustment_reason: 'unsupported_effort', stage: 'planner', model_name: 'gpt-5.6-luna',
+ }],
+ });
+ const store = H.stores.orchestration.useOrchestrationStore.getState();
+ store.adoptPlanEditor('approval-chat', 'live-turn', {
+ plan: store.plans['approval-chat\\u0000live-turn'], version: 'unchanged-v4',
+ edits: {disabled_step_ids: [], removed_document_ids: {}},
+ chat: [], history: [], next_before_revision: null, pending: null, busy: false,
+ });
+ H.mount('mount-b', 'OrchestrationPlanCard', {conversationId: 'approval-chat', turnId: 'live-turn'});
+ const originalFetch = window.fetch;
+ window.fetch = (url, options) => {
+ if (!String(url).endsWith('/api/v2/orchestration/run')) return originalFetch(url, options);
+ const stream = new ReadableStream({
+ start(controller) {
+ window.emitReasoningRunEvent = (event) => {
+ controller.enqueue(new TextEncoder().encode(`data: ${JSON.stringify(event)}\\n\\n`));
+ if (event.done) controller.close();
+ };
+ },
+ });
+ return Promise.resolve(new Response(stream, {headers: {'Content-Type': 'text/event-stream'}}));
+ };
+ void H.controller.approveAndRunPlan({conversationId: 'approval-chat', turnId: 'live-turn'});
+ }""")
+ page.wait_for_function("() => Boolean(window.emitReasoningRunEvent)")
+ correction = {
+ "type": "thought", "step_type": "orchestration_planning", "status": "info",
+ "reasoning_adjustments": [{
+ "requested_effort": "minimal", "effective_effort": None, "mode": "model_default",
+ "adjustment_reason": "provider_rejected", "stage": "planner", "model_name": "gpt-5.6-luna",
+ }],
+ }
+ page.evaluate("(event) => window.emitReasoningRunEvent(event)", correction)
+ notice = page.locator("#mount-b").get_by_role("status")
+ expect(notice).to_contain_text("Planner: Minimal could not be used for gpt-5.6-luna; using Model default.")
+ expect(notice).not_to_contain_text("using Low")
+ assert page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().streaming")
+ correction["reasoning_adjustments"][0].update(stage="answer", effective_effort="low", mode="explicit")
+ page.evaluate("(event) => window.emitReasoningRunEvent(event)", correction)
+ page.evaluate("(event) => window.emitReasoningRunEvent(event)", correction)
+ expect(notice.locator("p")).to_have_count(2)
+ expect(notice).to_contain_text("Answer: Minimal could not be used for gpt-5.6-luna; using Low.")
+ plan = page.evaluate("() => window.OrchHarness.stores.orchestration.useOrchestrationStore.getState().plans['approval-chat\\u0000live-turn']")
+ assert plan["revision"] == 4 and plan["edit_version"] == "unchanged-v4"
+ assert len(plan["reasoning_adjustments"]) == 2
+ page.evaluate("""() => window.emitReasoningRunEvent({
+ done: true, message_id: 'completed-answer',
+ reasoning_effort: null, requested_reasoning_effort: 'minimal',
+ reasoning_mode: 'model_default',
+ })""")
+ page.wait_for_function("() => !window.OrchHarness.stores.chat.useChatStore.getState().streaming")
+ metadata = page.evaluate("() => window.OrchHarness.stores.chat.useChatStore.getState().messages.find((message) => message.id === 'completed-answer').metadata")
+ assert metadata["reasoning_effort"] is None and metadata["reasoning_mode"] == "model_default"
+ assert metadata["requested_reasoning_effort"] == "minimal"
+ assert api.plans == []
+
+
+@pytest.mark.parametrize("original_requirements", [[], ["deep_research"], ["agent_invoke"]])
+def test_hydration_does_not_promote_model_selected_retrieval_to_manual_requirements(
+ approval_ui, original_requirements,
+):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ summary = {
+ "run_id": "hydrated-run", "conversation_id": "approval-chat", "turn_id": "hydrated-turn",
+ "status": "awaiting_approval", "user_message_id": "saved-question",
+ "created_at": "2026-09-07T12:00:00Z",
+ "plan_summary": {
+ "plan_id": "hydrated-plan", "run_id": "hydrated-run", "turn_id": "hydrated-turn",
+ "status": "awaiting_approval", "intent_summary": "Research chosen by the model", "step_count": 3,
+ },
+ }
+ original_seeds = {"required_capabilities": original_requirements, "web_search_enabled": False}
+ plan = {
+ "plan_id": "hydrated-plan", "run_id": "hydrated-run", "turn_id": "hydrated-turn",
+ "conversation_id": "approval-chat", "revision": 2,
+ "intent": {"summary": "Research chosen by the model", "complexity": "complex"},
+ "inputs": {"web": True, "documents": [], "required_capabilities": original_requirements},
+ "steps": [
+ {"step_id": "web", "capability_id": "web_search", "title": "Search"},
+ {"step_id": "deep", "capability_id": "deep_research", "title": "Read sources"},
+ {"step_id": "answer", "capability_id": "respond", "title": "Answer"},
+ ],
+ "approval": {"mode": "manual", "state": "pending", "timeout_seconds": 0},
+ "status": "awaiting_approval",
+ }
+ run_requests = []
+ page.route("**/api/v2/orchestration/runs?*", lambda route: route.fulfill(json={"runs": [summary]}))
+ page.route("**/api/v2/orchestration/runs/hydrated-run?*", lambda route: route.fulfill(
+ json={"run": {**summary, "plan": plan, "seeds": original_seeds}},
+ ))
+ page.route("**/api/v2/orchestration/runs/hydrated-run/steps?*", lambda route: route.fulfill(json={"steps": []}))
+
+ def complete(route):
+ run_requests.append(route.request.post_data_json)
+ route.fulfill(content_type="text/event-stream", body='data: {"done":true,"message_id":"hydrated-answer"}\n\n')
+
+ page.route("**/api/v2/orchestration/run", complete)
+ page.evaluate("""async () => {
+ const H = window.OrchHarness;
+ H.stores.chat.useChatStore.setState({
+ messages: [{id: 'saved-question', role: 'user', content: 'Find suitable information.'}],
+ });
+ await H.resume.resumeOrchestrationForConversation('approval-chat');
+ H.mount('mount-b', 'OrchestrationPlanCard', {conversationId: 'approval-chat', turnId: 'hydrated-turn'});
+ }""")
+ expect(page.locator("#mount-b")).to_contain_text("Research chosen by the model")
+ restored_requirements = page.evaluate("""() => {
+ const H = window.OrchHarness;
+ const plan = H.stores.orchestration.useOrchestrationStore.getState().plans['approval-chat\\u0000hydrated-turn'];
+ const narrowed = H.plan.applyPlanEdits(plan, {disabled_step_ids: ['web'], removed_document_ids: {}});
+ return {restored: plan.inputs.required_capabilities, narrowed: narrowed.inputs.required_capabilities};
+ }""")
+ assert restored_requirements == {"restored": original_requirements, "narrowed": original_requirements}
+ open_manual(page)
+ expect(page.get_by_title("Web", exact=True)).to_have_attribute("aria-pressed", "false")
+ expect(page.get_by_title("Deep research", exact=True)).to_have_attribute("aria-pressed", "false")
+ assert api.plans == [] and run_requests == []
+ with page.expect_response(lambda response: urlsplit(response.url).path == "/api/v2/orchestration/run"):
+ page.evaluate("() => { void window.OrchHarness.controller.approveAndRunPlan({conversationId: 'approval-chat', turnId: 'hydrated-turn'}); }")
+ request = run_requests[-1]
+ assert request["run_id"] == "hydrated-run" and request["plan_id"] == "hydrated-plan"
+ # The server uses the saved original selections under this run identity. The client must
+ # not replace them with flags inferred from inputs.web or model-authored retrieval steps.
+ assert not any(key in request for key in ("seeds", "required_capabilities", "web_search_enabled", "selected_document_ids"))
+ assert original_seeds == {"required_capabilities": original_requirements, "web_search_enabled": False}
+ assert api.plans == []
+
+
+def test_implicit_selected_documents_keep_user_provenance_without_widening_step_arguments(approval_ui):
+ open_page, api = approval_ui
+ configure(api)
+ page = open_page()
+ mount(page, api)
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ const plan = H.plan.normalizePlan({
+ plan_id: 'implicit-plan', run_id: 'implicit-run', turn_id: 'implicit-turn',
+ intent: {summary: 'Selected documents', complexity: 'simple'},
+ inputs: {
+ required_capabilities: ['document_search'], web: false,
+ documents: [
+ {document_id: 'selected-doc', display_name: 'Original report.pdf', selected_by_user: true},
+ {document_id: 'other-selected-doc', display_name: 'Second report.pdf', selected_by_user: true},
+ {document_id: 'model-doc', display_name: 'Model choice.pdf', selected_by_user: false},
+ ],
+ },
+ steps: [{step_id: 'docs', capability_id: 'document_search', arguments: {query: 'Summarize'}},
+ {step_id: 'answer', capability_id: 'respond'}],
+ approval: {mode: 'manual', state: 'pending'}, status: 'awaiting_approval',
+ });
+ window.implicitDocumentPlan = plan;
+ H.mount('mount-b', 'OrchestrationRunView', {conversationId: 'approval-chat', turnId: 'implicit-turn', previewPlan: plan});
+ }""")
+ view = page.locator("#mount-b")
+ expect(view.get_by_text("Original report.pdf", exact=True)).to_be_visible()
+ expect(view.get_by_text("Second report.pdf", exact=True)).to_be_visible()
+ expect(view.get_by_text("Model choice.pdf", exact=True)).to_have_count(0)
+ expect(view.get_by_text("yours", exact=True)).to_have_count(2)
+ assert page.evaluate("() => window.implicitDocumentPlan.steps[0].arguments") == {"query": "Summarize"}
+ page.evaluate("""() => {
+ const H = window.OrchHarness;
+ const plan = window.implicitDocumentPlan;
+ H.mount('mount-b', 'OrchestrationRunView', {
+ conversationId: 'approval-chat', turnId: 'implicit-turn',
+ previewPlan: {...plan, steps: [{...plan.steps[0], arguments: {document_ids: ['selected-doc']}}, plan.steps[1]]},
+ });
+ }""")
+ expect(view.get_by_text("Original report.pdf", exact=True)).to_be_visible()
+ expect(view.get_by_text("Second report.pdf", exact=True)).to_have_count(0)
diff --git a/ui_tests/test_v2_reasoning_plan_editor.py b/ui_tests/test_v2_reasoning_plan_editor.py
new file mode 100644
index 000000000..5e6a23c32
--- /dev/null
+++ b/ui_tests/test_v2_reasoning_plan_editor.py
@@ -0,0 +1,46 @@
+# test_v2_reasoning_plan_editor.py
+"""
+Real-editor regression for original selections and reasoning correction metadata.
+Version: 0.261.104
+Implemented in: 0.261.104
+
+Uses the existing editor harness and production CSS with mocked HTTP boundaries.
+The separate module keeps its Playwright lifetime independent of Composer fixtures.
+"""
+
+import pytest
+from playwright.sync_api import expect
+
+import test_v2_orchestration_plan_editor as editor_tests
+from test_v2_orchestration_plan_editor import ( # noqa: F401
+ connect_options, editor_assets, editor_browser, editor_ui,
+)
+
+pytestmark = pytest.mark.ui
+
+
+@pytest.mark.parametrize("requirements", [None, [], ["deep_research"], ["agent_invoke"]])
+def test_editor_preserves_original_requirements_and_reasoning_through_web_revision(editor_ui, requirements):
+ page, api = editor_ui
+ plan = api.add()
+ plan["inputs"]["web"] = True
+ if requirements is not None:
+ plan["inputs"]["required_capabilities"] = requirements
+ expected_requirements = requirements if requirements is not None else []
+ plan["reasoning_adjustments"] = [{
+ "requested_effort": "minimal", "effective_effort": "low", "mode": "explicit",
+ "adjustment_reason": "reasoning_effort_unsupported",
+ "model_name": "gpt-5.6-luna", "stage": "planner",
+ }]
+ editor_tests.mount(page, api)
+ dialog = editor_tests.open_editor(page)
+ editor_tests.ask(page, "Add a Web search for current information.")
+ editor_tests.wait_revision(page, 1)
+ current = editor_tests.state(page)
+ assert current["plan"]["inputs"]["required_capabilities"] == expected_requirements
+ assert current["editor"]["state"]["plan"]["inputs"]["required_capabilities"] == expected_requirements
+ assert current["plan"]["inputs"]["web"] is True
+ assert any(step["capability_id"] == "web_search" for step in current["plan"]["steps"])
+ assert current["plan"]["reasoning_adjustments"] == plan["reasoning_adjustments"]
+ expect(dialog.get_by_role("status").filter(has_text="Minimal could not be used")).to_be_visible()
+ assert current["plan"]["revision"] == 1