From 0a359d6ae72436e3ce316f9559458ad475fa65a6 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 06:37:34 -0400 Subject: [PATCH 1/9] Restore Custom model endpoint provider work for gap review Re-applies the changes from PR #1228, which was merged prematurely and reverted by PR #1431. This restores the feature onto current Development as a working base so the remaining provider-coverage gaps can be closed before it is proposed for merge again. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- application/single_app/functions_documents.py | 32 +- .../functions_model_endpoint_runtime.py | 185 ++++- .../functions_model_endpoint_types.py | 62 ++ .../functions_model_endpoint_validation.py | 312 +++++++ application/single_app/functions_settings.py | 105 ++- .../single_app/functions_workflow_runner.py | 20 +- .../single_app/model_endpoint_clients.py | 533 +++++++++++- .../single_app/route_backend_agents.py | 2 + application/single_app/route_backend_chats.py | 41 +- .../route_backend_conversation_export.py | 191 +++-- .../single_app/route_backend_models.py | 200 +++-- .../single_app/route_backend_settings.py | 30 +- .../route_frontend_admin_settings.py | 34 +- .../single_app/route_frontend_chats.py | 35 +- .../single_app/semantic_kernel_loader.py | 52 +- .../static/js/admin/admin_model_endpoints.js | 221 ++++- .../static/js/agent_modal_stepper.js | 4 +- .../single_app/static/js/agents_common.js | 14 +- .../static/js/chat/chat-messages.js | 14 +- .../static/js/chat/chat-model-selector.js | 22 +- .../js/workspace/workspace_model_endpoints.js | 377 +++++++-- .../templates/_multiendpoint_modal.html | 25 +- .../templates/admin/_panes/extraction.html | 4 +- .../admin/_panes/model-endpoints.html | 18 +- .../CUSTOM_MODEL_ENDPOINT_PROVIDER.md | 116 +++ .../MODEL_ENDPOINT_API_KEY_MANUAL_MODELS.md | 20 +- .../v0.241.001/WORKSPACE_MULTI_ENDPOINTS.md | 11 +- docs/explanation/release_notes.md | 11 + ..._admin_multi_endpoint_persistence_guard.py | 40 +- .../test_custom_model_endpoint_provider.py | 762 ++++++++++++++++++ ...nt_auto_metadata_extraction_consistency.py | 13 +- ...l_endpoint_management_cloud_environment.py | 23 +- .../test_new_foundry_streaming_runtime.py | 9 +- ...t_tabular_claude_model_endpoint_support.py | 13 +- .../test_workflow_model_core_capabilities.py | 5 +- .../test_workspace_multi_endpoints.py | 34 +- 37 files changed, 3196 insertions(+), 396 deletions(-) create mode 100644 application/single_app/functions_model_endpoint_types.py create mode 100644 application/single_app/functions_model_endpoint_validation.py create mode 100644 docs/explanation/features/CUSTOM_MODEL_ENDPOINT_PROVIDER.md create mode 100644 functional_tests/test_custom_model_endpoint_provider.py diff --git a/application/single_app/config.py b/application/single_app/config.py index a6bb61a3a..a97fabfc8 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.009" +VERSION = "0.261.010" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index a055c13b4..99f767106 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -37,6 +37,11 @@ from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers from functions_model_endpoint_runtime import MODEL_ENDPOINT_PROVIDER_ALLOWLIST, build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol import azure.cognitiveservices.speech as speechsdk _AUDIO_RUNTIME_CAPABILITIES_CACHE = None @@ -142,6 +147,9 @@ def _build_model_endpoint_client( api_version, deployment_name, *, + api_type='', + anthropic_version='', + allow_private_custom_endpoints=False, settings=None, endpoint_config=None, identity_context=None, @@ -152,6 +160,9 @@ def _build_model_endpoint_client( endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, settings=settings, endpoint_config=endpoint_config, identity_context=identity_context, @@ -191,14 +202,24 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): provider = str(endpoint_cfg.get("provider") or selection["provider"] or "aoai").lower() connection = endpoint_cfg.get("connection", {}) or {} auth_settings = endpoint_cfg.get("auth", {}) or {} - deployment = str(model_cfg.get("deploymentName") or model_cfg.get("deployment") or "").strip() + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) endpoint = str(connection.get("endpoint") or "").strip() api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() + api_type = get_model_endpoint_api_type(endpoint_cfg) + anthropic_version = str(connection.get("anthropic_version") or "").strip() + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.") - if not endpoint or not api_version or not deployment: - raise ValueError("Selected metadata extraction endpoint is missing endpoint, API version, or deployment configuration.") + if not endpoint or not deployment or ( + runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version + ): + raise ValueError("Selected metadata extraction endpoint is incomplete.") return _build_model_endpoint_client( auth_settings, @@ -206,6 +227,11 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): endpoint, api_version, deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get("allow_private_custom_model_endpoints", False) + ), settings=settings, endpoint_config=endpoint_cfg, identity_context=identity_context, diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 4deb4fa7c..f5b590c8f 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -1,13 +1,20 @@ # functions_model_endpoint_runtime.py """Runtime helpers for configured model endpoint clients and Semantic Kernel services.""" -from openai import AsyncOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI from azure.identity import ClientSecretCredential, DefaultAzureCredential, get_bearer_token_provider from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion from config import cognitive_services_scope from foundry_agent_runtime import resolve_authority from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import validate_custom_model_endpoint_url from functions_settings import resolve_model_endpoint_foundry_scope from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, @@ -15,14 +22,26 @@ MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, AnthropicSemanticKernelChatCompletion, build_anthropic_chat_client, + build_custom_openai_async_http_client, + build_custom_openai_sync_http_client, build_openai_style_chat_client, infer_model_endpoint_protocol, + normalize_custom_openai_base_url, normalize_openai_style_base_url, resolve_openai_style_request_api_version, + SanitizedCustomChatCompletionClient, + sanitize_custom_async_openai_client, ) -MODEL_ENDPOINT_PROVIDER_ALLOWLIST = {'aoai', 'aifoundry', 'new_foundry', 'anthropic', 'claude'} +MODEL_ENDPOINT_PROVIDER_ALLOWLIST = { + 'aoai', + 'aifoundry', + 'new_foundry', + 'anthropic', + 'claude', + MODEL_ENDPOINT_PROVIDER_CUSTOM, +} MODEL_CONTEXT_AUTH_FIELDS = ( 'type', 'tenant_id', @@ -62,9 +81,12 @@ def build_model_endpoint_context( endpoint=None, auth=None, api_version=None, + api_type=None, + anthropic_version=None, endpoint_id=None, model_id=None, model_deployment=None, + request_model=None, user_id=None, active_group_ids=None, ): @@ -73,9 +95,12 @@ def build_model_endpoint_context( 'provider': str(provider or '').strip().lower(), 'endpoint': str(endpoint or '').strip(), 'api_version': str(api_version or '').strip(), + 'api_type': str(api_type or '').strip().lower(), + 'anthropic_version': str(anthropic_version or '').strip(), 'endpoint_id': str(endpoint_id or '').strip(), 'model_id': str(model_id or '').strip(), 'model_deployment': str(model_deployment or '').strip(), + 'request_model': str(request_model or model_deployment or '').strip(), } normalized_user_id = str(user_id or '').strip() @@ -125,6 +150,9 @@ def build_model_endpoint_sync_chat_client( api_version, deployment_name='', *, + api_type='', + anthropic_version=DEFAULT_ANTHROPIC_VERSION, + allow_private_custom_endpoints=False, settings=None, endpoint_config=None, identity_context=None, @@ -137,8 +165,21 @@ def build_model_endpoint_sync_chat_client( identity_context=identity_context, ) normalized_provider = str(provider or 'aoai').strip().lower() - runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) + direct_custom = normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM + if direct_custom: + endpoint = validate_custom_model_endpoint_url( + endpoint, + allow_private=allow_private_custom_endpoints, + ) + runtime_protocol = infer_model_endpoint_protocol( + normalized_provider, + endpoint, + deployment_name, + api_type, + ) auth_type = str(auth_settings.get('type') or 'managed_identity').strip().lower() + if direct_custom and auth_type not in ('api_key', 'key'): + raise ValueError('Custom model endpoints support API key authentication only.') if auth_type in ('api_key', 'key'): api_key = auth_settings.get('api_key') @@ -148,6 +189,9 @@ def build_model_endpoint_sync_chat_client( return build_anthropic_chat_client( endpoint=endpoint, api_key=api_key, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: @@ -155,14 +199,25 @@ def build_model_endpoint_sync_chat_client( api_key, endpoint, api_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, default_headers=extra_headers, ), runtime_protocol - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=api_key, - default_headers=extra_headers or None, - ), runtime_protocol + client_kwargs = { + 'api_version': api_version, + 'azure_endpoint': endpoint, + 'api_key': api_key, + } + if extra_headers: + client_kwargs['default_headers'] = extra_headers + if direct_custom: + client_kwargs['http_client'] = build_custom_openai_sync_http_client( + allow_private=allow_private_custom_endpoints, + ) + client = AzureOpenAI(**client_kwargs) + if direct_custom: + client = SanitizedCustomChatCompletionClient(client) + return client, runtime_protocol credential = resolve_credential_for_model_endpoint_auth(auth_settings) scope = cognitive_services_scope @@ -210,11 +265,15 @@ def resolve_model_endpoint_from_context(settings, model_context): model_context = model_context if isinstance(model_context, dict) else {} requested_endpoint_id = str(model_context.get('endpoint_id') or '').strip() requested_model_id = str(model_context.get('model_id') or '').strip() - requested_deployment = str(model_context.get('model_deployment') or '').strip() + requested_model_name = str( + model_context.get('request_model') + or model_context.get('model_deployment') + or '' + ).strip() requested_provider = str(model_context.get('provider') or '').strip().lower() if not settings.get('enable_multi_model_endpoints', False): return None - if not (requested_endpoint_id or requested_model_id or requested_deployment): + if not (requested_endpoint_id or requested_model_id or requested_model_name): return None endpoints = [] @@ -252,11 +311,11 @@ def resolve_model_endpoint_from_context(settings, model_context): models = endpoint_cfg.get('models', []) or [] matched_model = None for model_cfg in models: - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() if requested_model_id and str(model_cfg.get('id') or '').strip() == requested_model_id: matched_model = model_cfg break - if requested_deployment and deployment == requested_deployment: + request_model = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + if requested_model_name and request_model == requested_model_name: matched_model = model_cfg break if not matched_model or not matched_model.get('enabled', True): @@ -296,18 +355,34 @@ def build_semantic_kernel_chat_service_for_model( provider = str(model_context.get('provider') or '').strip().lower() endpoint = str(model_context.get('endpoint') or '').strip() api_version = str(model_context.get('api_version') or '').strip() + api_type = str(model_context.get('api_type') or '').strip().lower() + anthropic_version = str( + model_context.get('anthropic_version') + or DEFAULT_ANTHROPIC_VERSION + ).strip() auth_settings = model_context.get('auth') if isinstance(model_context.get('auth'), dict) else {} - deployment_name = str(model_context.get('model_deployment') or gpt_model or '').strip() + request_model = str( + model_context.get('request_model') + or model_context.get('model_deployment') + or gpt_model + or '' + ).strip() if resolved_model_endpoint: provider = str(resolved_model_endpoint.get('provider') or provider or 'aoai').strip().lower() connection = resolved_model_endpoint.get('connection', {}) or {} endpoint = str(connection.get('endpoint') or endpoint).strip() + api_type = get_model_endpoint_api_type(resolved_model_endpoint) or api_type api_version = str( connection.get('openai_api_version') or connection.get('api_version') or api_version ).strip() + anthropic_version = str( + connection.get('anthropic_version') + or anthropic_version + or DEFAULT_ANTHROPIC_VERSION + ).strip() auth_settings = resolved_model_endpoint.get('auth', {}) or auth_settings resolved_models = resolved_model_endpoint.get('models', []) or [] requested_model_id = str(model_context.get('model_id') or '').strip() @@ -317,22 +392,42 @@ def build_semantic_kernel_chat_service_for_model( (model for model in resolved_models if str(model.get('id') or '').strip() == requested_model_id), None, ) - if matched_model is None and deployment_name: + if matched_model is None and request_model: matched_model = next( ( model for model in resolved_models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == deployment_name + if resolve_model_endpoint_request_model( + resolved_model_endpoint, + model, + ) == request_model ), None, ) if matched_model: - deployment_name = str( - matched_model.get('deploymentName') or matched_model.get('deployment') or deployment_name - ).strip() + request_model = resolve_model_endpoint_request_model( + resolved_model_endpoint, + matched_model, + ) - if provider and endpoint and deployment_name: - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) + if provider and endpoint and request_model: + direct_custom = provider == MODEL_ENDPOINT_PROVIDER_CUSTOM + allow_private_custom_endpoints = bool( + settings.get('allow_private_custom_model_endpoints', False) + ) + if direct_custom: + endpoint = validate_custom_model_endpoint_url( + endpoint, + allow_private=allow_private_custom_endpoints, + ) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + request_model, + api_type, + ) auth_type = str(auth_settings.get('type') or 'managed_identity').lower() + if direct_custom and auth_type not in ('api_key', 'key'): + raise ValueError('Custom model endpoints support API key authentication only.') extra_headers = build_model_endpoint_identity_headers( settings, endpoint_config=resolved_model_endpoint, @@ -345,29 +440,59 @@ def build_semantic_kernel_chat_service_for_model( if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: return AnthropicSemanticKernelChatCompletion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_key=api_key, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs = { 'api_key': api_key, - 'base_url': normalize_openai_style_base_url(endpoint), + 'base_url': ( + normalize_custom_openai_base_url(endpoint) + if direct_custom + else normalize_openai_style_base_url(endpoint) + ), } + if direct_custom: + client_kwargs['http_client'] = build_custom_openai_async_http_client( + allow_private=allow_private_custom_endpoints, + ) if extra_headers: client_kwargs['default_headers'] = extra_headers if request_api_version: client_kwargs['default_query'] = {'api-version': request_api_version} + async_client = AsyncOpenAI(**client_kwargs) + if direct_custom: + async_client = sanitize_custom_async_openai_client(async_client) return OpenAIChatCompletion( service_id=service_id, - ai_model_id=deployment_name, - async_client=AsyncOpenAI(**client_kwargs), + ai_model_id=request_model, + async_client=async_client, + ), runtime_protocol + if direct_custom: + async_client = AsyncAzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=api_key, + default_headers=extra_headers or None, + http_client=build_custom_openai_async_http_client( + allow_private=allow_private_custom_endpoints, + ), + ) + async_client = sanitize_custom_async_openai_client(async_client) + return AzureChatCompletion( + service_id=service_id, + deployment_name=request_model, + async_client=async_client, ), runtime_protocol return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_key=api_key, api_version=api_version, @@ -383,7 +508,7 @@ def build_semantic_kernel_chat_service_for_model( token = credential.get_token(scope).token return AnthropicSemanticKernelChatCompletion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, bearer_token=token, extra_headers=extra_headers, @@ -402,7 +527,7 @@ def build_semantic_kernel_chat_service_for_model( client_kwargs['default_query'] = {'api-version': request_api_version} return OpenAIChatCompletion( service_id=service_id, - ai_model_id=deployment_name, + ai_model_id=request_model, async_client=AsyncOpenAI(**client_kwargs), ), runtime_protocol @@ -410,7 +535,7 @@ def build_semantic_kernel_chat_service_for_model( try: return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_version=api_version, azure_ad_token_provider=token_provider, @@ -419,7 +544,7 @@ def build_semantic_kernel_chat_service_for_model( except TypeError: return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_version=api_version, ad_token_provider=token_provider, diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py new file mode 100644 index 000000000..9bab708c7 --- /dev/null +++ b/application/single_app/functions_model_endpoint_types.py @@ -0,0 +1,62 @@ +# functions_model_endpoint_types.py +"""Canonical provider, API type, and model identifier helpers.""" + +from typing import Any, Dict + + +MODEL_ENDPOINT_PROVIDER_CUSTOM = "custom" +MODEL_ENDPOINT_API_TYPE_OPENAI = "openai" +MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_API_TYPE_ANTHROPIC = "anthropic" +MODEL_ENDPOINT_CUSTOM_API_TYPES = { + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, +} +DEFAULT_ANTHROPIC_VERSION = "2023-06-01" + + +def normalize_model_endpoint_api_type(provider: Any, api_type: Any) -> str: + """Return a supported explicit API type for Custom endpoints.""" + normalized_provider = str(provider or "").strip().lower() + normalized_api_type = str(api_type or "").strip().lower().replace("-", "_") + if normalized_provider != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return "" + return normalized_api_type if normalized_api_type in MODEL_ENDPOINT_CUSTOM_API_TYPES else "" + + +def get_model_endpoint_api_type(endpoint: Any) -> str: + """Return the canonical explicit API type from an endpoint record.""" + if not isinstance(endpoint, dict): + return "" + return normalize_model_endpoint_api_type(endpoint.get("provider"), endpoint.get("api_type")) + + +def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: + """Resolve the model identifier that must be sent to the configured API.""" + endpoint_data: Dict[str, Any] = endpoint if isinstance(endpoint, dict) else {} + model_data: Dict[str, Any] = model if isinstance(model, dict) else {} + provider = str(endpoint_data.get("provider") or "aoai").strip().lower() + api_type = get_model_endpoint_api_type(endpoint_data) + + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + if api_type in { + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + }: + return str(model_data.get("modelName") or model_data.get("name") or "").strip() + if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or "" + ).strip() + return "" + + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or model_data.get("modelName") + or model_data.get("name") + or "" + ).strip() diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py new file mode 100644 index 000000000..3d081d5ad --- /dev/null +++ b/application/single_app/functions_model_endpoint_validation.py @@ -0,0 +1,312 @@ +# functions_model_endpoint_validation.py +"""Validation and outbound-network safety for Custom model endpoints.""" + +import ipaddress +import re +import socket +from typing import Any, Dict, Iterable +from urllib.parse import urlparse, urlunparse + +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) + + +CUSTOM_ENDPOINT_MAX_URL_LENGTH = 2048 +CUSTOM_ENDPOINT_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES = { + "instance-data.ec2.internal", + "localhost", + "localhost.localdomain", + "metadata.azure.com", + "metadata.google.internal", +} +CUSTOM_ENDPOINT_BLOCKED_IPS = { + ipaddress.ip_address("168.63.129.16"), + ipaddress.ip_address("169.254.169.254"), + ipaddress.ip_address("169.254.169.250"), + ipaddress.ip_address("169.254.169.251"), +} +CUSTOM_ENDPOINT_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("fc00::/7"), +) + + +class ModelEndpointValidationError(ValueError): + """Raised when a model endpoint configuration violates the saved policy.""" + + +def _is_ip_literal(hostname: str) -> bool: + try: + ipaddress.ip_address(hostname) + return True + except ValueError: + return False + + +def validate_custom_model_endpoint_address( + address: str, + *, + allow_private: bool = False, +) -> None: + """Validate one resolved Custom endpoint address against the outbound policy.""" + try: + ip_address = ipaddress.ip_address(address) + except ValueError as exc: + raise ModelEndpointValidationError( + "Custom endpoint hostname resolved to an invalid address." + ) from exc + + if ip_address in CUSTOM_ENDPOINT_BLOCKED_IPS: + raise ModelEndpointValidationError( + "Custom endpoint hostname resolves to a blocked platform address." + ) + if ip_address.is_loopback: + raise ModelEndpointValidationError( + "Custom endpoint hostname must not resolve to a loopback address." + ) + if ip_address.is_link_local: + raise ModelEndpointValidationError( + "Custom endpoint hostname must not resolve to a link-local address." + ) + if ip_address.is_multicast or ip_address.is_reserved or ip_address.is_unspecified: + raise ModelEndpointValidationError( + "Custom endpoint hostname must resolve to a usable network address." + ) + is_allowed_private_address = any( + ip_address in private_network + for private_network in CUSTOM_ENDPOINT_PRIVATE_NETWORKS + if ip_address.version == private_network.version + ) + if is_allowed_private_address: + if not allow_private: + raise ModelEndpointValidationError( + "Private Custom endpoint hosts are not enabled by the administrator." + ) + return + if not ip_address.is_global: + raise ModelEndpointValidationError( + "Custom endpoint hostname must resolve to a globally routable address." + ) + + +def resolve_custom_model_endpoint_addresses( + hostname: str, + port: int = 443, + *, + allow_private: bool = False, +) -> tuple[str, ...]: + """Resolve and validate every address before a Custom endpoint connection.""" + try: + resolved_addresses = socket.getaddrinfo( + hostname, + port, + type=socket.SOCK_STREAM, + ) + except socket.gaierror as exc: + raise ModelEndpointValidationError( + "Custom endpoint hostname could not be resolved." + ) from exc + + if not resolved_addresses: + raise ModelEndpointValidationError( + "Custom endpoint hostname did not resolve to an address." + ) + + validated_addresses = [] + seen_addresses = set() + for address_info in resolved_addresses: + address = address_info[4][0] + validate_custom_model_endpoint_address( + address, + allow_private=allow_private, + ) + if address not in seen_addresses: + seen_addresses.add(address) + validated_addresses.append(address) + return tuple(validated_addresses) + + +def validate_custom_model_endpoint_url( + endpoint: Any, + *, + allow_private: bool = False, +) -> str: + """Validate and normalize a Custom endpoint URL before an outbound request.""" + endpoint_text = str(endpoint or "").strip() + if not endpoint_text: + raise ModelEndpointValidationError("Custom endpoint URL is required.") + if len(endpoint_text) > CUSTOM_ENDPOINT_MAX_URL_LENGTH: + raise ModelEndpointValidationError("Custom endpoint URL is too long.") + + try: + parsed_endpoint = urlparse(endpoint_text) + port = parsed_endpoint.port + except ValueError as exc: + raise ModelEndpointValidationError("Custom endpoint URL is invalid.") from exc + + if parsed_endpoint.scheme.lower() != "https": + raise ModelEndpointValidationError("Custom endpoint URL must use HTTPS.") + if not parsed_endpoint.netloc or not parsed_endpoint.hostname: + raise ModelEndpointValidationError( + "Custom endpoint URL must include a fully qualified domain name." + ) + if parsed_endpoint.username or parsed_endpoint.password: + raise ModelEndpointValidationError( + "Custom endpoint URL must not include embedded credentials." + ) + if parsed_endpoint.query or parsed_endpoint.fragment: + raise ModelEndpointValidationError( + "Custom endpoint URL must not include a query string or fragment." + ) + + hostname = parsed_endpoint.hostname.strip().lower().rstrip(".") + try: + hostname = hostname.encode("idna").decode("ascii") + except UnicodeError as exc: + raise ModelEndpointValidationError( + "Custom endpoint hostname is invalid." + ) from exc + + if ( + hostname in CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES + or hostname.endswith(".localhost") + ): + raise ModelEndpointValidationError("Custom endpoint hostname is blocked.") + if _is_ip_literal(hostname) or "." not in hostname: + raise ModelEndpointValidationError( + "Custom endpoint URL must use a fully qualified domain name, not an IP address." + ) + if not allow_private and hostname.endswith((".internal", ".local")): + raise ModelEndpointValidationError( + "Private Custom endpoint hosts are not enabled by the administrator." + ) + + resolve_custom_model_endpoint_addresses( + hostname, + port or 443, + allow_private=allow_private, + ) + + normalized_netloc = hostname + if port and port != 443: + normalized_netloc = f"{hostname}:{port}" + return urlunparse(( + "https", + normalized_netloc, + parsed_endpoint.path or "", + "", + "", + "", + )).rstrip("/") + + +def _validate_version(value: Any, field_label: str) -> str: + normalized_value = str(value or "").strip() + if not CUSTOM_ENDPOINT_VERSION_PATTERN.fullmatch(normalized_value): + raise ModelEndpointValidationError( + f"{field_label} must contain only letters, numbers, dots, underscores, or hyphens." + ) + return normalized_value + + +def validate_custom_model_endpoint( + endpoint: Any, + settings: Dict[str, Any] | None = None, + *, + require_api_key: bool = True, +) -> None: + """Validate a normalized Custom endpoint record.""" + if not isinstance(endpoint, dict): + raise ModelEndpointValidationError("Custom endpoint configuration is invalid.") + if str(endpoint.get("provider") or "").strip().lower() != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return + + endpoint_name = str(endpoint.get("name") or "").strip() + if not endpoint_name: + raise ModelEndpointValidationError("Custom endpoint name is required.") + + api_type = get_model_endpoint_api_type(endpoint) + if not api_type: + raise ModelEndpointValidationError("Custom endpoint API type is not supported.") + + auth = endpoint.get("auth") if isinstance(endpoint.get("auth"), dict) else {} + auth_type = str(auth.get("type") or "").strip().lower() + if auth_type not in {"api_key", "key"}: + raise ModelEndpointValidationError( + "Custom endpoints support API key authentication only." + ) + if require_api_key and not auth.get("api_key"): + raise ModelEndpointValidationError("Custom endpoint API key is required.") + + connection = ( + endpoint.get("connection") + if isinstance(endpoint.get("connection"), dict) + else {} + ) + allow_private = bool((settings or {}).get("allow_private_custom_model_endpoints", False)) + connection["endpoint"] = validate_custom_model_endpoint_url( + connection.get("endpoint"), + allow_private=allow_private, + ) + + if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + _validate_version(connection.get("api_version"), "Azure OpenAI API version") + elif api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: + _validate_version( + connection.get("anthropic_version") or DEFAULT_ANTHROPIC_VERSION, + "Anthropic Version", + ) + + seen_model_names = set() + models: Iterable[Any] = endpoint.get("models") or [] + if not isinstance(models, list): + raise ModelEndpointValidationError("Custom endpoint models must be a list.") + if not models: + raise ModelEndpointValidationError( + "Custom endpoints require at least one manually configured model." + ) + for model in models: + if not isinstance(model, dict): + raise ModelEndpointValidationError("Custom endpoint model configuration is invalid.") + request_model = resolve_model_endpoint_request_model(endpoint, model) + if not request_model: + model_field = ( + "Deployment Name" + if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI + else "Model Name" + ) + raise ModelEndpointValidationError( + f"Custom endpoint models require {model_field}." + ) + normalized_model_name = request_model.casefold() + if normalized_model_name in seen_model_names: + raise ModelEndpointValidationError( + "Custom endpoint model names must be unique." + ) + seen_model_names.add(normalized_model_name) + + +def validate_custom_model_endpoints( + endpoints: Any, + settings: Dict[str, Any] | None = None, + *, + require_api_key: bool = True, +) -> None: + """Validate every Custom endpoint in an endpoint list.""" + if not isinstance(endpoints, list): + raise ModelEndpointValidationError("Model endpoints must be a list.") + for endpoint in endpoints: + validate_custom_model_endpoint( + endpoint, + settings, + require_api_key=require_api_key, + ) diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index d92b1e1b7..e2cb834cf 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -1,6 +1,7 @@ # functions_settings.py from functools import wraps +import logging from flask import g, has_request_context, jsonify, request, session @@ -21,6 +22,14 @@ normalize_model_endpoint_identity_header_value_type, ) from functions_mcp_server_config import INBOUND_MCP_SETTINGS_DEFAULTS, normalize_inbound_mcp_settings +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + normalize_model_endpoint_api_type, +) from functions_rate_limit import ( RATE_LIMIT_MESSAGE_DEFAULT, build_rate_limit_error_payload, @@ -1373,6 +1382,7 @@ def get_settings(use_cosmos=False, include_source=False): }, 'allow_user_agents': False, 'allow_user_custom_endpoints': False, + 'allow_private_custom_model_endpoints': False, 'allow_user_custom_agent_endpoints': False, 'allow_user_plugins': False, 'allow_user_workflows': False, @@ -2592,6 +2602,24 @@ def normalize_model_endpoint_auth_for_environment(endpoint_copy): provider = str(endpoint_copy.get("provider") or "").strip().lower() auth_type = str(auth.get("type") or "").strip().lower() + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + if auth.get("type") != "api_key": + auth["type"] = "api_key" + changed = True + for field_name in ( + "management_cloud", + "custom_authority", + "foundry_scope", + "tenant_id", + "client_id", + "client_secret", + "managed_identity_client_id", + ): + if field_name in auth: + auth.pop(field_name, None) + changed = True + return changed + current_cloud = normalize_model_endpoint_management_cloud(auth.get("management_cloud")) default_cloud = get_model_endpoint_management_cloud_for_environment() cloud_user_editable = is_model_endpoint_management_cloud_user_editable(provider, auth_type) @@ -2671,6 +2699,48 @@ def normalize_model_endpoints(endpoints): endpoint_copy.pop("has_api_key", None) endpoint_copy.pop("has_client_secret", None) connection = endpoint_copy.get("connection") or {} + provider = str(endpoint_copy.get("provider") or "aoai").strip().lower() + if endpoint_copy.get("provider") != provider: + endpoint_copy["provider"] = provider + changed = True + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + api_type = normalize_model_endpoint_api_type( + provider, + endpoint_copy.get("api_type"), + ) + if endpoint_copy.get("api_type") != api_type: + endpoint_copy["api_type"] = api_type + changed = True + if not isinstance(connection, dict): + connection = {} + changed = True + connection = json.loads(json.dumps(connection)) + if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + if "anthropic_version" in connection: + connection.pop("anthropic_version", None) + changed = True + elif api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: + anthropic_version = str( + connection.get("anthropic_version") + or DEFAULT_ANTHROPIC_VERSION + ).strip() + if connection.get("anthropic_version") != anthropic_version: + connection["anthropic_version"] = anthropic_version + changed = True + for field_name in ("api_version", "openai_api_version"): + if field_name in connection: + connection.pop(field_name, None) + changed = True + else: + for field_name in ( + "api_version", + "openai_api_version", + "anthropic_version", + ): + if field_name in connection: + connection.pop(field_name, None) + changed = True + endpoint_copy["connection"] = connection identity_header = normalize_model_endpoint_identity_header_override(endpoint_copy.get("identity_header")) if endpoint_copy.get("identity_header") != identity_header: endpoint_copy["identity_header"] = identity_header @@ -2691,10 +2761,38 @@ def normalize_model_endpoints(endpoints): models = endpoint_copy.get("models") or [] normalized_models = [] + custom_api_type = get_model_endpoint_api_type(endpoint_copy) for model in models: if not isinstance(model, dict): continue model_copy = json.loads(json.dumps(model)) + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + if custom_api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + deployment_name = str( + model_copy.get("deploymentName") + or model_copy.get("deployment") + or "" + ).strip() + if deployment_name and model_copy.get("deploymentName") != deployment_name: + model_copy["deploymentName"] = deployment_name + changed = True + for field_name in ("deployment", "modelName", "name"): + if field_name in model_copy: + model_copy.pop(field_name, None) + changed = True + else: + model_name = str( + model_copy.get("modelName") + or model_copy.get("name") + or "" + ).strip() + if model_name and model_copy.get("modelName") != model_name: + model_copy["modelName"] = model_name + changed = True + for field_name in ("deploymentName", "deployment", "name"): + if field_name in model_copy: + model_copy.pop(field_name, None) + changed = True if not model_copy.get("id"): model_id = ( model_copy.get("deploymentName") @@ -2740,7 +2838,12 @@ def normalize_model_endpoints(endpoints): def is_frontend_visible_model_endpoint_provider(provider): """Return whether the provider should be exposed in user-facing endpoint UIs.""" normalized_provider = (provider or "aoai").lower() - return normalized_provider in {"aoai", "aifoundry", "new_foundry"} + return normalized_provider in { + "aoai", + "aifoundry", + "new_foundry", + MODEL_ENDPOINT_PROVIDER_CUSTOM, + } def merge_model_endpoint_auth(existing_auth, incoming_auth): diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 4c3d09afc..363c28f85 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -163,6 +163,10 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_notifications import create_workflow_priority_notification from functions_workflow_alerts import ( build_workflow_alert_facts, @@ -6019,14 +6023,11 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group connection = resolved_endpoint.get('connection', {}) if isinstance(resolved_endpoint, dict) else {} auth = resolved_endpoint.get('auth', {}) if isinstance(resolved_endpoint, dict) else {} provider = str(resolved_endpoint.get('provider') or endpoint_cfg.get('provider') or 'aoai').strip().lower() - deployment_name = ( - model_cfg.get('deploymentName') - or model_cfg.get('deployment') - or model_cfg.get('displayName') - or model_id - ) - api_version = connection.get('api_version') or connection.get('openai_api_version') or settings.get('azure_openai_gpt_api_version') + deployment_name = resolve_model_endpoint_request_model(resolved_endpoint, model_cfg) + api_version = connection.get('api_version') or connection.get('openai_api_version') or '' endpoint = connection.get('endpoint') + api_type = get_model_endpoint_api_type(resolved_endpoint) + anthropic_version = connection.get('anthropic_version') or '' auth_type = str(auth.get('type') or 'api_key').strip().lower() auth_settings = { **auth, @@ -6041,6 +6042,11 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint, identity_context={'user_id': user_id}, diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index c40f8b9ee..5b1df07c8 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -7,8 +7,16 @@ from typing import Any, Dict, Iterable, Iterator, List from urllib.parse import urlparse +import anyio +import httpcore +import httpx import requests -from openai import OpenAI +from openai import ( + DEFAULT_CONNECTION_LIMITS, + DefaultAsyncHttpxClient, + DefaultHttpxClient, + OpenAI, +) from pydantic import Field from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration @@ -24,6 +32,18 @@ from semantic_kernel.exceptions.service_exceptions import ServiceInvalidExecutionSettingsError from functions_debug import debug_print +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + normalize_model_endpoint_api_type, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + resolve_custom_model_endpoint_addresses, +) MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" @@ -131,9 +151,27 @@ def endpoint_uses_openai_style_protocol(endpoint: Any) -> bool: ) -def infer_model_endpoint_protocol(provider: Any, endpoint: Any, deployment_name: Any = "") -> str: +def infer_model_endpoint_protocol( + provider: Any, + endpoint: Any, + deployment_name: Any = "", + api_type: Any = "", +) -> str: """Infer the runtime protocol from provider, endpoint path, and deployment name.""" normalized_provider = str(provider or "aoai").strip().lower() + if normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + normalized_api_type = normalize_model_endpoint_api_type( + normalized_provider, + api_type, + ) + if normalized_api_type == MODEL_ENDPOINT_API_TYPE_OPENAI: + return MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE + if normalized_api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + return MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI + if normalized_api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: + return MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + raise ValueError("Custom model endpoints require a supported API type.") + endpoint_path = get_endpoint_path(endpoint) if normalized_provider in ("anthropic", "claude"): @@ -173,13 +211,44 @@ def normalize_openai_style_base_url(raw_endpoint: Any) -> str: return endpoint.rstrip("/") + "/openai/v1/" -def normalize_anthropic_messages_url(raw_endpoint: Any) -> str: +def normalize_custom_openai_base_url(raw_endpoint: Any) -> str: + """Normalize a Custom OpenAI-compatible endpoint to its v1 base URL.""" + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ValueError("A Custom endpoint is required for OpenAI-compatible inference.") + + lowered_endpoint = endpoint.lower() + for suffix in ("/chat/completions", "/responses", "/models"): + if lowered_endpoint.endswith(suffix): + endpoint = endpoint[: -len(suffix)].rstrip("/") + lowered_endpoint = endpoint.lower() + break + + if lowered_endpoint.endswith("/v1"): + return endpoint.rstrip("/") + "/" + return endpoint.rstrip("/") + "/v1/" + + +def normalize_anthropic_messages_url( + raw_endpoint: Any, + *, + direct_custom: bool = False, +) -> str: """Normalize a Foundry endpoint to the Anthropic messages URL.""" endpoint = normalize_endpoint_text(raw_endpoint) if not endpoint: - raise ValueError("A Foundry endpoint is required for Anthropic inference.") + raise ValueError("An endpoint is required for Anthropic inference.") lowered_endpoint = endpoint.lower() + if direct_custom: + if lowered_endpoint.endswith("/v1/messages"): + return endpoint + if lowered_endpoint.endswith("/v1"): + return endpoint.rstrip("/") + "/messages" + if lowered_endpoint.endswith("/messages"): + return endpoint + return endpoint.rstrip("/") + "/v1/messages" + messages_index = lowered_endpoint.find("/anthropic/v1/messages") if messages_index >= 0: return endpoint[: messages_index + len("/anthropic/v1/messages")] @@ -231,36 +300,333 @@ def extract_chat_completion_response_text(response: Any) -> str: return normalize_chat_completion_text(getattr(message, "content", None)) +def _resolve_custom_connection_addresses(host, port, allow_private): + hostname = host.decode("ascii") if isinstance(host, bytes) else str(host) + try: + return resolve_custom_model_endpoint_addresses( + hostname, + port, + allow_private=allow_private, + ) + except ModelEndpointValidationError: + raise httpcore.ConnectError("Custom endpoint connection blocked.") from None + + +class _PinnedCustomEndpointSyncBackend(httpcore.NetworkBackend): + """Connect only to addresses returned by the validated DNS lookup.""" + + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.SyncBackend() + + def connect_tcp( + self, + host, + port, + timeout=None, + local_address=None, + socket_options=None, + ): + addresses = _resolve_custom_connection_addresses( + host, + port, + self._allow_private, + ) + last_error = None + for address in addresses: + try: + return self._backend.connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + if last_error: + raise last_error + raise httpcore.ConnectError("Custom endpoint connection failed.") + + def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + def sleep(self, seconds): + self._backend.sleep(seconds) + + +class _PinnedCustomEndpointAsyncBackend(httpcore.AsyncNetworkBackend): + """Async counterpart to the validated synchronous DNS backend.""" + + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.AnyIOBackend() + + async def connect_tcp( + self, + host, + port, + timeout=None, + local_address=None, + socket_options=None, + ): + addresses = await anyio.to_thread.run_sync( + _resolve_custom_connection_addresses, + host, + port, + self._allow_private, + ) + last_error = None + for address in addresses: + try: + return await self._backend.connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + if last_error: + raise last_error + raise httpcore.ConnectError("Custom endpoint connection failed.") + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + async def sleep(self, seconds): + await self._backend.sleep(seconds) + + +class _PinnedCustomEndpointHTTPTransport(httpx.HTTPTransport): + """HTTPX transport whose TCP connection uses the validated DNS results.""" + + def __init__(self, *, allow_private=False): + self._pool = httpcore.ConnectionPool( + ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointSyncBackend( + allow_private=allow_private, + ), + ) + + +class _PinnedCustomEndpointAsyncHTTPTransport(httpx.AsyncHTTPTransport): + """Async HTTPX transport whose TCP connection uses validated DNS results.""" + + def __init__(self, *, allow_private=False): + self._pool = httpcore.AsyncConnectionPool( + ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointAsyncBackend( + allow_private=allow_private, + ), + ) + + +def build_custom_openai_sync_http_client(*, allow_private=False): + """Return a no-redirect SDK transport pinned to validated DNS addresses.""" + return DefaultHttpxClient( + transport=_PinnedCustomEndpointHTTPTransport( + allow_private=allow_private, + ), + follow_redirects=False, + trust_env=False, + ) + + +def build_custom_openai_async_http_client(*, allow_private=False): + """Return an async no-redirect transport pinned to validated DNS addresses.""" + return DefaultAsyncHttpxClient( + transport=_PinnedCustomEndpointAsyncHTTPTransport( + allow_private=allow_private, + ), + follow_redirects=False, + trust_env=False, + ) + + def build_openai_style_chat_client( token_or_key: str, base_url: str, api_version: Any = "", default_headers: Dict[str, str] | None = None, + *, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, ): """Build an OpenAI-compatible chat client for Foundry data-plane endpoints.""" request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs: Dict[str, Any] = { "api_key": token_or_key, - "base_url": normalize_openai_style_base_url(base_url), + "base_url": ( + normalize_custom_openai_base_url(base_url) + if direct_custom + else normalize_openai_style_base_url(base_url) + ), } + if direct_custom: + client_kwargs["http_client"] = build_custom_openai_sync_http_client( + allow_private=allow_private_custom_endpoints, + ) if default_headers: client_kwargs["default_headers"] = default_headers if request_api_version: client_kwargs["default_query"] = {"api-version": request_api_version} - return OpenAIStyleChatCompletionClient(OpenAI(**client_kwargs)) + return OpenAIStyleChatCompletionClient( + OpenAI(**client_kwargs), + sanitize_errors=direct_custom, + ) class OpenAIStyleChatCompletionClient: """Small wrapper that makes OpenAI-compatible Foundry calls tolerant of Azure-only options.""" - def __init__(self, client: OpenAI): + def __init__(self, client: OpenAI, *, sanitize_errors: bool = False): self._client = client + self._sanitize_errors = sanitize_errors self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): request_kwargs = dict(kwargs) request_kwargs.pop("stream_options", None) - return self._client.chat.completions.create(**request_kwargs) + try: + response = self._client.chat.completions.create(**request_kwargs) + except Exception: + if self._sanitize_errors: + raise RuntimeError("Custom model request failed.") from None + raise + if self._sanitize_errors and request_kwargs.get("stream"): + return _SanitizedSyncIterator(response) + return response + + +class _SanitizedSyncIterator: + """Proxy a streaming response without exposing provider exception details.""" + + def __init__(self, iterator: Any): + self._iterator = iterator + self._items = iter(iterator) + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self._items) + except StopIteration: + raise + except Exception: + raise RuntimeError("Custom model stream failed.") from None + + def __enter__(self): + enter = getattr(self._iterator, "__enter__", None) + if callable(enter): + enter() + return self + + def __exit__(self, exc_type, exc_value, traceback): + exit_method = getattr(self._iterator, "__exit__", None) + if callable(exit_method): + return exit_method(exc_type, exc_value, traceback) + return False + + def close(self): + close_method = getattr(self._iterator, "close", None) + if callable(close_method): + return close_method() + return None + + def __getattr__(self, name: str): + return getattr(self._iterator, name) + + +class _SanitizedAsyncIterator: + """Proxy an async streaming response without exposing provider exception details.""" + + def __init__(self, iterator: Any): + self._iterator = iterator + self._items = iterator.__aiter__() + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return await self._items.__anext__() + except StopAsyncIteration: + raise + except Exception: + raise RuntimeError("Custom model stream failed.") from None + + async def __aenter__(self): + enter = getattr(self._iterator, "__aenter__", None) + if callable(enter): + await enter() + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + exit_method = getattr(self._iterator, "__aexit__", None) + if callable(exit_method): + return await exit_method(exc_type, exc_value, traceback) + return False + + async def close(self): + close_method = getattr(self._iterator, "close", None) + if callable(close_method): + result = close_method() + if asyncio.iscoroutine(result): + return await result + return None + + def __getattr__(self, name: str): + return getattr(self._iterator, name) + + +class SanitizedCustomChatCompletionClient: + """Expose an SDK chat client while replacing direct Custom provider errors.""" + + def __init__(self, client: Any): + self._client = client + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) + + def create(self, **kwargs: Any): + try: + response = self._client.chat.completions.create(**kwargs) + except Exception: + raise RuntimeError("Custom model request failed.") from None + if kwargs.get("stream"): + return _SanitizedSyncIterator(response) + return response + + def __getattr__(self, name: str): + return getattr(self._client, name) + + +def sanitize_custom_async_openai_client(client: Any): + """Replace async SDK chat errors with safe direct-Custom messages.""" + if getattr(client, "_simplechat_custom_errors_sanitized", False): + return client + + original_create = client.chat.completions.create + + async def sanitized_create(*args, **kwargs): + try: + response = await original_create(*args, **kwargs) + except Exception: + raise RuntimeError("Custom model request failed.") from None + if kwargs.get("stream"): + return _SanitizedAsyncIterator(response) + return response + + client.chat.completions.create = sanitized_create + client._simplechat_custom_errors_sanitized = True + return client def build_anthropic_chat_client( @@ -270,6 +636,9 @@ def build_anthropic_chat_client( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, ): """Build a chat-completions-shaped adapter over the Anthropic messages protocol.""" return AnthropicChatCompletionClient( @@ -278,6 +647,9 @@ def build_anthropic_chat_client( bearer_token=bearer_token, extra_headers=extra_headers, timeout=timeout, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, ) @@ -292,23 +664,38 @@ def __init__( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, ): - self.endpoint = normalize_anthropic_messages_url(endpoint) + self.endpoint = normalize_anthropic_messages_url( + endpoint, + direct_custom=direct_custom, + ) self.api_key = api_key self.bearer_token = bearer_token self.extra_headers = extra_headers or {} self.timeout = timeout + self.anthropic_version = str( + anthropic_version or DEFAULT_ANTHROPIC_VERSION + ).strip() + self.direct_custom = direct_custom + self.allow_private_custom_endpoints = allow_private_custom_endpoints self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): payload = self._build_payload(kwargs) stream = bool(kwargs.get("stream")) + if self.direct_custom: + return self._create_direct_custom(payload, stream=stream) + response = requests.post( self.endpoint, headers=self._build_headers(stream=stream), json=payload, timeout=(30, self.timeout), stream=stream, + allow_redirects=not self.direct_custom, ) if response.status_code >= 400: self._raise_response_error(response) @@ -318,16 +705,62 @@ def create(self, **kwargs: Any): return self._build_completion_response(response.json()) + def _create_direct_custom(self, payload, *, stream): + http_client = build_custom_openai_sync_http_client( + allow_private=self.allow_private_custom_endpoints, + ) + request = http_client.build_request( + "POST", + self.endpoint, + headers=self._build_headers(stream=stream), + json=payload, + timeout=httpx.Timeout(self.timeout, connect=30), + ) + try: + response = http_client.send( + request, + stream=stream, + follow_redirects=False, + ) + except Exception: + http_client.close() + raise RuntimeError("Custom Anthropic model request failed.") from None + + if response.status_code >= 400: + status_code = response.status_code + response.close() + http_client.close() + raise RuntimeError( + f"Custom Anthropic model request failed with status {status_code}." + ) + + if stream: + return self._iter_stream_chunks( + response, + http_client=http_client, + ) + + try: + return self._build_completion_response(response.json()) + except Exception: + raise RuntimeError( + "Custom Anthropic model returned an invalid response." + ) from None + finally: + response.close() + http_client.close() + def _build_headers(self, *, stream: bool = False) -> Dict[str, str]: headers = { "Content-Type": "application/json", "Accept": "text/event-stream" if stream else "application/json", - "anthropic-version": "2023-06-01", + "anthropic-version": self.anthropic_version, } if self.bearer_token: headers["Authorization"] = f"Bearer {self.bearer_token}" elif self.api_key: - headers["api-key"] = self.api_key + if not self.direct_custom: + headers["api-key"] = self.api_key headers["x-api-key"] = self.api_key else: raise ValueError("Anthropic model endpoints require an API key or bearer token.") @@ -340,7 +773,7 @@ def _build_headers(self, *, stream: bool = False) -> Dict[str, str]: def _build_payload(self, kwargs: Dict[str, Any]) -> Dict[str, Any]: model = str(kwargs.get("model") or "").strip() if not model: - raise ValueError("Anthropic model requests require a deployment name.") + raise ValueError("Anthropic model requests require a model name.") messages, system_prompt = self._convert_messages(kwargs.get("messages") or []) payload: Dict[str, Any] = { @@ -447,9 +880,14 @@ def _normalize_content(self, content: Any) -> str | List[Dict[str, Any]]: text_parts.append(item) elif isinstance(item, dict): item_type = item.get("type") - if item_type in ("text", "tool_use", "tool_result"): + if item_type in ("text", "image", "tool_use", "tool_result"): normalized_blocks.append(item) continue + if item_type == "image_url": + normalized_blocks.append( + self._convert_openai_image_block(item) + ) + continue text_value = item.get("text") if isinstance(text_value, str): text_parts.append(text_value) @@ -462,6 +900,33 @@ def _normalize_content(self, content: Any) -> str | List[Dict[str, Any]]: return "" return str(content) + def _convert_openai_image_block(self, image_block: Dict[str, Any]) -> Dict[str, Any]: + """Convert an OpenAI data-URL image block to Anthropic base64 content.""" + image_value = image_block.get("image_url") + image_url = ( + image_value.get("url") + if isinstance(image_value, dict) + else image_value + ) + image_url = str(image_url or "").strip() + if not image_url.startswith("data:") or ";base64," not in image_url: + raise ValueError( + "Anthropic image content requires a base64 data URL." + ) + + metadata, image_data = image_url.split(",", 1) + media_type = metadata[5:].split(";", 1)[0].strip().lower() + if not media_type.startswith("image/") or not image_data: + raise ValueError("Anthropic image content is invalid.") + return { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": image_data, + }, + } + def _content_to_text(self, content: Any) -> str: if isinstance(content, str): return content @@ -521,11 +986,20 @@ def _extract_response_parts(self, response_payload: Dict[str, Any]) -> tuple[str )) return "".join(text_parts), tool_calls - def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: + def _iter_stream_chunks( + self, + response, + *, + http_client=None, + ) -> Iterator[Any]: prompt_tokens = 0 completion_tokens = 0 try: - for raw_line in response.iter_lines(decode_unicode=True): + try: + response_lines = response.iter_lines(decode_unicode=True) + except TypeError: + response_lines = response.iter_lines() + for raw_line in response_lines: if not raw_line: continue if isinstance(raw_line, bytes): @@ -540,7 +1014,10 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: try: event_payload = json.loads(event_data) except json.JSONDecodeError: - debug_print(f"[MODEL_ENDPOINT] Ignoring invalid Anthropic stream payload: {event_data[:200]}") + if self.direct_custom: + debug_print("[MODEL_ENDPOINT] Ignoring invalid Custom Anthropic stream payload.") + else: + debug_print(f"[MODEL_ENDPOINT] Ignoring invalid Anthropic stream payload: {event_data[:200]}") continue event_type = event_payload.get("type") @@ -550,6 +1027,8 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: error_message = error_payload.get("message") or error_payload.get("type") or str(error_payload) else: error_message = str(error_payload or event_payload) + if self.direct_custom: + raise RuntimeError("Custom Anthropic model stream failed.") raise RuntimeError(f"Anthropic model stream failed: {error_message}") if event_type == "message_start": usage = event_payload.get("message", {}).get("usage", {}) @@ -569,8 +1048,14 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: prompt_tokens = int(usage.get("input_tokens") or prompt_tokens or 0) completion_tokens = int(usage.get("output_tokens") or completion_tokens or 0) continue + except Exception: + if self.direct_custom: + raise RuntimeError("Custom Anthropic model stream failed.") from None + raise finally: response.close() + if http_client is not None: + http_client.close() if prompt_tokens or completion_tokens: yield SimpleNamespace( @@ -594,6 +1079,10 @@ def _raise_response_error(self, response: requests.Response) -> None: else: error_message = str(error_payload or payload) + if self.direct_custom: + raise RuntimeError( + f"Custom Anthropic model request failed with status {response.status_code}." + ) raise RuntimeError( f"Anthropic model request failed with status {response.status_code}: {error_message}" ) @@ -609,6 +1098,9 @@ class AnthropicSemanticKernelChatCompletion(ChatCompletionClientBase): bearer_token: str = "" extra_headers: Dict[str, str] = Field(default_factory=dict) timeout: int = 90 + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION + direct_custom: bool = False + allow_private_custom_endpoints: bool = False prompt_execution_settings: OpenAIChatPromptExecutionSettings | None = Field(default=None) def __init__( @@ -621,6 +1113,9 @@ def __init__( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, ): super().__init__( ai_model_id=deployment_name, @@ -630,6 +1125,9 @@ def __init__( bearer_token=bearer_token, extra_headers=extra_headers or {}, timeout=timeout, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, ) def get_prompt_execution_settings_class(self): @@ -799,6 +1297,9 @@ def _build_client(self): bearer_token=self.bearer_token, extra_headers=self.extra_headers, timeout=self.timeout, + anthropic_version=self.anthropic_version, + direct_custom=self.direct_custom, + allow_private_custom_endpoints=self.allow_private_custom_endpoints, ) def _build_request_kwargs(self, chat_history, settings, *, stream: bool) -> Dict[str, Any]: diff --git a/application/single_app/route_backend_agents.py b/application/single_app/route_backend_agents.py index 30a967da0..9b03a9ff2 100644 --- a/application/single_app/route_backend_agents.py +++ b/application/single_app/route_backend_agents.py @@ -476,6 +476,8 @@ def _format_model_provider_label(provider): return 'Foundry (classic)' if normalized_provider == 'new_foundry': return 'New Foundry' + if normalized_provider == 'custom': + return 'Custom' return 'Azure OpenAI' diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index f403c2642..0bf133311 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -43,6 +43,10 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_mixed_source_orchestration import ( MixedSourceCancellationError, MixedSourceFinalizationError, @@ -13928,6 +13932,9 @@ def build_streaming_multi_endpoint_client( api_version, deployment_name='', *, + api_type='', + anthropic_version='', + allow_private_custom_endpoints=False, settings=None, endpoint_config=None, identity_context=None, @@ -13939,6 +13946,9 @@ def build_streaming_multi_endpoint_client( endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, settings=settings, endpoint_config=endpoint_config, identity_context=identity_context, @@ -14090,7 +14100,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ model_cfg = next( ( model for model in models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == requested_deployment + if resolve_model_endpoint_request_model(resolved_endpoint_cfg, model) == requested_deployment ), None, ) @@ -14122,10 +14132,12 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() + deployment = resolve_model_endpoint_request_model(resolved_endpoint_cfg, model_cfg) endpoint = str(connection.get('endpoint') or '').strip() api_version = str(connection.get('openai_api_version') or connection.get('api_version') or '').strip() - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + api_type = get_model_endpoint_api_type(resolved_endpoint_cfg) + anthropic_version = str(connection.get('anthropic_version') or '').strip() + runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment, api_type) model_icon = _normalize_model_icon_payload(model_cfg.get('icon')) model_response_length = normalize_model_response_length_from_model(model_cfg) model_behavior_name = _build_model_endpoint_behavior_name(model_cfg, deployment) @@ -14159,6 +14171,11 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ endpoint, api_version, deployment_name=deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint_cfg, identity_context={'user_id': user_id}, @@ -14166,7 +14183,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ debug_print( f"[STREAMING][Model Resolution] Resolved {selection_source} multi-endpoint model | " f"provider={provider} | endpoint_id={requested_endpoint_id} | model_id={model_cfg.get('id')} | " - f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol} | " + f"request_model={deployment} | api_version={api_version} | api_type={api_type} | protocol={runtime_protocol} | " f"response_length={model_response_length or ''} | " f"response_length_parameter={model_response_length_parameter or ''}" ) @@ -14177,6 +14194,8 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ endpoint, auth_settings, api_version, + api_type, + anthropic_version, requested_endpoint_id, str(model_cfg.get('id') or '').strip(), model_icon, @@ -16552,6 +16571,8 @@ def result_requires_message_reload(result: Any) -> bool: gpt_endpoint = None gpt_auth = None gpt_api_version = None + gpt_api_type = None + gpt_anthropic_version = None gpt_endpoint_id = None gpt_model_id = None gpt_model_icon = None @@ -16590,6 +16611,8 @@ def result_requires_message_reload(result: Any) -> bool: gpt_endpoint, gpt_auth, gpt_api_version, + gpt_api_type, + gpt_anthropic_version, gpt_endpoint_id, gpt_model_id, gpt_model_icon, @@ -16686,9 +16709,12 @@ def result_requires_message_reload(result: Any) -> bool: endpoint=gpt_endpoint, auth=gpt_auth, api_version=gpt_api_version, + api_type=gpt_api_type, + anthropic_version=gpt_anthropic_version, endpoint_id=gpt_endpoint_id or data.get('model_endpoint_id'), model_id=gpt_model_id or data.get('model_id'), model_deployment=gpt_model, + request_model=gpt_model, user_id=user_id, active_group_ids=active_group_ids, ) @@ -21006,6 +21032,8 @@ def collect_stream_response_conversation_metadata(): gpt_endpoint = None gpt_auth = None gpt_api_version = None + gpt_api_type = None + gpt_anthropic_version = None gpt_endpoint_id = None gpt_model_id = None gpt_model_icon = None @@ -21045,6 +21073,8 @@ def collect_stream_response_conversation_metadata(): gpt_endpoint, gpt_auth, gpt_api_version, + gpt_api_type, + gpt_anthropic_version, gpt_endpoint_id, gpt_model_id, gpt_model_icon, @@ -21124,9 +21154,12 @@ def collect_stream_response_conversation_metadata(): endpoint=gpt_endpoint, auth=gpt_auth, api_version=gpt_api_version, + api_type=gpt_api_type, + anthropic_version=gpt_anthropic_version, endpoint_id=gpt_endpoint_id or frontend_model_endpoint_id, model_id=gpt_model_id or frontend_model_id, model_deployment=gpt_model, + request_model=gpt_model, user_id=user_id, active_group_ids=active_group_ids, ) diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py index 56af5c671..e175c1007 100644 --- a/application/single_app/route_backend_conversation_export.py +++ b/application/single_app/route_backend_conversation_export.py @@ -52,6 +52,11 @@ ) from functions_settings import * from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_simplechat_operations import download_blob_content from functions_thoughts import get_thoughts_for_conversation from foundry_agent_runtime import resolve_authority @@ -1153,13 +1158,24 @@ def _get_summary_model_endpoint_candidates(settings: Dict[str, Any], user_id: st return candidates -def _summary_model_matches(model_cfg: Dict[str, Any], requested_model: str, requested_model_id: str) -> bool: +def _summary_model_matches( + endpoint_cfg: Dict[str, Any], + model_cfg: Dict[str, Any], + requested_model: str, + requested_model_id: str, +) -> bool: + request_model = '' + if _normalize_summary_model_value(endpoint_cfg.get('provider')).lower() == 'custom': + request_model = _normalize_summary_model_value( + resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + ) model_values = { _normalize_summary_model_value(model_cfg.get('id')), _normalize_summary_model_value(model_cfg.get('deploymentName')), _normalize_summary_model_value(model_cfg.get('deployment')), _normalize_summary_model_value(model_cfg.get('modelName')), _normalize_summary_model_value(model_cfg.get('name')), + request_model, } model_values.discard('') @@ -1177,7 +1193,12 @@ def _find_summary_endpoint_model( for model_cfg in models: if not isinstance(model_cfg, dict) or not model_cfg.get('enabled', True): continue - if _summary_model_matches(model_cfg, requested_model, requested_model_id): + if _summary_model_matches( + endpoint_cfg, + model_cfg, + requested_model, + requested_model_id, + ): return model_cfg return None @@ -1214,6 +1235,9 @@ def _build_summary_model_endpoint_client( api_version: str, deployment_name: str, *, + api_type: str = '', + anthropic_version: str = '', + allow_private_custom_endpoints: bool = False, settings: Dict[str, Any] = None, endpoint_config: Dict[str, Any] = None, identity_context: Dict[str, Any] = None, @@ -1224,55 +1248,105 @@ def _build_summary_model_endpoint_client( endpoint_config=endpoint_config, identity_context=identity_context, ) - auth_type = _normalize_summary_model_value(auth_settings.get('type') or 'managed_identity').lower() normalized_provider = _normalize_summary_model_value(provider or 'aoai').lower() - runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) + if normalized_provider != 'custom': + auth_type = _normalize_summary_model_value( + auth_settings.get('type') or 'managed_identity' + ).lower() + runtime_protocol = infer_model_endpoint_protocol( + normalized_provider, + endpoint, + deployment_name, + ) + + if auth_type in ('api_key', 'key'): + api_key = auth_settings.get('api_key') + if not api_key: + raise ValueError('Selected summary model endpoint is missing an API key.') + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: + return build_anthropic_chat_client( + endpoint=endpoint, + api_key=api_key, + extra_headers=extra_headers, + ) + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: + return build_openai_style_chat_client( + api_key, + endpoint, + api_version, + default_headers=extra_headers, + ) + return AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=api_key, + default_headers=extra_headers or None, + ) + + if auth_type == 'service_principal': + credential = ClientSecretCredential( + tenant_id=auth_settings.get('tenant_id'), + client_id=auth_settings.get('client_id'), + client_secret=auth_settings.get('client_secret'), + authority=resolve_authority(auth_settings), + ) + else: + managed_identity_client_id = auth_settings.get( + 'managed_identity_client_id' + ) or None + credential = DefaultAzureCredential( + managed_identity_client_id=managed_identity_client_id + ) + + scope = cognitive_services_scope + if ( + normalized_provider in ('aifoundry', 'new_foundry') + or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI + ): + scope = _resolve_summary_foundry_scope_for_auth( + auth_settings, + endpoint=endpoint, + ) - if auth_type in ('api_key', 'key'): - api_key = auth_settings.get('api_key') - if not api_key: - raise ValueError('Selected summary model endpoint is missing an API key.') if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - return build_anthropic_chat_client(endpoint=endpoint, api_key=api_key, extra_headers=extra_headers) + token = credential.get_token(scope).token + return build_anthropic_chat_client( + endpoint=endpoint, + bearer_token=token, + extra_headers=extra_headers, + ) + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - return build_openai_style_chat_client(api_key, endpoint, api_version, default_headers=extra_headers) + token = credential.get_token(scope).token + return build_openai_style_chat_client( + token, + endpoint, + api_version, + default_headers=extra_headers, + ) + + token_provider = get_bearer_token_provider(credential, scope) return AzureOpenAI( api_version=api_version, azure_endpoint=endpoint, - api_key=api_key, + azure_ad_token_provider=token_provider, default_headers=extra_headers or None, ) - if auth_type == 'service_principal': - credential = ClientSecretCredential( - tenant_id=auth_settings.get('tenant_id'), - client_id=auth_settings.get('client_id'), - client_secret=auth_settings.get('client_secret'), - authority=resolve_authority(auth_settings), - ) - else: - managed_identity_client_id = auth_settings.get('managed_identity_client_id') or None - credential = DefaultAzureCredential(managed_identity_client_id=managed_identity_client_id) - - scope = cognitive_services_scope - if normalized_provider in ('aifoundry', 'new_foundry') or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: - scope = _resolve_summary_foundry_scope_for_auth(auth_settings, endpoint=endpoint) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - token = credential.get_token(scope).token - return build_anthropic_chat_client(endpoint=endpoint, bearer_token=token, extra_headers=extra_headers) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - token = credential.get_token(scope).token - return build_openai_style_chat_client(token, endpoint, api_version, default_headers=extra_headers) - - token_provider = get_bearer_token_provider(credential, scope) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - azure_ad_token_provider=token_provider, - default_headers=extra_headers or None, + client, _ = build_model_endpoint_sync_chat_client( + auth_settings, + provider, + endpoint, + api_version, + deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, + settings=settings, + endpoint_config=endpoint_config, + identity_context=identity_context, ) + return client def _resolve_summary_multi_endpoint_client( @@ -1336,12 +1410,34 @@ def _resolve_summary_multi_endpoint_client( provider = _normalize_summary_model_value(resolved_endpoint_cfg.get('provider') or requested_provider or 'aoai').lower() connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = _normalize_summary_model_value( - model_cfg.get('deploymentName') or model_cfg.get('deployment') or model_cfg.get('id') - ) + if provider == 'custom': + deployment = resolve_model_endpoint_request_model( + resolved_endpoint_cfg, + model_cfg, + ) + else: + deployment = _normalize_summary_model_value( + model_cfg.get('deploymentName') + or model_cfg.get('deployment') + or model_cfg.get('modelName') + or model_cfg.get('name') + ) endpoint = _normalize_summary_model_value(connection.get('endpoint')) api_version = _normalize_summary_model_value(connection.get('openai_api_version') or connection.get('api_version')) - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + api_type = ( + get_model_endpoint_api_type(resolved_endpoint_cfg) + if provider == 'custom' + else '' + ) + anthropic_version = _normalize_summary_model_value( + connection.get('anthropic_version') + ) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) missing_required_config = not endpoint or not deployment or ( runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version @@ -1357,6 +1453,11 @@ def _resolve_summary_multi_endpoint_client( endpoint, api_version, deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint_cfg, identity_context={'user_id': user_id}, @@ -1364,7 +1465,7 @@ def _resolve_summary_multi_endpoint_client( debug_print( f"[SUMMARY][Model Resolution] Resolved {selection_source} multi-endpoint model | " f"provider={provider} | endpoint_id={endpoint_id} | model_id={model_cfg.get('id')} | " - f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol}" + f"request_model={deployment} | api_version={api_version} | api_type={api_type} | protocol={runtime_protocol}" ) return gpt_client, deployment diff --git a/application/single_app/route_backend_models.py b/application/single_app/route_backend_models.py index 652b1558f..148560792 100644 --- a/application/single_app/route_backend_models.py +++ b/application/single_app/route_backend_models.py @@ -7,6 +7,18 @@ from functions_governance import ensure_governance_access from functions_group import assert_group_role, get_group_model_endpoints, require_active_group, update_group_model_endpoints from functions_keyvault import SecretReturnType, keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_get_helper, keyvault_model_endpoint_save_helper +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_model_endpoint, + validate_custom_model_endpoints, +) from functions_settings import * from foundry_agent_runtime import FoundryAgentUserAuthenticationRequired, list_foundry_agents_from_endpoint, list_foundry_workflows_from_endpoint, list_new_foundry_agents_from_endpoint, resolve_foundry_project_base, resolve_foundry_project_api_version, build_project_credential, resolve_authority from functions_appinsights import log_event @@ -169,7 +181,39 @@ def resolve_request_endpoint_payload(payload, scope="global"): # Persisted non-admin endpoints must resolve from stored configuration only. merged_payload = merge_model_endpoint_payload(persisted_endpoint, {}) if "model" in payload: - merged_payload["model"] = payload.get("model") + requested_model = payload.get("model") + if not isinstance(requested_model, dict): + raise LookupError("Model endpoint model not found.") + requested_model_id = str(requested_model.get("id") or "").strip() + requested_model_name = resolve_model_endpoint_request_model( + persisted_endpoint, + requested_model, + ) + persisted_model = next( + ( + model + for model in (persisted_endpoint.get("models") or []) + if isinstance(model, dict) + and model.get("enabled", True) + and ( + ( + requested_model_id + and str(model.get("id") or "").strip() == requested_model_id + ) + or ( + requested_model_name + and resolve_model_endpoint_request_model( + persisted_endpoint, + model, + ) == requested_model_name + ) + ) + ), + None, + ) + if not persisted_model: + raise LookupError("Model endpoint model not found.") + merged_payload["model"] = persisted_model else: merged_payload = merge_model_endpoint_payload(persisted_endpoint or {}, payload) @@ -268,54 +312,31 @@ def build_legacy_aoai_discovery_auth_settings(): "client_secret": MICROSOFT_PROVIDER_AUTHENTICATION_SECRET, } - def build_inference_client(endpoint, api_version, auth_settings, provider="aoai", deployment_name=""): - auth_type = (auth_settings.get("type") or "managed_identity").lower() - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) - if auth_type == "api_key": - api_key = auth_settings.get("api_key") - if not api_key: - raise ValueError("API key is required for API key authentication.") - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - return build_anthropic_chat_client(endpoint=endpoint, api_key=api_key) - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - return build_openai_style_chat_client(api_key, endpoint, api_version) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=api_key - ) - - if auth_type == "service_principal": - authority_override = resolve_authority(auth_settings) - credential = ClientSecretCredential( - tenant_id=auth_settings.get("tenant_id"), - client_id=auth_settings.get("client_id"), - client_secret=auth_settings.get("client_secret"), - authority=authority_override - ) - else: - managed_identity_client_id = auth_settings.get("managed_identity_client_id") or None - credential = DefaultAzureCredential(managed_identity_client_id=managed_identity_client_id) - - scope = cognitive_services_scope - if provider in ("aifoundry", "new_foundry") or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: - scope = resolve_foundry_scope(auth_settings) - log_models_debug(f"Inference token scope={scope} provider={provider} protocol={runtime_protocol}") - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - token = credential.get_token(scope).token - return build_anthropic_chat_client(endpoint=endpoint, bearer_token=token) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - token = credential.get_token(scope).token - return build_openai_style_chat_client(token, endpoint, api_version) - - token_provider = get_bearer_token_provider(credential, scope) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - azure_ad_token_provider=token_provider + def build_inference_client( + endpoint, + api_version, + auth_settings, + provider="aoai", + deployment_name="", + api_type="", + anthropic_version=DEFAULT_ANTHROPIC_VERSION, + ): + client, runtime_protocol = build_model_endpoint_sync_chat_client( + auth_settings, + provider, + endpoint, + api_version, + deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + get_settings().get("allow_private_custom_model_endpoints", False) + ), ) + log_models_debug( + f"Inference client provider={provider} protocol={runtime_protocol}" + ) + return client def fetch_foundry_project_deployments(endpoint, api_version, auth_settings, project_name=None): if not endpoint: @@ -373,6 +394,12 @@ def handle_fetch_model_list(scope="global"): f" resource_group_present={bool(management.get('resource_group'))}" ) + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + return build_safe_error_response( + "Model discovery is not available for Custom endpoints. Add models manually.", + 400, + ) + if provider in ("aifoundry", "new_foundry"): endpoint = connection.get("endpoint") api_version = connection.get("project_api_version") or connection.get("api_version") or "v1" @@ -464,23 +491,48 @@ def handle_test_model_connection(scope="global"): endpoint = connection.get("endpoint") or "" api_version = connection.get("openai_api_version") or connection.get("api_version") or "" - deployment_name = model.get("deploymentName") or "" - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) + api_type = get_model_endpoint_api_type(data) + anthropic_version = ( + connection.get("anthropic_version") + or DEFAULT_ANTHROPIC_VERSION + ) + request_model = resolve_model_endpoint_request_model(data, model) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + request_model, + api_type, + ) auth_type = (auth_settings.get("type") or "managed_identity").lower() log_models_debug( "Test model request" f" provider={provider} auth_type={auth_type}" - f" endpoint={endpoint} deployment={deployment_name}" + f" endpoint={endpoint} model={request_model}" ) - if not endpoint or not deployment_name: - return jsonify({"error": "Endpoint and deployment name are required."}), 400 + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + validation_endpoint = dict(data) + validation_endpoint["models"] = [model] + validate_custom_model_endpoint( + validation_endpoint, + get_settings(), + ) - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version: - return jsonify({"error": "Endpoint, API version, and deployment name are required."}), 400 + if not endpoint or not request_model: + return jsonify({"error": "Endpoint and model identifier are required."}), 400 - if provider not in ("aoai", "aifoundry", "new_foundry", "anthropic", "claude"): + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version: + return jsonify({"error": "Endpoint, API version, and model identifier are required."}), 400 + + if provider not in ( + "aoai", + "aifoundry", + "new_foundry", + "anthropic", + "claude", + MODEL_ENDPOINT_PROVIDER_CUSTOM, + ): return jsonify({"error": "Model provider not found."}), 400 gpt_client = build_inference_client( @@ -488,10 +540,12 @@ def handle_test_model_connection(scope="global"): api_version, auth_settings, provider=provider, - deployment_name=deployment_name, + deployment_name=request_model, + api_type=api_type, + anthropic_version=anthropic_version, ) response = gpt_client.chat.completions.create( - model=deployment_name, + model=request_model, messages=[{"role": "user", "content": "Testing access."}] ) @@ -820,6 +874,16 @@ def save_user_model_endpoints(): merged = merge_model_endpoints_with_existing(incoming, existing) normalized, _ = normalize_model_endpoints(merged) + try: + validate_custom_model_endpoints(normalized, get_settings()) + except ModelEndpointValidationError as exc: + log_models_exception( + "Personal model endpoint validation failed", + exc, + extra={"scope": "user"}, + level=logging.WARNING, + ) + return build_safe_error_response(str(exc), 400) existing_by_id = { endpoint.get("id"): endpoint for endpoint in existing @@ -861,7 +925,10 @@ def save_user_model_endpoints(): keyvault_model_endpoint_delete_helper(endpoint, endpoint_id, scope="user") update_user_settings(user_id, {"personal_model_endpoints": saved_endpoints}) - return jsonify({"success": True}) + return jsonify({ + "success": True, + "endpoints": sanitize_model_endpoints_for_frontend(saved_endpoints), + }) @bp.route('/api/group/model-endpoints', methods=['GET']) @@ -924,6 +991,16 @@ def save_group_model_endpoints(): merged = merge_model_endpoints_with_existing(incoming, existing) normalized, _ = normalize_model_endpoints(merged) + try: + validate_custom_model_endpoints(normalized, get_settings()) + except ModelEndpointValidationError as exc: + log_models_exception( + "Group model endpoint validation failed", + exc, + extra={"scope": "group"}, + level=logging.WARNING, + ) + return build_safe_error_response(str(exc), 400) existing_by_id = { endpoint.get("id"): endpoint for endpoint in existing @@ -965,7 +1042,10 @@ def save_group_model_endpoints(): keyvault_model_endpoint_delete_helper(endpoint, endpoint_id, scope="group") update_group_model_endpoints(group_id, saved_endpoints) - return jsonify({"success": True}) + return jsonify({ + "success": True, + "endpoints": sanitize_model_endpoints_for_frontend(saved_endpoints), + }) @bp.route('/api/models/foundry/agents', methods=['POST']) diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 9f582f7ec..9b5d5ea59 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -11,6 +11,10 @@ resolve_model_endpoint_from_context, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_activity_logging import ( log_admin_feedback_email_submission, log_general_admin_action, @@ -1395,6 +1399,7 @@ def _test_multimodal_vision_connection(payload): # Create a simple test image (1x1 red pixel PNG) test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" + is_custom_model_endpoint = False try: multi_endpoint_selection = payload.get('multi_endpoint') if isinstance(payload.get('multi_endpoint'), dict) else None @@ -1414,6 +1419,9 @@ def _test_multimodal_vision_connection(payload): resolved_endpoint = resolve_model_endpoint_from_context(settings, model_context) if not resolved_endpoint: return jsonify({'error': 'Selected vision model endpoint could not be resolved from saved settings'}), 400 + is_custom_model_endpoint = ( + str(resolved_endpoint.get('provider') or '').strip().lower() == 'custom' + ) resolved_models = resolved_endpoint.get('models', []) or [] matched_model = next( @@ -1427,18 +1435,14 @@ def _test_multimodal_vision_connection(payload): matched_model = next( ( model for model in resolved_models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == model_context['model_deployment'] + if resolve_model_endpoint_request_model(resolved_endpoint, model) == model_context['model_deployment'] ), None, ) if not matched_model: return jsonify({'error': 'Selected vision model could not be resolved from saved settings'}), 400 - vision_model = str( - matched_model.get('deploymentName') - or matched_model.get('deployment') - or model_context['model_deployment'] - ).strip() + vision_model = resolve_model_endpoint_request_model(resolved_endpoint, matched_model) vision_model_name = str(matched_model.get('modelName') or vision_model).strip() connection = resolved_endpoint.get('connection', {}) or {} gpt_client, _ = build_model_endpoint_sync_chat_client( @@ -1447,6 +1451,11 @@ def _test_multimodal_vision_connection(payload): connection.get('endpoint'), connection.get('openai_api_version') or connection.get('api_version'), deployment_name=vision_model, + api_type=get_model_endpoint_api_type(resolved_endpoint), + anthropic_version=connection.get('anthropic_version') or '', + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint, identity_context=identity_context, @@ -1548,6 +1557,15 @@ def _test_multimodal_vision_connection(payload): }), 200 except Exception as e: + if is_custom_model_endpoint: + log_event( + "[MODEL_ENDPOINT] Custom vision model test failed", + extra={"exception_type": type(e).__name__}, + level=logging.WARNING, + ) + return jsonify({ + 'error': 'The Custom vision model test failed. Review the endpoint and model configuration.' + }), 500 return jsonify({'error': f'Vision test failed: {str(e)}'}), 500 def get_index_client() -> SearchIndexClient: diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 0c9be0b50..c5ccd6938 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -9,6 +9,11 @@ from flask import current_app, jsonify, request from functions_keyvault import keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_save_helper, redact_model_endpoint_secret_values +from functions_model_endpoint_types import resolve_model_endpoint_request_model +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_model_endpoints, +) from functions_settings import * from functions_content_safety import normalize_content_safety_violation_message from functions_rate_limit import normalize_rate_limit_message @@ -780,6 +785,8 @@ def admin_settings(): settings['allow_user_agents'] = False if 'allow_user_custom_endpoints' not in settings: settings['allow_user_custom_endpoints'] = settings.get('allow_user_custom_agent_endpoints', False) + if 'allow_private_custom_model_endpoints' not in settings: + settings['allow_private_custom_model_endpoints'] = False if 'allow_user_plugins' not in settings: settings['allow_user_plugins'] = False if 'allow_user_workflows' not in settings: @@ -1720,6 +1727,23 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul parsed_model_endpoints = merge_model_endpoints_with_existing(parsed_model_endpoints, existing_model_endpoints) parsed_model_endpoints, _ = normalize_model_endpoints(parsed_model_endpoints) + custom_endpoint_validation_settings = dict(settings) + custom_endpoint_validation_settings['allow_private_custom_model_endpoints'] = ( + form_data.get('allow_private_custom_model_endpoints') == 'on' + ) + try: + validate_custom_model_endpoints( + parsed_model_endpoints, + custom_endpoint_validation_settings, + ) + except ModelEndpointValidationError as exc: + log_event( + "[MODEL_ENDPOINT] Custom model endpoint validation failed", + extra={"exception_type": type(exc).__name__}, + level=logging.WARNING, + ) + flash(str(exc), 'danger') + return redirect(url_for('frontend_admin_settings.admin_settings')) existing_endpoints_by_id = { endpoint.get('id'): endpoint @@ -1882,9 +1906,10 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul if endpoint_provider: normalized_metadata_model_selection['provider'] = endpoint_provider metadata_extraction_model_deployment = str( - model_cfg.get('deploymentName') - or model_cfg.get('deployment') - or '' + resolve_model_endpoint_request_model( + endpoint_cfg, + model_cfg, + ) ).strip() else: normalized_metadata_model_selection = { @@ -2453,6 +2478,9 @@ def is_valid_url(url): 'gpt_model': gpt_model_obj, 'enable_multi_model_endpoints': enable_multi_model_endpoints, 'model_endpoints': parsed_model_endpoints, + 'allow_private_custom_model_endpoints': ( + form_data.get('allow_private_custom_model_endpoints') == 'on' + ), 'model_endpoint_identity_header_enabled': model_endpoint_identity_header_enabled, 'model_endpoint_identity_header_name': model_endpoint_identity_header_name, 'model_endpoint_identity_header_value_type': model_endpoint_identity_header_value_type, diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 4e700465b..b4a2caee6 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -5,6 +5,7 @@ from functions_authentication import * from functions_content import * from functions_settings import * +from functions_model_endpoint_types import resolve_model_endpoint_request_model from functions_agent_catalog import build_accessible_agent_catalog from functions_ai_notice import get_ai_notice_config, is_ai_notice_dismissed from functions_collaboration import ( @@ -465,15 +466,22 @@ def serialize_option(option): selection_key = _normalize_chat_model_value(option.get('selection_key')) model_id = _normalize_chat_model_value(option.get('model_id')) display_name = _normalize_chat_model_value( - option.get('display_name') or option.get('deployment_name') or option.get('model_id') + option.get('display_name') + or option.get('request_model') + or option.get('deployment_name') + or option.get('model_id') ) or 'Select a Model' deployment_name = _normalize_chat_model_value(option.get('deployment_name')) + request_model = _normalize_chat_model_value( + option.get('request_model') or deployment_name + ) scope_type = _normalize_chat_model_value(option.get('scope_type')) scope_name = _normalize_chat_model_value(option.get('scope_name')) search_parts = [ display_name, model_id, + request_model, deployment_name, scope_name or scope_type, ] @@ -481,6 +489,7 @@ def serialize_option(option): 'selection_key': selection_key, 'model_id': model_id, 'display_name': display_name, + 'request_model': request_model, 'deployment_name': deployment_name, 'endpoint_id': _normalize_chat_model_value(option.get('endpoint_id')), 'provider': _normalize_chat_model_value(option.get('provider')), @@ -488,23 +497,28 @@ def serialize_option(option): 'scope_id': _normalize_chat_model_value(option.get('scope_id')), 'scope_name': scope_name, 'icon': option.get('icon') if isinstance(option.get('icon'), dict) else {}, - 'option_value': deployment_name or model_id or selection_key, + 'option_value': request_model or deployment_name or model_id or selection_key, 'search_text': ' '.join(part for part in search_parts if part), } def sort_key(option): scope_type = _normalize_chat_model_value(option.get('scope_type')) display_name = _normalize_chat_model_value( - option.get('display_name') or option.get('deployment_name') or option.get('model_id') + option.get('display_name') + or option.get('request_model') + or option.get('deployment_name') + or option.get('model_id') ).lower() scope_name = _normalize_chat_model_value(option.get('scope_name')).lower() model_id = _normalize_chat_model_value(option.get('model_id')).lower() deployment_name = _normalize_chat_model_value(option.get('deployment_name')).lower() + request_model = _normalize_chat_model_value(option.get('request_model')).lower() return ( scope_order.get(scope_type, 99), scope_name, display_name, model_id, + request_model, deployment_name, ) @@ -526,7 +540,13 @@ def sort_key(option): if normalized_preferred_model_deployment: for option in sorted_options: deployment_name = _normalize_chat_model_value(option.get('deployment_name')) - if deployment_name == normalized_preferred_model_deployment: + request_model = _normalize_chat_model_value( + option.get('request_model') or deployment_name + ) + if ( + deployment_name == normalized_preferred_model_deployment + or request_model == normalized_preferred_model_deployment + ): return serialize_option(option) return serialize_option(sorted_options[0]) @@ -556,13 +576,15 @@ def append_models(endpoints, scope_type, scope_id=None, scope_name=None): model_id = model.get('id') or model.get('deploymentName') or model.get('deployment') or model.get('modelName') or model.get('name') or '' deployment_name = model.get('deploymentName') or model.get('deployment') or '' - display_name = model.get('displayName') or model.get('modelName') or deployment_name or model.get('name') or model_id - selection_key = f"{scope_type}:{scope_id or ''}:{endpoint_id}:{model_id or deployment_name}" + request_model = resolve_model_endpoint_request_model(endpoint, model) + display_name = model.get('displayName') or model.get('modelName') or request_model or deployment_name or model.get('name') or model_id + selection_key = f"{scope_type}:{scope_id or ''}:{endpoint_id}:{model_id or deployment_name or request_model}" catalog.append({ 'selection_key': selection_key, 'model_id': model_id, 'display_name': display_name, + 'request_model': request_model, 'deployment_name': deployment_name, 'endpoint_id': endpoint_id, 'provider': provider, @@ -780,6 +802,7 @@ def chats(): multi_endpoint_models.append({ "id": model.get("id"), "display_name": model.get("displayName") or model.get("deploymentName") or model.get("modelName") or "", + "request_model": resolve_model_endpoint_request_model(endpoint, model), "deployment_name": model.get("deploymentName") or "", "endpoint_id": endpoint.get("id"), "provider": endpoint.get("provider"), diff --git a/application/single_app/semantic_kernel_loader.py b/application/single_app/semantic_kernel_loader.py index ec7ba2b6a..2c38cad7c 100644 --- a/application/single_app/semantic_kernel_loader.py +++ b/application/single_app/semantic_kernel_loader.py @@ -27,6 +27,11 @@ from semantic_kernel_plugins.chart_plugin import ChartPlugin from semantic_kernel_plugins.tabular_processing_plugin import TabularProcessingPlugin from functions_settings import get_settings, get_user_settings, is_tabular_processing_enabled, resolve_model_endpoint_foundry_scope +from functions_model_endpoint_runtime import build_semantic_kernel_chat_service_for_model +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from foundry_agent_runtime import ( AzureAIFoundryChatCompletionAgent, AzureAIFoundryNewChatCompletionAgent, @@ -165,6 +170,7 @@ def resolve_agent_endpoint_protocol(agent_config): agent_config.get("model_provider") or agent_config.get("provider") or "aoai", agent_config.get("endpoint"), agent_config.get("deployment"), + agent_config.get("api_type"), ) @@ -179,11 +185,31 @@ def resolve_agent_endpoint_token(agent_config): return "" -def create_model_endpoint_chat_completion_service(agent_config, service_id): +def create_model_endpoint_chat_completion_service(agent_config, service_id, settings=None): """Create the correct Semantic Kernel chat service for an endpoint-bound agent.""" if not agent_config.get("endpoint") or not agent_config.get("deployment"): return None + provider = str( + agent_config.get("model_provider") or agent_config.get("provider") or "aoai" + ).strip().lower() + if provider == "custom": + chat_service, _ = build_semantic_kernel_chat_service_for_model( + agent_config["deployment"], + settings or {}, + service_id=service_id, + model_context={ + "provider": provider, + "endpoint": agent_config["endpoint"], + "api_version": agent_config.get("api_version") or "", + "api_type": agent_config.get("api_type") or "", + "anthropic_version": agent_config.get("anthropic_version") or "", + "auth": agent_config.get("auth") or {}, + "request_model": agent_config["deployment"], + }, + ) + return chat_service + runtime_protocol = resolve_agent_endpoint_protocol(agent_config) token_or_key = resolve_agent_endpoint_token(agent_config) if not token_or_key: @@ -576,13 +602,15 @@ def resolve_multi_endpoint_agent_binding(endpoint_candidates, endpoint_id, model provider = (endpoint_cfg.get("provider") or "aoai").lower() connection = endpoint_cfg.get("connection", {}) or {} auth = endpoint_cfg.get("auth", {}) or {} - deployment = model_cfg.get("deploymentName") or model_cfg.get("deployment") or "" + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) api_version = connection.get("openai_api_version") or connection.get("api_version") endpoint = connection.get("endpoint") return { "provider": provider, "endpoint": endpoint, "api_version": api_version, + "api_type": get_model_endpoint_api_type(endpoint_cfg), + "anthropic_version": connection.get("anthropic_version") or "", "deployment": deployment, "auth": auth, "model": model_cfg, @@ -808,7 +836,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): if not per_user_enabled: try: token_provider = None - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -816,13 +844,15 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): deployment = multi_endpoint_config.get("deployment") api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" - if auth_type != "api_key": + if auth_type not in ("api_key", "key"): token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) return { "endpoint": endpoint, "key": key, "deployment": deployment, "api_version": api_version, + "api_type": multi_endpoint_config.get("api_type") or "", + "anthropic_version": multi_endpoint_config.get("anthropic_version") or "", "instructions": agent.get("instructions", ""), "actions_to_load": agent.get("actions_to_load", []), "additional_settings": agent.get("additional_settings", {}), @@ -843,6 +873,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "auth": auth, } if global_apim_enabled: g_apim = get_global_apim() @@ -885,7 +916,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): can_use_agent_endpoints = allow_custom_agent_endpoints user_apim_allowed = user_apim_enabled and can_use_agent_endpoints - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -894,13 +925,15 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" token_provider = None - if auth_type != "api_key": + if auth_type not in ("api_key", "key"): token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) result = { "endpoint": endpoint, "key": key, "deployment": deployment, "api_version": api_version, + "api_type": multi_endpoint_config.get("api_type") or "", + "anthropic_version": multi_endpoint_config.get("anthropic_version") or "", "instructions": agent.get("instructions", ""), "actions_to_load": agent.get("actions_to_load", []), "additional_settings": agent.get("additional_settings", {}), @@ -921,6 +954,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "auth": auth, } return result @@ -1797,7 +1831,7 @@ def load_single_agent_for_kernel(kernel, agent_cfg, settings, context_obj, redis apim_enabled = settings.get("enable_gpt_apim", False) def create_chat_completion_service(): - return create_model_endpoint_chat_completion_service(agent_config, service_id) + return create_model_endpoint_chat_completion_service(agent_config, service_id, settings) if agent_type in {"aifoundry", "new_foundry", "foundry_workflow"}: if agent_type == "foundry_workflow": @@ -2925,7 +2959,7 @@ def load_semantic_kernel(kernel: Kernel, settings): }, level=logging.INFO ) - chat_service = create_model_endpoint_chat_completion_service(agent_config, service_id) + chat_service = create_model_endpoint_chat_completion_service(agent_config, service_id, settings) if should_apply_prompt_settings(orchestrator_config, settings): if orchestrator_config.get('max_completion_tokens', -1) > 0: print(f"[SK_LOADER] Using {orchestrator_config['max_completion_tokens']} max_completion_tokens for {orchestrator_config['name']}") @@ -3023,7 +3057,7 @@ def load_semantic_kernel(kernel: Kernel, settings): }, level=logging.INFO ) - chat_service = create_model_endpoint_chat_completion_service(orchestrator_config, service_id) + chat_service = create_model_endpoint_chat_completion_service(orchestrator_config, service_id, settings) if should_apply_prompt_settings(agent_config, settings): if agent_config.get('max_completion_tokens', -1) > 0: print(f"[SK_LOADER] Using {agent_config['max_completion_tokens']} max_completion_tokens for {agent_config['name']}") diff --git a/application/single_app/static/js/admin/admin_model_endpoints.js b/application/single_app/static/js/admin/admin_model_endpoints.js index d62f67114..8e19851a6 100644 --- a/application/single_app/static/js/admin/admin_model_endpoints.js +++ b/application/single_app/static/js/admin/admin_model_endpoints.js @@ -43,6 +43,8 @@ const endpointModal = endpointModalEl && window.bootstrap ? bootstrap.Modal.getO const endpointIdInput = document.getElementById("model-endpoint-id"); const endpointNameInput = document.getElementById("model-endpoint-name"); const endpointProviderSelect = document.getElementById("model-endpoint-provider"); +const endpointApiTypeGroup = document.getElementById("model-endpoint-api-type-group"); +const endpointApiTypeSelect = document.getElementById("model-endpoint-api-type"); const endpointUrlInput = document.getElementById("model-endpoint-endpoint"); const endpointUrlLabel = document.getElementById("model-endpoint-endpoint-label"); const endpointUrlHelp = document.getElementById("model-endpoint-endpoint-help"); @@ -54,6 +56,8 @@ const endpointProjectApiVersionCustomInput = document.getElementById("model-endp const endpointOpenAiApiVersionGroup = document.getElementById("model-endpoint-openai-api-version-group"); const endpointOpenAiApiVersionInput = document.getElementById("model-endpoint-openai-api-version"); const endpointOpenAiApiVersionCustomInput = document.getElementById("model-endpoint-openai-api-version-custom"); +const endpointAnthropicVersionGroup = document.getElementById("model-endpoint-anthropic-version-group"); +const endpointAnthropicVersionInput = document.getElementById("model-endpoint-anthropic-version"); const endpointSubscriptionGroup = document.getElementById("model-endpoint-subscription-group"); const endpointResourceGroup = document.getElementById("model-endpoint-resource-group-group"); const endpointSubscriptionInput = document.getElementById("model-endpoint-subscription-id"); @@ -66,6 +70,7 @@ const endpointCustomAuthorityInput = document.getElementById("model-endpoint-cus const endpointFoundryScopeGroup = document.getElementById("model-endpoint-foundry-scope-group"); const endpointFoundryScopeInput = document.getElementById("model-endpoint-foundry-scope"); const apiKeyNote = document.getElementById("model-endpoint-api-key-note"); +const apiKeyNoteText = document.getElementById("model-endpoint-api-key-note-text"); const miTypeGroup = document.getElementById("model-endpoint-mi-type-group"); const miClientGroup = document.getElementById("model-endpoint-mi-client-group"); @@ -108,6 +113,7 @@ let migrationSelectedKeys = new Set(); const DEFAULT_AOAI_OPENAI_API_VERSION = "2024-05-01-preview"; const DEFAULT_FOUNDRY_OPENAI_API_VERSION = "v1"; const DEFAULT_FOUNDRY_PROJECT_API_VERSION = "v1"; +const DEFAULT_ANTHROPIC_VERSION = "2023-06-01"; const CUSTOM_VERSION_VALUE = "custom"; const IDENTITY_HEADER_MODES = new Set(["inherit", "enabled", "disabled"]); const IDENTITY_HEADER_VALUE_TYPES = new Set(["", "user_oid_tenant_id", "user_oid", "user_upn_tenant_id", "user_upn"]); @@ -149,6 +155,40 @@ function isFoundryProvider(provider) { return provider === "aifoundry" || provider === "new_foundry"; } +function isCustomProvider(provider = endpointProviderSelect?.value) { + return provider === "custom"; +} + +function getCustomApiType() { + return endpointApiTypeSelect?.value || "openai"; +} + +function customApiTypeUsesModelName(apiType = getCustomApiType()) { + return apiType === "openai" || apiType === "anthropic"; +} + +function getModelRequestName(model) { + if (isCustomProvider() && customApiTypeUsesModelName()) { + return String(model?.modelName || "").trim(); + } + return String(model?.deploymentName || model?.deployment || "").trim(); +} + +function setModelRequestName(model, value) { + const requestName = String(value || "").trim(); + if (isCustomProvider() && customApiTypeUsesModelName()) { + model.modelName = requestName; + delete model.deploymentName; + delete model.deployment; + return; + } + model.deploymentName = requestName; + if (isCustomProvider()) { + delete model.modelName; + delete model.name; + } +} + function endpointIncludesProject(endpoint) { return String(endpoint || "").toLowerCase().includes("/api/projects/"); } @@ -198,9 +238,13 @@ function syncEndpointCopyForProvider() { : "Endpoint Fully Qualified Domain Name (FQDN)"; } if (endpointUrlHelp) { - endpointUrlHelp.textContent = isFoundryProvider(provider) - ? "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name." - : "For Azure OpenAI, paste the resource endpoint."; + if (isFoundryProvider(provider)) { + endpointUrlHelp.textContent = "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name."; + } else if (isCustomProvider(provider)) { + endpointUrlHelp.textContent = "Enter the HTTPS FQDN for the Custom endpoint."; + } else { + endpointUrlHelp.textContent = "For Azure OpenAI, paste the resource endpoint."; + } } } @@ -392,6 +436,9 @@ function formatProviderLabel(provider) { if (provider === "new_foundry") { return "New Foundry"; } + if (provider === "custom") { + return "Custom"; + } return "Azure OpenAI"; } @@ -417,7 +464,7 @@ function syncOpenAiApiVersionForProvider() { return; } - if (!currentValue) { + if (!currentValue || currentValue === DEFAULT_FOUNDRY_OPENAI_API_VERSION) { setSelectedVersionValue( endpointOpenAiApiVersionInput, endpointOpenAiApiVersionCustomInput, @@ -776,18 +823,32 @@ function handleMetadataExtractionModelChange() { } function updateAuthVisibility() { - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const modelsPlaceholder = document.getElementById("model-endpoint-models-placeholder"); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + if (customProvider && endpointAuthTypeSelect) { + endpointAuthTypeSelect.value = "api_key"; + } + if (endpointAuthTypeSelect) { + endpointAuthTypeSelect.disabled = customProvider; + } + setElementVisibility(endpointApiTypeGroup, customProvider); + + const apiType = getCustomApiType(); + const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; - const isFoundry = isFoundryProvider(provider); + const isFoundry = !customProvider && isFoundryProvider(provider); + const showOpenAiVersion = !customProvider || apiType === "azure_openai"; + const showAnthropicVersion = customProvider && apiType === "anthropic"; const projectNameFromEndpoint = syncProjectNameFromEndpoint(); syncEndpointCopyForProvider(); syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, true); - setElementVisibility(endpointSubscriptionGroup, provider === "aoai" && !isApiKey); - setElementVisibility(endpointResourceGroup, provider === "aoai" && !isApiKey); + setElementVisibility(endpointOpenAiApiVersionGroup, showOpenAiVersion); + setElementVisibility(endpointAnthropicVersionGroup, showAnthropicVersion); + setElementVisibility(endpointSubscriptionGroup, !customProvider && provider === "aoai" && !isApiKey); + setElementVisibility(endpointResourceGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); setElementVisibility(miClientGroup, authType === "managed_identity" && (miTypeSelect?.value === "user_assigned")); setElementVisibility(tenantGroup, authType === "service_principal"); @@ -797,9 +858,27 @@ function updateAuthVisibility() { setElementVisibility(endpointManagementCloudGroup, authType === "service_principal" && isFoundry); setElementVisibility(endpointCustomAuthorityGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); setElementVisibility(endpointFoundryScopeGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); - setElementVisibility(apiKeyNote, authType === "api_key"); - setElementVisibility(addModelBtn, authType === "api_key"); - setElementVisibility(fetchBtn, authType !== "api_key"); + setElementVisibility(apiKeyNote, customProvider || authType === "api_key"); + setElementVisibility(addModelBtn, customProvider || authType === "api_key"); + setElementVisibility(fetchBtn, !customProvider && authType !== "api_key"); + + if (customProvider) { + if (apiKeyNoteText) { + apiKeyNoteText.textContent = "Custom endpoints use API key authentication and manual model entry. Model discovery is unavailable."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = "Add a model manually."; + } + } else { + if (apiKeyNoteText) { + apiKeyNoteText.textContent = "API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = authType === "api_key" + ? "Add a model manually, or switch authentication to discover deployments." + : "Fetch models or add a model manually."; + } + } } function resetModal() { @@ -809,6 +888,7 @@ function resetModal() { if (endpointIdInput) endpointIdInput.value = ""; if (endpointNameInput) endpointNameInput.value = ""; if (endpointProviderSelect) endpointProviderSelect.value = "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = "openai"; if (endpointUrlInput) endpointUrlInput.value = ""; if (endpointProjectInput) endpointProjectInput.value = ""; setSelectedVersionValue( @@ -821,6 +901,7 @@ function resetModal() { endpointOpenAiApiVersionCustomInput, getDefaultOpenAiApiVersion("aoai") ); + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = DEFAULT_ANTHROPIC_VERSION; if (endpointSubscriptionInput) endpointSubscriptionInput.value = ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = "managed_identity"; @@ -840,7 +921,7 @@ function resetModal() { if (endpointIdentityValueTypeSelect) endpointIdentityValueTypeSelect.value = ""; modalModels = []; - if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; + if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; updateAuthVisibility(); } @@ -856,6 +937,7 @@ function openModalForEndpoint(endpoint) { if (endpointIdInput) endpointIdInput.value = endpoint.id || ""; if (endpointNameInput) endpointNameInput.value = endpoint.name || ""; if (endpointProviderSelect) endpointProviderSelect.value = endpoint.provider || "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = endpoint.api_type || "openai"; if (endpointUrlInput) endpointUrlInput.value = endpoint.connection?.endpoint || ""; if (endpointProjectInput) endpointProjectInput.value = endpoint.connection?.project_name || ""; setSelectedVersionValue( @@ -868,6 +950,9 @@ function openModalForEndpoint(endpoint) { endpointOpenAiApiVersionCustomInput, endpoint.connection?.openai_api_version || endpoint.connection?.api_version || getDefaultOpenAiApiVersion(endpoint.provider || "aoai") ); + if (endpointAnthropicVersionInput) { + endpointAnthropicVersionInput.value = endpoint.connection?.anthropic_version || DEFAULT_ANTHROPIC_VERSION; + } if (endpointSubscriptionInput) endpointSubscriptionInput.value = endpoint.management?.subscription_id || ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = endpoint.management?.resource_group || ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = endpoint.auth?.type || "managed_identity"; @@ -1202,7 +1287,7 @@ function renderModalModels(models) { } if (!models || !models.length) { - modelsListEl.innerHTML = "

No models loaded yet.

"; + modelsListEl.innerHTML = "

No models loaded yet.

"; return; } @@ -1210,12 +1295,15 @@ function renderModalModels(models) { models.forEach((model) => { const wrapper = document.createElement("div"); wrapper.className = "border rounded p-2 mb-2"; - const deploymentName = model.deploymentName || ""; + const requestName = getModelRequestName(model); const modelName = model.modelName || ""; - const displayName = model.displayName || deploymentName; + const displayName = model.displayName || requestName; const description = model.description || ""; const responseLength = getModelResponseLength(model); - const deploymentReadonly = model.isDiscovered ? "readonly" : ""; + const requestNameReadonly = model.isDiscovered && !isCustomProvider(); + const requestNameLabel = isCustomProvider() && customApiTypeUsesModelName() + ? "Model Name" + : "Deployment Name"; const modelId = model.id || generateId(); model.id = modelId; @@ -1226,8 +1314,8 @@ function renderModalModels(models) { checkbox.dataset.modelId = modelId; checkbox.checked = !!model.enabled; const checkboxLabel = createElement("label", "form-check-label"); - checkboxLabel.appendChild(document.createTextNode(deploymentName)); - if (modelName) { + checkboxLabel.appendChild(document.createTextNode(requestName)); + if (!isCustomProvider() && modelName) { checkboxLabel.appendChild(document.createTextNode(" ")); const modelNameLabel = createElement("span", "text-muted"); modelNameLabel.textContent = `(${modelName})`; @@ -1238,8 +1326,8 @@ function renderModalModels(models) { const fieldsRow = createElement("div", "row g-2"); const deploymentCol = createElement("div", "col-md-4"); - deploymentCol.appendChild(createSmallLabel("Deployment Name")); - deploymentCol.appendChild(createModelTextInput(modelId, "deploymentNameFor", deploymentName, Boolean(deploymentReadonly))); + deploymentCol.appendChild(createSmallLabel(requestNameLabel)); + deploymentCol.appendChild(createModelTextInput(modelId, "requestModelFor", requestName, requestNameReadonly)); const displayCol = createElement("div", "col-md-4"); displayCol.appendChild(createSmallLabel("Display Name")); displayCol.appendChild(createModelTextInput(modelId, "displayNameFor", displayName)); @@ -1298,7 +1386,7 @@ function collectModalModels() { const updated = modalModels.map((model) => ({ ...model })); updated.forEach((model) => { const checkbox = modelsListEl.querySelector(`input[data-model-id="${model.id}"]`); - const deploymentInput = modelsListEl.querySelector(`input[data-deployment-name-for="${model.id}"]`); + const requestModelInput = modelsListEl.querySelector(`input[data-request-model-for="${model.id}"]`); const displayInput = modelsListEl.querySelector(`input[data-display-name-for="${model.id}"]`); const descriptionInput = modelsListEl.querySelector(`input[data-description-for="${model.id}"]`); const responseLengthInput = modelsListEl.querySelector(`input[data-response-length-for="${model.id}"]`); @@ -1308,7 +1396,7 @@ function collectModalModels() { throw new Error("Response length must be a positive whole number."); } model.enabled = checkbox ? checkbox.checked : model.enabled; - model.deploymentName = deploymentInput ? deploymentInput.value.trim() : model.deploymentName; + setModelRequestName(model, requestModelInput ? requestModelInput.value : getModelRequestName(model)); model.displayName = displayInput ? displayInput.value.trim() : model.displayName; model.icon = iconEditor ? getIconPayload(iconEditor, MODEL_ICON_CONTROL_CONFIG) : model.icon || {}; model.description = descriptionInput ? descriptionInput.value.trim() : model.description; @@ -1323,16 +1411,17 @@ function collectModalModels() { async function testModelConnection(model) { const payload = buildEndpointPayload(); - if (!payload || !model?.deploymentName) { - showToast("Model deployment name is required for testing.", "warning"); + const requestModel = getModelRequestName(model); + if (!payload || !requestModel) { + showToast(`${isCustomProvider() && customApiTypeUsesModelName() ? "Model" : "Deployment"} name is required for testing.`, "warning"); return; } + const testModel = {}; + setModelRequestName(testModel, requestModel); const requestBody = { ...payload, - model: { - deploymentName: model.deploymentName - } + model: testModel }; try { @@ -1353,6 +1442,10 @@ async function testModelConnection(model) { } async function fetchModels() { + if (isCustomProvider()) { + showToast("Model discovery is unavailable for Custom endpoints. Add models manually.", "warning"); + return; + } const payload = buildEndpointPayload(); if (!payload) { return; @@ -1418,6 +1511,8 @@ function buildEndpointPayload() { const name = endpointNameInput.value.trim(); const endpoint = endpointUrlInput.value.trim(); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + const apiType = getCustomApiType(); const projectNameFromEndpoint = isFoundryProvider(provider) ? syncProjectNameFromEndpoint() : ""; const projectName = projectNameFromEndpoint || endpointProjectInput?.value.trim() || ""; const projectApiVersion = getSelectedVersionValue( @@ -1432,7 +1527,7 @@ function buildEndpointPayload() { ); const subscriptionId = endpointSubscriptionInput?.value.trim() || ""; const resourceGroup = endpointResourceGroupInput?.value.trim() || ""; - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const authType = customProvider ? "api_key" : (endpointAuthTypeSelect?.value || "managed_identity"); const existingEndpoint = modelEndpoints.find((savedEndpoint) => savedEndpoint.id === endpointId); const identityHeader = normalizeEndpointIdentityHeaderOverride({ mode: endpointIdentityModeSelect?.value || "inherit", @@ -1440,8 +1535,18 @@ function buildEndpointPayload() { value_type: endpointIdentityValueTypeSelect?.value || "" }); - if (!name || !endpoint || !openAiApiVersion) { - showToast("Endpoint name, URL, and OpenAI API version are required.", "warning"); + if (!name || !endpoint) { + showToast("Endpoint name and URL are required.", "warning"); + return null; + } + + if (customProvider && !/^https:\/\//i.test(endpoint)) { + showToast("Custom endpoint URLs must use HTTPS.", "warning"); + return null; + } + + if ((!customProvider || apiType === "azure_openai") && !openAiApiVersion) { + showToast("OpenAI API version is required.", "warning"); return null; } @@ -1460,7 +1565,7 @@ function buildEndpointPayload() { return null; } - const auth = { + let auth = { type: authType, managed_identity_type: miTypeSelect?.value || "system_assigned", managed_identity_client_id: miClientIdInput?.value.trim() || "", @@ -1472,6 +1577,12 @@ function buildEndpointPayload() { custom_authority: endpointCustomAuthorityInput?.value.trim() || "", foundry_scope: endpointFoundryScopeInput?.value.trim() || "" }; + if (customProvider) { + auth = { + type: "api_key", + api_key: apiKeyInput?.value.trim() || "" + }; + } const hasStoredApiKey = authType === "api_key" && Boolean(existingEndpoint?.has_api_key); const hasStoredClientSecret = authType === "service_principal" && Boolean(existingEndpoint?.has_client_secret); @@ -1497,17 +1608,21 @@ function buildEndpointPayload() { return null; } - const management = provider === "aoai" ? { + const management = !customProvider && provider === "aoai" ? { subscription_id: subscriptionId, resource_group: resourceGroup } : {}; - const connection = { - endpoint, - openai_api_version: openAiApiVersion - }; + const connection = { endpoint }; + if (customProvider && apiType === "azure_openai") { + connection.api_version = openAiApiVersion; + } else if (customProvider && apiType === "anthropic") { + connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; + } else if (!customProvider) { + connection.openai_api_version = openAiApiVersion; + } - if (isFoundryProvider(provider)) { + if (!customProvider && isFoundryProvider(provider)) { connection.project_api_version = projectApiVersion; if (projectName) { connection.project_name = projectName; @@ -1517,6 +1632,7 @@ function buildEndpointPayload() { return { id: endpointId, provider, + ...(customProvider ? { api_type: apiType } : {}), name, connection, management, @@ -1543,6 +1659,7 @@ function saveEndpoint() { id: endpointId, name: payload.name, provider: payload.provider, + ...(payload.api_type ? { api_type: payload.api_type } : {}), enabled: endpointModalEl?.dataset.duplicateDisabledDefault === 'true' ? false : (existingEndpoint ? existingEndpoint.enabled !== false : true), @@ -1574,16 +1691,16 @@ function saveEndpoint() { } function addManualModel() { - modalModels.push({ + const model = { id: generateId(), - deploymentName: "", - modelName: "", displayName: "", icon: {}, description: "", enabled: true, isDiscovered: false - }); + }; + setModelRequestName(model, ""); + modalModels.push(model); renderModalModels(modalModels); } @@ -2072,8 +2189,28 @@ function init() { } if (endpointProviderSelect) { endpointProviderSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the endpoint provider.", "danger"); + return; + } + syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); updateAuthVisibility(); + }); + } + if (endpointApiTypeSelect) { + endpointApiTypeSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the API type.", "danger"); + return; + } syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); + updateAuthVisibility(); }); } if (endpointUrlInput) { diff --git a/application/single_app/static/js/agent_modal_stepper.js b/application/single_app/static/js/agent_modal_stepper.js index 782ad5f3c..83ada36b5 100644 --- a/application/single_app/static/js/agent_modal_stepper.js +++ b/application/single_app/static/js/agent_modal_stepper.js @@ -4777,7 +4777,9 @@ export class AgentModalStepper { // Using global model - need to set at least one deployment field // We'll use the selected model as the deployment name for now if (formData.model) { - const deploymentName = selectedModelOption?.dataset?.deploymentName || formData.model; + const deploymentName = selectedModelOption?.dataset?.requestModel + || selectedModelOption?.dataset?.deploymentName + || formData.model; formData.azure_openai_gpt_deployment = deploymentName; } } diff --git a/application/single_app/static/js/agents_common.js b/application/single_app/static/js/agents_common.js index 169e4ca7c..21947dab0 100644 --- a/application/single_app/static/js/agents_common.js +++ b/application/single_app/static/js/agents_common.js @@ -704,17 +704,22 @@ export function getAvailableModels({ apimEnabled, settings, agent }) { return; } const endpointId = endpoint.id || ''; + const apiType = (endpoint.api_type || '').toLowerCase(); const endpointModels = endpoint.models || []; endpointModels.forEach(model => { if (!model || model.enabled === false) return; const modelId = model.id || model.deploymentName || model.deployment || model.modelName || model.name || ''; const deploymentName = model.deploymentName || model.deployment || ''; const modelName = model.modelName || model.name || ''; - const displayName = model.displayName || deploymentName || modelName || modelId; + const requestModel = provider === 'custom' && ['openai', 'anthropic'].includes(apiType) + ? modelName + : deploymentName || modelName; + const displayName = model.displayName || requestModel || modelId; if (!displayName) return; models.push({ id: modelId, - deployment: deploymentName, + deployment: requestModel, + request_model: requestModel, name: modelName, display_name: displayName, endpoint_id: endpointId, @@ -832,7 +837,10 @@ export function populateGlobalModelDropdown(selectEl, models, selectedModel) { if (model.deployment) { opt.dataset.deploymentName = model.deployment; } - if (selectedModel && (model.name === selectedModel || model.deployment === selectedModel || model.id === selectedModel)) { + if (model.request_model) { + opt.dataset.requestModel = model.request_model; + } + if (selectedModel && (model.name === selectedModel || model.request_model === selectedModel || model.deployment === selectedModel || model.id === selectedModel)) { opt.selected = true; } selectEl.appendChild(opt); diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index ffd0c8504..97b839e0b 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -6755,7 +6755,10 @@ function getCurrentModelSelection() { modelId = selectedOption?.dataset?.modelId || selectedOption?.value || null; modelEndpointId = selectedOption?.dataset?.endpointId || null; modelProvider = selectedOption?.dataset?.provider || null; - modelDeployment = selectedOption?.dataset?.deploymentName || null; + modelDeployment = selectedOption?.dataset?.requestModel + || selectedOption?.value + || selectedOption?.dataset?.deploymentName + || null; modelIcon = parseSafeJsonObject(selectedOption?.dataset?.modelIcon || ''); } @@ -6916,7 +6919,14 @@ function buildCollaborativeModelTarget(option = {}) { return null; } - const modelDeployment = String(dataset.deploymentName || option.deployment_name || option.value || '').trim() || null; + const modelDeployment = String( + dataset.requestModel + || option.request_model + || dataset.deploymentName + || option.deployment_name + || option.value + || '' + ).trim() || null; const modelId = String(dataset.modelId || option.model_id || option.value || '').trim() || null; const modelEndpointId = String(dataset.endpointId || option.endpoint_id || '').trim() || null; const modelProvider = String(dataset.provider || option.provider || '').trim() || null; diff --git a/application/single_app/static/js/chat/chat-model-selector.js b/application/single_app/static/js/chat/chat-model-selector.js index 5b96f2456..b339aa5a5 100644 --- a/application/single_app/static/js/chat/chat-model-selector.js +++ b/application/single_app/static/js/chat/chat-model-selector.js @@ -44,13 +44,14 @@ function getSortedGroups() { } function getModelDisplayName(option) { - return (option.display_name || option.model_id || option.deployment_name || 'Unnamed Model').trim() || 'Unnamed Model'; + return (option.display_name || option.request_model || option.model_id || option.deployment_name || 'Unnamed Model').trim() || 'Unnamed Model'; } function getModelSearchText(option, sectionLabel) { return [ getModelDisplayName(option), option.model_id || '', + option.request_model || '', option.deployment_name || '', sectionLabel, ].join(' ').trim(); @@ -71,7 +72,7 @@ function getModelOptionLabel(option, duplicateCounts) { return displayName; } - return `${displayName} (${option.deployment_name || option.model_id || 'model'})`; + return `${displayName} (${option.request_model || option.deployment_name || option.model_id || 'model'})`; } function getKnownGroupIds() { @@ -260,6 +261,7 @@ function getSelectionSnapshot() { value: null, selectionKey: null, modelId: null, + requestModel: null, deploymentName: null, }; } @@ -269,6 +271,7 @@ function getSelectionSnapshot() { value: modelSelect.value || null, selectionKey: selectedOption?.dataset?.selectionKey || null, modelId: selectedOption?.dataset?.modelId || null, + requestModel: selectedOption?.dataset?.requestModel || null, deploymentName: selectedOption?.dataset?.deploymentName || null, }; } @@ -316,14 +319,20 @@ function resolveSelectedSelectionKey(options, restoreOptions = {}) { } if (preferredModelDeployment) { - const deploymentOption = matchBy(option => option.deployment_name === preferredModelDeployment); + const deploymentOption = matchBy(option => ( + option.request_model === preferredModelDeployment + || option.deployment_name === preferredModelDeployment + )); if (deploymentOption) { return deploymentOption.selection_key; } } - if (preserveCurrentSelection && currentSelection?.deploymentName) { - const currentDeploymentOption = matchBy(option => option.deployment_name === currentSelection.deploymentName); + if (preserveCurrentSelection && (currentSelection?.requestModel || currentSelection?.deploymentName)) { + const currentDeploymentOption = matchBy(option => ( + option.request_model === currentSelection.requestModel + || option.deployment_name === currentSelection.deploymentName + )); if (currentDeploymentOption) { return currentDeploymentOption.selection_key; } @@ -376,11 +385,12 @@ function rebuildModelOptions(sections, restoreOptions = {}) { section.options.forEach(option => { const modelOption = document.createElement('option'); - modelOption.value = option.deployment_name || option.model_id || option.selection_key; + modelOption.value = option.request_model || option.deployment_name || option.model_id || option.selection_key; modelOption.textContent = option.optionLabel; modelOption.dataset.selectionKey = option.selection_key || ''; modelOption.dataset.modelId = option.model_id || ''; modelOption.dataset.displayName = option.display_name || ''; + modelOption.dataset.requestModel = option.request_model || ''; modelOption.dataset.deploymentName = option.deployment_name || ''; modelOption.dataset.endpointId = option.endpoint_id || ''; modelOption.dataset.provider = option.provider || ''; diff --git a/application/single_app/static/js/workspace/workspace_model_endpoints.js b/application/single_app/static/js/workspace/workspace_model_endpoints.js index 3658f5d94..f4a4c790d 100644 --- a/application/single_app/static/js/workspace/workspace_model_endpoints.js +++ b/application/single_app/static/js/workspace/workspace_model_endpoints.js @@ -14,6 +14,8 @@ const endpointModal = endpointModalEl && window.bootstrap ? bootstrap.Modal.getO const endpointIdInput = document.getElementById("model-endpoint-id"); const endpointNameInput = document.getElementById("model-endpoint-name"); const endpointProviderSelect = document.getElementById("model-endpoint-provider"); +const endpointApiTypeGroup = document.getElementById("model-endpoint-api-type-group"); +const endpointApiTypeSelect = document.getElementById("model-endpoint-api-type"); const endpointUrlInput = document.getElementById("model-endpoint-endpoint"); const endpointUrlLabel = document.getElementById("model-endpoint-endpoint-label"); const endpointUrlHelp = document.getElementById("model-endpoint-endpoint-help"); @@ -25,6 +27,8 @@ const endpointProjectApiVersionCustomInput = document.getElementById("model-endp const endpointOpenAiApiVersionGroup = document.getElementById("model-endpoint-openai-api-version-group"); const endpointOpenAiApiVersionInput = document.getElementById("model-endpoint-openai-api-version"); const endpointOpenAiApiVersionCustomInput = document.getElementById("model-endpoint-openai-api-version-custom"); +const endpointAnthropicVersionGroup = document.getElementById("model-endpoint-anthropic-version-group"); +const endpointAnthropicVersionInput = document.getElementById("model-endpoint-anthropic-version"); const endpointSubscriptionGroup = document.getElementById("model-endpoint-subscription-group"); const endpointResourceGroup = document.getElementById("model-endpoint-resource-group-group"); const endpointSubscriptionInput = document.getElementById("model-endpoint-subscription-id"); @@ -37,6 +41,7 @@ const endpointCustomAuthorityInput = document.getElementById("model-endpoint-cus const endpointFoundryScopeGroup = document.getElementById("model-endpoint-foundry-scope-group"); const endpointFoundryScopeInput = document.getElementById("model-endpoint-foundry-scope"); const apiKeyNote = document.getElementById("model-endpoint-api-key-note"); +const apiKeyNoteText = document.getElementById("model-endpoint-api-key-note-text"); const miTypeGroup = document.getElementById("model-endpoint-mi-type-group"); const miClientGroup = document.getElementById("model-endpoint-mi-client-group"); @@ -70,6 +75,7 @@ let modalModels = []; const DEFAULT_AOAI_OPENAI_API_VERSION = "2024-05-01-preview"; const DEFAULT_FOUNDRY_OPENAI_API_VERSION = "v1"; const DEFAULT_FOUNDRY_PROJECT_API_VERSION = "v1"; +const DEFAULT_ANTHROPIC_VERSION = "2023-06-01"; const CUSTOM_VERSION_VALUE = "custom"; const MODEL_ICON_CLASS_PATTERN = /^bi-[a-z0-9][a-z0-9-]{0,80}$/; const MODEL_ICON_CONTROL_CONFIG = Object.freeze({ @@ -124,6 +130,40 @@ function isFoundryProvider(provider) { return provider === "aifoundry" || provider === "new_foundry"; } +function isCustomProvider(provider = endpointProviderSelect?.value) { + return provider === "custom"; +} + +function getCustomApiType() { + return endpointApiTypeSelect?.value || "openai"; +} + +function customApiTypeUsesModelName(apiType = getCustomApiType()) { + return apiType === "openai" || apiType === "anthropic"; +} + +function getModelRequestName(model) { + if (isCustomProvider() && customApiTypeUsesModelName()) { + return String(model?.modelName || "").trim(); + } + return String(model?.deploymentName || model?.deployment || "").trim(); +} + +function setModelRequestName(model, value) { + const requestName = String(value || "").trim(); + if (isCustomProvider() && customApiTypeUsesModelName()) { + model.modelName = requestName; + delete model.deploymentName; + delete model.deployment; + return; + } + model.deploymentName = requestName; + if (isCustomProvider()) { + delete model.modelName; + delete model.name; + } +} + function endpointIncludesProject(endpoint) { return String(endpoint || "").toLowerCase().includes("/api/projects/"); } @@ -173,9 +213,13 @@ function syncEndpointCopyForProvider() { : "Endpoint Fully Qualified Domain Name (FQDN)"; } if (endpointUrlHelp) { - endpointUrlHelp.textContent = isFoundryProvider(provider) - ? "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name." - : "For Azure OpenAI, paste the resource endpoint."; + if (isFoundryProvider(provider)) { + endpointUrlHelp.textContent = "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name."; + } else if (isCustomProvider(provider)) { + endpointUrlHelp.textContent = "Enter the HTTPS FQDN for the Custom endpoint."; + } else { + endpointUrlHelp.textContent = "For Azure OpenAI, paste the resource endpoint."; + } } } @@ -242,7 +286,7 @@ function syncOpenAiApiVersionForProvider() { return; } - if (!currentValue) { + if (!currentValue || currentValue === DEFAULT_FOUNDRY_OPENAI_API_VERSION) { setSelectedVersionValue( endpointOpenAiApiVersionInput, endpointOpenAiApiVersionCustomInput, @@ -258,6 +302,9 @@ function formatProviderLabel(provider) { if (provider === "new_foundry") { return "New Foundry"; } + if (provider === "custom") { + return "Custom"; + } return "Azure OpenAI"; } @@ -318,18 +365,30 @@ function renderEndpoints() { } function updateAuthVisibility() { - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const modelsPlaceholder = document.getElementById("model-endpoint-models-placeholder"); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + if (customProvider && endpointAuthTypeSelect) { + endpointAuthTypeSelect.value = "api_key"; + } + if (endpointAuthTypeSelect) { + endpointAuthTypeSelect.disabled = customProvider; + } + setElementVisibility(endpointApiTypeGroup, customProvider); + + const apiType = getCustomApiType(); + const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; - const isFoundry = isFoundryProvider(provider); + const isFoundry = !customProvider && isFoundryProvider(provider); const projectNameFromEndpoint = syncProjectNameFromEndpoint(); syncEndpointCopyForProvider(); syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, true); - setElementVisibility(endpointSubscriptionGroup, provider === "aoai" && !isApiKey); - setElementVisibility(endpointResourceGroup, provider === "aoai" && !isApiKey); + setElementVisibility(endpointOpenAiApiVersionGroup, !customProvider || apiType === "azure_openai"); + setElementVisibility(endpointAnthropicVersionGroup, customProvider && apiType === "anthropic"); + setElementVisibility(endpointSubscriptionGroup, !customProvider && provider === "aoai" && !isApiKey); + setElementVisibility(endpointResourceGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); setElementVisibility(miClientGroup, authType === "managed_identity" && (miTypeSelect?.value === "user_assigned")); setElementVisibility(tenantGroup, authType === "service_principal"); @@ -339,15 +398,28 @@ function updateAuthVisibility() { setElementVisibility(endpointManagementCloudGroup, authType === "service_principal" && isFoundry); setElementVisibility(endpointCustomAuthorityGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); setElementVisibility(endpointFoundryScopeGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); - setElementVisibility(apiKeyNote, authType === "api_key"); - setElementVisibility(addModelBtn, authType === "api_key"); - setElementVisibility(fetchBtn, authType !== "api_key"); + setElementVisibility(apiKeyNote, customProvider || authType === "api_key"); + setElementVisibility(addModelBtn, customProvider || authType === "api_key"); + setElementVisibility(fetchBtn, !customProvider && authType !== "api_key"); + if (apiKeyNoteText) { + apiKeyNoteText.textContent = customProvider + ? "Custom endpoints use API key authentication and manual model entry. Model discovery is unavailable." + : "API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = customProvider + ? "Add a model manually." + : (authType === "api_key" + ? "Add a model manually, or switch authentication to discover deployments." + : "Fetch models or add a model manually."); + } } function resetModal() { if (endpointIdInput) endpointIdInput.value = ""; if (endpointNameInput) endpointNameInput.value = ""; if (endpointProviderSelect) endpointProviderSelect.value = "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = "openai"; if (endpointUrlInput) endpointUrlInput.value = ""; if (endpointProjectInput) endpointProjectInput.value = ""; setSelectedVersionValue( @@ -360,6 +432,7 @@ function resetModal() { endpointOpenAiApiVersionCustomInput, getDefaultOpenAiApiVersion("aoai") ); + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = DEFAULT_ANTHROPIC_VERSION; if (endpointSubscriptionInput) endpointSubscriptionInput.value = ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = "managed_identity"; @@ -376,7 +449,7 @@ function resetModal() { if (apiKeyInput) apiKeyInput.placeholder = ""; modalModels = []; - if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; + if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; updateAuthVisibility(); } @@ -392,6 +465,7 @@ function openModalForEndpoint(endpoint) { if (endpointIdInput) endpointIdInput.value = endpoint.id || ""; if (endpointNameInput) endpointNameInput.value = endpoint.name || ""; if (endpointProviderSelect) endpointProviderSelect.value = endpoint.provider || "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = endpoint.api_type || "openai"; if (endpointUrlInput) endpointUrlInput.value = endpoint.connection?.endpoint || ""; if (endpointProjectInput) endpointProjectInput.value = endpoint.connection?.project_name || ""; setSelectedVersionValue( @@ -404,6 +478,9 @@ function openModalForEndpoint(endpoint) { endpointOpenAiApiVersionCustomInput, endpoint.connection?.openai_api_version || endpoint.connection?.api_version || getDefaultOpenAiApiVersion(endpoint.provider || "aoai") ); + if (endpointAnthropicVersionInput) { + endpointAnthropicVersionInput.value = endpoint.connection?.anthropic_version || DEFAULT_ANTHROPIC_VERSION; + } if (endpointSubscriptionInput) endpointSubscriptionInput.value = endpoint.management?.subscription_id || ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = endpoint.management?.resource_group || ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = endpoint.auth?.type || "managed_identity"; @@ -458,6 +535,44 @@ function createModelTextInput(modelId, datasetKey, value, disabled = false) { return input; } +function normalizeModelResponseLength(value) { + const valueText = String(value ?? "").trim(); + if (!valueText) { + return ""; + } + if (!/^\d+$/.test(valueText)) { + return null; + } + + const parsedValue = Number.parseInt(valueText, 10); + return parsedValue > 0 ? parsedValue : null; +} + +function getModelResponseLength(model) { + return normalizeModelResponseLength( + model.responseLength + ?? model.response_length + ?? model.maxTokens + ?? model.max_tokens + ?? model.maxCompletionTokens + ?? model.max_completion_tokens + ); +} + +function createModelResponseLengthInput(modelId, value) { + const input = document.createElement("input"); + input.type = "number"; + input.className = "form-control form-control-sm"; + input.min = "1"; + input.step = "1"; + input.placeholder = "Optional"; + input.dataset.responseLengthFor = modelId; + input.id = getModelIconDomId(modelId, "response-length"); + input.value = value || ""; + input.setAttribute("aria-describedby", getModelIconDomId(modelId, "response-length-help")); + return input; +} + function getModelIconDomId(modelId, suffix) { const safeModelId = String(modelId || "model").replace(/[^A-Za-z0-9_-]/g, "-"); return `model-${safeModelId}-${suffix}`; @@ -596,7 +711,7 @@ function renderModalModels(models) { } if (!models || !models.length) { - modelsListEl.innerHTML = "

No models loaded yet.

"; + modelsListEl.innerHTML = "

No models loaded yet.

"; return; } @@ -604,10 +719,14 @@ function renderModalModels(models) { models.forEach((model) => { const wrapper = document.createElement("div"); wrapper.className = "border rounded p-2 mb-2"; - const deploymentName = model.deploymentName || ""; + const requestName = getModelRequestName(model); const modelName = model.modelName || ""; - const displayName = model.displayName || deploymentName; + const displayName = model.displayName || requestName; const description = model.description || ""; + const responseLength = getModelResponseLength(model); + const requestNameLabel = isCustomProvider() && customApiTypeUsesModelName() + ? "Model Name" + : "Deployment Name"; const modelId = model.id || generateId(); model.id = modelId; @@ -624,18 +743,29 @@ function renderModalModels(models) { const fieldsRow = createElement("div", "row g-2 mt-2"); const deploymentCol = createElement("div", "col-md-4"); - deploymentCol.appendChild(createSmallLabel("Deployment")); - deploymentCol.appendChild(createModelTextInput(modelId, "deploymentNameFor", deploymentName)); + deploymentCol.appendChild(createSmallLabel(requestNameLabel)); + deploymentCol.appendChild(createModelTextInput(modelId, "requestModelFor", requestName)); const displayCol = createElement("div", "col-md-4"); displayCol.appendChild(createSmallLabel("Display Name")); displayCol.appendChild(createModelTextInput(modelId, "displayNameFor", displayName)); - const modelNameCol = createElement("div", "col-md-4"); - modelNameCol.appendChild(createSmallLabel("Model Name")); - const modelNameInput = createModelTextInput(modelId, "modelNameFor", modelName, true); - modelNameCol.appendChild(modelNameInput); + const responseLengthCol = createElement("div", "col-md-4"); + const responseLengthLabel = createSmallLabel("Response Length"); + responseLengthLabel.htmlFor = getModelIconDomId(modelId, "response-length"); + responseLengthCol.appendChild(responseLengthLabel); + responseLengthCol.appendChild(createModelResponseLengthInput(modelId, responseLength)); + const responseLengthHelp = createElement("div", "form-text"); + responseLengthHelp.id = getModelIconDomId(modelId, "response-length-help"); + responseLengthHelp.textContent = "Optional output token ceiling for standard chat responses."; + responseLengthCol.appendChild(responseLengthHelp); fieldsRow.appendChild(deploymentCol); fieldsRow.appendChild(displayCol); - fieldsRow.appendChild(modelNameCol); + fieldsRow.appendChild(responseLengthCol); + if (!isCustomProvider()) { + const modelNameCol = createElement("div", "col-md-4"); + modelNameCol.appendChild(createSmallLabel("Model Name")); + modelNameCol.appendChild(createModelTextInput(modelId, "modelNameFor", modelName, true)); + fieldsRow.appendChild(modelNameCol); + } const descriptionWrapper = createElement("div", "mt-2"); descriptionWrapper.appendChild(createSmallLabel("Description")); @@ -650,10 +780,27 @@ function renderModalModels(models) { iconWrapper.appendChild(createSmallLabel("Icon")); iconWrapper.appendChild(createModelIconEditor(model, modelId)); + const actions = createElement("div", "d-flex gap-2 mt-2"); + const testButton = document.createElement("button"); + testButton.type = "button"; + testButton.className = "btn btn-sm btn-outline-secondary"; + testButton.dataset.action = "test-model"; + testButton.dataset.modelId = modelId; + testButton.textContent = "Test Connection"; + const removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "btn btn-sm btn-outline-danger"; + removeButton.dataset.action = "remove-model"; + removeButton.dataset.modelId = modelId; + removeButton.textContent = "Remove"; + actions.appendChild(testButton); + actions.appendChild(removeButton); + wrapper.appendChild(checkWrapper); wrapper.appendChild(fieldsRow); wrapper.appendChild(descriptionWrapper); wrapper.appendChild(iconWrapper); + wrapper.appendChild(actions); fragment.appendChild(wrapper); }); @@ -670,31 +817,42 @@ function collectModalModels() { const updated = modalModels.map((model) => ({ ...model })); updated.forEach((model) => { const checkbox = modelsListEl.querySelector(`input[data-model-id="${model.id}"]`); - const deploymentInput = modelsListEl.querySelector(`input[data-deployment-name-for="${model.id}"]`); + const requestModelInput = modelsListEl.querySelector(`input[data-request-model-for="${model.id}"]`); const displayInput = modelsListEl.querySelector(`input[data-display-name-for="${model.id}"]`); const descriptionInput = modelsListEl.querySelector(`textarea[data-description-for="${model.id}"]`); + const responseLengthInput = modelsListEl.querySelector(`input[data-response-length-for="${model.id}"]`); const iconEditor = findModelEditor(model.id); + const responseLength = responseLengthInput ? normalizeModelResponseLength(responseLengthInput.value) : ""; + if (responseLength === null) { + throw new Error("Response length must be a positive whole number."); + } model.enabled = checkbox ? checkbox.checked : model.enabled; - model.deploymentName = deploymentInput ? deploymentInput.value.trim() : model.deploymentName; + setModelRequestName(model, requestModelInput ? requestModelInput.value : getModelRequestName(model)); model.displayName = displayInput ? displayInput.value.trim() : model.displayName; model.icon = iconEditor ? getIconPayload(iconEditor, MODEL_ICON_CONTROL_CONFIG) : model.icon || {}; model.description = descriptionInput ? descriptionInput.value.trim() : model.description; + if (responseLength) { + model.responseLength = responseLength; + } else { + delete model.responseLength; + } }); return updated; } async function testModelConnection(model) { const payload = buildEndpointPayload(); - if (!payload || !model?.deploymentName) { - showToast("Model deployment name is required for testing.", "warning"); + const requestModel = getModelRequestName(model); + if (!payload || !requestModel) { + showToast(`${isCustomProvider() && customApiTypeUsesModelName() ? "Model" : "Deployment"} name is required for testing.`, "warning"); return; } + const testModel = {}; + setModelRequestName(testModel, requestModel); const requestBody = { ...payload, - model: { - deploymentName: model.deploymentName - } + model: testModel }; try { @@ -715,6 +873,10 @@ async function testModelConnection(model) { } async function fetchModels() { + if (isCustomProvider()) { + showToast("Model discovery is unavailable for Custom endpoints. Add models manually.", "warning"); + return; + } const payload = buildEndpointPayload(); if (!payload) { return; @@ -780,6 +942,8 @@ function buildEndpointPayload() { const name = endpointNameInput.value.trim(); const endpoint = endpointUrlInput.value.trim(); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + const apiType = getCustomApiType(); const projectNameFromEndpoint = isFoundryProvider(provider) ? syncProjectNameFromEndpoint() : ""; const projectName = projectNameFromEndpoint || endpointProjectInput?.value.trim() || ""; const projectApiVersion = getSelectedVersionValue( @@ -794,11 +958,21 @@ function buildEndpointPayload() { ); const subscriptionId = endpointSubscriptionInput?.value.trim() || ""; const resourceGroup = endpointResourceGroupInput?.value.trim() || ""; - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const authType = customProvider ? "api_key" : (endpointAuthTypeSelect?.value || "managed_identity"); const existingEndpoint = workspaceEndpoints.find((savedEndpoint) => savedEndpoint.id === endpointId); - if (!name || !endpoint || !openAiApiVersion) { - showToast("Endpoint name, URL, and OpenAI API version are required.", "warning"); + if (!name || !endpoint) { + showToast("Endpoint name and URL are required.", "warning"); + return null; + } + + if (customProvider && !/^https:\/\//i.test(endpoint)) { + showToast("Custom endpoint URLs must use HTTPS.", "warning"); + return null; + } + + if ((!customProvider || apiType === "azure_openai") && !openAiApiVersion) { + showToast("OpenAI API version is required.", "warning"); return null; } @@ -817,7 +991,7 @@ function buildEndpointPayload() { return null; } - const auth = { + let auth = { type: authType, managed_identity_type: miTypeSelect?.value || "system_assigned", managed_identity_client_id: miClientIdInput?.value.trim() || "", @@ -829,6 +1003,12 @@ function buildEndpointPayload() { custom_authority: endpointCustomAuthorityInput?.value.trim() || "", foundry_scope: endpointFoundryScopeInput?.value.trim() || "" }; + if (customProvider) { + auth = { + type: "api_key", + api_key: apiKeyInput?.value.trim() || "" + }; + } const hasStoredApiKey = authType === "api_key" && Boolean(existingEndpoint?.has_api_key); const hasStoredClientSecret = authType === "service_principal" && Boolean(existingEndpoint?.has_client_secret); @@ -854,17 +1034,21 @@ function buildEndpointPayload() { return null; } - const management = provider === "aoai" ? { + const management = !customProvider && provider === "aoai" ? { subscription_id: subscriptionId, resource_group: resourceGroup } : {}; - const connection = { - endpoint, - openai_api_version: openAiApiVersion - }; + const connection = { endpoint }; + if (customProvider && apiType === "azure_openai") { + connection.api_version = openAiApiVersion; + } else if (customProvider && apiType === "anthropic") { + connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; + } else if (!customProvider) { + connection.openai_api_version = openAiApiVersion; + } - if (isFoundryProvider(provider)) { + if (!customProvider && isFoundryProvider(provider)) { connection.project_api_version = projectApiVersion; if (projectName) { connection.project_name = projectName; @@ -874,6 +1058,7 @@ function buildEndpointPayload() { return { id: endpointId, provider, + ...(customProvider ? { api_type: apiType } : {}), name, connection, management, @@ -881,7 +1066,8 @@ function buildEndpointPayload() { }; } -function saveEndpoint() { +async function saveEndpoint() { + const previousEndpoints = [...workspaceEndpoints]; try { const payload = buildEndpointPayload(); if (!payload) { @@ -899,7 +1085,8 @@ function saveEndpoint() { id: endpointId, name: payload.name, provider: payload.provider, - enabled: true, + ...(payload.api_type ? { api_type: payload.api_type } : {}), + enabled: existingEndpoint ? existingEndpoint.enabled !== false : true, auth: payload.auth, connection: payload.connection, management: payload.management, @@ -915,41 +1102,60 @@ function saveEndpoint() { workspaceEndpoints.push(endpointData); } - persistEndpoints(); + await persistEndpoints(); renderEndpoints(); endpointModal.hide(); showToast("Endpoint saved successfully.", "success"); } catch (error) { + workspaceEndpoints = previousEndpoints; console.error("Error saving endpoint", error); showToast(error.message || "Failed to save endpoint.", "danger"); } } -function persistEndpoints() { - fetch(endpointsApi, { +async function persistEndpoints() { + const response = await fetch(endpointsApi, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ endpoints: workspaceEndpoints }) - }).catch((error) => { - console.error("Failed to save endpoints", error); - showToast("Failed to save endpoints.", "danger"); }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.error || "Failed to save endpoints."); + } + if (Array.isArray(data.endpoints)) { + workspaceEndpoints = [...data.endpoints]; + } } -function toggleEndpoint(endpointId) { +async function toggleEndpoint(endpointId) { const endpoint = workspaceEndpoints.find((item) => item.id === endpointId); if (!endpoint) { return; } + const previousEnabled = endpoint.enabled; endpoint.enabled = !endpoint.enabled; - persistEndpoints(); - renderEndpoints(); + try { + await persistEndpoints(); + renderEndpoints(); + } catch (error) { + endpoint.enabled = previousEnabled; + console.error("Failed to update endpoint", error); + showToast(error.message || "Failed to update endpoint.", "danger"); + } } -function deleteEndpoint(endpointId) { +async function deleteEndpoint(endpointId) { + const previousEndpoints = workspaceEndpoints; workspaceEndpoints = workspaceEndpoints.filter((item) => item.id !== endpointId); - persistEndpoints(); - renderEndpoints(); + try { + await persistEndpoints(); + renderEndpoints(); + } catch (error) { + workspaceEndpoints = previousEndpoints; + console.error("Failed to delete endpoint", error); + showToast(error.message || "Failed to delete endpoint.", "danger"); + } } function handleTableClick(event) { @@ -979,18 +1185,48 @@ function handleTableClick(event) { function addManualModel() { modalModels = collectModalModels(); - modalModels.push({ + const model = { id: generateId(), - deploymentName: "", - modelName: "", displayName: "", icon: {}, description: "", enabled: true - }); + }; + setModelRequestName(model, ""); + modalModels.push(model); renderModalModels(modalModels); } +function handleModelListClick(event) { + const button = event.target.closest("button[data-action]"); + if (!button) { + return; + } + + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the model.", "danger"); + return; + } + + const modelId = button.dataset.modelId; + const model = modalModels.find((item) => item.id === modelId); + if (!model) { + return; + } + + if (button.dataset.action === "remove-model") { + modalModels = modalModels.filter((item) => item.id !== modelId); + renderModalModels(modalModels); + return; + } + + if (button.dataset.action === "test-model") { + testModelConnection(model); + } +} + function escapeHtml(value) { if (!value) return ""; return value.replace(/[&<>"']/g, (char) => ({ @@ -1054,8 +1290,29 @@ function initialize() { if (endpointProviderSelect) { endpointProviderSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the endpoint provider.", "danger"); + return; + } + syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); updateAuthVisibility(); + }); + } + + if (endpointApiTypeSelect) { + endpointApiTypeSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the API type.", "danger"); + return; + } syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); + updateAuthVisibility(); }); } @@ -1094,6 +1351,10 @@ function initialize() { if (addModelBtn) { addModelBtn.addEventListener("click", addManualModel); } + + if (modelsListEl) { + modelsListEl.addEventListener("click", handleModelListClick); + } } if (document.readyState === "loading") { diff --git a/application/single_app/templates/_multiendpoint_modal.html b/application/single_app/templates/_multiendpoint_modal.html index ed14026bc..5facba8e8 100644 --- a/application/single_app/templates/_multiendpoint_modal.html +++ b/application/single_app/templates/_multiendpoint_modal.html @@ -21,6 +21,7 @@
Identity se
  • Azure OpenAI: assign Reader plus Cognitive Services OpenAI User on the Azure OpenAI resource when using managed identity or service principal model discovery.
  • Foundry (classic): assign Foundry User, or Azure AI User where older role names still appear, on the target Foundry project or backing resource.
  • New Foundry: use the same Foundry project access as classic Foundry, then select the New Foundry provider and the project endpoint in this modal.
    • +
  • Custom: select an API type, enter an HTTPS endpoint and API key, then add models manually.
  • Provider setup: for Foundry project model endpoints, keep OpenAI API Version at endpoint default v1. Use separate endpoints when Grok, Meta/Llama, DeepSeek, OpenAI-compatible, or other model families need different project settings, auth, or manual deployment rows.
  • API Key: use for inference-only endpoints or APIM paths. Model and Foundry project discovery requires managed identity or service principal RBAC.
    • @@ -38,11 +39,21 @@
    Identity se +
    - For APIM, choose the matching provider with API key auth. If using classic Foundry, use Foundry (classic). If using the application-based runtime, use New Foundry. + Choose Custom for a manually configured API type and model list.
    +
    + + +
    The API type controls request paths, model identifiers, and headers for this Custom endpoint.
    +
    @@ -73,6 +84,11 @@
    Identity se
    For Foundry project endpoints, Project API Version controls discovery and usually stays v1. OpenAI API Version controls the normalized /openai/v1 inference client and should stay Endpoint default (v1); the /v1 path does not allow an api-version query. Split model families into separate endpoints when they need different project settings or auth. Claude deployments are detected from the model name and use the Anthropic messages protocol.
    +
    + + +
    Sent as the anthropic-version request header.
    +
    @@ -144,7 +160,7 @@
    Identity se
    - API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry. + API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry.
    @@ -241,6 +257,11 @@
    Provider selection
    Application-based Foundry runtime, New Foundry agents, and project model deployments. Use the New Foundry project endpoint. Set Project API Version for discovery, usually v1, and keep OpenAI API Version at endpoint default v1 for the normalized /openai/v1 inference path. + + Custom + Manually configured OpenAI API, Azure OpenAI API, or Anthropic models. + Use an HTTPS endpoint with API key authentication, select the API type, and add models manually. +
    diff --git a/application/single_app/templates/admin/_panes/extraction.html b/application/single_app/templates/admin/_panes/extraction.html index 0a12f26b3..890fa93e9 100644 --- a/application/single_app/templates/admin/_panes/extraction.html +++ b/application/single_app/templates/admin/_panes/extraction.html @@ -681,14 +681,14 @@
    {% for endpoint in settings.model_endpoints if endpoint.enabled %} {% for m in endpoint.models if m.enabled %} {% if is_vision_capable_model is defined and is_vision_capable_model(m) %} - {% set option_value = m.deploymentName %} + {% set option_value = m.modelName or m.deploymentName %} {% endif %} {% endfor %} diff --git a/application/single_app/templates/admin/_panes/model-endpoints.html b/application/single_app/templates/admin/_panes/model-endpoints.html index 5c9683ad4..68b833677 100644 --- a/application/single_app/templates/admin/_panes/model-endpoints.html +++ b/application/single_app/templates/admin/_panes/model-endpoints.html @@ -9,7 +9,7 @@
    Model Endpoints

    - Manage multiple AI model endpoints (Azure OpenAI and Azure AI Foundry). When enabled, model selection in chat is driven by these endpoints. + Manage Azure OpenAI, Foundry, New Foundry, and Custom model endpoints. When enabled, model selection in chat is driven by these endpoints.

    {% if not settings.enable_multi_model_endpoints %} @@ -19,6 +19,22 @@
    {% endif %} +
    + + +
    + Custom endpoints require HTTPS. When disabled, hosts resolving to private addresses are rejected; loopback, link-local, metadata, and direct IP targets are always rejected. +
    +
    + {% if settings.enable_semantic_kernel %}
    diff --git a/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_PROVIDER.md b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_PROVIDER.md new file mode 100644 index 000000000..3d3c3b35e --- /dev/null +++ b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_PROVIDER.md @@ -0,0 +1,116 @@ +# Custom Model Endpoint Provider + +## Overview and Purpose + +The Custom provider lets administrators and authorized workspace owners configure chat models through a supported API contract without changing the provider label shown throughout SimpleChat. Custom endpoints are available in global, personal, and group model endpoint scopes and use manual model entry. + +**Implemented in version: 0.250.172** + +**Issue:** [#1222](https://github.com/microsoft/simplechat/issues/1222) + +## Dependencies + +- Multi-model endpoint management +- Existing global, personal, and group endpoint governance +- Existing Key Vault model-endpoint secret storage +- OpenAI Python client and the existing Anthropic Messages adapter + +## Technical Specifications + +### Architecture + +Custom endpoints persist `provider: "custom"` and one explicit `api_type`. The API type is authoritative at runtime; endpoint paths and model names do not change the selected protocol. + +| API Type | Model Identifier | Version Field | Request Contract | +|---|---|---|---| +| OpenAI API (`openai`) | `modelName` | None | OpenAI-compatible Chat Completions under `/v1/` | +| Azure OpenAI API (`azure_openai`) | `deploymentName` | `connection.api_version` | Azure OpenAI Chat Completions | +| Anthropic (`anthropic`) | `modelName` | `connection.anthropic_version` | Anthropic Messages under `/v1/messages` | + +Every model keeps a stable SimpleChat `id` for selection and authorization. The stable ID is never sent as the provider model identifier. + +### Authentication and Secret Storage + +- API key is the only supported Custom authentication type. +- API keys use the existing model-endpoint Key Vault flow when Key Vault secret storage is enabled. +- Frontend endpoint payloads contain only `has_api_key`; they never contain a stored key. +- Existing blank-on-edit behavior preserves a stored API key. + +### Endpoint Safety + +- Custom endpoint URLs must use HTTPS and a fully qualified DNS hostname. +- Embedded credentials, query strings, fragments, direct IP literals, and single-label hosts are rejected. +- Loopback, link-local, metadata/platform, multicast, reserved, and unspecified addresses are always rejected. +- Private addresses are rejected unless an administrator enables **Allow private Custom endpoint hosts**. +- DNS and URL policy are checked when configuration is saved and again before runtime client construction. +- Each direct Custom connection is pinned to the addresses from its validated DNS lookup, preventing a second DNS resolution from redirecting the request to a blocked address. +- Direct Custom requests do not follow redirects. +- Provider response bodies and raw provider exceptions are not returned for direct Custom Anthropic failures. + +### API Endpoints + +- `POST /api/models/test-model` +- `POST /api/user/models/test-model` +- `POST /api/group/models/test-model` +- `GET|POST /api/user/model-endpoints` +- `GET|POST /api/group/model-endpoints` + +Model discovery endpoints deliberately reject Custom providers before network dispatch. Models must be entered with **Add Model**. + +### Configuration + +- `enable_multi_model_endpoints`: enables endpoint-backed model selection. +- `allow_user_custom_endpoints`: allows authorized personal endpoint management. +- `allow_group_custom_endpoints`: allows authorized group endpoint management. +- `allow_private_custom_model_endpoints`: permits private DNS results for Custom endpoints while retaining the always-blocked address classes. + +### File Structure + +- Canonical types: `application/single_app/functions_model_endpoint_types.py` +- Validation and URL policy: `application/single_app/functions_model_endpoint_validation.py` +- Runtime construction: `application/single_app/functions_model_endpoint_runtime.py` +- Protocol adapters: `application/single_app/model_endpoint_clients.py` +- Save and test routes: `application/single_app/route_backend_models.py` +- Shared modal: `application/single_app/templates/_multiendpoint_modal.html` +- Admin editor: `application/single_app/static/js/admin/admin_model_endpoints.js` +- Workspace editor: `application/single_app/static/js/workspace/workspace_model_endpoints.js` + +## Usage Instructions + +### Configure an Endpoint + +1. Open Model Endpoints in Admin Settings, Personal Workspace, or Group Workspace. +2. Add an endpoint and choose **Custom**. +3. Select **OpenAI API**, **Azure OpenAI API**, or **Anthropic**. +4. Enter the HTTPS endpoint and API key. +5. For Azure OpenAI API, enter the API version. For Anthropic, confirm or change the Anthropic Version. +6. Select **Add Model** and enter a Model Name or Deployment Name as indicated. +7. Optionally set Display Name, Response Length, Description, Icon, and Enabled state. +8. Test the model connection, then save the endpoint. + +### Scope and Governance + +Global endpoints remain controlled by administrators. Personal and group endpoints continue to use their existing feature flags, role checks, governance decisions, active-group checks, endpoint/model IDs, and Key Vault scope. Runtime requests resolve the saved endpoint and model instead of trusting client-supplied connection details. + +## Testing and Validation + +- `functional_tests/test_custom_model_endpoint_provider.py` +- Existing model endpoint normalization, protocol, Key Vault, workspace, streaming, summary, metadata, multimodal, and route-policy regressions +- JavaScript syntax checks for both endpoint editors and the chat model selector +- Python compilation checks for all modified runtime and route modules + +## Performance Considerations + +Custom model discovery is disabled, so configuration does not perform model-list requests. URL validation performs DNS resolution during save and runtime construction, and the protected transport resolves again when opening a connection so it can pin the validated addresses. Runtime latency and model capability depend on the configured API. + +## Known Limitations + +- API key is the only Custom authentication type. +- Models are entered manually; model discovery is unavailable. +- Supported inference contracts are OpenAI-compatible Chat Completions, Azure OpenAI Chat Completions, and Anthropic Messages. +- Custom endpoints do not add embeddings, image generation, OpenAI Responses, arbitrary headers, or non-HTTPS transport. +- A configured model can only use features supported by its selected API contract. + +## Version Reference + +The application version was updated in `application/single_app/config.py` to **0.250.172**. diff --git a/docs/explanation/features/v0.241.001/MODEL_ENDPOINT_API_KEY_MANUAL_MODELS.md b/docs/explanation/features/v0.241.001/MODEL_ENDPOINT_API_KEY_MANUAL_MODELS.md index 709920959..81866f892 100644 --- a/docs/explanation/features/v0.241.001/MODEL_ENDPOINT_API_KEY_MANUAL_MODELS.md +++ b/docs/explanation/features/v0.241.001/MODEL_ENDPOINT_API_KEY_MANUAL_MODELS.md @@ -1,4 +1,4 @@ -# Model Endpoint API Key Manual Models (v0.236.019) +# Model Endpoint API Key Manual Models (v0.250.172) ## Overview and Purpose Adds manual model entry for API key-authenticated endpoints, with per-model connection tests and guidance to prefer identity-based discovery. @@ -6,6 +6,8 @@ Adds manual model entry for API key-authenticated endpoints, with per-model conn ## Version Implemented Fixed/Implemented in version: **0.236.019** +Updated in version: **0.250.172** + ## Dependencies - Admin model endpoint modal - Backend model test endpoint @@ -16,6 +18,8 @@ Fixed/Implemented in version: **0.236.019** - API key endpoints skip discovery and allow manual model entries. - Each model row supports per-model connection testing. - Service principal auth includes management cloud and custom authority inputs. +- The Custom provider uses API-key-only authentication and always uses manual model entry. +- Custom OpenAI API and Anthropic models use Model Name; Custom Azure OpenAI API models use Deployment Name. ### API Endpoints - `/api/models/test-model` — tests a specific model deployment using the endpoint settings. @@ -25,8 +29,9 @@ Fixed/Implemented in version: **0.236.019** - `auth.custom_authority` — custom authority URL for service principal auth. ### File Structure -- Modal UI: application/single_app/templates/admin_settings.html +- Modal UI: application/single_app/templates/_multiendpoint_modal.html - Modal logic: application/single_app/static/js/admin/admin_model_endpoints.js +- Workspace modal logic: application/single_app/static/js/workspace/workspace_model_endpoints.js - Backend test endpoint: application/single_app/route_backend_models.py ## Usage Instructions @@ -35,6 +40,12 @@ Fixed/Implemented in version: **0.236.019** 2. Use Add Model to enter deployment name, display name, and description. 3. Use the per-model Test Connection button to verify access. +### Custom Provider Flow +1. Choose Provider: Custom. +2. Select OpenAI API, Azure OpenAI API, or Anthropic. +3. Enter the HTTPS endpoint and API key. +4. Add models manually using the type-specific Model Name or Deployment Name field. + ### Service Principal Flow 1. Choose Authentication Type: Service Principal. 2. Select Management Cloud (Public/Government/Custom). @@ -42,9 +53,12 @@ Fixed/Implemented in version: **0.236.019** ## Testing and Validation - Functional test: functional_tests/test_model_endpoints_api_key_manual_models.py +- Functional test: functional_tests/test_custom_model_endpoint_provider.py ## Known Limitations - API key auth supports inference only; discovery requires identity-based auth. +- Custom endpoints never use discovery. ## Reference to Config Version Update -- Version updated in application/single_app/config.py to **0.236.019**. +- Initial version updated in application/single_app/config.py to **0.236.019**. +- Custom provider update in application/single_app/config.py: **0.250.172**. diff --git a/docs/explanation/features/v0.241.001/WORKSPACE_MULTI_ENDPOINTS.md b/docs/explanation/features/v0.241.001/WORKSPACE_MULTI_ENDPOINTS.md index c0578df49..04654989b 100644 --- a/docs/explanation/features/v0.241.001/WORKSPACE_MULTI_ENDPOINTS.md +++ b/docs/explanation/features/v0.241.001/WORKSPACE_MULTI_ENDPOINTS.md @@ -5,7 +5,7 @@ Workspace multi-endpoint management extends the admin multi-endpoint system to p **Implemented in version: 0.236.045** -**Updated in version: 0.242.071** +**Updated in version: 0.250.172** ## Dependencies - Global model endpoints configured in admin settings @@ -19,6 +19,8 @@ Workspace multi-endpoint management extends the admin multi-endpoint system to p - Group endpoints are stored on group documents under `model_endpoints`. - Agent modal requests a combined, sanitized endpoint list for model selection. - Foundry agent lookup uses endpoint IDs to resolve authentication and list agents. +- Custom endpoints use an explicit OpenAI API, Azure OpenAI API, or Anthropic contract and manual model entry. +- Runtime calls resolve the saved endpoint and model by scope; client-supplied connection details do not replace stored personal or group configuration. ### API Endpoints - `GET /api/user/model-endpoints` / `POST /api/user/model-endpoints` @@ -32,6 +34,8 @@ Workspace multi-endpoint management extends the admin multi-endpoint system to p ### Configuration - Global toggle: `enable_multi_model_endpoints` in [application/single_app/config.py](application/single_app/config.py) - Workspace endpoints stored per user and per group +- `allow_user_custom_endpoints` and `allow_group_custom_endpoints` control personal and group endpoint management. +- `allow_private_custom_model_endpoints` is an administrator-controlled network policy shared by all Custom endpoint scopes. ### File Structure - Frontend templates: [application/single_app/templates/workspace.html](application/single_app/templates/workspace.html), [application/single_app/templates/group_workspaces.html](application/single_app/templates/group_workspaces.html), [application/single_app/templates/_agent_modal.html](application/single_app/templates/_agent_modal.html) @@ -44,6 +48,8 @@ Workspace multi-endpoint management extends the admin multi-endpoint system to p 2. Users open Personal Workspace or Group Workspace and add endpoints under the new Workspace/Group Model Endpoints card. 3. In the agent modal, select a model from the combined endpoint list. +For a Custom endpoint, choose its API Type, enter the HTTPS endpoint and API key, and add each model manually. OpenAI API and Anthropic use Model Name; Azure OpenAI API uses Deployment Name. + Use the **Setup Guide** button in the endpoint table or Model Endpoint modal for in-product RBAC reminders. For Azure OpenAI, Foundry (classic), or New Foundry managed identity and service principal setup, see [Configure Model Endpoint Identity]({{ '/how-to/model_endpoint_identity_setup/' | relative_url }}). The same RBAC guidance applies to global, personal, and group-scoped endpoints. ### Foundry Agent Import @@ -63,7 +69,8 @@ Use the **Setup Guide** button in the endpoint table or Model Endpoint modal for - Verify Foundry agent list import using configured endpoints. ## Performance Considerations -- Model discovery uses on-demand API calls to Azure/Foundry endpoints. +- Model discovery uses on-demand API calls to Azure/Foundry endpoints. Custom endpoints do not perform discovery. ## Known Limitations - Workspace endpoints require configured credentials; only stored secrets are used for runtime resolution. +- Custom endpoints support API-key authentication and manual chat-model entry only. diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index bcbde2f12..b07dc95ce 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,16 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.010)** + +#### New Features + +* **Custom Model Endpoint Provider** + * Added manually configured Custom endpoints for OpenAI API, Azure OpenAI API, and Anthropic chat models across global, personal, and group scopes. + * Added type-specific model identifiers, API-key authentication, connection testing, response-length controls, and Anthropic Version support without model discovery. + * Enforced HTTPS, DNS/address safety with connection-time address pinning, runtime URL revalidation, redirect refusal, Key Vault secret handling, and an administrator-controlled private-host policy. + * (Ref: #1222, Custom model endpoints, `functions_model_endpoint_runtime.py`, `_multiendpoint_modal.html`, `CUSTOM_MODEL_ENDPOINT_PROVIDER.md`) + ### **(v0.261.009)** #### Bug Fixes @@ -18,6 +28,7 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Ver * This preserves SQL ODBC and Playwright Chromium runtime packaging while avoiding path-type collisions against evolving base-image filesystem layouts. * (Ref: `Dockerfile`, `test_sql_container_odbc_runtime.py`, `test_deep_research_chromium_build_opt_out.py`, [Distroless Runtime Overlay Path Fix](fixes/DISTROLESS_RUNTIME_OVERLAY_PATH_FIX.md)) + ### **(v0.261.007)** #### Bug Fixes diff --git a/functional_tests/test_admin_multi_endpoint_persistence_guard.py b/functional_tests/test_admin_multi_endpoint_persistence_guard.py index 967c26fa7..91ec579e1 100644 --- a/functional_tests/test_admin_multi_endpoint_persistence_guard.py +++ b/functional_tests/test_admin_multi_endpoint_persistence_guard.py @@ -1,16 +1,15 @@ -#!/usr/bin/env python3 # test_admin_multi_endpoint_persistence_guard.py +#!/usr/bin/env python3 """ Functional test for admin multi-endpoint persistence guard. -Version: 0.239.199 -Implemented in: 0.239.199 +Version: 0.250.172 +Implemented in: 0.239.199; updated in 0.250.172 This test ensures that once multi-endpoint model management is enabled, admin settings saves preserve it even if the checkbox is omitted from later form posts, and that the backend save helper enforces the same one-way behavior. """ -import importlib import json import os import sys @@ -27,6 +26,10 @@ sys.path.append(ROOT_DIR) sys.path.append(SINGLE_APP_ROOT) +from test_model_endpoint_normalization_backend import ( + _load_functions_settings_module as load_functions_settings_module, +) + def read_file(path): with open(path, 'r', encoding='utf-8') as file_handle: @@ -42,34 +45,7 @@ def _restore_modules(original_modules): def _load_functions_settings_module(): - config_stub = types.ModuleType('config') - config_stub.json = json - config_stub.re = __import__('re') - config_stub.WORD_CHUNK_SIZE = 400 - config_stub.video_indexer_endpoint = '' - config_stub.cosmos_settings_container = types.SimpleNamespace(upsert_item=lambda item: item) - - appinsights_stub = types.ModuleType('functions_appinsights') - appinsights_stub.log_event = lambda *args, **kwargs: None - - cache_stub = types.ModuleType('app_settings_cache') - cache_stub.get_settings_cache = lambda: None - cache_stub.update_settings_cache = lambda settings: None - - original_modules = {} - for module_name, module_stub in { - 'config': config_stub, - 'functions_appinsights': appinsights_stub, - 'app_settings_cache': cache_stub, - }.items(): - original_modules[module_name] = sys.modules.get(module_name) - sys.modules[module_name] = module_stub - - module_name = 'application.single_app.functions_settings' - original_modules[module_name] = sys.modules.get(module_name) - sys.modules.pop(module_name, None) - module = importlib.import_module(module_name) - return module, original_modules + return load_functions_settings_module() def test_admin_settings_route_preserves_enabled_multi_endpoint_flag(): diff --git a/functional_tests/test_custom_model_endpoint_provider.py b/functional_tests/test_custom_model_endpoint_provider.py new file mode 100644 index 000000000..5a446046e --- /dev/null +++ b/functional_tests/test_custom_model_endpoint_provider.py @@ -0,0 +1,762 @@ +# test_custom_model_endpoint_provider.py +#!/usr/bin/env python3 +""" +Functional test for the Custom model endpoint provider. +Version: 0.250.172 +Implemented in: 0.250.172 + +This test validates canonical model identifiers, API-type precedence, Custom +endpoint URL safety, direct Anthropic request behavior, normalization, secret +sanitization, and the admin/workspace UI contract without network traffic. +""" + +import asyncio +import importlib +import socket +import sys +import types +from pathlib import Path +from unittest.mock import patch + + +ROOT = Path(__file__).resolve().parents[1] +APP_DIR = ROOT / "application" / "single_app" +sys.path.insert(0, str(APP_DIR)) +sys.path.insert(0, str(ROOT)) + +from functions_model_endpoint_types import ( # noqa: E402 + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_OPENAI, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import ( # noqa: E402 + ModelEndpointValidationError, + validate_custom_model_endpoint, + validate_custom_model_endpoint_url, +) +from model_endpoint_clients import ( # noqa: E402 + MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + _PinnedCustomEndpointAsyncBackend, + _PinnedCustomEndpointSyncBackend, + AnthropicChatCompletionClient, + SanitizedCustomChatCompletionClient, + build_custom_openai_async_http_client, + build_custom_openai_sync_http_client, + infer_model_endpoint_protocol, + normalize_anthropic_messages_url, + normalize_custom_openai_base_url, + sanitize_custom_async_openai_client, +) +from test_model_endpoint_normalization_backend import ( # noqa: E402 + _load_functions_settings_module, + _restore_modules, +) + + +PUBLIC_ADDRESS_INFO = [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("93.184.216.34", 443), + ) +] + + +def assert_validation_error(callable_value, expected_message): + """Assert a configuration is rejected with a stable, user-safe message.""" + try: + callable_value() + except ModelEndpointValidationError as exc: + assert expected_message in str(exc) + return + raise AssertionError(f"Expected ModelEndpointValidationError containing {expected_message!r}") + + +def build_custom_endpoint(api_type, model, connection=None): + """Build a valid Custom endpoint record for validation tests.""" + endpoint_connection = {"endpoint": "https://models.example.com"} + endpoint_connection.update(connection or {}) + return { + "id": f"custom-{api_type}", + "name": "Custom Models", + "provider": "custom", + "api_type": api_type, + "enabled": True, + "auth": {"type": "api_key", "api_key": "test-key"}, + "connection": endpoint_connection, + "models": [{"id": "stable-model-id", "enabled": True, **model}], + } + + +def load_model_endpoint_runtime_module(): + """Load the runtime helper without initializing the application config.""" + config_stub = types.ModuleType("config") + config_stub.cognitive_services_scope = "https://cognitiveservices.azure.com/.default" + + foundry_runtime_stub = types.ModuleType("foundry_agent_runtime") + foundry_runtime_stub.resolve_authority = lambda auth_settings: None + + settings_stub = types.ModuleType("functions_settings") + settings_stub.resolve_model_endpoint_foundry_scope = ( + lambda auth_settings, endpoint=None: "https://ai.azure.com/.default" + ) + + original_modules = {} + for module_name, module_stub in { + "config": config_stub, + "foundry_agent_runtime": foundry_runtime_stub, + "functions_settings": settings_stub, + }.items(): + original_modules[module_name] = sys.modules.get(module_name) + sys.modules[module_name] = module_stub + + original_modules["functions_model_endpoint_runtime"] = sys.modules.get( + "functions_model_endpoint_runtime" + ) + sys.modules.pop("functions_model_endpoint_runtime", None) + module = importlib.import_module("functions_model_endpoint_runtime") + return module, original_modules + + +def test_request_model_resolution_and_protocol_precedence(): + """Ensure stable IDs and model-name heuristics never override Custom API type.""" + openai_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"modelName": "claude-compatible-model", "deploymentName": "wrong-deployment"}, + ) + anthropic_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + {"modelName": "vendor-model"}, + ) + azure_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + {"deploymentName": "azure-deployment", "modelName": "wrong-model"}, + {"api_version": "2024-05-01-preview"}, + ) + + assert resolve_model_endpoint_request_model( + openai_endpoint, + openai_endpoint["models"][0], + ) == "claude-compatible-model" + assert resolve_model_endpoint_request_model( + anthropic_endpoint, + anthropic_endpoint["models"][0], + ) == "vendor-model" + assert resolve_model_endpoint_request_model( + azure_endpoint, + azure_endpoint["models"][0], + ) == "azure-deployment" + + assert infer_model_endpoint_protocol( + "custom", + "https://models.example.com", + "claude-compatible-model", + MODEL_ENDPOINT_API_TYPE_OPENAI, + ) == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE + assert infer_model_endpoint_protocol( + "custom", + "https://models.example.com", + "gpt-compatible-name", + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + ) == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + assert infer_model_endpoint_protocol( + "custom", + "https://models.example.com/openai/v1", + "claude-name", + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + ) == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI + assert infer_model_endpoint_protocol( + "new_foundry", + "https://eastus.services.ai.azure.com/api/projects/example", + "claude-sonnet", + ) == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + + +def test_custom_endpoint_url_policy(): + """Ensure Custom URLs enforce HTTPS, DNS, and address-class policy.""" + with patch("functions_model_endpoint_validation.socket.getaddrinfo", return_value=PUBLIC_ADDRESS_INFO): + assert validate_custom_model_endpoint_url( + "https://Models.Example.com/custom/" + ) == "https://models.example.com/custom" + + for endpoint, expected_message in ( + ("http://models.example.com", "must use HTTPS"), + ("https://user:password@models.example.com", "embedded credentials"), + ("https://models.example.com?key=value", "query string or fragment"), + ("https://127.0.0.1", "not an IP address"), + ("https://single-label", "fully qualified domain name"), + ("https://localhost", "hostname is blocked"), + ): + assert_validation_error( + lambda endpoint=endpoint: validate_custom_model_endpoint_url(endpoint), + expected_message, + ) + + private_address_info = [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("10.20.30.40", 443), + ) + ] + with patch( + "functions_model_endpoint_validation.socket.getaddrinfo", + return_value=private_address_info, + ): + assert_validation_error( + lambda: validate_custom_model_endpoint_url("https://private.example.com"), + "not enabled", + ) + assert validate_custom_model_endpoint_url( + "https://private.example.com", + allow_private=True, + ) == "https://private.example.com" + + loopback_address_info = [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("127.0.0.1", 443), + ) + ] + with patch( + "functions_model_endpoint_validation.socket.getaddrinfo", + return_value=loopback_address_info, + ): + assert_validation_error( + lambda: validate_custom_model_endpoint_url( + "https://loopback.example.com", + allow_private=True, + ), + "loopback", + ) + + shared_address_info = [ + ( + socket.AF_INET, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + "", + ("100.64.0.1", 443), + ) + ] + with patch( + "functions_model_endpoint_validation.socket.getaddrinfo", + return_value=shared_address_info, + ): + assert_validation_error( + lambda: validate_custom_model_endpoint_url( + "https://shared.example.com", + allow_private=True, + ), + "globally routable", + ) + + with patch( + "functions_model_endpoint_validation.socket.getaddrinfo", + side_effect=socket.gaierror(), + ): + assert_validation_error( + lambda: validate_custom_model_endpoint_url("https://missing.example.com"), + "could not be resolved", + ) + + +def test_custom_endpoint_configuration_validation(): + """Validate required type-specific fields and manual model uniqueness.""" + openai_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"modelName": "openai-model"}, + ) + azure_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + {"deploymentName": "azure-deployment"}, + {"api_version": "2024-05-01-preview"}, + ) + anthropic_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + {"modelName": "anthropic-model"}, + {"anthropic_version": "2023-06-01"}, + ) + + with patch("functions_model_endpoint_validation.socket.getaddrinfo", return_value=PUBLIC_ADDRESS_INFO): + validate_custom_model_endpoint(openai_endpoint) + validate_custom_model_endpoint(azure_endpoint) + validate_custom_model_endpoint(anthropic_endpoint) + + missing_version = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + {"deploymentName": "azure-deployment"}, + ) + assert_validation_error( + lambda: validate_custom_model_endpoint(missing_version), + "Azure OpenAI API version", + ) + + wrong_model_field = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"deploymentName": "deployment-only"}, + ) + assert_validation_error( + lambda: validate_custom_model_endpoint(wrong_model_field), + "Model Name", + ) + + wrong_azure_model_field = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + {"modelName": "model-only"}, + {"api_version": "2024-05-01-preview"}, + ) + assert_validation_error( + lambda: validate_custom_model_endpoint(wrong_azure_model_field), + "Deployment Name", + ) + + no_models = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"modelName": "unused"}, + ) + no_models["models"] = [] + assert_validation_error( + lambda: validate_custom_model_endpoint(no_models), + "at least one manually configured model", + ) + + duplicate_models = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + {"modelName": "duplicate-model"}, + ) + duplicate_models["models"].append({ + "id": "another-stable-id", + "modelName": "DUPLICATE-MODEL", + "enabled": True, + }) + assert_validation_error( + lambda: validate_custom_model_endpoint(duplicate_models), + "must be unique", + ) + + +def test_custom_client_paths_headers_and_redirect_policy(): + """Ensure direct Custom adapters use provider paths, headers, and no redirects.""" + assert normalize_custom_openai_base_url( + "https://models.example.com" + ) == "https://models.example.com/v1/" + assert normalize_custom_openai_base_url( + "https://models.example.com/v1/chat/completions" + ) == "https://models.example.com/v1/" + assert normalize_anthropic_messages_url( + "https://models.example.com", + direct_custom=True, + ) == "https://models.example.com/v1/messages" + + client = AnthropicChatCompletionClient( + endpoint="https://models.example.com", + api_key="test-key", + anthropic_version="2024-01-01", + direct_custom=True, + ) + headers = client._build_headers() + assert headers["x-api-key"] == "test-key" + assert headers["anthropic-version"] == "2024-01-01" + assert "api-key" not in headers + assert "Authorization" not in headers + + image_payload = client._build_payload({ + "model": "anthropic-model", + "messages": [{ + "role": "user", + "content": [{ + "type": "image_url", + "image_url": {"url": "data:image/png;base64,aW1hZ2U="}, + }], + }], + }) + assert image_payload["messages"][0]["content"][0] == { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aW1hZ2U=", + }, + } + + class FakeResponse: + status_code = 200 + closed = False + + def json(self): + return { + "id": "message-1", + "content": [{"type": "text", "text": "ok"}], + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def close(self): + self.closed = True + + class FakeHttpClient: + def __init__(self, response): + self.response = response + self.closed = False + self.send_kwargs = None + + def build_request(self, *args, **kwargs): + return (args, kwargs) + + def send(self, request, **kwargs): + self.send_kwargs = kwargs + return self.response + + def close(self): + self.closed = True + + fake_response = FakeResponse() + fake_http_client = FakeHttpClient(fake_response) + with patch( + "model_endpoint_clients.build_custom_openai_sync_http_client", + return_value=fake_http_client, + ): + client.create( + model="anthropic-model", + messages=[{"role": "user", "content": "test"}], + ) + assert fake_http_client.send_kwargs["follow_redirects"] is False + assert fake_response.closed is True + assert fake_http_client.closed is True + + class FakeErrorResponse: + status_code = 401 + closed = False + + def close(self): + self.closed = True + + fake_error_response = FakeErrorResponse() + fake_error_client = FakeHttpClient(fake_error_response) + with patch( + "model_endpoint_clients.build_custom_openai_sync_http_client", + return_value=fake_error_client, + ): + try: + client.create( + model="anthropic-model", + messages=[{"role": "user", "content": "test"}], + ) + except RuntimeError as exc: + assert "provider secret response" not in str(exc) + assert "status 401" in str(exc) + else: + raise AssertionError("Expected the direct Anthropic client to surface a safe error") + assert fake_error_response.closed is True + assert fake_error_client.closed is True + + class FailingCompletions: + @staticmethod + def create(**kwargs): + raise ValueError("provider secret response") + + fake_sync_client = types.SimpleNamespace( + chat=types.SimpleNamespace(completions=FailingCompletions()) + ) + safe_sync_client = SanitizedCustomChatCompletionClient(fake_sync_client) + try: + safe_sync_client.chat.completions.create(model="test") + except RuntimeError as exc: + assert str(exc) == "Custom model request failed." + assert exc.__cause__ is None + else: + raise AssertionError("Expected direct Custom SDK errors to be sanitized") + + class FailingAsyncCompletions: + @staticmethod + async def create(**kwargs): + raise ValueError("provider secret response") + + fake_async_client = types.SimpleNamespace( + chat=types.SimpleNamespace(completions=FailingAsyncCompletions()) + ) + sanitize_custom_async_openai_client(fake_async_client) + + async def assert_safe_async_error(): + try: + await fake_async_client.chat.completions.create(model="test") + except RuntimeError as exc: + assert str(exc) == "Custom model request failed." + assert exc.__cause__ is None + return + raise AssertionError("Expected direct Custom async SDK errors to be sanitized") + + asyncio.run(assert_safe_async_error()) + + sync_http_client = build_custom_openai_sync_http_client() + async_http_client = build_custom_openai_async_http_client() + try: + assert sync_http_client.follow_redirects is False + assert async_http_client.follow_redirects is False + finally: + sync_http_client.close() + asyncio.run(async_http_client.aclose()) + + sync_backend = _PinnedCustomEndpointSyncBackend(allow_private=True) + sync_connections = [] + + class FakeSyncBackend: + @staticmethod + def connect_tcp(host, port, **kwargs): + sync_connections.append((host, port)) + return "sync-stream" + + sync_backend._backend = FakeSyncBackend() + with patch( + "model_endpoint_clients.resolve_custom_model_endpoint_addresses", + return_value=("93.184.216.34",), + ) as resolve_addresses: + assert sync_backend.connect_tcp("models.example.com", 443) == "sync-stream" + resolve_addresses.assert_called_once_with( + "models.example.com", + 443, + allow_private=True, + ) + assert sync_connections == [("93.184.216.34", 443)] + + async_backend = _PinnedCustomEndpointAsyncBackend(allow_private=False) + async_connections = [] + + class FakeAsyncBackend: + @staticmethod + async def connect_tcp(host, port, **kwargs): + async_connections.append((host, port)) + return "async-stream" + + async_backend._backend = FakeAsyncBackend() + + async def assert_async_dns_pinning(): + with patch( + "model_endpoint_clients.resolve_custom_model_endpoint_addresses", + return_value=("93.184.216.34",), + ): + stream = await async_backend.connect_tcp("models.example.com", 443) + assert stream == "async-stream" + + asyncio.run(assert_async_dns_pinning()) + assert async_connections == [("93.184.216.34", 443)] + + +def test_custom_runtime_client_construction(): + """Ensure shared sync and Semantic Kernel builders honor explicit Custom types.""" + runtime, original_modules = load_model_endpoint_runtime_module() + try: + with patch.object( + runtime, + "validate_custom_model_endpoint_url", + return_value="https://models.example.com", + ) as validate_url: + openai_client, openai_protocol = runtime.build_model_endpoint_sync_chat_client( + {"type": "api_key", "api_key": "test-key"}, + "custom", + "https://models.example.com", + "", + deployment_name="claude-compatible-model", + api_type=MODEL_ENDPOINT_API_TYPE_OPENAI, + allow_private_custom_endpoints=True, + ) + assert openai_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE + assert str(openai_client._client.base_url) == "https://models.example.com/v1/" + validate_url.assert_called_with( + "https://models.example.com", + allow_private=True, + ) + openai_client._client.close() + + azure_client, azure_protocol = runtime.build_model_endpoint_sync_chat_client( + {"type": "api_key", "api_key": "test-key"}, + "custom", + "https://models.example.com", + "2024-05-01-preview", + deployment_name="azure-deployment", + api_type=MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + ) + assert azure_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI + azure_client.close() + + anthropic_client, anthropic_protocol = runtime.build_model_endpoint_sync_chat_client( + {"type": "api_key", "api_key": "test-key"}, + "custom", + "https://models.example.com", + "", + deployment_name="anthropic-model", + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + anthropic_version="2024-01-01", + allow_private_custom_endpoints=True, + ) + assert anthropic_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + assert anthropic_client.direct_custom is True + assert anthropic_client.anthropic_version == "2024-01-01" + assert anthropic_client.allow_private_custom_endpoints is True + + openai_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"modelName": "openai-model"}, + ) + openai_service, openai_service_protocol = ( + runtime.build_semantic_kernel_chat_service_for_model( + "stable-model-id", + {"allow_private_custom_model_endpoints": True}, + model_context={"model_id": "stable-model-id"}, + resolved_model_endpoint=openai_endpoint, + ) + ) + assert openai_service_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE + assert openai_service.ai_model_id == "openai-model" + + anthropic_endpoint = build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + {"modelName": "anthropic-model"}, + {"anthropic_version": "2024-01-01"}, + ) + service, service_protocol = runtime.build_semantic_kernel_chat_service_for_model( + "stable-model-id", + {"allow_private_custom_model_endpoints": True}, + model_context={"model_id": "stable-model-id"}, + resolved_model_endpoint=anthropic_endpoint, + ) + assert service_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC + assert service.ai_model_id == "anthropic-model" + assert service.direct_custom is True + assert service.allow_private_custom_endpoints is True + finally: + sys.modules.pop("functions_model_endpoint_runtime", None) + _restore_modules(original_modules) + + +def test_custom_endpoint_normalization_and_sanitization(): + """Ensure canonical persistence uses the right model field and strips API keys.""" + functions_settings, original_modules = _load_functions_settings_module() + try: + endpoints = [ + build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_OPENAI, + {"modelName": "openai-model", "deploymentName": "remove-me"}, + ), + build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + {"deploymentName": "azure-deployment", "modelName": "remove-me"}, + {"api_version": "2024-05-01-preview"}, + ), + build_custom_endpoint( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + {"modelName": "anthropic-model"}, + {}, + ), + ] + normalized, changed = functions_settings.normalize_model_endpoints(endpoints) + assert changed is True + assert normalized[0]["models"][0]["modelName"] == "openai-model" + assert "deploymentName" not in normalized[0]["models"][0] + assert normalized[1]["models"][0]["deploymentName"] == "azure-deployment" + assert "modelName" not in normalized[1]["models"][0] + assert normalized[2]["connection"]["anthropic_version"] == "2023-06-01" + assert "api_version" not in normalized[0]["connection"] + assert "anthropic_version" not in normalized[1]["connection"] + + sanitized = functions_settings.sanitize_model_endpoints_for_frontend(normalized) + assert len(sanitized) == 3 + assert all(endpoint["provider"] == "custom" for endpoint in sanitized) + assert all(endpoint["has_api_key"] is True for endpoint in sanitized) + assert all("api_key" not in endpoint["auth"] for endpoint in sanitized) + finally: + _restore_modules(original_modules) + + +def test_custom_endpoint_ui_contract(): + """Ensure both endpoint editors expose the same safe Custom workflow.""" + modal = (APP_DIR / "templates" / "_multiendpoint_modal.html").read_text(encoding="utf-8") + admin_template = "\n".join( + [(APP_DIR / "templates" / "admin_settings.html").read_text(encoding="utf-8")] + + [ + pane.read_text(encoding="utf-8") + for pane in sorted((APP_DIR / "templates" / "admin" / "_panes").glob("*.html")) + ] + ) + admin_js = ( + APP_DIR / "static" / "js" / "admin" / "admin_model_endpoints.js" + ).read_text(encoding="utf-8") + workspace_js = ( + APP_DIR / "static" / "js" / "workspace" / "workspace_model_endpoints.js" + ).read_text(encoding="utf-8") + agents_common_js = ( + APP_DIR / "static" / "js" / "agents_common.js" + ).read_text(encoding="utf-8") + agent_stepper_js = ( + APP_DIR / "static" / "js" / "agent_modal_stepper.js" + ).read_text(encoding="utf-8") + backend = (APP_DIR / "route_backend_models.py").read_text(encoding="utf-8") + agent_backend = (APP_DIR / "route_backend_agents.py").read_text(encoding="utf-8") + + assert '' in modal + assert 'id="model-endpoint-api-type"' in modal + assert '' in modal + assert '' in modal + assert '' in modal + assert 'id="model-endpoint-anthropic-version"' in modal + assert 'name="allow_private_custom_model_endpoints"' in admin_template + + for script in (admin_js, workspace_js): + assert "Custom endpoints use API key authentication and manual model entry." in script + assert "Model discovery is unavailable for Custom endpoints." in script + assert "api_type" in script + assert "anthropic_version" in script + assert "customApiTypeUsesModelName" in script + assert "dataset.responseLengthFor" in script + assert "model.responseLength = responseLength" in script + + assert "if (!response.ok)" in workspace_js + assert "provider == MODEL_ENDPOINT_PROVIDER_CUSTOM" in backend + assert "Model discovery is not available for Custom endpoints." in backend + assert "persisted_model = next(" in backend + assert "request_model: requestModel" in agents_common_js + assert "selectedModelOption?.dataset?.requestModel" in agent_stepper_js + assert "normalized_provider == 'custom'" in agent_backend + + +def run_tests(): + """Run all Custom endpoint functional checks.""" + tests = [ + test_request_model_resolution_and_protocol_precedence, + test_custom_endpoint_url_policy, + test_custom_endpoint_configuration_validation, + test_custom_client_paths_headers_and_redirect_policy, + test_custom_runtime_client_construction, + test_custom_endpoint_normalization_and_sanitization, + test_custom_endpoint_ui_contract, + ] + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + try: + test() + print("Test passed") + results.append(True) + except Exception as exc: + print(f"Test failed: {exc}") + import traceback + + traceback.print_exc() + results.append(False) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + return all(results) + + +if __name__ == "__main__": + raise SystemExit(0 if run_tests() else 1) diff --git a/functional_tests/test_document_auto_metadata_extraction_consistency.py b/functional_tests/test_document_auto_metadata_extraction_consistency.py index 7210de5ff..1efb20489 100644 --- a/functional_tests/test_document_auto_metadata_extraction_consistency.py +++ b/functional_tests/test_document_auto_metadata_extraction_consistency.py @@ -2,8 +2,9 @@ # test_document_auto_metadata_extraction_consistency.py """ Functional test for document auto metadata extraction consistency. -Version: 0.241.111 +Version: 0.250.172 Implemented in: 0.241.110 +Updated in: 0.250.172 This test ensures upload processing runs final metadata extraction consistently for all supported file types and preserves public workspace scope for media files. @@ -11,15 +12,14 @@ import ast import os -import re import sys +from test_support.versioning import assert_app_version_at_least + ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SINGLE_APP_ROOT = os.path.join(ROOT_DIR, 'application', 'single_app') FUNCTIONS_DOCUMENTS_FILE = os.path.join(SINGLE_APP_ROOT, 'functions_documents.py') -CONFIG_FILE = os.path.join(SINGLE_APP_ROOT, 'config.py') - def read_file(path): with open(path, 'r', encoding='utf-8') as file_handle: @@ -182,10 +182,7 @@ def test_config_version_bumped_for_auto_metadata_fix(): """Verify config.py version was bumped for this fix.""" print('Testing config version bump...') - config_source = read_file(CONFIG_FILE) - version_match = re.search(r'VERSION = "([0-9.]+)"', config_source) - assert version_match, 'Could not find VERSION in config.py' - assert version_match.group(1) == '0.241.111', 'Expected config.py version 0.241.111' + assert_app_version_at_least("0.241.111") print('Config version bump passed') return True diff --git a/functional_tests/test_model_endpoint_management_cloud_environment.py b/functional_tests/test_model_endpoint_management_cloud_environment.py index cb182d42c..a8bb547b7 100644 --- a/functional_tests/test_model_endpoint_management_cloud_environment.py +++ b/functional_tests/test_model_endpoint_management_cloud_environment.py @@ -1,8 +1,8 @@ # test_model_endpoint_management_cloud_environment.py """ Functional test for model endpoint management cloud environment normalization. -Version: 0.250.004 -Implemented in: 0.250.004 +Version: 0.261.010 +Implemented in: 0.250.004; updated in 0.250.172, 0.261.010 This test ensures model endpoint normalization derives non-editable management cloud settings from AZURE_ENVIRONMENT and preserves explicit service principal @@ -53,6 +53,9 @@ def load_functions_settings_module(): cache_stub.get_settings_cache = lambda: None cache_stub.update_settings_cache = lambda settings: None + content_safety_stub = types.ModuleType("functions_content_safety") + content_safety_stub.CONTENT_SAFETY_VIOLATION_MESSAGE_DEFAULT = "Content safety policy violation." + throughput_stub = types.ModuleType("functions_cosmos_throughput") throughput_stub.get_default_cosmos_throughput_settings = lambda: {} @@ -62,6 +65,13 @@ def load_functions_settings_module(): icon_utils_stub = types.ModuleType("functions_icon_utils") icon_utils_stub.normalize_icon_payload = lambda icon, field_name=None: icon or {} + latest_features_stub = types.ModuleType("functions_latest_features_nav") + latest_features_stub.LATEST_FEATURES_HIDDEN_VERSION_SETTING = "latest_features_hidden_version" + + mcp_stub = types.ModuleType("functions_mcp_server_config") + mcp_stub.INBOUND_MCP_SETTINGS_DEFAULTS = {} + mcp_stub.normalize_inbound_mcp_settings = lambda settings: None + service_health_stub = types.ModuleType("functions_service_health") service_health_stub.get_default_service_health = lambda: {} @@ -75,9 +85,12 @@ def load_functions_settings_module(): "config": config_stub, "functions_appinsights": appinsights_stub, "app_settings_cache": cache_stub, + "functions_content_safety": content_safety_stub, "functions_cosmos_throughput": throughput_stub, "functions_document_actions": document_actions_stub, "functions_icon_utils": icon_utils_stub, + "functions_latest_features_nav": latest_features_stub, + "functions_mcp_server_config": mcp_stub, "functions_service_health": service_health_stub, "support_menu_config": support_menu_stub, "functions_settings": None, @@ -176,7 +189,7 @@ def test_service_principal_preserves_explicit_cross_cloud_selection(): functions_settings, original_modules = load_functions_settings_module() restore = run_with_environment(functions_settings, "usgovernment") try: - endpoint, changed = normalize_single_endpoint(functions_settings, { + endpoint, _ = normalize_single_endpoint(functions_settings, { "id": "public-foundry-sp", "provider": "new_foundry", "enabled": True, @@ -186,12 +199,16 @@ def test_service_principal_preserves_explicit_cross_cloud_selection(): }, "models": [], }) + # Normalization adds backend-owned defaults on first pass, so idempotency is + # what proves the explicit cross-cloud selection is never rewritten. + renormalized, changed = normalize_single_endpoint(functions_settings, endpoint) finally: restore() restore_modules(original_modules) assert changed is False assert endpoint["auth"]["management_cloud"] == "public" + assert renormalized["auth"]["management_cloud"] == "public" def test_missing_service_principal_cloud_defaults_to_environment(): diff --git a/functional_tests/test_new_foundry_streaming_runtime.py b/functional_tests/test_new_foundry_streaming_runtime.py index 3d1c08e41..9bf8cab4f 100644 --- a/functional_tests/test_new_foundry_streaming_runtime.py +++ b/functional_tests/test_new_foundry_streaming_runtime.py @@ -2,8 +2,9 @@ #!/usr/bin/env python3 """ Functional test for new Foundry REST streaming runtime. -Version: 0.239.205 +Version: 0.250.172 Implemented in: 0.239.177 +Updated in: 0.250.172 This test ensures that new Foundry application discovery stays REST-based, that the runtime exposes a streaming executor, and that the chat stream route @@ -12,6 +13,8 @@ from pathlib import Path +from test_support.versioning import assert_app_version_at_least + ROOT = Path(__file__).resolve().parents[1] @@ -34,8 +37,6 @@ def test_new_foundry_streaming_runtime() -> None: runtime_path = ROOT / "application" / "single_app" / "foundry_agent_runtime.py" chats_path = ROOT / "application" / "single_app" / "route_backend_chats.py" models_path = ROOT / "application" / "single_app" / "route_backend_models.py" - config_path = ROOT / "application" / "single_app" / "config.py" - assert_contains(runtime_path, "async def execute_new_foundry_agent_stream(") assert_contains(runtime_path, '"stream": stream') assert_contains(runtime_path, "stream=True,") @@ -49,7 +50,7 @@ def test_new_foundry_streaming_runtime() -> None: assert_contains(chats_path, "response = loop.run_until_complete(agent_stream.__anext__())") assert_not_contains(chats_path, "chunks, stream_usage = loop.run_until_complete(stream_agent_async())") - assert_contains(config_path, 'VERSION = "0.239.205"') + assert_app_version_at_least("0.239.205") print("✅ New Foundry REST streaming runtime verified.") diff --git a/functional_tests/test_tabular_claude_model_endpoint_support.py b/functional_tests/test_tabular_claude_model_endpoint_support.py index 0785b0973..24bb75d10 100644 --- a/functional_tests/test_tabular_claude_model_endpoint_support.py +++ b/functional_tests/test_tabular_claude_model_endpoint_support.py @@ -2,8 +2,9 @@ #!/usr/bin/env python3 """ Functional test for tabular Claude model endpoint support. -Version: 0.241.186 +Version: 0.250.172 Implemented in: 0.241.186 +Updated in: 0.250.172 This test ensures tabular analysis and generated tabular exports preserve the selected Claude/Anthropic model endpoint context, use provider-aware Semantic @@ -15,13 +16,13 @@ import sys from pathlib import Path +from test_support.versioning import assert_app_version_at_least + REPO_ROOT = Path(__file__).resolve().parents[1] APP_ROOT = REPO_ROOT / "application" / "single_app" CHAT_ROUTE = APP_ROOT / "route_backend_chats.py" RUNTIME_HELPER = APP_ROOT / "functions_model_endpoint_runtime.py" -CONFIG = APP_ROOT / "config.py" - def read_text(path): """Read source text as UTF-8.""" @@ -90,7 +91,8 @@ def test_claude_tabular_uses_direct_planner_fallback(): def test_runtime_helper_supports_claude_sk_services(): """Validate runtime helper can build Anthropic SK services from context.""" source_text = read_text(RUNTIME_HELPER) - assert_contains(source_text, "MODEL_ENDPOINT_PROVIDER_ALLOWLIST = {'aoai', 'aifoundry', 'new_foundry', 'anthropic', 'claude'}", "Claude provider allowlist") + assert_contains(source_text, "'claude'", "Claude provider allowlist") + assert_contains(source_text, "MODEL_ENDPOINT_PROVIDER_CUSTOM", "Custom provider allowlist") assert_contains(source_text, "resolve_model_endpoint_from_context", "model context re-resolution") assert_contains(source_text, "AnthropicSemanticKernelChatCompletion", "Anthropic SK adapter") assert_contains(source_text, "sanitize_model_endpoint_auth_for_context", "non-secret auth context") @@ -111,8 +113,7 @@ def test_summary_helpers_are_anthropic_message_safe(): def test_version_bumped_for_fix(): """Validate config.py version was bumped for the fix.""" - source_text = read_text(CONFIG) - assert_contains(source_text, 'VERSION = "0.241.186"', "fix version") + assert_app_version_at_least("0.241.186") def main(): diff --git a/functional_tests/test_workflow_model_core_capabilities.py b/functional_tests/test_workflow_model_core_capabilities.py index 9101f2156..78daed5d4 100644 --- a/functional_tests/test_workflow_model_core_capabilities.py +++ b/functional_tests/test_workflow_model_core_capabilities.py @@ -1,9 +1,9 @@ # test_workflow_model_core_capabilities.py """ Functional test for Direct Model workflow core capabilities. -Version: 0.250.064 +Version: 0.250.172 Implemented in: 0.250.063 -Enhanced in: 0.250.064 +Enhanced in: 0.250.064; updated in 0.250.172 This test ensures new Direct Model workflows bind their saved model selection to a Semantic Kernel service and pass the kernel to auto-invoked core tools. @@ -69,6 +69,7 @@ def load_model_core_helpers(): helper_names = { "_workflow_model_chat_capabilities_enabled", "_build_workflow_model_context", + "_resolve_workflow_conversation_context", "_workflow_model_core_execution_context", "_execute_model_workflow_with_core_capabilities", "_execute_model_workflow", diff --git a/functional_tests/test_workspace_multi_endpoints.py b/functional_tests/test_workspace_multi_endpoints.py index df5a7f0ab..431f7a125 100644 --- a/functional_tests/test_workspace_multi_endpoints.py +++ b/functional_tests/test_workspace_multi_endpoints.py @@ -1,8 +1,8 @@ # test_workspace_multi_endpoints.py """ Functional test for workspace multi-endpoint routing. -Version: 0.239.155 -Implemented in: 0.239.155 +Version: 0.250.172 +Implemented in: 0.239.155; updated in 0.250.172 This test ensures that workspace multi-endpoint payloads are sanitized and that agent payloads accept multi-endpoint selection fields. @@ -10,9 +10,6 @@ import sys import os -import importlib -import json -import types repo_root = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) single_app_root = os.path.join(repo_root, "application", "single_app") @@ -20,6 +17,9 @@ sys.path.append(single_app_root) from application.single_app.functions_agent_payload import sanitize_agent_payload +from test_model_endpoint_normalization_backend import ( + _load_functions_settings_module as load_functions_settings_module, +) def _restore_modules(original_modules): @@ -31,29 +31,7 @@ def _restore_modules(original_modules): def _load_functions_settings_module(): - config_stub = types.ModuleType("config") - config_stub.json = json - - appinsights_stub = types.ModuleType("functions_appinsights") - appinsights_stub.log_event = lambda *args, **kwargs: None - - cache_stub = types.ModuleType("app_settings_cache") - cache_stub.get_settings_cache = lambda: None - cache_stub.update_settings_cache = lambda settings: None - - original_modules = {} - for module_name, module_stub in { - "config": config_stub, - "functions_appinsights": appinsights_stub, - "app_settings_cache": cache_stub, - }.items(): - original_modules[module_name] = sys.modules.get(module_name) - sys.modules[module_name] = module_stub - - original_modules["application.single_app.functions_settings"] = sys.modules.get("application.single_app.functions_settings") - sys.modules.pop("application.single_app.functions_settings", None) - module = importlib.import_module("application.single_app.functions_settings") - return module, original_modules + return load_functions_settings_module() def test_model_endpoint_sanitization(): From dd18f41554281ab0f835423c3ff4d3f5f32756da Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 07:38:55 -0400 Subject: [PATCH 2/9] Resolve model capabilities from the catalog instead of the model name SimpleChat decided what a model could do by pattern-matching its name, so any model the shipped catalog did not know about -- an on-premises or customer supplied model reached through a Custom endpoint -- silently got the wrong answer for vision, tool calling, streaming, and reasoning. A model named "corp-llm-v2" was treated as having no capabilities at all. Capability answers now resolve through per-model override, endpoint override, catalog entry, then the original name heuristic, so the catalog can be incomplete without blocking an administrator and unknown models behave exactly as they did before. Two matching rules keep this honest. "family" is never used to match, because members of one family disagree on capabilities: the phi-4 family holds both the multimodal and the text-only Phi models, and each gpt-5.x family holds a non-vision "-chat" member. And a longer identifier prefix wins, with a digit-leading remainder rejected as a version continuation rather than a variant, so gpt-5.1-chat-v2 resolves to gpt-5.1-chat and gpt-5.3 no longer resolves to gpt-5. The catalog gains supportsStreaming and reasoning for all 65 existing models and 10 Google Gemini records, a provider it did not cover at all. Claude, Llama 4, and Phi-4 multimodal are now correctly recognised as vision-capable, and the gpt-5.x "-chat" variants correctly as not. A published JSON schema plus a coverage test means a malformed record fails a test rather than degrading capability answers at runtime. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_model_capabilities.py | 335 ++++++++++- .../static/json/model_capabilities.json | 539 +++++++++++++++--- .../schemas/model_capabilities.schema.json | 150 +++++ .../features/MODEL_CAPABILITY_CATALOG.md | 205 +++++++ docs/explanation/release-notes/index.md | 96 ++++ docs/explanation/release_notes.md | 11 + ...est_model_capability_catalog_resolution.py | 261 +++++++++ 8 files changed, 1518 insertions(+), 81 deletions(-) create mode 100644 application/single_app/static/json/schemas/model_capabilities.schema.json create mode 100644 docs/explanation/features/MODEL_CAPABILITY_CATALOG.md create mode 100644 functional_tests/test_model_capability_catalog_resolution.py diff --git a/application/single_app/config.py b/application/single_app/config.py index a97fabfc8..64cd23643 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.010" +VERSION = "0.261.011" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_capabilities.py b/application/single_app/functions_model_capabilities.py index aecb13dab..9520bba65 100644 --- a/application/single_app/functions_model_capabilities.py +++ b/application/single_app/functions_model_capabilities.py @@ -1,12 +1,27 @@ # functions_model_capabilities.py +"""Model capability resolution backed by the SimpleChat model capability catalog. +Capability answers resolve through a precedence chain so that a model which is not +present in the shipped catalog -- a customer's on-premises or bespoke model -- can +still be described accurately instead of being guessed at from its name: + + per-model override -> endpoint override -> catalog entry -> name heuristic + +Only stdlib imports are used here on purpose. This module sits below the settings, +logging, and route layers, so pulling those in would risk import cycles. +""" + +import json +import os import re +import threading from collections.abc import Mapping MODEL_IDENTIFIER_SEPARATOR_PATTERN = re.compile(r"[\s_.]+") GPT_VISION_MODEL_PATTERN = re.compile(r"(?:^|-)gpt-(?:[5-9]|\d{2,})(?:-|$)") O_SERIES_MODEL_PATTERN = re.compile(r"(?:^|-)o\d+(?:-|$)") +REASONING_MODEL_PATTERN = re.compile(r"(?:^|-)(?:o\d+|gpt-(?:[5-9]|\d{2,}))(?:-|$)") MODEL_IDENTIFIER_FIELDS = ( "modelName", "displayName", @@ -15,6 +30,37 @@ "name", ) +CATALOG_RELATIVE_PATH = ("static", "json", "model_capabilities.json") + +CAPABILITY_PROCESSES_IMAGES = "processesImages" +CAPABILITY_TOOL_CALLING = "toolCalling" +CAPABILITY_STRUCTURED_OUTPUT = "structuredOutput" +CAPABILITY_SUPPORTS_STREAMING = "supportsStreaming" +CAPABILITY_REASONING = "reasoning" + +CAPABILITY_FIELD_NAMES = ( + "processesText", + "generatesText", + CAPABILITY_PROCESSES_IMAGES, + "generatesImages", + "processesAudio", + "generatesAudio", + "processesVideo", + "generatesVideo", + "processesBinaryFiles", + "optimizedForCoding", + CAPABILITY_TOOL_CALLING, + CAPABILITY_STRUCTURED_OUTPUT, + CAPABILITY_SUPPORTS_STREAMING, + CAPABILITY_REASONING, +) + +CATALOG_CONTEXT_LIMIT_FIELDS = ("inputTokenLimit", "contextWindow", "maxInputTokens") +CATALOG_OUTPUT_LIMIT_FIELDS = ("outputTokenLimit", "maxOutputTokens", "maxCompletionTokens") + +_CATALOG_LOCK = threading.Lock() +_CATALOG_CACHE = None + def _normalize_model_identifier(value): return MODEL_IDENTIFIER_SEPARATOR_PATTERN.sub( @@ -23,10 +69,154 @@ def _normalize_model_identifier(value): ) -def is_vision_capable_model_name(*model_names): - """Return whether any supplied identifier names a supported vision model.""" - for model_name in model_names: - normalized_name = _normalize_model_identifier(model_name) +def get_model_capability_catalog_path(): + """Return the absolute path of the shipped model capability catalog.""" + return os.path.join(os.path.dirname(__file__), *CATALOG_RELATIVE_PATH) + + +def reset_model_capability_catalog_cache(): + """Clear the cached catalog so a later read picks the file up again.""" + global _CATALOG_CACHE + with _CATALOG_LOCK: + _CATALOG_CACHE = None + + +def load_model_capability_catalog(): + """Return the parsed catalog, caching it after the first successful read.""" + global _CATALOG_CACHE + with _CATALOG_LOCK: + if _CATALOG_CACHE is not None: + return _CATALOG_CACHE + try: + with open(get_model_capability_catalog_path(), "r", encoding="utf-8") as catalog_file: + catalog = json.load(catalog_file) + except (OSError, json.JSONDecodeError): + catalog = {} + if not isinstance(catalog, dict): + catalog = {} + _CATALOG_CACHE = catalog + return _CATALOG_CACHE + + +def get_model_capability_catalog_records(): + """Return every model record defined by the catalog.""" + catalog = load_model_capability_catalog() + return [record for record in catalog.get("models") or [] if isinstance(record, dict)] + + +def _get_record_field(record, field_name): + if isinstance(record, Mapping): + return record.get(field_name) + return getattr(record, field_name, None) + + +def _iter_model_identifiers(model): + """Yield every normalized identifier that could name the supplied model.""" + if model is None: + return + if isinstance(model, str): + normalized = _normalize_model_identifier(model) + if normalized: + yield normalized + return + for field_name in MODEL_IDENTIFIER_FIELDS: + normalized = _normalize_model_identifier(_get_record_field(model, field_name)) + if normalized: + yield normalized + + +def _iter_catalog_record_identifiers(record): + """Yield every normalized identifier a catalog record answers to. + + "family" is deliberately excluded. It is a grouping attribute rather than an + identifier, and members of one family disagree on capabilities -- "phi-4" + covers both the multimodal and the text-only Phi models, and the "gpt-5.x" + families each contain a non-vision "-chat" member. Matching on it would let a + model inherit a sibling's capabilities. + """ + for field_name in ("id", "displayName"): + normalized = _normalize_model_identifier(record.get(field_name)) + if normalized: + yield normalized + aliases = record.get("aliases") + if isinstance(aliases, (list, tuple)): + for alias in aliases: + normalized = _normalize_model_identifier(alias) + if normalized: + yield normalized + + +def _is_variant_suffix_match(requested_identifier, record_identifier): + """Return whether requested is a variant of record rather than a later version. + + Identifier normalization collapses "." and "-" to the same separator, so + "gpt-5.3" becomes "gpt-5-3" and would otherwise look like a suffixed variant of + "gpt-5". A remainder that starts with a digit is a version continuation, not a + variant, so it is rejected. A remainder starting with a letter -- the "eastus" + in "gpt-5.6-sol-eastus", or the "mini" in "gpt-4o-mini" -- is a real variant. + """ + prefix = f"{record_identifier}-" + if not requested_identifier.startswith(prefix): + return False + remainder = requested_identifier[len(prefix):] + return bool(remainder) and not remainder[0].isdigit() + + +def find_model_catalog_record(model): + """Return the catalog record naming this model, or None when it is unknown. + + An exact identifier match always wins. Otherwise the longest matching + identifier prefix wins, so a deployment named "gpt-5.6-sol-eastus" resolves to + "gpt-5.6-sol", and "gpt-5.1-chat-v2" resolves to "gpt-5.1-chat" rather than to + the shorter, and differently capable, "gpt-5.1". + """ + requested_identifiers = list(_iter_model_identifiers(model)) + if not requested_identifiers: + return None + + requested_identifier_set = set(requested_identifiers) + best_prefix_match = None + best_prefix_length = 0 + for record in get_model_capability_catalog_records(): + record_identifiers = list(_iter_catalog_record_identifiers(record)) + if requested_identifier_set.intersection(record_identifiers): + return record + for record_identifier in record_identifiers: + if len(record_identifier) <= best_prefix_length: + continue + for requested_identifier in requested_identifiers: + if _is_variant_suffix_match(requested_identifier, record_identifier): + best_prefix_match = record + best_prefix_length = len(record_identifier) + break + return best_prefix_match + + +def _read_declared_capabilities(source): + """Return the explicit capability map declared on a model or endpoint record.""" + if source is None: + return {} + capabilities = _get_record_field(source, "capabilities") + if not isinstance(capabilities, Mapping): + return {} + declared = {} + for capability_name, capability_value in capabilities.items(): + if isinstance(capability_value, bool): + declared[str(capability_name)] = capability_value + return declared + + +def _heuristic_capability(capability_name, model): + """Return the legacy name-based answer for the capabilities that have one.""" + if capability_name == CAPABILITY_PROCESSES_IMAGES: + return _heuristic_is_vision_capable(model) + if capability_name == CAPABILITY_REASONING: + return _heuristic_is_reasoning_model(model) + return None + + +def _heuristic_is_vision_capable(model): + for normalized_name in _iter_model_identifiers(model): if ( "vision" in normalized_name or "gpt-4o" in normalized_name @@ -36,18 +226,139 @@ def is_vision_capable_model_name(*model_names): or O_SERIES_MODEL_PATTERN.search(normalized_name) ): return True + return False + +def _heuristic_is_reasoning_model(model): + for normalized_name in _iter_model_identifiers(model): + if REASONING_MODEL_PATTERN.search(normalized_name) or "gpt-5" in normalized_name: + return True return False -def is_vision_capable_model(model): +def resolve_model_capability(capability_name, model=None, endpoint=None, default=None): + """Resolve one capability through the override, catalog, then heuristic chain.""" + declared_model_capabilities = _read_declared_capabilities(model) + if capability_name in declared_model_capabilities: + return declared_model_capabilities[capability_name] + + declared_endpoint_capabilities = _read_declared_capabilities(endpoint) + if capability_name in declared_endpoint_capabilities: + return declared_endpoint_capabilities[capability_name] + + catalog_record = find_model_catalog_record(model) + if catalog_record is not None: + catalog_capabilities = catalog_record.get("capabilities") + if isinstance(catalog_capabilities, Mapping): + catalog_value = catalog_capabilities.get(capability_name) + if isinstance(catalog_value, bool): + return catalog_value + + heuristic_value = _heuristic_capability(capability_name, model) + if heuristic_value is not None: + return heuristic_value + return default + + +def resolve_model_capabilities(model=None, endpoint=None): + """Return every known capability for a model as a name to boolean-or-None map.""" + return { + capability_name: resolve_model_capability(capability_name, model, endpoint) + for capability_name in CAPABILITY_FIELD_NAMES + } + + +def _read_token_limit(record, field_names): + for field_name in field_names: + value = _get_record_field(record, field_name) + try: + normalized_value = int(value) + except (TypeError, ValueError): + continue + if normalized_value > 0: + return normalized_value + return None + + +def resolve_model_token_limits(model=None, endpoint=None): + """Return the (context, output) token limits for a model, or None when unknown.""" + for source in (model, endpoint): + if source is None or isinstance(source, str): + continue + context_limit = _read_token_limit(source, CATALOG_CONTEXT_LIMIT_FIELDS) + output_limit = _read_token_limit(source, CATALOG_OUTPUT_LIMIT_FIELDS) + if context_limit or output_limit: + return context_limit, output_limit + + catalog_record = find_model_catalog_record(model) + if catalog_record is None: + return None, None + return ( + _read_token_limit(catalog_record, CATALOG_CONTEXT_LIMIT_FIELDS), + _read_token_limit(catalog_record, CATALOG_OUTPUT_LIMIT_FIELDS), + ) + + +def resolve_model_output_token_limit(model=None, endpoint=None, default=None): + """Return the output token limit for a model, falling back to the supplied default.""" + _, output_limit = resolve_model_token_limits(model, endpoint) + return output_limit or default + + +def is_vision_capable_model_name(*model_names): + """Return whether any supplied identifier names a supported vision model.""" + for model_name in model_names: + if model_name in (None, ""): + continue + if resolve_model_capability(CAPABILITY_PROCESSES_IMAGES, model_name, default=False): + return True + + return False + + +def is_vision_capable_model(model, endpoint=None): """Return whether a model record or identifier names a supported vision model.""" - if isinstance(model, str): - return is_vision_capable_model_name(model) + return bool( + resolve_model_capability( + CAPABILITY_PROCESSES_IMAGES, + model, + endpoint, + default=False, + ) + ) + + +def is_reasoning_model(model, endpoint=None): + """Return whether a model uses reasoning-style response length parameters.""" + return bool( + resolve_model_capability( + CAPABILITY_REASONING, + model, + endpoint, + default=False, + ) + ) - if isinstance(model, Mapping): - model_names = [model.get(field_name) for field_name in MODEL_IDENTIFIER_FIELDS] - else: - model_names = [getattr(model, field_name, None) for field_name in MODEL_IDENTIFIER_FIELDS] - return is_vision_capable_model_name(*model_names) \ No newline at end of file +def supports_streaming(model=None, endpoint=None): + """Return whether a model can stream. Unknown models are assumed to stream.""" + return bool( + resolve_model_capability( + CAPABILITY_SUPPORTS_STREAMING, + model, + endpoint, + default=True, + ) + ) + + +def supports_tool_calling(model=None, endpoint=None, default=True): + """Return whether a model supports tool or function calling.""" + return bool( + resolve_model_capability( + CAPABILITY_TOOL_CALLING, + model, + endpoint, + default=default, + ) + ) diff --git a/application/single_app/static/json/model_capabilities.json b/application/single_app/static/json/model_capabilities.json index d44d19ad7..b2a016d59 100644 --- a/application/single_app/static/json/model_capabilities.json +++ b/application/single_app/static/json/model_capabilities.json @@ -1,6 +1,6 @@ { "$schema": "https://simplechat.local/schemas/model-capabilities.schema.json", - "schemaVersion": 1, + "schemaVersion": 2, "lastUpdated": "2026-08-04", "description": "SimpleChat model capability catalog. Capability flags remain data-only; optional model token-limit fields are consumed by durable tabular batch planning when present.", "capabilityFields": { @@ -15,14 +15,17 @@ "processesBinaryFiles": "Accepts uploaded files or binary/document payloads through the provider API.", "optimizedForCoding": "Documented or positioned for coding, agentic software tasks, code generation, or code understanding.", "toolCalling": "Supports function/tool calling or provider-equivalent tool use.", - "structuredOutput": "Supports structured outputs, JSON-schema outputs, or provider-equivalent constrained structured responses." + "structuredOutput": "Supports structured outputs, JSON-schema outputs, or provider-equivalent constrained structured responses.", + "supportsStreaming": "Supports incremental token streaming for chat responses. SimpleChat wraps models without streaming support so they still deliver through the stream.", + "reasoning": "Performs extended reasoning or thinking before responding." }, "coverageNotes": [ "OpenAI coverage starts at GPT-5.0 model families and includes Azure OpenAI GPT-5.x model IDs that SimpleChat commonly sees through Foundry.", "Claude coverage includes current, legacy, deprecated, and recently retired Claude models that fall within the requested two-year window.", "Meta coverage focuses on public Llama model families with clear model cards for text, vision, and coding support.", "xAI coverage includes Grok chat/coding models plus documented Imagine and Voice model SKUs.", - "Microsoft coverage includes public Phi and MAI model cards with clear capability statements." + "Microsoft coverage includes public Phi and MAI model cards with clear capability statements.", + "Google coverage includes the generally available Gemini chat model tiers that expose generateContent and streamGenerateContent." ], "sources": [ { @@ -167,7 +170,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Frontier GPT-5.6 tier; text and image input with text output."], "sourceIds": ["openai-gpt5-6", "azure-openai-gpt5"] @@ -192,7 +197,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["GPT-5.6 lower-latency tier; Azure catalog documents text and image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -217,7 +224,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["GPT-5.6 smallest tier; Azure catalog documents text and image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -242,7 +251,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents reasoning, Responses API, structured outputs, text and image processing, and tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -267,7 +278,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Azure catalog describes this preview model as Chat Completions/Responses with structured outputs and tools."], "sourceIds": ["azure-openai-gpt5"] @@ -292,7 +305,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus tools."], "sourceIds": ["azure-openai-gpt5"] @@ -317,7 +332,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API, text and image processing, and functions/tools."], "sourceIds": ["azure-openai-gpt5"] @@ -342,7 +359,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus parallel tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -367,7 +386,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus parallel tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -392,7 +413,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents optimization for Codex CLI and Codex VS Code extension."], "sourceIds": ["azure-openai-gpt5"] @@ -417,7 +440,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog does not list image processing for this entry."], "sourceIds": ["azure-openai-gpt5"] @@ -442,7 +467,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents optimization for Codex CLI and Codex VS Code extension."], "sourceIds": ["azure-openai-gpt5"] @@ -467,7 +494,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus tools."], "sourceIds": ["azure-openai-gpt5"] @@ -492,7 +521,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog does not list image processing for this entry."], "sourceIds": ["azure-openai-gpt5"] @@ -517,7 +548,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5-1", "azure-openai-gpt5"] @@ -542,7 +575,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog documents tools and structured outputs but not image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -567,7 +602,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -592,7 +629,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -617,7 +656,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only, Codex optimization, and xhigh reasoning effort."], "sourceIds": ["azure-openai-gpt5"] @@ -642,7 +683,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -667,7 +710,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API, text and image processing, and functions/tools."], "sourceIds": ["azure-openai-gpt5"] @@ -692,7 +737,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -717,7 +764,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -742,7 +791,9 @@ "processesBinaryFiles": true, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI positions this as fastest and cost-efficient for summarization and classification."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -767,7 +818,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Azure catalog explicitly lists input as text/image and output as text only."], "sourceIds": ["azure-openai-gpt5"] @@ -792,7 +845,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic describes this as next-generation intelligence for long-running agents; current Claude models support text and image input with text output."], "sourceIds": ["anthropic-models"] @@ -817,7 +872,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Invitation-only Project Glasswing model sharing Fable 5 specs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -842,7 +899,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic positions Opus 5 for complex agentic coding and enterprise work."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -867,7 +926,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic positions Sonnet 5 as a speed/intelligence balance."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -892,7 +953,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -917,7 +980,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -942,7 +1007,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -967,7 +1034,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.5 model listed in Anthropic lifecycle docs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -992,7 +1061,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Sonnet 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1017,7 +1088,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Sonnet 4.5 model listed in Anthropic lifecycle docs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1042,7 +1115,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Fastest current Claude model; current Claude models support vision."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1067,7 +1142,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Deprecated; scheduled retirement listed by Anthropic."], "sourceIds": ["anthropic-deprecations"] @@ -1092,7 +1169,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Recently retired; included for two-year Claude coverage."], "sourceIds": ["anthropic-deprecations"] @@ -1117,7 +1196,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Recently retired; included for two-year Claude coverage."], "sourceIds": ["anthropic-deprecations"] @@ -1142,7 +1223,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1167,7 +1250,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1192,7 +1277,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1217,7 +1304,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text and image input with multilingual text and code output."], "sourceIds": ["meta-llama4"] @@ -1242,7 +1331,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text and image input with multilingual text and code output."], "sourceIds": ["meta-llama4"] @@ -1267,7 +1358,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text input and multilingual text/code output; Transformers examples document tool use."], "sourceIds": ["meta-llama33"] @@ -1292,7 +1385,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes text + image input with text output."], "sourceIds": ["meta-llama32-vision"] @@ -1317,7 +1412,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes the 11B and 90B Llama 3.2 Vision sizes as text + image input with text output."], "sourceIds": ["meta-llama32-vision"] @@ -1342,7 +1439,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes Code Llama as text-only input/output designed for code synthesis and understanding."], "sourceIds": ["meta-codellama"] @@ -1367,7 +1466,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["xAI describes Grok 4.5 as text,image -> text and recommends it for code and agentic software tasks."], "sourceIds": ["xai-grok45", "xai-models"] @@ -1392,7 +1493,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI text API pricing; xAI overview documents image input constraints for image-input models."], "sourceIds": ["xai-models"] @@ -1417,7 +1520,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Listed in xAI text API pricing as a reasoning model."], "sourceIds": ["xai-models"] @@ -1442,7 +1547,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI text API pricing as a non-reasoning model."], "sourceIds": ["xai-models"] @@ -1467,7 +1574,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI model pricing; Grok 4.5 page aliases grok-build-latest to Grok 4.5."], "sourceIds": ["xai-models", "xai-grok45"] @@ -1492,7 +1601,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI model pricing as a multi-agent model."], "sourceIds": ["xai-models"] @@ -1517,7 +1628,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI documents modalities as text,image -> image."], "sourceIds": ["xai-imagine-image"] @@ -1542,7 +1655,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI overview lists this as an Imagine image model."], "sourceIds": ["xai-models", "xai-imagine-image"] @@ -1567,7 +1682,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI documents modalities as text,image -> video."], "sourceIds": ["xai-imagine-video"] @@ -1592,7 +1709,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI overview lists this as an Imagine video model."], "sourceIds": ["xai-models", "xai-imagine-video"] @@ -1617,7 +1736,9 @@ "processesBinaryFiles": true, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["xAI Voice API documents speech-to-speech, speech-to-text, and text-to-speech powered by Grok."], "sourceIds": ["xai-voice", "xai-models"] @@ -1642,7 +1763,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card says it processes text, image, and audio inputs, generates text outputs, and supports multi-image or video clip summarization."], "sourceIds": ["microsoft-phi4-multimodal"] @@ -1667,7 +1790,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card documents instruction following and function calling for a text model."], "sourceIds": ["microsoft-phi4-mini"] @@ -1692,7 +1817,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Model card says Phi-4 reasoning is trained for math, science, and coding skills with text input and text output."], "sourceIds": ["microsoft-phi4-reasoning"] @@ -1717,7 +1844,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Model card positions this compact model for math reasoning, not general multimodal or coding use."], "sourceIds": ["microsoft-phi4-mini"] @@ -1742,7 +1871,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes visual and text input, multi-image comparison, and video clip summarization."], "sourceIds": ["microsoft-phi35-vision"] @@ -1767,10 +1898,282 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes broad text generation, reasoning, problem solving, code generation, and code comprehension."], "sourceIds": ["microsoft-mai-ds-r1"] + }, + { + "id": "gemini-3.8-flash", + "provider": "google", + "displayName": "Gemini 3.8 Flash", + "aliases": [], + "family": "gemini-3.8", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Current Gemini 3.8 Flash tier; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.7-flash", + "provider": "google", + "displayName": "Gemini 3.7 Flash", + "aliases": [], + "family": "gemini-3.7", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.7 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.6-flash", + "provider": "google", + "displayName": "Gemini 3.6 Flash", + "aliases": [], + "family": "gemini-3.6", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.6 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.5-flash", + "provider": "google", + "displayName": "Gemini 3.5 Flash", + "aliases": [], + "family": "gemini-3.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.5 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.5-flash-lite", + "provider": "google", + "displayName": "Gemini 3.5 Flash-Lite", + "aliases": [], + "family": "gemini-3.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Cost-optimized Gemini 3.5 Flash-Lite tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.1-pro-preview", + "provider": "google", + "displayName": "Gemini 3.1 Pro Preview", + "aliases": ["gemini-3.1-pro"], + "family": "gemini-3.1", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.1 Pro preview tier; strongest Gemini 3.1 reasoning and coding tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-pro", + "provider": "google", + "displayName": "Gemini 2.5 Pro", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 2.5 Pro; multimodal input with text output and thinking support."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-flash", + "provider": "google", + "displayName": "Gemini 2.5 Flash", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 2.5 Flash; balanced multimodal tier with thinking support."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-flash-lite", + "provider": "google", + "displayName": "Gemini 2.5 Flash-Lite", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Cost-optimized Gemini 2.5 Flash-Lite tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.0-flash", + "provider": "google", + "displayName": "Gemini 2.0 Flash", + "aliases": [], + "family": "gemini-2.0", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false + }, + "notes": ["Gemini 2.0 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] } ] } diff --git a/application/single_app/static/json/schemas/model_capabilities.schema.json b/application/single_app/static/json/schemas/model_capabilities.schema.json new file mode 100644 index 000000000..0043f3040 --- /dev/null +++ b/application/single_app/static/json/schemas/model_capabilities.schema.json @@ -0,0 +1,150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://simplechat.local/schemas/model-capabilities.schema.json", + "title": "SimpleChat model capability catalog", + "description": "Schema for static/json/model_capabilities.json. The catalog is the source of truth for per-model capability answers, so that models are described by data rather than guessed at from their names.", + "type": "object", + "required": ["schemaVersion", "capabilityFields", "models"], + "properties": { + "$schema": { + "type": "string" + }, + "schemaVersion": { + "type": "integer", + "minimum": 2 + }, + "lastUpdated": { + "type": ["string", "null"] + }, + "description": { + "type": "string" + }, + "capabilityFields": { + "type": "object", + "description": "Human-readable description of every capability flag a model record may declare.", + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "coverageNotes": { + "type": "array", + "items": { + "type": "string" + } + }, + "sources": { + "type": "array", + "items": { + "type": "object" + } + }, + "models": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/modelRecord" + } + } + }, + "$defs": { + "modelRecord": { + "type": "object", + "required": [ + "id", + "provider", + "displayName", + "aliases", + "family", + "lifecycle", + "capabilities" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "displayName": { + "type": "string", + "minLength": 1 + }, + "aliases": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "family": { + "type": "string", + "minLength": 1 + }, + "lifecycle": { + "type": "string", + "enum": [ + "current", + "preview", + "limited-availability", + "legacy", + "deprecated", + "retired" + ] + }, + "releaseDate": { + "type": ["string", "null"] + }, + "capabilities": { + "$ref": "#/$defs/capabilityMap" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "sourceIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTokenLimit": { + "type": ["integer", "null"], + "minimum": 1 + }, + "outputTokenLimit": { + "type": ["integer", "null"], + "minimum": 1 + } + }, + "additionalProperties": false + }, + "capabilityMap": { + "type": "object", + "description": "Every flag is required so that a model is never silently missing a capability answer.", + "required": [ + "processesText", + "generatesText", + "processesImages", + "generatesImages", + "processesAudio", + "generatesAudio", + "processesVideo", + "generatesVideo", + "processesBinaryFiles", + "optimizedForCoding", + "toolCalling", + "structuredOutput", + "supportsStreaming", + "reasoning" + ], + "additionalProperties": { + "type": "boolean" + } + } + } +} diff --git a/docs/explanation/features/MODEL_CAPABILITY_CATALOG.md b/docs/explanation/features/MODEL_CAPABILITY_CATALOG.md new file mode 100644 index 000000000..40e568e18 --- /dev/null +++ b/docs/explanation/features/MODEL_CAPABILITY_CATALOG.md @@ -0,0 +1,205 @@ +# Model Capability Catalog + +## Overview + +SimpleChat needs to know what each model can do in order to decide whether to offer +it as a vision model, whether it can be given tools, whether it can stream, and +whether it uses reasoning-style request parameters. + +Those answers used to be derived by pattern-matching the model's name. That works +for well-known Azure OpenAI deployments and fails for everything else. A model +reached through a Custom endpoint — an on-premises gateway, a customer's own +fine-tune, or a provider SimpleChat has not shipped support for — was named +whatever the customer chose to name it, so every capability question silently +returned the wrong answer. + +The model capability catalog makes those answers data-driven, and lets an +administrator override them per model or per endpoint. + +**Implemented in version: 0.261.011** + +## Dependencies + +- `jsonschema` (already a SimpleChat dependency) for catalog validation in tests. +- No new runtime dependencies. The resolver uses only the standard library so it + can sit below the settings, logging, and route layers without import cycles. + +## Architecture + +### Resolution order + +Every capability question resolves through this chain, stopping at the first +source that gives a definite answer: + +1. **Per-model override** — a `capabilities` map on the model record. +2. **Endpoint override** — a `capabilities` map on the endpoint record. +3. **Catalog entry** — the matching record in `model_capabilities.json`. +4. **Name heuristic** — the original pattern matching, kept as a last resort so + models absent from the catalog behave exactly as they did before. + +This ordering means the catalog can be wrong or incomplete without blocking an +administrator, and an unknown model degrades to previous behaviour rather than to +"no capabilities". + +### Catalog matching + +A model record is matched to a catalog record by normalized identifier. The +model's `modelName`, `displayName`, `deploymentName`, `deployment`, and `name` +fields are each compared against the catalog record's `id`, `displayName`, and +`aliases`. + +Two rules keep matching honest: + +**`family` is never used for matching.** It is a grouping attribute, and members +of one family disagree on capabilities. The `phi-4` family contains both the +multimodal and the text-only Phi models; each `gpt-5.x` family contains a +non-vision `-chat` member. Matching on family would let a model inherit a +sibling's capabilities. + +**A longer identifier prefix wins, and a version continuation is not a prefix +match.** A deployment named `gpt-5.6-sol-eastus` resolves to `gpt-5.6-sol`, and +`gpt-5.1-chat-v2` resolves to `gpt-5.1-chat` rather than the shorter and +differently capable `gpt-5.1`. Because identifier normalization turns `.` into +`-`, `gpt-5.3` would otherwise look like a variant of `gpt-5`; a remainder that +starts with a digit is treated as a different version, not a variant, so that +match is rejected. + +## Configuration + +### Catalog file + +`application/single_app/static/json/model_capabilities.json` + +```json +{ + "schemaVersion": 2, + "capabilityFields": { "...": "description of each flag" }, + "models": [ + { + "id": "gemini-3.8-flash", + "provider": "google", + "displayName": "Gemini 3.8 Flash", + "aliases": [], + "family": "gemini-3.8", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["..."], + "sourceIds": ["google-gemini-api"] + } + ] +} +``` + +The catalog is validated against +`application/single_app/static/json/schemas/model_capabilities.schema.json`. +Every capability flag is required on every record, so a model can never be +silently missing an answer. + +### Capability flags + +| Flag | Meaning | +|---|---| +| `processesText` / `generatesText` | Accepts text input / produces text output | +| `processesImages` / `generatesImages` | Accepts image input / produces image output | +| `processesAudio` / `generatesAudio` | Accepts audio input / produces audio output | +| `processesVideo` / `generatesVideo` | Accepts video input / produces video output | +| `processesBinaryFiles` | Accepts uploaded files or binary document payloads | +| `optimizedForCoding` | Documented or positioned for coding and agentic software tasks | +| `toolCalling` | Supports function/tool calling or the provider equivalent | +| `structuredOutput` | Supports JSON-schema or equivalent constrained output | +| `supportsStreaming` | Supports incremental token streaming for chat responses | +| `reasoning` | Performs extended reasoning or thinking before responding | + +### Overriding a capability + +To describe a model the catalog does not know about, add a `capabilities` map to +the model record on the endpoint. Only the flags you specify are overridden; +everything else continues to resolve through the chain. + +```json +{ + "id": "corp-llm", + "modelName": "corp-llm-v2", + "enabled": true, + "capabilities": { + "processesImages": false, + "toolCalling": true, + "supportsStreaming": false + } +} +``` + +An endpoint-level `capabilities` map applies the same way to every model on that +endpoint, and is outranked by a per-model map. + +## Usage + +```python +from functions_model_capabilities import ( + is_vision_capable_model, + is_reasoning_model, + supports_streaming, + supports_tool_calling, + resolve_model_capabilities, + resolve_model_output_token_limit, +) + +is_vision_capable_model(model_record, endpoint_record) +supports_streaming(model_record, endpoint_record) # unknown models default to True +supports_tool_calling(model_record, endpoint_record) # unknown models default to True +resolve_model_capabilities(model_record, endpoint_record) # every flag at once +``` + +`supports_streaming` and `supports_tool_calling` default to `True` for unknown +models, so an undescribed model is not needlessly downgraded. `is_vision_capable_model` +defaults to `False`, matching the previous behaviour of the vision model selector. + +## Maintaining the catalog + +The catalog is a maintained JSON file rather than an admin-managed surface. To add +a model, add a record and run the coverage test. There is deliberately no admin UI +for it yet; administrators who need a one-off answer use the per-model override +above instead. + +Google coverage was added in this version and includes the generally available +Gemini chat tiers that expose `generateContent` and `streamGenerateContent`. + +## Testing and validation + +`functional_tests/test_model_capability_catalog_resolution.py` covers: + +- the shipped catalog validating against its JSON schema, with unique model ids; +- family isolation — every member of a family whose members disagree on vision + resolving to its own value, and a bare family name matching no specific model; +- longest-prefix matching, including the version-continuation rejection; +- override precedence for per-model and endpoint-level overrides; +- heuristic fallback for models absent from the catalog; +- Google models resolving from the catalog rather than the name heuristics. + +## Known limitations + +- The catalog carries no token-limit fields yet. `resolve_model_token_limits` + reads `inputTokenLimit` and `outputTokenLimit` when present and returns `None` + otherwise, so callers must supply their own default. +- `reasoning` records whether a model performs extended reasoning. It is + deliberately not yet wired to the request-parameter switch that chooses between + `max_tokens` and `max_completion_tokens`, because that choice depends on the + wire protocol rather than the model. +- Capability overrides are read from the endpoint and model records but are not + yet editable in the Admin Settings endpoint editor. diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index 510f205a1..0f3dd25a5 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,13 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.011 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.010 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.009 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.007 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.006 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.005 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.004 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.003 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.002 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.001 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | @@ -71,6 +78,81 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.011)** + +#### New Features + +* **Model Capabilities Now Come From The Model Catalog Instead Of The Model's Name** + * SimpleChat previously worked out what a model could do by pattern-matching its name, so a model the catalog did not know about — an on-premises or customer-supplied model reached through a Custom endpoint — silently received wrong answers for vision, tool calling, streaming, and reasoning. A model named `corp-llm-v2` was treated as having no capabilities at all. + * Capability answers now resolve through a precedence chain: a per-model override, then an endpoint-level override, then the shipped model catalog, then the original name heuristics. Administrators can describe a model the catalog has never heard of without waiting for a catalog update, and models absent from the catalog behave exactly as before. + * The catalog gained `supportsStreaming` and `reasoning` flags for every model, and now covers Google Gemini, which had no entries at all. Claude, Llama 4, and Phi-4 multimodal models are correctly recognised as vision-capable for the first time; the `-chat` variants of the GPT-5.x families are correctly recognised as not vision-capable. + * The catalog is now validated against a published JSON schema, so a malformed or incomplete model record fails a test rather than silently degrading capability answers at runtime. + * (Ref: `functions_model_capabilities.py`, `model_capabilities.json`, `model_capabilities.schema.json`, `MODEL_CAPABILITY_CATALOG.md`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + +### **(v0.261.010)** + +#### New Features + +* **Custom Model Endpoint Provider** + * Added manually configured Custom endpoints for OpenAI API, Azure OpenAI API, and Anthropic chat models across global, personal, and group scopes. + * Added type-specific model identifiers, API-key authentication, connection testing, response-length controls, and Anthropic Version support without model discovery. + * Enforced HTTPS, DNS/address safety with connection-time address pinning, runtime URL revalidation, redirect refusal, Key Vault secret handling, and an administrator-controlled private-host policy. + * (Ref: #1222, Custom model endpoints, `functions_model_endpoint_runtime.py`, `_multiendpoint_modal.html`, `CUSTOM_MODEL_ENDPOINT_PROVIDER.md`) + +### **(v0.261.009)** + +#### Bug Fixes + +* **Shared Workspace File Approvals Are Visible To Approvers Again** + * Fixed document access index candidate selection for workspace scope projections so pending-approval records are considered alongside already granted records. + * Shared files staged for approval are intentionally not granted yet, so filtering only on `access_granted = true` could hide those files from approval experiences even though they were eligible for review. + * The projection query now includes `approval_status = not_approved` rows while still requiring current-version projection records. + * (Ref: `functions_document_access_index.py`, [Workspace Shared File Approval Visibility Fix](fixes/WORKSPACE_SHARED_FILE_APPROVAL_VISIBILITY_FIX.md)) + +* **Distroless Runtime Copy No Longer Fails On `/usr/lib64` Overlay Conflicts** + * Fixed Docker BuildKit failures where `COPY --from=builder /odbc-runtime/ /` or `COPY --from=builder /playwright-runtime/ /` could abort with `cannot copy to non-directory ... /usr/lib64` when the distroless base exposes `/usr/lib64` as a non-directory entry. + * Updated runtime staging to copy native shared libraries into `/odbc-runtime/usr/lib` and `/playwright-runtime/usr/lib` while continuing to source candidates from both `/usr/lib64` and `/usr/lib` in the builder stage. + * This preserves SQL ODBC and Playwright Chromium runtime packaging while avoiding path-type collisions against evolving base-image filesystem layouts. + * (Ref: `Dockerfile`, `test_sql_container_odbc_runtime.py`, `test_deep_research_chromium_build_opt_out.py`, [Distroless Runtime Overlay Path Fix](fixes/DISTROLESS_RUNTIME_OVERLAY_PATH_FIX.md)) + +### **(v0.261.007)** + +#### Bug Fixes + +* **Markdown Retry Helper Return Contract Clarified** + * Added an explicit defensive exception at the end of the Markdown `OrderedDict` retry helper so static analysis no longer sees a possible implicit `None` return. + * Runtime behavior is unchanged for normal success and retry-exhaustion paths. + * (Ref: `functions_documents.py`, [Markdown Retry Return Contract Fix](fixes/MARKDOWN_RETRY_RETURN_CONTRACT_FIX.md)) + +### **(v0.261.006)** + +#### Bug Fixes + +* **Markdown Uploads Retry Transient OrderedDict Parser Failures** + * Markdown document processing now retries the known transient `OrderedDict mutated during iteration` parser failure before marking a document failed. + * The retry is limited to this specific Markdown failure signature, so unrelated parsing, validation, or service errors still fail normally with their original error. + * (Ref: Markdown upload processing, `functions_documents.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) + +### **(v0.261.005)** + +#### User Interface Enhancements + +* **Workspace Upload Progress Now Separates Request Status From Document Processing Status** + * The temporary upload summary no longer labels unconfirmed browser upload requests as final document failures. This avoids misleading summaries such as `Uploaded 77/204, Failed: 127` when the document list later shows that most documents were queued and processed successfully. + * Personal, group, and public workspace uploads now use `Queued` for confirmed upload requests and direct users to the refreshed document list for final processing status. + * (Ref: workspace upload progress summary, `workspace-documents.js`, `public_workspace.js`, `group_workspaces.html`, [Workspace Upload Status Counter Fix](fixes/WORKSPACE_UPLOAD_STATUS_COUNTER_FIX.md)) + +### **(v0.261.004)** + +#### Bug Fixes + +* **Large Workspace Uploads No Longer Fail On Search Write Gate Contention** + * Fixed partial failures when uploading many small Markdown, JSON, or YAML files to personal, group, or public workspaces at once. Document processing could fail with a message that the Data Management Search write gate changed too often to reserve a write slot. + * The shared write gate now waits within the existing request timeout budget, briefly backs off after transient Cosmos ETag conflicts, and serializes Search writes inside each worker process. This prevents local upload threads from stampeding the same gate document while preserving the migration freeze protection for Azure AI Search writes. + * Markdown processing now batches its chunk embeddings and Search upload instead of reserving the gate once per chunk, which reduces contention and avoids the intermittent `OrderedDict mutated during iteration` failures seen during concurrent Markdown ingestion. + * Added regression coverage for repeated transient gate conflicts, local worker serialization, and Markdown use of the batch chunk writer. + * (Ref: `functions_data_management_search_write_fence.py`, `functions_documents.py`, `test_data_management_search_write_fence.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) + ### **(v0.261.003)** #### Bug Fixes @@ -137,3 +219,17 @@ This page includes the latest release notes inline. Older release sections are s * Page and slide counts are left uncapped here, since how much text a page holds is not known until extraction runs. They are bounded when the chunk is indexed instead. * No shipping default changed. Only custom overrides that could never have been indexed are affected. * (Ref: `get_chunk_size_cap`, `get_chunk_size_config`, Document Extraction settings, `admin_settings.js`) + +* **Logout No Longer Redirects To A Missing Easy Auth Endpoint** + * Logout could redirect to `/.auth/logout?post_logout_redirect_uri=%2Flogin` and return a 404 on Azure App Service deployments that were not actually serving App Service Easy Auth. This affected production deployments as well as development ones. + * The root cause was Easy Auth detection treating the manually configured `WEBSITE_AUTH_AAD_ALLOWED_TENANTS` application setting as proof that Easy Auth was intercepting requests. SimpleChat's own advanced environment variable guidance instructs operators to set that value by hand, so it was never a reliable signal. + * Detection now relies only on the `X-MS-CLIENT-PRINCIPAL` request headers that App Service Easy Auth injects on requests it actually intercepts, so deployments genuinely behind Easy Auth still clear the upstream platform session, and everyone else gets a clean local logout. + * Idle-timeout logout uses the same local logout path, so automatic session expiration follows the corrected behavior as well. + * (Ref: `route_frontend_authentication.py`, `_use_app_service_easy_auth_logout`, `test_app_service_easy_auth_logout.py`, [Easy Auth Logout Detection Fix](fixes/EASY_AUTH_LOGOUT_DETECTION_FIX.md)) + +#### New Features + +* **Opt-Out For App Service Easy Auth Logout** + * Added the `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT` environment variable for deployments where Easy Auth is genuinely active but the platform `/.auth/logout` endpoint is not reachable on the public host, such as when a custom domain or gateway does not route `/.auth/*` to the App Service origin. + * Setting it to `true` keeps logout on the local path instead of redirecting to the platform endpoint. Logout routing decisions are now also traced through debug logging, so `FLASK_DEBUG=1` shows which path was taken and why. + * (Ref: `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT`, `config.py`, `example.env`, [Running SimpleChat Locally](running_simplechat_locally.md)) diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index b07dc95ce..a2c517352 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.011)** + +#### New Features + +* **Model Capabilities Now Come From The Model Catalog Instead Of The Model's Name** + * SimpleChat previously worked out what a model could do by pattern-matching its name, so a model the catalog did not know about — an on-premises or customer-supplied model reached through a Custom endpoint — silently received wrong answers for vision, tool calling, streaming, and reasoning. A model named `corp-llm-v2` was treated as having no capabilities at all. + * Capability answers now resolve through a precedence chain: a per-model override, then an endpoint-level override, then the shipped model catalog, then the original name heuristics. Administrators can describe a model the catalog has never heard of without waiting for a catalog update, and models absent from the catalog behave exactly as before. + * The catalog gained `supportsStreaming` and `reasoning` flags for every model, and now covers Google Gemini, which had no entries at all. Claude, Llama 4, and Phi-4 multimodal models are correctly recognised as vision-capable for the first time; the `-chat` variants of the GPT-5.x families are correctly recognised as not vision-capable. + * The catalog is now validated against a published JSON schema, so a malformed or incomplete model record fails a test rather than silently degrading capability answers at runtime. + * (Ref: `functions_model_capabilities.py`, `model_capabilities.json`, `model_capabilities.schema.json`, `MODEL_CAPABILITY_CATALOG.md`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.010)** #### New Features diff --git a/functional_tests/test_model_capability_catalog_resolution.py b/functional_tests/test_model_capability_catalog_resolution.py new file mode 100644 index 000000000..e1e12d953 --- /dev/null +++ b/functional_tests/test_model_capability_catalog_resolution.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +""" +Functional test for the catalog-backed model capability resolver. +Version: 0.261.011 +Implemented in: 0.261.011 + +Model capability answers used to be guessed from the model's name, so a model the +catalog did not know about -- an on-premises or customer-supplied model reached +through a Custom endpoint -- silently received wrong answers for vision, tool +calling, streaming, and reasoning. + +These tests ensure that: + * the shipped catalog validates against its JSON schema, + * capability answers resolve through override -> endpoint -> catalog -> heuristic, + * catalog matching never lets a model inherit a sibling's capabilities through a + shared "family" value, + * a longer, more specific identifier prefix wins over a shorter one, + * models absent from the catalog still fall back to the legacy heuristics. +""" + +import json +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + ) +) + +from test_support.versioning import assert_app_version_at_least + +import functions_model_capabilities as capabilities + + +CATALOG_PATH = capabilities.get_model_capability_catalog_path() +SCHEMA_PATH = os.path.join( + os.path.dirname(CATALOG_PATH), + "schemas", + "model_capabilities.schema.json", +) + + +def test_catalog_matches_schema(): + """The shipped catalog must validate against its own schema.""" + print("Testing model capability catalog schema...") + try: + import jsonschema + + with open(SCHEMA_PATH, "r", encoding="utf-8") as schema_file: + schema = json.load(schema_file) + with open(CATALOG_PATH, "r", encoding="utf-8") as catalog_file: + catalog = json.load(catalog_file) + + jsonschema.validate(instance=catalog, schema=schema) + + assert catalog["schemaVersion"] >= 2, "Catalog must be schemaVersion 2 or later." + + model_ids = [record["id"] for record in catalog["models"]] + assert len(model_ids) == len(set(model_ids)), "Catalog model ids must be unique." + + providers = {record["provider"] for record in catalog["models"]} + assert "google" in providers, "Catalog must cover Google models." + + print(f"Catalog validated: {len(model_ids)} models, providers={sorted(providers)}") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_family_does_not_leak_capabilities(): + """A shared 'family' must never let one model inherit a sibling's capabilities.""" + print("Testing family isolation...") + try: + with open(CATALOG_PATH, "r", encoding="utf-8") as catalog_file: + catalog = json.load(catalog_file) + + families = {} + for record in catalog["models"]: + families.setdefault(record["family"], []).append(record) + + mixed_families = [ + family + for family, records in families.items() + if len({r["capabilities"]["processesImages"] for r in records}) > 1 + ] + assert mixed_families, ( + "Expected at least one family whose members disagree on vision, " + "otherwise this test proves nothing." + ) + + for family in mixed_families: + for record in families[family]: + resolved = capabilities.is_vision_capable_model(record["id"]) + expected = record["capabilities"]["processesImages"] + assert resolved == expected, ( + f"{record['id']} in family '{family}' resolved vision={resolved}, " + f"catalog says {expected}" + ) + + # A bare family name must not resolve to any member of that family. + for family in mixed_families: + if family not in {record["id"] for record in catalog["models"]}: + assert capabilities.find_model_catalog_record(family) is None, ( + f"Bare family name '{family}' must not match a specific model." + ) + + print(f"Family isolation held for: {sorted(mixed_families)}") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_longest_prefix_wins(): + """A deployment suffix must resolve to the most specific catalog record.""" + print("Testing longest-prefix identifier matching...") + try: + record = capabilities.find_model_catalog_record("gpt-5.1-chat-v2") + assert record is not None, "Expected a catalog match for gpt-5.1-chat-v2." + assert record["id"] == "gpt-5.1-chat", ( + f"gpt-5.1-chat-v2 must resolve to gpt-5.1-chat, got {record['id']}" + ) + assert capabilities.is_vision_capable_model("gpt-5.1-chat-v2") is False + + record = capabilities.find_model_catalog_record("gpt-5.6-sol-eastus") + assert record is not None and record["id"] == "gpt-5.6-sol" + + print("Longest-prefix matching passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_override_precedence(): + """Per-model and endpoint overrides must outrank the catalog and heuristics.""" + print("Testing capability override precedence...") + try: + # gpt-4o is vision-capable by heuristic and catalog; an explicit override wins. + model = {"modelName": "gpt-4o", "capabilities": {"processesImages": False}} + assert capabilities.is_vision_capable_model(model) is False, ( + "Per-model override must outrank the catalog." + ) + + # An endpoint-level override applies when the model declares nothing. + endpoint = {"capabilities": {"processesImages": True}} + unknown_model = {"modelName": "corp-llm-v2"} + assert capabilities.is_vision_capable_model(unknown_model, endpoint) is True, ( + "Endpoint override must apply to an otherwise unknown model." + ) + + # The model override still wins over the endpoint override. + conflicting_model = {"modelName": "corp-llm-v2", "capabilities": {"processesImages": False}} + assert capabilities.is_vision_capable_model(conflicting_model, endpoint) is False + + print("Override precedence passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_unknown_model_falls_back_to_heuristics(): + """Models absent from the catalog keep the legacy name-based answers.""" + print("Testing heuristic fallback for unknown models...") + try: + assert capabilities.find_model_catalog_record("corp-llm-v2") is None + assert capabilities.is_vision_capable_model("corp-llm-v2") is False + assert capabilities.is_vision_capable_model("my-vision-model") is True + assert capabilities.is_vision_capable_model("gpt-4o") is True + + # Streaming defaults to True for unknown models so nothing is wrongly wrapped. + assert capabilities.supports_streaming("corp-llm-v2") is True + # But an explicit declaration is honoured. + assert capabilities.supports_streaming( + {"modelName": "corp-llm-v2", "capabilities": {"supportsStreaming": False}} + ) is False + + print("Heuristic fallback passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_google_models_resolve(): + """Gemini models must resolve from the catalog rather than the name heuristics.""" + print("Testing Google model resolution...") + try: + record = capabilities.find_model_catalog_record("gemini-3.8-flash") + assert record is not None, "gemini-3.8-flash must be in the catalog." + assert record["provider"] == "google" + + # The legacy heuristics answer False for every Gemini capability. + assert capabilities.is_vision_capable_model("gemini-3.8-flash") is True + assert capabilities.supports_tool_calling("gemini-2.5-pro") is True + assert capabilities.supports_streaming("gemini-2.5-flash") is True + + print("Google model resolution passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_version_bumped(): + """The capability resolver ships at or after its implementation version.""" + print("Testing config version...") + try: + assert_app_version_at_least("0.261.011") + print("Config version check passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +if __name__ == "__main__": + tests = [ + test_catalog_matches_schema, + test_family_does_not_leak_capabilities, + test_longest_prefix_wins, + test_override_precedence, + test_unknown_model_falls_back_to_heuristics, + test_google_models_resolve, + test_version_bumped, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + results.append(test()) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) From 88e5224202f99d6dca528caa824b2db98bc0d3de Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 07:51:42 -0400 Subject: [PATCH 3/9] Make Custom endpoint API types a registry, and add Google Gemini Custom endpoints supported exactly three API types, each hard-coded in five places: the allowlist, the request-model resolver, the protocol inference chain, the admin template option list, and the admin JavaScript. Adding a provider meant editing all five, and missing one failed silently. An API type is now one declarative record carrying its wire protocol, model identifier field, URL policy, accepted auth types, and version field. The admin option list, model identifier label, version fields, validation, and protocol inference all derive from it. The three existing API types are unchanged and an unregistered api_type is still refused. Google Gemini is registered, reached through its OpenAI-compatible surface so it keeps running on the validated-DNS pinned transport rather than needing a vendor SDK with its own transport. This also fixes URL construction for any surface that already carries a version segment. SimpleChat appended /v1 unconditionally, turning Gemini's .../v1beta/openai base into .../v1beta/openai/v1 and a 404. The URL policy is now per provider, so /v1 is appended only where it belongs. The registry reaches the browser as server-rendered inline JSON, through the modal's data-api-types attribute and a window helper in base.html for scripts that build model lists without the endpoint editor present. No hard-coded api_type comparison remains in the admin or workspace scripts, which the existing UI contract test now asserts. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/app.py | 2 + application/single_app/config.py | 2 +- .../functions_model_endpoint_providers.py | 184 ++++++++++++ .../functions_model_endpoint_runtime.py | 4 +- .../functions_model_endpoint_types.py | 51 ++-- .../functions_model_endpoint_validation.py | 21 +- .../single_app/model_endpoint_clients.py | 43 ++- .../static/js/admin/admin_model_endpoints.js | 50 +++- .../single_app/static/js/agents_common.js | 2 +- .../js/workspace/workspace_model_endpoints.js | 50 +++- .../templates/_multiendpoint_modal.html | 11 +- application/single_app/templates/base.html | 7 + .../MODEL_ENDPOINT_PROVIDER_REGISTRY.md | 161 ++++++++++ docs/explanation/release-notes/index.md | 13 + docs/explanation/release_notes.md | 12 + .../test_custom_model_endpoint_provider.py | 19 +- .../test_model_endpoint_provider_registry.py | 281 ++++++++++++++++++ 17 files changed, 845 insertions(+), 68 deletions(-) create mode 100644 application/single_app/functions_model_endpoint_providers.py create mode 100644 docs/explanation/features/MODEL_ENDPOINT_PROVIDER_REGISTRY.md create mode 100644 functional_tests/test_model_endpoint_provider_registry.py diff --git a/application/single_app/app.py b/application/single_app/app.py index e91d04577..08902a770 100644 --- a/application/single_app/app.py +++ b/application/single_app/app.py @@ -105,6 +105,7 @@ from route_plugin_logging import bpl as plugin_logging_bp from functions_custom_pages import get_custom_pages_nav from functions_debug import debug_print +from functions_model_endpoint_providers import get_model_endpoint_provider_ui_options from functions_terms_of_use import has_terms_of_use_acceptance from functions_mcp_server_auth import inbound_mcp_required_blueprint @@ -612,6 +613,7 @@ def inject_settings(): idle_timeout_enabled=idle_timeout_enabled, idle_timeout_minutes=idle_timeout_minutes, idle_warning_minutes=idle_warning_minutes, + model_endpoint_api_types=get_model_endpoint_provider_ui_options(), mcp_ui_enabled=is_mcp_ui_enabled() ) diff --git a/application/single_app/config.py b/application/single_app/config.py index 64cd23643..42be1c9c5 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.011" +VERSION = "0.261.012" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py new file mode 100644 index 000000000..85283dd12 --- /dev/null +++ b/application/single_app/functions_model_endpoint_providers.py @@ -0,0 +1,184 @@ +# functions_model_endpoint_providers.py +"""Registry of Custom model endpoint API types. + +Custom endpoints used to support exactly three API types, hard-coded in five +places: the api-type allowlist, the request-model resolver, the protocol +inference if-chain, the admin template's option list, and the admin JavaScript. +Adding a provider meant editing all five and hoping none were missed. + +This module makes an API type a single declarative record. A provider entry +carries everything the rest of the application needs to know: which wire protocol +to speak, which field names the model identifier, how to turn the configured URL +into a request URL, which auth types are accepted, and which optional version +field applies. + +Transport tiers +--------------- +Providers are tiered by whether SimpleChat can control the outbound connection: + + Tier A reached through an OpenAI-compatible or Anthropic HTTP surface, so the + request goes through the validated-DNS pinned transport. + Tier B would require a vendor SDK with its own transport (gRPC or botocore), + which the pinned transport cannot wrap. + +Only Tier A providers are registered here. Google Gemini is reachable at Tier A +through its OpenAI-compatible surface, so it does not need a Tier B entry. + +Only stdlib imports are used so this module can sit below the client, runtime, +validation, and route layers without creating import cycles. +""" + +from typing import Any, Dict, Tuple + + +MODEL_ENDPOINT_PROVIDER_CUSTOM = "custom" + +MODEL_ENDPOINT_API_TYPE_OPENAI = "openai" +MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_API_TYPE_ANTHROPIC = "anthropic" +MODEL_ENDPOINT_API_TYPE_GEMINI = "gemini" + +# Wire protocols. These mirror the MODEL_ENDPOINT_PROTOCOL_* values in +# model_endpoint_clients, which imports them from here to keep one definition. +MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE = "openai_style" +MODEL_ENDPOINT_PROTOCOL_ANTHROPIC = "anthropic" + +# How the configured endpoint URL becomes a request URL. +URL_POLICY_APPEND_V1_IF_MISSING = "append_v1_if_missing" +URL_POLICY_AS_GIVEN = "as_given" +URL_POLICY_AZURE_DEPLOYMENT = "azure_deployment" +URL_POLICY_ANTHROPIC_MESSAGES = "anthropic_messages" + +# Which model record field carries the identifier sent on the wire. +MODEL_IDENTIFIER_MODEL_NAME = "model_name" +MODEL_IDENTIFIER_DEPLOYMENT_NAME = "deployment_name" + +DEFAULT_ANTHROPIC_VERSION = "2023-06-01" + +AUTH_TYPE_API_KEY = "api_key" + + +class ModelEndpointProvider: + """One Custom endpoint API type and everything the app needs to know about it.""" + + def __init__( + self, + api_type: str, + display_name: str, + protocol: str, + model_identifier: str, + url_policy: str, + *, + auth_types: Tuple[str, ...] = (AUTH_TYPE_API_KEY,), + requires_api_version: bool = False, + version_field: str = "", + default_version: str = "", + supports_streaming: bool = True, + supports_tools: bool = True, + description: str = "", + ): + self.api_type = api_type + self.display_name = display_name + self.protocol = protocol + self.model_identifier = model_identifier + self.url_policy = url_policy + self.auth_types = auth_types + self.requires_api_version = requires_api_version + self.version_field = version_field + self.default_version = default_version + self.supports_streaming = supports_streaming + self.supports_tools = supports_tools + self.description = description + + @property + def uses_model_name(self) -> bool: + """Return whether this API type names models rather than deployments.""" + return self.model_identifier == MODEL_IDENTIFIER_MODEL_NAME + + def to_ui_option(self) -> Dict[str, Any]: + """Return the descriptor the admin UI needs to render and drive this type.""" + return { + "value": self.api_type, + "label": self.display_name, + "usesModelName": self.uses_model_name, + "requiresApiVersion": self.requires_api_version, + "versionField": self.version_field, + "defaultVersion": self.default_version, + "authTypes": list(self.auth_types), + "description": self.description, + } + + +MODEL_ENDPOINT_PROVIDERS: Tuple[ModelEndpointProvider, ...] = ( + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_OPENAI, + display_name="OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_APPEND_V1_IF_MISSING, + description=( + "OpenAI and any OpenAI-compatible surface, including gateways, " + "vLLM, and LiteLLM." + ), + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + display_name="Azure OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + model_identifier=MODEL_IDENTIFIER_DEPLOYMENT_NAME, + url_policy=URL_POLICY_AZURE_DEPLOYMENT, + requires_api_version=True, + version_field="api_version", + description="An Azure OpenAI resource addressed by deployment name.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + display_name="Anthropic", + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_ANTHROPIC_MESSAGES, + version_field="anthropic_version", + default_version=DEFAULT_ANTHROPIC_VERSION, + description="Anthropic's messages API, direct or through a gateway.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_GEMINI, + display_name="Google Gemini (OpenAI-compatible)", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + # Gemini's compatible surface already ends in /v1beta/openai/, so appending + # /v1 would produce a 404. The URL is used exactly as configured. + url_policy=URL_POLICY_AS_GIVEN, + description=( + "Google Gemini through its OpenAI-compatible surface, normally " + "https://generativelanguage.googleapis.com/v1beta/openai/." + ), + ), +) + +MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE: Dict[str, ModelEndpointProvider] = { + provider.api_type: provider for provider in MODEL_ENDPOINT_PROVIDERS +} + +MODEL_ENDPOINT_CUSTOM_API_TYPES = frozenset(MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE) + + +def normalize_api_type_value(api_type: Any) -> str: + """Return an api_type string in canonical form.""" + return str(api_type or "").strip().lower().replace("-", "_") + + +def get_model_endpoint_provider(api_type: Any) -> ModelEndpointProvider | None: + """Return the registered provider for an api_type, or None when unsupported.""" + return MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE.get(normalize_api_type_value(api_type)) + + +def get_model_endpoint_provider_ui_options() -> list: + """Return every registered API type as an admin UI descriptor.""" + return [provider.to_ui_option() for provider in MODEL_ENDPOINT_PROVIDERS] + + +def is_supported_custom_api_type(api_type: Any) -> bool: + """Return whether an api_type names a registered provider.""" + return normalize_api_type_value(api_type) in MODEL_ENDPOINT_CUSTOM_API_TYPES diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index f5b590c8f..956eb4906 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -27,6 +27,7 @@ build_openai_style_chat_client, infer_model_endpoint_protocol, normalize_custom_openai_base_url, + resolve_custom_openai_base_url, normalize_openai_style_base_url, resolve_openai_style_request_api_version, SanitizedCustomChatCompletionClient, @@ -202,6 +203,7 @@ def build_model_endpoint_sync_chat_client( direct_custom=direct_custom, allow_private_custom_endpoints=allow_private_custom_endpoints, default_headers=extra_headers, + api_type=api_type, ), runtime_protocol client_kwargs = { 'api_version': api_version, @@ -453,7 +455,7 @@ def build_semantic_kernel_chat_service_for_model( client_kwargs = { 'api_key': api_key, 'base_url': ( - normalize_custom_openai_base_url(endpoint) + resolve_custom_openai_base_url(endpoint, api_type) if direct_custom else normalize_openai_style_base_url(endpoint) ), diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py index 9bab708c7..6a310cb0d 100644 --- a/application/single_app/functions_model_endpoint_types.py +++ b/application/single_app/functions_model_endpoint_types.py @@ -1,27 +1,33 @@ # functions_model_endpoint_types.py -"""Canonical provider, API type, and model identifier helpers.""" +"""Canonical provider, API type, and model identifier helpers. -from typing import Any, Dict +The supported API types and their per-type behaviour live in +functions_model_endpoint_providers. This module keeps the long-standing helper +names that the rest of the application imports, and delegates the decisions to +the registry so an API type is declared in exactly one place. +""" +from typing import Any, Dict -MODEL_ENDPOINT_PROVIDER_CUSTOM = "custom" -MODEL_ENDPOINT_API_TYPE_OPENAI = "openai" -MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI = "azure_openai" -MODEL_ENDPOINT_API_TYPE_ANTHROPIC = "anthropic" -MODEL_ENDPOINT_CUSTOM_API_TYPES = { - MODEL_ENDPOINT_API_TYPE_OPENAI, - MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, +from functions_model_endpoint_providers import ( + DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_API_TYPE_ANTHROPIC, -} -DEFAULT_ANTHROPIC_VERSION = "2023-06-01" + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_GEMINI, + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_CUSTOM_API_TYPES, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_provider, + normalize_api_type_value, +) def normalize_model_endpoint_api_type(provider: Any, api_type: Any) -> str: """Return a supported explicit API type for Custom endpoints.""" normalized_provider = str(provider or "").strip().lower() - normalized_api_type = str(api_type or "").strip().lower().replace("-", "_") if normalized_provider != MODEL_ENDPOINT_PROVIDER_CUSTOM: return "" + normalized_api_type = normalize_api_type_value(api_type) return normalized_api_type if normalized_api_type in MODEL_ENDPOINT_CUSTOM_API_TYPES else "" @@ -37,21 +43,18 @@ def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: endpoint_data: Dict[str, Any] = endpoint if isinstance(endpoint, dict) else {} model_data: Dict[str, Any] = model if isinstance(model, dict) else {} provider = str(endpoint_data.get("provider") or "aoai").strip().lower() - api_type = get_model_endpoint_api_type(endpoint_data) if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: - if api_type in { - MODEL_ENDPOINT_API_TYPE_OPENAI, - MODEL_ENDPOINT_API_TYPE_ANTHROPIC, - }: + registered_provider = get_model_endpoint_provider(get_model_endpoint_api_type(endpoint_data)) + if registered_provider is None: + return "" + if registered_provider.uses_model_name: return str(model_data.get("modelName") or model_data.get("name") or "").strip() - if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: - return str( - model_data.get("deploymentName") - or model_data.get("deployment") - or "" - ).strip() - return "" + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or "" + ).strip() return str( model_data.get("deploymentName") diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py index 3d081d5ad..dde795475 100644 --- a/application/single_app/functions_model_endpoint_validation.py +++ b/application/single_app/functions_model_endpoint_validation.py @@ -7,6 +7,7 @@ from typing import Any, Dict, Iterable from urllib.parse import urlparse, urlunparse +from functions_model_endpoint_providers import get_model_endpoint_provider from functions_model_endpoint_types import ( DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_API_TYPE_ANTHROPIC, @@ -237,6 +238,7 @@ def validate_custom_model_endpoint( api_type = get_model_endpoint_api_type(endpoint) if not api_type: raise ModelEndpointValidationError("Custom endpoint API type is not supported.") + registered_provider = get_model_endpoint_provider(api_type) auth = endpoint.get("auth") if isinstance(endpoint.get("auth"), dict) else {} auth_type = str(auth.get("type") or "").strip().lower() @@ -258,13 +260,12 @@ def validate_custom_model_endpoint( allow_private=allow_private, ) - if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: - _validate_version(connection.get("api_version"), "Azure OpenAI API version") - elif api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: - _validate_version( - connection.get("anthropic_version") or DEFAULT_ANTHROPIC_VERSION, - "Anthropic Version", - ) + if registered_provider is not None and registered_provider.version_field: + version_value = connection.get(registered_provider.version_field) + if registered_provider.default_version: + version_value = version_value or registered_provider.default_version + if registered_provider.requires_api_version or version_value: + _validate_version(version_value, f"{registered_provider.display_name} version") seen_model_names = set() models: Iterable[Any] = endpoint.get("models") or [] @@ -280,9 +281,9 @@ def validate_custom_model_endpoint( request_model = resolve_model_endpoint_request_model(endpoint, model) if not request_model: model_field = ( - "Deployment Name" - if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI - else "Model Name" + "Model Name" + if registered_provider is not None and registered_provider.uses_model_name + else "Deployment Name" ) raise ModelEndpointValidationError( f"Custom endpoint models require {model_field}." diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index 5b1df07c8..c65352384 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -32,6 +32,11 @@ from semantic_kernel.exceptions.service_exceptions import ServiceInvalidExecutionSettingsError from functions_debug import debug_print +from functions_model_endpoint_providers import ( + URL_POLICY_APPEND_V1_IF_MISSING, + URL_POLICY_AS_GIVEN, + get_model_endpoint_provider, +) from functions_model_endpoint_types import ( DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_API_TYPE_ANTHROPIC, @@ -160,17 +165,12 @@ def infer_model_endpoint_protocol( """Infer the runtime protocol from provider, endpoint path, and deployment name.""" normalized_provider = str(provider or "aoai").strip().lower() if normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: - normalized_api_type = normalize_model_endpoint_api_type( - normalized_provider, - api_type, + registered_provider = get_model_endpoint_provider( + normalize_model_endpoint_api_type(normalized_provider, api_type) ) - if normalized_api_type == MODEL_ENDPOINT_API_TYPE_OPENAI: - return MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE - if normalized_api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: - return MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI - if normalized_api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: - return MODEL_ENDPOINT_PROTOCOL_ANTHROPIC - raise ValueError("Custom model endpoints require a supported API type.") + if registered_provider is None: + raise ValueError("Custom model endpoints require a supported API type.") + return registered_provider.protocol endpoint_path = get_endpoint_path(endpoint) @@ -229,6 +229,26 @@ def normalize_custom_openai_base_url(raw_endpoint: Any) -> str: return endpoint.rstrip("/") + "/v1/" +def resolve_custom_openai_base_url(raw_endpoint: Any, api_type: Any = "") -> str: + """Resolve a Custom endpoint base URL using the registered provider's URL policy. + + Appending "/v1" is correct for OpenAI and OpenAI-compatible gateways, but wrong + for surfaces that already carry their own version segment. Google Gemini's + compatible base ends in "/v1beta/openai/", and appending "/v1" to it produces a + 404, so that provider declares the as-given policy instead. + """ + provider = get_model_endpoint_provider(api_type) + url_policy = provider.url_policy if provider else URL_POLICY_APPEND_V1_IF_MISSING + + if url_policy == URL_POLICY_AS_GIVEN: + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ValueError("A Custom endpoint is required for OpenAI-compatible inference.") + return endpoint.rstrip("/") + "/" + + return normalize_custom_openai_base_url(raw_endpoint) + + def normalize_anthropic_messages_url( raw_endpoint: Any, *, @@ -459,13 +479,14 @@ def build_openai_style_chat_client( *, direct_custom: bool = False, allow_private_custom_endpoints: bool = False, + api_type: Any = "", ): """Build an OpenAI-compatible chat client for Foundry data-plane endpoints.""" request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs: Dict[str, Any] = { "api_key": token_or_key, "base_url": ( - normalize_custom_openai_base_url(base_url) + resolve_custom_openai_base_url(base_url, api_type) if direct_custom else normalize_openai_style_base_url(base_url) ), diff --git a/application/single_app/static/js/admin/admin_model_endpoints.js b/application/single_app/static/js/admin/admin_model_endpoints.js index 8e19851a6..693cb66b9 100644 --- a/application/single_app/static/js/admin/admin_model_endpoints.js +++ b/application/single_app/static/js/admin/admin_model_endpoints.js @@ -159,12 +159,49 @@ function isCustomProvider(provider = endpointProviderSelect?.value) { return provider === "custom"; } +// The API type registry is rendered server-side from +// functions_model_endpoint_providers so the option list, the model identifier +// field, and the version field are declared in exactly one place. +let customApiTypeRegistry = null; + +function getCustomApiTypeRegistry() { + if (customApiTypeRegistry) { + return customApiTypeRegistry; + } + customApiTypeRegistry = {}; + try { + const rawRegistry = endpointApiTypeSelect?.dataset?.apiTypes; + if (rawRegistry) { + JSON.parse(rawRegistry).forEach((apiType) => { + customApiTypeRegistry[apiType.value] = apiType; + }); + } + } catch (error) { + console.error("Unable to parse the model endpoint API type registry.", error); + } + return customApiTypeRegistry; +} + +function getCustomApiTypeDescriptor(apiType = getCustomApiType()) { + return getCustomApiTypeRegistry()[apiType] || null; +} + function getCustomApiType() { return endpointApiTypeSelect?.value || "openai"; } function customApiTypeUsesModelName(apiType = getCustomApiType()) { - return apiType === "openai" || apiType === "anthropic"; + const descriptor = getCustomApiTypeDescriptor(apiType); + return descriptor ? Boolean(descriptor.usesModelName) : true; +} + +function customApiTypeRequiresApiVersion(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return Boolean(descriptor?.requiresApiVersion); +} + +function customApiTypeVersionField(apiType = getCustomApiType()) { + return getCustomApiTypeDescriptor(apiType)?.versionField || ""; } function getModelRequestName(model) { @@ -838,8 +875,8 @@ function updateAuthVisibility() { const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; const isFoundry = !customProvider && isFoundryProvider(provider); - const showOpenAiVersion = !customProvider || apiType === "azure_openai"; - const showAnthropicVersion = customProvider && apiType === "anthropic"; + const showOpenAiVersion = !customProvider || customApiTypeRequiresApiVersion(apiType); + const showAnthropicVersion = customProvider && customApiTypeVersionField(apiType) === "anthropic_version"; const projectNameFromEndpoint = syncProjectNameFromEndpoint(); syncEndpointCopyForProvider(); syncVersionCustomVisibility(); @@ -1545,7 +1582,7 @@ function buildEndpointPayload() { return null; } - if ((!customProvider || apiType === "azure_openai") && !openAiApiVersion) { + if ((!customProvider || customApiTypeRequiresApiVersion(apiType)) && !openAiApiVersion) { showToast("OpenAI API version is required.", "warning"); return null; } @@ -1614,9 +1651,10 @@ function buildEndpointPayload() { } : {}; const connection = { endpoint }; - if (customProvider && apiType === "azure_openai") { + const versionField = customProvider ? customApiTypeVersionField(apiType) : ""; + if (customProvider && versionField === "api_version") { connection.api_version = openAiApiVersion; - } else if (customProvider && apiType === "anthropic") { + } else if (customProvider && versionField === "anthropic_version") { connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; } else if (!customProvider) { connection.openai_api_version = openAiApiVersion; diff --git a/application/single_app/static/js/agents_common.js b/application/single_app/static/js/agents_common.js index 21947dab0..01b71de73 100644 --- a/application/single_app/static/js/agents_common.js +++ b/application/single_app/static/js/agents_common.js @@ -711,7 +711,7 @@ export function getAvailableModels({ apimEnabled, settings, agent }) { const modelId = model.id || model.deploymentName || model.deployment || model.modelName || model.name || ''; const deploymentName = model.deploymentName || model.deployment || ''; const modelName = model.modelName || model.name || ''; - const requestModel = provider === 'custom' && ['openai', 'anthropic'].includes(apiType) + const requestModel = provider === 'custom' && window.simplechatCustomApiTypeUsesModelName?.(apiType) ? modelName : deploymentName || modelName; const displayName = model.displayName || requestModel || modelId; diff --git a/application/single_app/static/js/workspace/workspace_model_endpoints.js b/application/single_app/static/js/workspace/workspace_model_endpoints.js index f4a4c790d..02154f45f 100644 --- a/application/single_app/static/js/workspace/workspace_model_endpoints.js +++ b/application/single_app/static/js/workspace/workspace_model_endpoints.js @@ -134,12 +134,49 @@ function isCustomProvider(provider = endpointProviderSelect?.value) { return provider === "custom"; } +// The API type registry is rendered server-side from +// functions_model_endpoint_providers so the option list, the model identifier +// field, and the version field are declared in exactly one place. +let customApiTypeRegistry = null; + +function getCustomApiTypeRegistry() { + if (customApiTypeRegistry) { + return customApiTypeRegistry; + } + customApiTypeRegistry = {}; + try { + const rawRegistry = endpointApiTypeSelect?.dataset?.apiTypes; + if (rawRegistry) { + JSON.parse(rawRegistry).forEach((apiType) => { + customApiTypeRegistry[apiType.value] = apiType; + }); + } + } catch (error) { + console.error("Unable to parse the model endpoint API type registry.", error); + } + return customApiTypeRegistry; +} + +function getCustomApiTypeDescriptor(apiType = getCustomApiType()) { + return getCustomApiTypeRegistry()[apiType] || null; +} + function getCustomApiType() { return endpointApiTypeSelect?.value || "openai"; } function customApiTypeUsesModelName(apiType = getCustomApiType()) { - return apiType === "openai" || apiType === "anthropic"; + const descriptor = getCustomApiTypeDescriptor(apiType); + return descriptor ? Boolean(descriptor.usesModelName) : true; +} + +function customApiTypeRequiresApiVersion(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return Boolean(descriptor?.requiresApiVersion); +} + +function customApiTypeVersionField(apiType = getCustomApiType()) { + return getCustomApiTypeDescriptor(apiType)?.versionField || ""; } function getModelRequestName(model) { @@ -385,8 +422,8 @@ function updateAuthVisibility() { syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, !customProvider || apiType === "azure_openai"); - setElementVisibility(endpointAnthropicVersionGroup, customProvider && apiType === "anthropic"); + setElementVisibility(endpointOpenAiApiVersionGroup, !customProvider || customApiTypeRequiresApiVersion(apiType)); + setElementVisibility(endpointAnthropicVersionGroup, customProvider && customApiTypeVersionField(apiType) === "anthropic_version"); setElementVisibility(endpointSubscriptionGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(endpointResourceGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); @@ -971,7 +1008,7 @@ function buildEndpointPayload() { return null; } - if ((!customProvider || apiType === "azure_openai") && !openAiApiVersion) { + if ((!customProvider || customApiTypeRequiresApiVersion(apiType)) && !openAiApiVersion) { showToast("OpenAI API version is required.", "warning"); return null; } @@ -1040,9 +1077,10 @@ function buildEndpointPayload() { } : {}; const connection = { endpoint }; - if (customProvider && apiType === "azure_openai") { + const versionField = customProvider ? customApiTypeVersionField(apiType) : ""; + if (customProvider && versionField === "api_version") { connection.api_version = openAiApiVersion; - } else if (customProvider && apiType === "anthropic") { + } else if (customProvider && versionField === "anthropic_version") { connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; } else if (!customProvider) { connection.openai_api_version = openAiApiVersion; diff --git a/application/single_app/templates/_multiendpoint_modal.html b/application/single_app/templates/_multiendpoint_modal.html index 5facba8e8..9c3d0bae0 100644 --- a/application/single_app/templates/_multiendpoint_modal.html +++ b/application/single_app/templates/_multiendpoint_modal.html @@ -47,12 +47,13 @@
    Identity se
    - + {% for api_type in model_endpoint_api_types %} + + {% endfor %} -
    The API type controls request paths, model identifiers, and headers for this Custom endpoint.
    +
    The API type controls request paths, model identifiers, and headers for this Custom endpoint.
    diff --git a/application/single_app/templates/base.html b/application/single_app/templates/base.html index f5899222a..fc59212e7 100644 --- a/application/single_app/templates/base.html +++ b/application/single_app/templates/base.html @@ -691,6 +691,13 @@
    diff --git a/docs/explanation/features/MODEL_ENDPOINT_PROVIDER_REGISTRY.md b/docs/explanation/features/MODEL_ENDPOINT_PROVIDER_REGISTRY.md index b862443d3..17d3aa463 100644 --- a/docs/explanation/features/MODEL_ENDPOINT_PROVIDER_REGISTRY.md +++ b/docs/explanation/features/MODEL_ENDPOINT_PROVIDER_REGISTRY.md @@ -69,11 +69,40 @@ produces `…/v1beta/openai/v1/` and a 404. | Policy | Behaviour | |---|---| -| `append_v1_if_missing` | Strip a trailing operation path, then append `/v1` unless the base already ends in `/v1` | +| `append_v1_if_missing` | Append `/v1` only when the URL does not already name the API surface | | `as_given` | Use the configured URL exactly, normalizing only the trailing slash | | `azure_deployment` | Pass to the Azure OpenAI SDK as the resource endpoint | | `anthropic_messages` | Normalize to the Anthropic `/v1/messages` URL | +`append_v1_if_missing` does not append when either of these is true: + +- **The last path segment is already a version**, matching `v` followed by digits + and optional qualifiers — `v1`, `v2`, `v1beta`, `v1alpha`. +- **The URL is a full operation URL**, ending in `/chat/completions`, + `/responses`, or `/models`. Such a URL states the base exactly, so the + operation suffix is stripped and the remainder is used as given. + +Worked examples: + +| Configured | Resolved | +|---|---| +| `https://api.openai.com` | `https://api.openai.com/v1/` | +| `https://api.gen.ai.mil/v1` | `https://api.gen.ai.mil/v1/` | +| `https://gw.example.com/api/v2` | `https://gw.example.com/api/v2/` | +| `https://apim.example.com/inference/chat/completions` | `https://apim.example.com/inference/` | +| `https://generativelanguage.googleapis.com/v1beta/openai` | unchanged (`as_given`) | + +### The exact-URL escape hatch + +Some gateways mount the OpenAI surface at a path SimpleChat cannot infer, such as +`https://gw.example.com/llm/openai`, where the API may live at that path or at +`…/openai/v1`. Rather than guess, the endpoint editor offers **Use this URL +exactly as entered**, stored as `connection.url_mode = "exact"`, which forces the +`as_given` policy for any API type. + +Because the resolved URL is otherwise invisible, **Test Connection reports the +URL that was actually called**, so a rewrite is always verifiable. + ### Transport tiers Providers are tiered by whether SimpleChat can control the outbound connection: @@ -145,6 +174,9 @@ third-party or CDN asset is involved. - unregistered API types still being rejected, including by protocol inference; - Gemini's base URL not gaining a second `/v1`, while plain OpenAI keeps the appending behaviour; +- the append rule leaving existing version segments and full operation URLs + alone, across seven real endpoint shapes; +- the exact-URL escape hatch disabling rewriting for any API type; - every UI descriptor carrying the fields the admin JavaScript reads. `functional_tests/test_custom_model_endpoint_provider.py` additionally asserts diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index 7667e9dc2..caa71bf09 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.013 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.012 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.011 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -80,6 +81,17 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.014)** + +#### Bug Fixes + +* **Custom Endpoint URLs Are No Longer Rewritten Into 404s** + * SimpleChat appended `/v1` to every Custom OpenAI-compatible endpoint, even when the configured URL already said where the API lived. A gateway at `https://apim.example.com/inference/chat/completions` was called at `https://apim.example.com/inference/v1/`, and any base carrying its own version segment, such as `/v1beta` or `/v2`, was broken the same way. + * `/v1` is now appended only when the URL does not already name the API surface. A path whose last segment is a version is left alone, and a full operation URL is treated as stating the base exactly. + * Added a **Use this URL exactly as entered** option for gateways that serve the API at a path SimpleChat cannot infer. + * **Test Connection now reports the URL that was actually called.** URL normalization rewrites the configured endpoint, and that rewrite was previously invisible, so a misdirected request looked identical to a correct one. + * (Ref: `model_endpoint_clients.py`, `route_backend_models.py`, `_multiendpoint_modal.html`, `admin_model_endpoints.js`, `workspace_model_endpoints.js`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.013)** #### Bug Fixes diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 816f7d6fd..d03adce6a 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.014)** + +#### Bug Fixes + +* **Custom Endpoint URLs Are No Longer Rewritten Into 404s** + * SimpleChat appended `/v1` to every Custom OpenAI-compatible endpoint, even when the configured URL already said where the API lived. A gateway at `https://apim.example.com/inference/chat/completions` was called at `https://apim.example.com/inference/v1/`, and any base carrying its own version segment, such as `/v1beta` or `/v2`, was broken the same way. + * `/v1` is now appended only when the URL does not already name the API surface. A path whose last segment is a version is left alone, and a full operation URL is treated as stating the base exactly. + * Added a **Use this URL exactly as entered** option for gateways that serve the API at a path SimpleChat cannot infer. + * **Test Connection now reports the URL that was actually called.** URL normalization rewrites the configured endpoint, and that rewrite was previously invisible, so a misdirected request looked identical to a correct one. + * (Ref: `model_endpoint_clients.py`, `route_backend_models.py`, `_multiendpoint_modal.html`, `admin_model_endpoints.js`, `workspace_model_endpoints.js`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.013)** #### Bug Fixes diff --git a/functional_tests/test_model_endpoint_provider_registry.py b/functional_tests/test_model_endpoint_provider_registry.py index 576863ecc..ef2bfa184 100644 --- a/functional_tests/test_model_endpoint_provider_registry.py +++ b/functional_tests/test_model_endpoint_provider_registry.py @@ -41,6 +41,7 @@ URL_POLICY_AS_GIVEN, get_model_endpoint_provider, get_model_endpoint_provider_ui_options, + normalize_custom_endpoint_url_mode, ) from functions_model_endpoint_types import ( normalize_model_endpoint_api_type, @@ -247,11 +248,74 @@ def test_ui_descriptors_are_complete(): return False +def test_url_policy_respects_existing_version_and_operation_paths(): + """"/v1" must be appended only when the URL does not already say where the API lives.""" + print("Testing URL append policy...") + try: + cases = [ + # (configured URL, expected resolved base) + ("https://api.openai.com/v1", "https://api.openai.com/v1/"), + ("https://api.openai.com", "https://api.openai.com/v1/"), + ("https://api.gen.ai.mil", "https://api.gen.ai.mil/v1/"), + ("https://vllm.corp.example.com/v1", "https://vllm.corp.example.com/v1/"), + # A version segment already names the API surface. + ("https://gw.example.com/api/v2", "https://gw.example.com/api/v2/"), + ("https://x.example.com/v1beta", "https://x.example.com/v1beta/"), + # A full operation URL states the base exactly, so it must not gain /v1. + ( + "https://apim.example.com/inference/chat/completions", + "https://apim.example.com/inference/", + ), + ] + for configured, expected in cases: + resolved = resolve_custom_openai_base_url(configured, MODEL_ENDPOINT_API_TYPE_OPENAI) + assert resolved == expected, ( + f"{configured} resolved to {resolved}, expected {expected}" + ) + + print(f"URL append policy correct for {len(cases)} endpoint shapes") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_exact_url_mode_escape_hatch(): + """url_mode "exact" must disable rewriting for any API type.""" + print("Testing exact URL escape hatch...") + try: + assert normalize_custom_endpoint_url_mode("exact") == "exact" + assert normalize_custom_endpoint_url_mode("") == "auto" + assert normalize_custom_endpoint_url_mode("nonsense") == "auto" + + gateway = "https://gw.example.com/llm/openai" + # Auto mode appends, because the path does not name a version. + assert resolve_custom_openai_base_url( + gateway, MODEL_ENDPOINT_API_TYPE_OPENAI + ) == f"{gateway}/v1/" + # Exact mode leaves it alone. + assert resolve_custom_openai_base_url( + gateway, MODEL_ENDPOINT_API_TYPE_OPENAI, "exact" + ) == f"{gateway}/" + + print("Exact URL escape hatch passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + def test_version_bumped(): """The provider registry ships at or after its implementation version.""" print("Testing config version...") try: - assert_app_version_at_least("0.261.012") + assert_app_version_at_least("0.261.014") print("Config version check passed") return True except Exception as e: @@ -268,6 +332,8 @@ def test_version_bumped(): test_every_registered_provider_is_reachable, test_unregistered_api_type_is_rejected, test_gemini_base_url_is_not_mangled, + test_url_policy_respects_existing_version_and_operation_paths, + test_exact_url_mode_escape_hatch, test_ui_descriptors_are_complete, test_version_bumped, ] From 934a557750300588db41535d69c7bd1762305b85 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 08:16:19 -0400 Subject: [PATCH 6/9] Deliver completed responses as a real stream, not one blob SimpleChat supports one response mode, streaming, but some responses cannot be streamed by the model. Tool calling is the clearest case: a tool call has to arrive complete, so the request is made without streaming. That path already pushed the completed answer through the streaming interface, but as a single chunk, so the user saw nothing and then everything at once, which reads as a hang. Agents and plugins rely on tool calling, so this was a common path rather than an exotic one. A completed answer is now split at word boundaries and delivered progressively. Chunking is lossless, which matters because the frontend accumulates chunks by concatenation: the tokenizer preserves every character, including runs of whitespace and newlines, so the reassembled text is identical to the original. Function calls cannot be partially delivered, so non-text items ride on the final message alongside the finish reason and usage metadata. Emitting metadata exactly once keeps token usage from being multiplied by the chunk count. This also restores usage reporting on streaming responses. stream_options is how an OpenAI-compatible stream reports token usage, and it was stripped from every request, including from endpoints that accept it. Support is now declared per provider and the option is dropped only for surfaces that reject it. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_model_endpoint_providers.py | 5 + .../single_app/model_endpoint_clients.py | 109 ++++++- .../features/SYNTHETIC_STREAMING.md | 113 +++++++ docs/explanation/release-notes/index.md | 58 +--- docs/explanation/release-notes/v0.261.md | 46 ++- docs/explanation/release_notes.md | 11 + ...stom_model_endpoint_synthetic_streaming.py | 277 ++++++++++++++++++ 8 files changed, 570 insertions(+), 51 deletions(-) create mode 100644 docs/explanation/features/SYNTHETIC_STREAMING.md create mode 100644 functional_tests/test_custom_model_endpoint_synthetic_streaming.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 18c541ec5..084347a10 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.014" +VERSION = "0.261.015" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py index bd5829756..e7a987fff 100644 --- a/application/single_app/functions_model_endpoint_providers.py +++ b/application/single_app/functions_model_endpoint_providers.py @@ -88,6 +88,7 @@ def __init__( default_version: str = "", supports_streaming: bool = True, supports_tools: bool = True, + supports_stream_options: bool = False, description: str = "", ): self.api_type = api_type @@ -101,6 +102,7 @@ def __init__( self.default_version = default_version self.supports_streaming = supports_streaming self.supports_tools = supports_tools + self.supports_stream_options = supports_stream_options self.description = description @property @@ -129,6 +131,9 @@ def to_ui_option(self) -> Dict[str, Any]: protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, model_identifier=MODEL_IDENTIFIER_MODEL_NAME, url_policy=URL_POLICY_APPEND_V1_IF_MISSING, + # OpenAI accepts stream_options.include_usage, which is how a streaming + # response reports token usage. Providers that reject it keep the default. + supports_stream_options=True, description=( "OpenAI and any OpenAI-compatible surface, including gateways, " "vLLM, and LiteLLM." diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index 357b1bba3..66a900123 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -319,10 +319,44 @@ def resolve_openai_style_request_api_version(raw_api_version: Any) -> str: return "" +SYNTHETIC_STREAM_CHUNK_CHARACTERS = 24 +_SYNTHETIC_STREAM_TOKEN_PATTERN = re.compile(r"\S+\s*|\s+") + + +def iter_synthetic_stream_text_chunks( + text: Any, + chunk_characters: int = SYNTHETIC_STREAM_CHUNK_CHARACTERS, +) -> Iterator[str]: + """Split a completed response into stream-sized chunks at word boundaries. + + SimpleChat only supports streaming responses, so a provider or code path that + can only return a completed answer still has to deliver it through the stream. + Emitting the whole answer as one chunk technically satisfies that, but the user + sees nothing and then everything at once, which reads as a hang. + + Chunking is lossless: concatenating every chunk reproduces the original text + exactly, including its whitespace, because the frontend accumulates chunks. + """ + normalized_text = str(text or "") + if not normalized_text: + return + if chunk_characters < 1: + yield normalized_text + return + + buffer = "" + for token in _SYNTHETIC_STREAM_TOKEN_PATTERN.findall(normalized_text): + buffer += token + if len(buffer) >= chunk_characters: + yield buffer + buffer = "" + if buffer: + yield buffer + + def normalize_chat_completion_text(content: Any) -> str: """Normalize text content returned by OpenAI-compatible chat responses.""" - if content is None: - return "" + if content is None: return "" if isinstance(content, str): return content if isinstance(content, (list, tuple)): @@ -557,11 +591,16 @@ def __init__( self._sanitize_errors = sanitize_errors self._api_type = api_type self._request_url = request_url + provider = get_model_endpoint_provider(api_type) if sanitize_errors else None + self._supports_stream_options = bool(provider and provider.supports_stream_options) self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): request_kwargs = dict(kwargs) - request_kwargs.pop("stream_options", None) + # stream_options is how a streaming response reports token usage. It is + # dropped only for surfaces that reject it, rather than for everyone. + if not self._supports_stream_options: + request_kwargs.pop("stream_options", None) try: response = self._client.chat.completions.create(**request_kwargs) except Exception as exc: @@ -1361,11 +1400,18 @@ async def _inner_get_streaming_chat_message_contents( function_invoke_attempt: int = 0, ): if getattr(settings, "tools", None): + # Tool calling is answered without streaming, because a tool call has + # to arrive complete. The completed answer is still delivered through + # the stream, chunked so it reads like one. request_kwargs = self._build_request_kwargs(chat_history, settings, stream=False) client = self._build_client() response = await asyncio.to_thread(client.chat.completions.create, **request_kwargs) for message in self._create_chat_message_contents_from_response(response): - yield [self._to_streaming_chat_message_content(message, function_invoke_attempt)] + for streaming_message in self._iter_synthetic_stream_messages( + message, + function_invoke_attempt, + ): + yield [streaming_message] return request_kwargs = self._build_request_kwargs(chat_history, settings, stream=True) @@ -1410,6 +1456,61 @@ async def _inner_get_streaming_chat_message_contents( ) ] + def _iter_synthetic_stream_messages( + self, + message: ChatMessageContent, + function_invoke_attempt: int, + ) -> Iterator[StreamingChatMessageContent]: + """Deliver a completed message through the streaming interface, in chunks. + + Text is split so the response arrives progressively. Non-text items, such + as function calls, must arrive whole, so they ride on the final message + alongside the finish reason and usage metadata. Emitting metadata only + once keeps token usage from being counted per chunk. + """ + text_items = [item for item in message.items or [] if isinstance(item, TextContent)] + other_items = [item for item in message.items or [] if not isinstance(item, TextContent)] + + combined_text = "".join(item.text or "" for item in text_items) + text_chunks = list(iter_synthetic_stream_text_chunks(combined_text)) + + # Every chunk before the last carries text only. + for chunk_text in text_chunks[:-1]: + yield StreamingChatMessageContent( + role=message.role, + items=[StreamingTextContent( + choice_index=0, + text=chunk_text, + ai_model_id=self.ai_model_id, + )], + choice_index=0, + ai_model_id=self.ai_model_id, + function_invoke_attempt=function_invoke_attempt, + ) + + final_items: List[Any] = [] + if text_chunks: + final_items.append(StreamingTextContent( + choice_index=0, + text=text_chunks[-1], + ai_model_id=self.ai_model_id, + inner_content=text_items[-1].inner_content if text_items else None, + metadata=text_items[-1].metadata if text_items else {}, + encoding=text_items[-1].encoding if text_items else None, + )) + final_items.extend(other_items) + + yield StreamingChatMessageContent( + role=message.role, + items=final_items, + choice_index=0, + ai_model_id=self.ai_model_id, + inner_content=message.inner_content, + metadata=message.metadata, + finish_reason=message.finish_reason, + function_invoke_attempt=function_invoke_attempt, + ) + def _to_streaming_chat_message_content( self, message: ChatMessageContent, diff --git a/docs/explanation/features/SYNTHETIC_STREAMING.md b/docs/explanation/features/SYNTHETIC_STREAMING.md new file mode 100644 index 000000000..f01e48c73 --- /dev/null +++ b/docs/explanation/features/SYNTHETIC_STREAMING.md @@ -0,0 +1,113 @@ +# Synthetic Streaming For Completed Responses + +## Overview + +SimpleChat supports one response mode: streaming. The non-streaming chat path is +legacy and is being retired. + +Some responses cannot be streamed by the model. The clearest case is tool +calling: a tool call has to arrive complete before it can be executed, so the +request is made without streaming. Some providers also expose no streaming +surface at all. + +Those responses still have to reach the browser through the stream. Synthetic +streaming is how a completed answer is delivered as a stream. + +**Implemented in version: 0.261.015** + +## The problem it solves + +The behaviour already existed in miniature. The Anthropic adapter's tool-calling +path took a completed response and pushed it through the streaming interface — +but as a single chunk. + +That satisfies the interface and fails the user. The response arrives as nothing, +nothing, nothing, then the entire answer at once. For a long answer that reads as +a hang, and because SimpleChat leans heavily on agents and plugins, this was a +common path rather than an exotic one. + +## Architecture + +### Chunking + +`iter_synthetic_stream_text_chunks(text, chunk_characters=24)` splits completed +text at word boundaries into stream-sized chunks. + +The critical property is that **chunking is lossless**. The frontend accumulates +chunks by string concatenation, so any lost or duplicated character would corrupt +the rendered answer. The splitter tokenizes with `\S+\s*|\s+`, which preserves +every character including runs of whitespace and newlines, so concatenating every +chunk reproduces the input exactly. + +```python +"".join(iter_synthetic_stream_text_chunks(text)) == text # always true +``` + +Short answers stay a single chunk, because there is nothing to animate. + +### Message assembly + +`_iter_synthetic_stream_messages` turns one completed `ChatMessageContent` into +several `StreamingChatMessageContent` messages: + +| Content | Placement | +|---|---| +| Text | Split across every chunk | +| Function calls and other non-text items | The final message only | +| `finish_reason` | The final message only | +| `metadata`, including usage | The final message only | + +Non-text items ride on the final message because a function call cannot be +partially delivered. The finish reason and metadata are emitted exactly once so +that token usage is not multiplied by the number of chunks. + +### Streaming usage reporting + +`stream_options` is how an OpenAI-compatible streaming response reports token +usage. It was previously stripped from every OpenAI-compatible request, which +suppressed usage reporting for all of them, including endpoints that support it. + +Support is now declared per provider through `supports_stream_options` on the +provider registry entry, and the option is dropped only for surfaces that reject +it. OpenAI declares support; other providers keep the conservative default until +their support is confirmed. + +## Usage + +Any adapter that must return a completed answer through the streaming interface +should use the chunker rather than yielding one message: + +```python +from model_endpoint_clients import iter_synthetic_stream_text_chunks + +for chunk_text in iter_synthetic_stream_text_chunks(completed_text): + yield build_streaming_message(chunk_text) +``` + +The provider registry's `supports_streaming` flag records whether an API type can +stream natively, so a future adapter can decide whether to wrap. + +## Testing and validation + +`functional_tests/test_custom_model_endpoint_synthetic_streaming.py` covers: + +- lossless chunking across eight samples, including empty input, leading and + trailing whitespace, embedded newlines, a 200-character unbroken token, and + multi-byte Unicode including an emoji; +- a long answer being split into several chunks while a short one stays single; +- function calls surviving as exactly one item, on the final chunk, with the text + still reconstructing exactly; +- the finish reason and usage metadata appearing only on the final chunk; +- `stream_options` no longer being dropped unconditionally. + +## Known limitations + +- The terminal `done` event guarantee on the server's SSE route is unchanged in + this version. The frontend errors if a stream ends without one, so that + guarantee is worth making explicit on every failure path. +- The legacy non-streaming `/api/chat` route still exists and is still reachable + through the compatibility bridge. It is expected to be deprecated and removed + in a later release. +- `supports_streaming` is declared on provider registry entries but is not yet + used to wrap a non-streaming provider automatically, because every currently + registered provider streams natively. diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index caa71bf09..855ae2510 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.015 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.013 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.012 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -31,7 +32,7 @@ This page includes the latest release notes inline. Older release sections are s | v0.261.005 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.004 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.003 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.261.002 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.002 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.001 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.260.025 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.024 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | @@ -81,6 +82,17 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.015)** + +#### Bug Fixes + +* **Tool-Calling Responses Now Actually Stream** + * SimpleChat only supports streaming responses, but tool calling cannot stream, because a tool call has to arrive whole. The completed answer was delivered through the streaming interface as a single chunk, so the user saw nothing at all and then the entire response at once, which reads as a hang. Because agents and plugins rely on tool calling, this was a common path rather than an edge case. + * A completed answer is now split into chunks at word boundaries and delivered progressively, so it reads like a real stream. Chunking is lossless — the reassembled text is byte-for-byte identical. + * Tool calls still arrive whole, on the final chunk, alongside the finish reason and usage metadata. Emitting metadata once means token usage is no longer at risk of being counted per chunk. + * **Streaming responses can report token usage again.** `stream_options` was stripped from every OpenAI-compatible request, which suppressed usage reporting for all of them. It is now dropped only for surfaces that reject it. + * (Ref: `model_endpoint_clients.py`, `functions_model_endpoint_providers.py`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.014)** #### Bug Fixes @@ -227,47 +239,3 @@ This page includes the latest release notes inline. Older release sections are s * Action configuration panes were captured without saving any action, so every credential field shows only its placeholder text and no tenant values were recorded. Where an admin settings pane already held real values, those fields were replaced with example values before capture and the page was reloaded without saving. * Replaced the generated placeholder alt text on every filled slot with a description of what the reader actually learns from the image. * (Ref: `docs/images/admin/`, `docs/images/reference/`, `docs/images/guides/`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) - -### **(v0.261.002)** - -#### User Interface Enhancements - -* **Inbound MCP Enablement Guidance** - * Added a visible **Inbound MCP** tab state for deployments where the preview admin UI is disabled by the missing `ENABLE_MCP_UI=true` App Service application setting. - * The disabled-state card explains how to enable the preview UI while making clear that the inbound MCP runtime remains off until an admin turns on **Enable inbound MCP server** after authentication, client allowlist, source, and governance prerequisites are ready. - * (Ref: `admin/_panes/inbound-mcp.html`, `admin_settings_nav.py`, [#1364](https://github.com/microsoft/simplechat/issues/1364)) - -#### Bug Fixes - -* **Delegated Governance New Policy Modal Opens On Split Governance Tabs** - * Fixed the delegated item governance **New Policy** button so it opens the policy editor after Admin Settings governance was split into Feature Governance, Policies, and MCP Governance tabs. - * Updated governance quick links to target the correct split tab panes instead of the retired aggregate Governance pane. - * (Ref: `admin_governance.js`, delegated item policy editor, [#1362](https://github.com/microsoft/simplechat/issues/1362)) - -* **Large Markdown Files No Longer Fail To Upload** - * Uploading a Markdown file could fail with `Failed processing Markdown file ...` and take down the whole document, not just the oversized part of it. Long pages with a big section under a single heading, such as a release notes file, were the usual trigger. - * Markdown was the only ingestion path with no maximum chunk size. Its splitter divided the file on headings, and the step afterwards only ever merged chunks that were **too small** — nothing split a chunk that was too large. A heading with no subheading beneath it therefore became one chunk as large as all the text under it, which the embedding model refused. - * Lowering **Markdown (words)** in Admin Settings did not work around this, because that value was only ever used as a minimum. It is now a real target, so the setting behaves the way its name implies. - * Sections are now split to the configured size, a character limit is applied after merging to catch content such as tables and code blocks that take up more of the model's budget than their word count suggests, and a final safeguard keeps any remaining outlier inside the limit. That safeguard trims only the text used to build the chunk's search vector — the chunk itself is still stored in full, so citations and content are unaffected. - * (Ref: `process_md`, `save_chunks`, `functions_content.py`, `functions_documents.py`) - -* **Chunk Size Limits Now Respect Their Unit** - * Chunk sizes are configured per file type in words, characters, or pages, but a single shared limit was applied to all of them. That let a word-based field be set to 16,384 words — far more than can be indexed — while implying the value was valid. - * Word and character fields now have separate limits, both derived from the embedding model's context window, and the Document Extraction tab shows the current values. A value above its limit is reduced on save and the page names the fields it changed. - * Page and slide counts are left uncapped here, since how much text a page holds is not known until extraction runs. They are bounded when the chunk is indexed instead. - * No shipping default changed. Only custom overrides that could never have been indexed are affected. - * (Ref: `get_chunk_size_cap`, `get_chunk_size_config`, Document Extraction settings, `admin_settings.js`) - -* **Logout No Longer Redirects To A Missing Easy Auth Endpoint** - * Logout could redirect to `/.auth/logout?post_logout_redirect_uri=%2Flogin` and return a 404 on Azure App Service deployments that were not actually serving App Service Easy Auth. This affected production deployments as well as development ones. - * The root cause was Easy Auth detection treating the manually configured `WEBSITE_AUTH_AAD_ALLOWED_TENANTS` application setting as proof that Easy Auth was intercepting requests. SimpleChat's own advanced environment variable guidance instructs operators to set that value by hand, so it was never a reliable signal. - * Detection now relies only on the `X-MS-CLIENT-PRINCIPAL` request headers that App Service Easy Auth injects on requests it actually intercepts, so deployments genuinely behind Easy Auth still clear the upstream platform session, and everyone else gets a clean local logout. - * Idle-timeout logout uses the same local logout path, so automatic session expiration follows the corrected behavior as well. - * (Ref: `route_frontend_authentication.py`, `_use_app_service_easy_auth_logout`, `test_app_service_easy_auth_logout.py`, [Easy Auth Logout Detection Fix](fixes/EASY_AUTH_LOGOUT_DETECTION_FIX.md)) - -#### New Features - -* **Opt-Out For App Service Easy Auth Logout** - * Added the `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT` environment variable for deployments where Easy Auth is genuinely active but the platform `/.auth/logout` endpoint is not reachable on the public host, such as when a custom domain or gateway does not route `/.auth/*` to the App Service origin. - * Setting it to `true` keeps logout on the local path instead of redirecting to the platform endpoint. Logout routing decisions are now also traced through debug logging, so `FLASK_DEBUG=1` shows which path was taken and why. - * (Ref: `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT`, `config.py`, `example.env`, [Running SimpleChat Locally](running_simplechat_locally.md)) diff --git a/docs/explanation/release-notes/v0.261.md b/docs/explanation/release-notes/v0.261.md index 90389ba31..addf486c1 100644 --- a/docs/explanation/release-notes/v0.261.md +++ b/docs/explanation/release-notes/v0.261.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.261 series" -description: "SimpleChat release notes for 0.261.001." +description: "SimpleChat release notes for 0.261.002 – 0.261.001." section: "Reference" layout: page --- @@ -11,6 +11,50 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.261.002)** + +#### User Interface Enhancements + +* **Inbound MCP Enablement Guidance** + * Added a visible **Inbound MCP** tab state for deployments where the preview admin UI is disabled by the missing `ENABLE_MCP_UI=true` App Service application setting. + * The disabled-state card explains how to enable the preview UI while making clear that the inbound MCP runtime remains off until an admin turns on **Enable inbound MCP server** after authentication, client allowlist, source, and governance prerequisites are ready. + * (Ref: `admin/_panes/inbound-mcp.html`, `admin_settings_nav.py`, [#1364](https://github.com/microsoft/simplechat/issues/1364)) + +#### Bug Fixes + +* **Delegated Governance New Policy Modal Opens On Split Governance Tabs** + * Fixed the delegated item governance **New Policy** button so it opens the policy editor after Admin Settings governance was split into Feature Governance, Policies, and MCP Governance tabs. + * Updated governance quick links to target the correct split tab panes instead of the retired aggregate Governance pane. + * (Ref: `admin_governance.js`, delegated item policy editor, [#1362](https://github.com/microsoft/simplechat/issues/1362)) + +* **Large Markdown Files No Longer Fail To Upload** + * Uploading a Markdown file could fail with `Failed processing Markdown file ...` and take down the whole document, not just the oversized part of it. Long pages with a big section under a single heading, such as a release notes file, were the usual trigger. + * Markdown was the only ingestion path with no maximum chunk size. Its splitter divided the file on headings, and the step afterwards only ever merged chunks that were **too small** — nothing split a chunk that was too large. A heading with no subheading beneath it therefore became one chunk as large as all the text under it, which the embedding model refused. + * Lowering **Markdown (words)** in Admin Settings did not work around this, because that value was only ever used as a minimum. It is now a real target, so the setting behaves the way its name implies. + * Sections are now split to the configured size, a character limit is applied after merging to catch content such as tables and code blocks that take up more of the model's budget than their word count suggests, and a final safeguard keeps any remaining outlier inside the limit. That safeguard trims only the text used to build the chunk's search vector — the chunk itself is still stored in full, so citations and content are unaffected. + * (Ref: `process_md`, `save_chunks`, `functions_content.py`, `functions_documents.py`) + +* **Chunk Size Limits Now Respect Their Unit** + * Chunk sizes are configured per file type in words, characters, or pages, but a single shared limit was applied to all of them. That let a word-based field be set to 16,384 words — far more than can be indexed — while implying the value was valid. + * Word and character fields now have separate limits, both derived from the embedding model's context window, and the Document Extraction tab shows the current values. A value above its limit is reduced on save and the page names the fields it changed. + * Page and slide counts are left uncapped here, since how much text a page holds is not known until extraction runs. They are bounded when the chunk is indexed instead. + * No shipping default changed. Only custom overrides that could never have been indexed are affected. + * (Ref: `get_chunk_size_cap`, `get_chunk_size_config`, Document Extraction settings, `admin_settings.js`) + +* **Logout No Longer Redirects To A Missing Easy Auth Endpoint** + * Logout could redirect to `/.auth/logout?post_logout_redirect_uri=%2Flogin` and return a 404 on Azure App Service deployments that were not actually serving App Service Easy Auth. This affected production deployments as well as development ones. + * The root cause was Easy Auth detection treating the manually configured `WEBSITE_AUTH_AAD_ALLOWED_TENANTS` application setting as proof that Easy Auth was intercepting requests. SimpleChat's own advanced environment variable guidance instructs operators to set that value by hand, so it was never a reliable signal. + * Detection now relies only on the `X-MS-CLIENT-PRINCIPAL` request headers that App Service Easy Auth injects on requests it actually intercepts, so deployments genuinely behind Easy Auth still clear the upstream platform session, and everyone else gets a clean local logout. + * Idle-timeout logout uses the same local logout path, so automatic session expiration follows the corrected behavior as well. + * (Ref: `route_frontend_authentication.py`, `_use_app_service_easy_auth_logout`, `test_app_service_easy_auth_logout.py`, [Easy Auth Logout Detection Fix](fixes/EASY_AUTH_LOGOUT_DETECTION_FIX.md)) + +#### New Features + +* **Opt-Out For App Service Easy Auth Logout** + * Added the `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT` environment variable for deployments where Easy Auth is genuinely active but the platform `/.auth/logout` endpoint is not reachable on the public host, such as when a custom domain or gateway does not route `/.auth/*` to the App Service origin. + * Setting it to `true` keeps logout on the local path instead of redirecting to the platform endpoint. Logout routing decisions are now also traced through debug logging, so `FLASK_DEBUG=1` shows which path was taken and why. + * (Ref: `DISABLE_APP_SERVICE_EASY_AUTH_LOGOUT`, `config.py`, `example.env`, [Running SimpleChat Locally](running_simplechat_locally.md)) + {% raw %} ### **(v0.261.001)** diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index d03adce6a..b2206b5b7 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,17 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.015)** + +#### Bug Fixes + +* **Tool-Calling Responses Now Actually Stream** + * SimpleChat only supports streaming responses, but tool calling cannot stream, because a tool call has to arrive whole. The completed answer was delivered through the streaming interface as a single chunk, so the user saw nothing at all and then the entire response at once, which reads as a hang. Because agents and plugins rely on tool calling, this was a common path rather than an edge case. + * A completed answer is now split into chunks at word boundaries and delivered progressively, so it reads like a real stream. Chunking is lossless — the reassembled text is byte-for-byte identical. + * Tool calls still arrive whole, on the final chunk, alongside the finish reason and usage metadata. Emitting metadata once means token usage is no longer at risk of being counted per chunk. + * **Streaming responses can report token usage again.** `stream_options` was stripped from every OpenAI-compatible request, which suppressed usage reporting for all of them. It is now dropped only for surfaces that reject it. + * (Ref: `model_endpoint_clients.py`, `functions_model_endpoint_providers.py`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.014)** #### Bug Fixes diff --git a/functional_tests/test_custom_model_endpoint_synthetic_streaming.py b/functional_tests/test_custom_model_endpoint_synthetic_streaming.py new file mode 100644 index 000000000..3238c0191 --- /dev/null +++ b/functional_tests/test_custom_model_endpoint_synthetic_streaming.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +""" +Functional test for synthetic streaming of completed model responses. +Version: 0.261.015 +Implemented in: 0.261.015 + +SimpleChat only supports streaming responses; the non-streaming path is legacy. +Some providers and code paths can only return a completed answer -- notably tool +calling, where a tool call has to arrive whole -- so that answer still has to be +delivered through the stream. + +The previous implementation did deliver it through the streaming interface, but +as a single chunk, so the user saw nothing and then everything at once. Because +SimpleChat leans on agents and plugins, that path is common rather than exotic. + +These tests ensure that: + * chunking is lossless, since the frontend accumulates chunks, + * a completed answer is delivered as several chunks rather than one blob, + * non-text items such as function calls still arrive whole, + * the finish reason and usage metadata appear exactly once, on the final chunk, + so token usage is not multiplied by the number of chunks. +""" + +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + ) +) + +from test_support.versioning import assert_app_version_at_least + +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.contents.utils.finish_reason import FinishReason + +from model_endpoint_clients import ( + AnthropicSemanticKernelChatCompletion, + iter_synthetic_stream_text_chunks, +) + + +SAMPLE_TEXTS = [ + "Hello world, this is a synthetic stream that should arrive in several pieces.", + "short", + "", + " leading and trailing ", + "line one\nline two\n\nline four", + "a" * 200, + "word " * 40, + "Unicode: caf\u00e9 na\u00efve \u4e2d\u6587 \U0001F600 done", +] + + +def _build_service(): + return AnthropicSemanticKernelChatCompletion( + service_id="test", + deployment_name="claude-opus-5", + endpoint="https://api.anthropic.com", + api_key="test-key", + ) + + +def test_chunking_is_lossless(): + """Concatenating every chunk must reproduce the original text exactly.""" + print("Testing lossless chunking...") + try: + for text in SAMPLE_TEXTS: + chunks = list(iter_synthetic_stream_text_chunks(text)) + rejoined = "".join(chunks) + assert rejoined == text, ( + f"Chunking lost or altered content.\n in : {text!r}\n out: {rejoined!r}" + ) + + # Empty input yields nothing rather than an empty chunk. + assert list(iter_synthetic_stream_text_chunks("")) == [] + assert list(iter_synthetic_stream_text_chunks(None)) == [] + + # A non-positive chunk size must not loop or lose content. + assert list(iter_synthetic_stream_text_chunks("abc", 0)) == ["abc"] + + print(f"Chunking lossless across {len(SAMPLE_TEXTS)} samples") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_long_answer_is_split_into_several_chunks(): + """A completed answer must not arrive as a single blob.""" + print("Testing multi-chunk delivery...") + try: + text = ( + "The answer arrives in several pieces so that it reads like a real " + "stream rather than appearing all at once after a long pause." + ) + chunks = list(iter_synthetic_stream_text_chunks(text)) + assert len(chunks) > 1, "A long answer must be split into multiple chunks." + assert "".join(chunks) == text + + # A short answer stays a single chunk; there is nothing to animate. + assert len(list(iter_synthetic_stream_text_chunks("ok"))) == 1 + + print(f"Long answer split into {len(chunks)} chunks") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_function_calls_survive_synthetic_streaming(): + """Tool calls must arrive whole, on the final chunk.""" + print("Testing function call preservation...") + try: + service = _build_service() + text = "Calling a tool now, after a reasonably long preamble to force chunking." + message = ChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + TextContent(text=text), + FunctionCallContent(id="call-1", name="do_thing", arguments="{}"), + ], + finish_reason=FinishReason.TOOL_CALLS, + ) + + streamed = list(service._iter_synthetic_stream_messages(message, 0)) + assert len(streamed) > 1, "Expected the preamble to be chunked." + + function_calls = [ + item + for streamed_message in streamed + for item in streamed_message.items + if isinstance(item, FunctionCallContent) + ] + assert len(function_calls) == 1, ( + f"Expected exactly one function call, got {len(function_calls)}" + ) + assert function_calls[0].name == "do_thing" + + # The function call must ride on the final message, not an earlier one. + final_items = streamed[-1].items + assert any(isinstance(item, FunctionCallContent) for item in final_items) + + # The text must still reconstruct exactly. + assert "".join(str(m) for m in streamed) == text + + print("Function calls preserved on the final chunk") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_finish_reason_and_metadata_appear_once(): + """Usage metadata must not be multiplied by the number of chunks.""" + print("Testing terminal metadata placement...") + try: + service = _build_service() + message = ChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[TextContent(text="A long enough answer to be split into chunks here.")], + finish_reason=FinishReason.STOP, + metadata={"usage": {"prompt_tokens": 10, "completion_tokens": 20}}, + ) + + streamed = list(service._iter_synthetic_stream_messages(message, 0)) + assert len(streamed) > 1, "Expected multiple chunks for this answer." + + finish_reasons = [m.finish_reason for m in streamed] + assert finish_reasons[-1] == FinishReason.STOP + assert all(reason is None for reason in finish_reasons[:-1]), ( + f"Finish reason leaked onto non-final chunks: {finish_reasons}" + ) + + carries_usage = [bool(m.metadata) for m in streamed] + assert carries_usage[-1] is True + assert not any(carries_usage[:-1]), ( + f"Usage metadata repeated across chunks: {carries_usage}" + ) + + print("Finish reason and usage metadata appear once, on the final chunk") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_stream_options_are_kept_where_supported(): + """stream_options must be dropped only for surfaces that reject it.""" + print("Testing stream_options handling...") + try: + from functions_model_endpoint_providers import ( + MODEL_ENDPOINT_API_TYPE_OPENAI, + get_model_endpoint_provider, + ) + + openai_provider = get_model_endpoint_provider(MODEL_ENDPOINT_API_TYPE_OPENAI) + assert openai_provider.supports_stream_options is True, ( + "OpenAI accepts stream_options.include_usage, which reports token usage." + ) + + source = open( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + "model_endpoint_clients.py", + ), + encoding="utf-8", + ).read() + assert ( + 'request_kwargs = dict(kwargs)\n request_kwargs.pop("stream_options", None)' + not in source + ), "stream_options must no longer be dropped unconditionally." + assert "if not self._supports_stream_options:" in source + + print("stream_options retained where the provider supports it") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_version_bumped(): + """Synthetic streaming ships at or after its implementation version.""" + print("Testing config version...") + try: + assert_app_version_at_least("0.261.015") + print("Config version check passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +if __name__ == "__main__": + tests = [ + test_chunking_is_lossless, + test_long_answer_is_split_into_several_chunks, + test_function_calls_survive_synthetic_streaming, + test_finish_reason_and_metadata_appear_once, + test_stream_options_are_kept_where_supported, + test_version_bumped, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + results.append(test()) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) From 26c3344b9e901a9b976c624553989559d5fca482 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 08:28:49 -0400 Subject: [PATCH 7/9] Make on-premises Custom model endpoints usable The gate named "allow private Custom endpoint hosts" did not permit the two address forms on-premises deployments actually use. An IP address and a short single-label host name were both rejected even with the gate enabled, and both were refused with a message claiming the URL was an IP address, which was simply wrong for the short host name and named no setting that would allow it. Even once an address was accepted, the connection could not complete. The outbound transport trusts only certifi public roots and deliberately ignores SSL_CERT_FILE, so a gateway presenting an internally issued certificate could never be validated. On-premises inference was unreachable in practice. With the gate enabled, IP addresses, short host names, and private-range hosts are now accepted, and administrators can name a CA bundle to trust an internal issuer. Ambient environment variables are still ignored, so widening trust stays a recorded configuration decision, and a bundle that cannot be loaded fails the request rather than silently falling back to public roots. Plaintext HTTP gets its own second gate for isolated networks, labelled with its consequence, and requires the private-hosts gate as well. None of this weakens the outbound protections. Loopback, link-local, and cloud metadata addresses stay blocked with every gate enabled, and addresses are still revalidated at connection time. Saving an endpoint no longer requires the host to resolve from the application tier, so configuration can be seeded or restored ahead of connectivity. Only resolution is tolerated; policy violations are still refused at save time, and are reported as policy violations rather than resolution failures. Egress proxies remain unsupported on purpose: with a proxy the CONNECT target is resolved by the proxy, so connection-time pinning would protect nothing, and replacing it with a host allowlist is a security decision rather than plumbing. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_model_endpoint_runtime.py | 16 + .../functions_model_endpoint_validation.py | 100 ++++-- application/single_app/functions_settings.py | 2 + .../single_app/model_endpoint_clients.py | 48 ++- .../route_frontend_admin_settings.py | 9 + .../admin/_panes/model-endpoints.html | 35 +- .../features/ON_PREM_MODEL_ENDPOINTS.md | 127 ++++++++ docs/explanation/release-notes/index.md | 52 +-- docs/explanation/release-notes/v0.261.md | 39 ++- docs/explanation/release_notes.md | 12 + .../test_custom_model_endpoint_on_prem.py | 300 ++++++++++++++++++ .../test_custom_model_endpoint_provider.py | 3 +- 13 files changed, 678 insertions(+), 67 deletions(-) create mode 100644 docs/explanation/features/ON_PREM_MODEL_ENDPOINTS.md create mode 100644 functional_tests/test_custom_model_endpoint_on_prem.py diff --git a/application/single_app/config.py b/application/single_app/config.py index 084347a10..dd9553058 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.015" +VERSION = "0.261.016" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 89f058613..17dec8fb4 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -156,6 +156,8 @@ def build_model_endpoint_sync_chat_client( url_mode='', anthropic_version=DEFAULT_ANTHROPIC_VERSION, allow_private_custom_endpoints=False, + allow_insecure_custom_endpoints=False, + custom_endpoint_ca_bundle_path='', settings=None, endpoint_config=None, identity_context=None, @@ -173,6 +175,7 @@ def build_model_endpoint_sync_chat_client( endpoint = validate_custom_model_endpoint_url( endpoint, allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, ) runtime_protocol = infer_model_endpoint_protocol( normalized_provider, @@ -195,6 +198,7 @@ def build_model_endpoint_sync_chat_client( anthropic_version=anthropic_version, direct_custom=direct_custom, allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: @@ -207,6 +211,7 @@ def build_model_endpoint_sync_chat_client( default_headers=extra_headers, api_type=api_type, url_mode=url_mode, + ca_bundle_path=custom_endpoint_ca_bundle_path, ), runtime_protocol client_kwargs = { 'api_version': api_version, @@ -218,6 +223,7 @@ def build_model_endpoint_sync_chat_client( if direct_custom: client_kwargs['http_client'] = build_custom_openai_sync_http_client( allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, ) client = AzureOpenAI(**client_kwargs) if direct_custom: @@ -427,10 +433,17 @@ def build_semantic_kernel_chat_service_for_model( allow_private_custom_endpoints = bool( settings.get('allow_private_custom_model_endpoints', False) ) + allow_insecure_custom_endpoints = bool( + settings.get('allow_insecure_custom_model_endpoints', False) + ) + custom_endpoint_ca_bundle_path = str( + settings.get('custom_model_endpoint_ca_bundle_path') or '' + ).strip() if direct_custom: endpoint = validate_custom_model_endpoint_url( endpoint, allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, ) runtime_protocol = infer_model_endpoint_protocol( provider, @@ -459,6 +472,7 @@ def build_semantic_kernel_chat_service_for_model( anthropic_version=anthropic_version, direct_custom=direct_custom, allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: @@ -474,6 +488,7 @@ def build_semantic_kernel_chat_service_for_model( if direct_custom: client_kwargs['http_client'] = build_custom_openai_async_http_client( allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, ) if extra_headers: client_kwargs['default_headers'] = extra_headers @@ -499,6 +514,7 @@ def build_semantic_kernel_chat_service_for_model( default_headers=extra_headers or None, http_client=build_custom_openai_async_http_client( allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, ), ) async_client = sanitize_custom_async_openai_client( diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py index dde795475..886d1c63d 100644 --- a/application/single_app/functions_model_endpoint_validation.py +++ b/application/single_app/functions_model_endpoint_validation.py @@ -45,6 +45,16 @@ class ModelEndpointValidationError(ValueError): """Raised when a model endpoint configuration violates the saved policy.""" +class ModelEndpointUnresolvableError(ModelEndpointValidationError): + """Raised when a Custom endpoint hostname cannot be resolved right now. + + This is distinct from a policy violation. A name that does not resolve yet is + tolerable when saving configuration -- the environment may not be reachable + from the application tier at configuration time -- but a policy violation + never is. + """ + + def _is_ip_literal(hostname: str) -> bool: try: ipaddress.ip_address(hostname) @@ -113,12 +123,12 @@ def resolve_custom_model_endpoint_addresses( type=socket.SOCK_STREAM, ) except socket.gaierror as exc: - raise ModelEndpointValidationError( + raise ModelEndpointUnresolvableError( "Custom endpoint hostname could not be resolved." ) from exc if not resolved_addresses: - raise ModelEndpointValidationError( + raise ModelEndpointUnresolvableError( "Custom endpoint hostname did not resolve to an address." ) @@ -140,8 +150,21 @@ def validate_custom_model_endpoint_url( endpoint: Any, *, allow_private: bool = False, + allow_insecure: bool = False, + require_resolvable: bool = True, ) -> str: - """Validate and normalize a Custom endpoint URL before an outbound request.""" + """Validate and normalize a Custom endpoint URL before an outbound request. + + ``allow_private`` is the administrator's on-premises gate. With it enabled, an + endpoint may be an IP literal, a single-label host, or a private-range + address, because those are how on-premises inference is normally addressed. + Connect-time address validation still applies on every request. + + ``require_resolvable`` is set to False on the configuration save path so that + an endpoint can be configured, seeded, or restored from backup before the + application tier can resolve it. The connect-time check is what actually + protects the request, and it always runs. + """ endpoint_text = str(endpoint or "").strip() if not endpoint_text: raise ModelEndpointValidationError("Custom endpoint URL is required.") @@ -154,11 +177,19 @@ def validate_custom_model_endpoint_url( except ValueError as exc: raise ModelEndpointValidationError("Custom endpoint URL is invalid.") from exc - if parsed_endpoint.scheme.lower() != "https": + scheme = parsed_endpoint.scheme.lower() + if scheme == "http": + if not (allow_private and allow_insecure): + raise ModelEndpointValidationError( + "Custom endpoint URL must use HTTPS. Plaintext HTTP requires the " + "administrator to enable both private hosts and insecure endpoints." + ) + elif scheme != "https": raise ModelEndpointValidationError("Custom endpoint URL must use HTTPS.") + if not parsed_endpoint.netloc or not parsed_endpoint.hostname: raise ModelEndpointValidationError( - "Custom endpoint URL must include a fully qualified domain name." + "Custom endpoint URL must include a host name." ) if parsed_endpoint.username or parsed_endpoint.password: raise ModelEndpointValidationError( @@ -182,26 +213,51 @@ def validate_custom_model_endpoint_url( or hostname.endswith(".localhost") ): raise ModelEndpointValidationError("Custom endpoint hostname is blocked.") - if _is_ip_literal(hostname) or "." not in hostname: - raise ModelEndpointValidationError( - "Custom endpoint URL must use a fully qualified domain name, not an IP address." - ) - if not allow_private and hostname.endswith((".internal", ".local")): + + is_ip_literal = _is_ip_literal(hostname) + is_single_label = not is_ip_literal and "." not in hostname + + if is_ip_literal: + if not allow_private: + raise ModelEndpointValidationError( + "Custom endpoint URL must use a fully qualified domain name. " + "Enable private Custom endpoint hosts to use an IP address." + ) + # An IP literal skips DNS entirely, so validate the address directly. + validate_custom_model_endpoint_address(hostname, allow_private=True) + elif is_single_label: + if not allow_private: + raise ModelEndpointValidationError( + "Custom endpoint URL must use a fully qualified domain name. " + "Enable private Custom endpoint hosts to use a short host name." + ) + elif not allow_private and hostname.endswith((".internal", ".local")): raise ModelEndpointValidationError( "Private Custom endpoint hosts are not enabled by the administrator." ) - resolve_custom_model_endpoint_addresses( - hostname, - port or 443, - allow_private=allow_private, - ) - + if not is_ip_literal: + default_port = 80 if scheme == "http" else 443 + try: + resolve_custom_model_endpoint_addresses( + hostname, + port or default_port, + allow_private=allow_private, + ) + except ModelEndpointUnresolvableError: + # A name that does not resolve yet is only fatal when the caller needs + # it resolvable now. Policy violations are a different exception and + # always propagate. The connect-time check re-resolves on every + # request, so nothing is skipped by tolerating this here. + if require_resolvable: + raise + + default_port = 80 if scheme == "http" else 443 normalized_netloc = hostname - if port and port != 443: + if port and port != default_port: normalized_netloc = f"{hostname}:{port}" return urlunparse(( - "https", + scheme, normalized_netloc, parsed_endpoint.path or "", "", @@ -254,10 +310,16 @@ def validate_custom_model_endpoint( if isinstance(endpoint.get("connection"), dict) else {} ) - allow_private = bool((settings or {}).get("allow_private_custom_model_endpoints", False)) + endpoint_settings = settings or {} + allow_private = bool(endpoint_settings.get("allow_private_custom_model_endpoints", False)) + allow_insecure = bool(endpoint_settings.get("allow_insecure_custom_model_endpoints", False)) connection["endpoint"] = validate_custom_model_endpoint_url( connection.get("endpoint"), allow_private=allow_private, + allow_insecure=allow_insecure, + # Configuration may be saved before the application tier can resolve the + # host, so saving does not require the name to resolve right now. + require_resolvable=False, ) if registered_provider is not None and registered_provider.version_field: diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index e2cb834cf..fa3deb68f 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -1383,6 +1383,8 @@ def get_settings(use_cosmos=False, include_source=False): 'allow_user_agents': False, 'allow_user_custom_endpoints': False, 'allow_private_custom_model_endpoints': False, + 'allow_insecure_custom_model_endpoints': False, + 'custom_model_endpoint_ca_bundle_path': '', 'allow_user_custom_agent_endpoints': False, 'allow_user_plugins': False, 'allow_user_workflows': False, diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index 66a900123..ad478c699 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -4,6 +4,7 @@ import json import asyncio import re +import ssl from types import SimpleNamespace from typing import Any, Dict, Iterable, Iterator, List from urllib.parse import urlparse @@ -487,12 +488,34 @@ async def sleep(self, seconds): await self._backend.sleep(seconds) +def build_custom_endpoint_ssl_context(ca_bundle_path: Any = ""): + """Return the TLS context for Custom endpoint requests. + + The default context trusts only certifi's public roots, and deliberately does + not read SSL_CERT_FILE, so ambient environment variables cannot silently widen + what SimpleChat trusts. That leaves an on-premises gateway with an + enterprise-issued certificate untrustable, so an administrator may name a CA + bundle explicitly. Naming a bundle is an explicit decision, not an ambient one. + """ + bundle_path = str(ca_bundle_path or "").strip() + if bundle_path: + try: + return ssl.create_default_context(cafile=bundle_path) + except (OSError, ssl.SSLError): + # A missing or unreadable bundle must not silently fall back to a + # weaker context, so the failure is surfaced to the caller. + raise ModelEndpointValidationError( + "The configured Custom endpoint CA bundle could not be loaded." + ) from None + return httpx.create_ssl_context(verify=True, trust_env=False) + + class _PinnedCustomEndpointHTTPTransport(httpx.HTTPTransport): """HTTPX transport whose TCP connection uses the validated DNS results.""" - def __init__(self, *, allow_private=False): + def __init__(self, *, allow_private=False, ca_bundle_path=""): self._pool = httpcore.ConnectionPool( - ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path), max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, @@ -505,9 +528,9 @@ def __init__(self, *, allow_private=False): class _PinnedCustomEndpointAsyncHTTPTransport(httpx.AsyncHTTPTransport): """Async HTTPX transport whose TCP connection uses validated DNS results.""" - def __init__(self, *, allow_private=False): + def __init__(self, *, allow_private=False, ca_bundle_path=""): self._pool = httpcore.AsyncConnectionPool( - ssl_context=httpx.create_ssl_context(verify=True, trust_env=False), + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path), max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, @@ -517,22 +540,24 @@ def __init__(self, *, allow_private=False): ) -def build_custom_openai_sync_http_client(*, allow_private=False): +def build_custom_openai_sync_http_client(*, allow_private=False, ca_bundle_path=""): """Return a no-redirect SDK transport pinned to validated DNS addresses.""" return DefaultHttpxClient( transport=_PinnedCustomEndpointHTTPTransport( allow_private=allow_private, + ca_bundle_path=ca_bundle_path, ), follow_redirects=False, trust_env=False, ) -def build_custom_openai_async_http_client(*, allow_private=False): +def build_custom_openai_async_http_client(*, allow_private=False, ca_bundle_path=""): """Return an async no-redirect transport pinned to validated DNS addresses.""" return DefaultAsyncHttpxClient( transport=_PinnedCustomEndpointAsyncHTTPTransport( allow_private=allow_private, + ca_bundle_path=ca_bundle_path, ), follow_redirects=False, trust_env=False, @@ -549,6 +574,7 @@ def build_openai_style_chat_client( allow_private_custom_endpoints: bool = False, api_type: Any = "", url_mode: Any = "", + ca_bundle_path: Any = "", ): """Build an OpenAI-compatible chat client for Foundry data-plane endpoints.""" request_api_version = resolve_openai_style_request_api_version(api_version) @@ -563,6 +589,7 @@ def build_openai_style_chat_client( if direct_custom: client_kwargs["http_client"] = build_custom_openai_sync_http_client( allow_private=allow_private_custom_endpoints, + ca_bundle_path=ca_bundle_path, ) if default_headers: client_kwargs["default_headers"] = default_headers @@ -798,6 +825,7 @@ def build_anthropic_chat_client( anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, direct_custom: bool = False, allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): """Build a chat-completions-shaped adapter over the Anthropic messages protocol.""" return AnthropicChatCompletionClient( @@ -809,6 +837,7 @@ def build_anthropic_chat_client( anthropic_version=anthropic_version, direct_custom=direct_custom, allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, ) @@ -826,6 +855,7 @@ def __init__( anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, direct_custom: bool = False, allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): self.endpoint = normalize_anthropic_messages_url( endpoint, @@ -840,6 +870,7 @@ def __init__( ).strip() self.direct_custom = direct_custom self.allow_private_custom_endpoints = allow_private_custom_endpoints + self.custom_endpoint_ca_bundle_path = custom_endpoint_ca_bundle_path self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): @@ -867,6 +898,7 @@ def create(self, **kwargs: Any): def _create_direct_custom(self, payload, *, stream): http_client = build_custom_openai_sync_http_client( allow_private=self.allow_private_custom_endpoints, + ca_bundle_path=self.custom_endpoint_ca_bundle_path, ) request = http_client.build_request( "POST", @@ -1289,6 +1321,7 @@ class AnthropicSemanticKernelChatCompletion(ChatCompletionClientBase): anthropic_version: str = DEFAULT_ANTHROPIC_VERSION direct_custom: bool = False allow_private_custom_endpoints: bool = False + custom_endpoint_ca_bundle_path: str = "" prompt_execution_settings: OpenAIChatPromptExecutionSettings | None = Field(default=None) def __init__( @@ -1304,6 +1337,7 @@ def __init__( anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, direct_custom: bool = False, allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): super().__init__( ai_model_id=deployment_name, @@ -1316,6 +1350,7 @@ def __init__( anthropic_version=anthropic_version, direct_custom=direct_custom, allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, ) def get_prompt_execution_settings_class(self): @@ -1550,6 +1585,7 @@ def _build_client(self): anthropic_version=self.anthropic_version, direct_custom=self.direct_custom, allow_private_custom_endpoints=self.allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=self.custom_endpoint_ca_bundle_path, ) def _build_request_kwargs(self, chat_history, settings, *, stream: bool) -> Dict[str, Any]: diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index c5ccd6938..025468a23 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -1731,6 +1731,9 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul custom_endpoint_validation_settings['allow_private_custom_model_endpoints'] = ( form_data.get('allow_private_custom_model_endpoints') == 'on' ) + custom_endpoint_validation_settings['allow_insecure_custom_model_endpoints'] = ( + form_data.get('allow_insecure_custom_model_endpoints') == 'on' + ) try: validate_custom_model_endpoints( parsed_model_endpoints, @@ -2481,6 +2484,12 @@ def is_valid_url(url): 'allow_private_custom_model_endpoints': ( form_data.get('allow_private_custom_model_endpoints') == 'on' ), + 'allow_insecure_custom_model_endpoints': ( + form_data.get('allow_insecure_custom_model_endpoints') == 'on' + ), + 'custom_model_endpoint_ca_bundle_path': ( + form_data.get('custom_model_endpoint_ca_bundle_path', '').strip() + ), 'model_endpoint_identity_header_enabled': model_endpoint_identity_header_enabled, 'model_endpoint_identity_header_name': model_endpoint_identity_header_name, 'model_endpoint_identity_header_value_type': model_endpoint_identity_header_value_type, diff --git a/application/single_app/templates/admin/_panes/model-endpoints.html b/application/single_app/templates/admin/_panes/model-endpoints.html index 68b833677..ca2accd85 100644 --- a/application/single_app/templates/admin/_panes/model-endpoints.html +++ b/application/single_app/templates/admin/_panes/model-endpoints.html @@ -31,7 +31,40 @@
    Allow private Custom endpoint hosts
    - Custom endpoints require HTTPS. When disabled, hosts resolving to private addresses are rejected; loopback, link-local, metadata, and direct IP targets are always rejected. + Enable this for on-premises inference. It permits IP addresses, short host names, and hosts resolving to private ranges. Loopback, link-local, and cloud metadata addresses are always rejected, and every address is revalidated when the connection is made. +
    +
    + +
    + + +
    + Prompts and API keys travel unencrypted. Only for an isolated network where TLS cannot be terminated. Requires private Custom endpoint hosts to also be enabled. +
    +
    + +
    + + +
    + Custom endpoints trust only public certificate authorities, and deliberately ignore ambient environment variables so nothing can silently widen that trust. Name a PEM bundle here to trust an internally issued certificate, which an on-premises gateway normally uses.
    diff --git a/docs/explanation/features/ON_PREM_MODEL_ENDPOINTS.md b/docs/explanation/features/ON_PREM_MODEL_ENDPOINTS.md new file mode 100644 index 000000000..c3a18221e --- /dev/null +++ b/docs/explanation/features/ON_PREM_MODEL_ENDPOINTS.md @@ -0,0 +1,127 @@ +# On-Premises Custom Model Endpoints + +## Overview + +SimpleChat can reach a model API running inside your own network — an on-premises +gateway, a self-hosted vLLM or LiteLLM deployment, or an air-gapped inference +appliance — through a Custom model endpoint. + +That path exists because not every deployment can send prompts to a public cloud +API. It is off by default, because pointing an application at arbitrary internal +addresses is exactly the shape of a server-side request forgery, so each relaxation +is a deliberate administrator decision. + +**Implemented in version: 0.261.016** + +## What was blocking on-premises use + +The gate for private hosts existed but did not permit the two address forms +on-premises deployments actually use: + +| Endpoint | Before | After (gate enabled) | +|---|---|---| +| `https://10.20.30.40/v1` | Rejected | Accepted | +| `https://10.20.30.40:8443/v1` | Rejected | Accepted | +| `https://llm-gateway/v1` | Rejected | Accepted | +| `https://llm.corp.internal/v1` | Rejected | Accepted | +| `http://llm.corp.example.com/v1` | Rejected | Accepted with the second gate | + +Both rejections also used the same message — that the URL had to be a fully +qualified domain name "not an IP address" — which was simply wrong for a short +host name, and did not say which setting would allow it. + +Even once an address was accepted, TLS made the connection impossible. The +outbound transport trusts only certifi's public roots and deliberately ignores +`SSL_CERT_FILE` and `SSL_CERT_DIR`, so a gateway presenting an internally issued +certificate could never be validated. + +## Settings + +| Setting | Default | Effect | +|---|---|---| +| `allow_private_custom_model_endpoints` | off | Permits IP addresses, short host names, and hosts resolving to private ranges | +| `allow_insecure_custom_model_endpoints` | off | Permits plaintext `http://`. Requires the private-hosts gate as well | +| `custom_model_endpoint_ca_bundle_path` | empty | Path to a PEM bundle used to validate Custom endpoint certificates | + +All three are on the **Model Endpoints** admin tab. + +## What stays blocked + +Enabling every gate does not disable the outbound protections. These are always +refused, because they are the targets that make request forgery useful: + +- Loopback addresses, including `127.0.0.1` and `localhost` +- Link-local addresses +- Cloud instance metadata endpoints — `169.254.169.254`, `168.63.129.16`, + `metadata.google.internal`, `metadata.azure.com`, `instance-data.ec2.internal` +- Multicast, reserved, and unspecified addresses +- URLs carrying embedded credentials, a query string, or a fragment +- UNIX domain sockets +- HTTP redirects + +Address validation also runs again at connection time, on the addresses the +connection actually uses, so a DNS answer that changes between configuration and +request cannot redirect the connection. + +## Trusting an internal certificate authority + +Set **Custom endpoint CA bundle path** to a PEM file readable by the application: + +``` +/etc/ssl/certs/internal-ca.pem +``` + +Two properties are deliberate: + +- **Ambient environment variables are still ignored.** Setting `SSL_CERT_FILE` in + the environment does not change what SimpleChat trusts. Widening trust is a + configuration decision, recorded in settings, not an ambient one. +- **A bundle that cannot be loaded is an error.** A missing or unreadable file + fails the request rather than silently falling back to public roots, so a + typo cannot quietly downgrade what is being validated. + +## Plaintext HTTP + +`allow_insecure_custom_model_endpoints` permits `http://` endpoints, and requires +the private-hosts gate as well. + +Prompts, responses, and the API key all travel unencrypted. This exists for +isolated networks where TLS genuinely cannot be terminated; prefer the CA bundle +setting and keep TLS wherever it is possible. + +## Configuring before connectivity exists + +Saving an endpoint no longer requires the host name to resolve from the +application tier. Configuration can be seeded, scripted, or restored from backup +before the network path exists. + +Only name resolution is tolerated at save time. A policy violation — a blocked +address, a bad scheme, embedded credentials — is still refused when saving, and +the connection-time check is unchanged, so nothing is skipped. + +## Testing and validation + +`functional_tests/test_custom_model_endpoint_on_prem.py` covers: + +- default-deny behaviour with the gate off, across five address forms; +- acceptance of IP literals, ports, short host names, and `.internal` names with + the gate on; +- plaintext HTTP requiring its own second gate rather than riding on the first; +- loopback, link-local, and cloud metadata staying blocked with every gate on; +- the short-host-name rejection message no longer claiming the URL is an IP + address, and naming the setting that would allow it; +- the CA bundle being honoured explicitly, ambient environment variables still + being ignored, and a missing bundle failing rather than falling back; +- save-time tolerance of an unresolvable name, while a policy violation is still + refused and reported as a policy violation rather than a resolution failure. + +## Known limitations + +- **Egress proxies are not supported.** This is deliberate rather than + incomplete. With an HTTP proxy the client issues `CONNECT host:443` and the + *proxy* resolves DNS, so connection-time address pinning would provide no + protection at all. Supporting a proxy means replacing that control with an + administrator-managed host allowlist, which is a security decision rather than + a plumbing change, so the proxy is refused instead of silently unprotected. +- Client certificate authentication (mTLS) to the endpoint is not yet supported; + the CA bundle setting validates the server, not the client. diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index 855ae2510..e5c87f954 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.016 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.015 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.013 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -31,7 +32,7 @@ This page includes the latest release notes inline. Older release sections are s | v0.261.006 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.005 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.004 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.261.003 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.003 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.002 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.001 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.260.025 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | @@ -82,6 +83,18 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.016)** + +#### New Features + +* **On-Premises Custom Model Endpoints Now Work** + * The administrator gate named "allow private Custom endpoint hosts" did not actually permit the two most common on-premises address forms. An IP address such as `https://10.20.30.40/v1` and a short host name such as `https://llm-gateway/v1` were both rejected even with the gate enabled, and both were refused with a message claiming the URL was an IP address, which was wrong for the short host name. + * With the gate enabled, IP addresses, short host names, and hosts resolving to private ranges are now accepted. Loopback, link-local, and cloud metadata addresses remain rejected regardless of any setting, and every address is still revalidated at connection time. + * **Added a CA bundle setting.** Custom endpoints trust only public certificate authorities and deliberately ignore ambient environment variables, so an on-premises gateway using an internally issued certificate previously could not be trusted at all. An administrator can now name a PEM bundle. A bundle that cannot be loaded fails loudly rather than silently falling back to weaker trust. + * **Added a separate plaintext HTTP gate** for isolated networks where TLS cannot be terminated. It requires the private-hosts gate as well, and is labelled with its consequence: prompts and API keys travel unencrypted. + * Saving an endpoint no longer requires the host name to resolve from the application tier, so configuration can be seeded or restored from backup ahead of connectivity. Policy violations are still refused at save time, and the connection-time check is unchanged. + * (Ref: `functions_model_endpoint_validation.py`, `model_endpoint_clients.py`, `allow_insecure_custom_model_endpoints`, `custom_model_endpoint_ca_bundle_path`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.015)** #### Bug Fixes @@ -202,40 +215,3 @@ This page includes the latest release notes inline. Older release sections are s * Markdown processing now batches its chunk embeddings and Search upload instead of reserving the gate once per chunk, which reduces contention and avoids the intermittent `OrderedDict mutated during iteration` failures seen during concurrent Markdown ingestion. * Added regression coverage for repeated transient gate conflicts, local worker serialization, and Markdown use of the batch chunk writer. * (Ref: `functions_data_management_search_write_fence.py`, `functions_documents.py`, `test_data_management_search_write_fence.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) - -### **(v0.261.003)** - -#### Bug Fixes - -* **Broken Documentation Links Repaired** - * Clicking the upgrade guide, Docker customization, or enterprise networking links from the repository README or the deployer READMEs led to a "page not found". Those pages were reorganized from `docs/how-to/.md` to `docs/guides/.md`, and the site kept redirects, but redirects do not apply when browsing files on GitHub. No documentation was ever lost, only mislinked. - * Repaired 46 broken relative links in total: 12 in the README and deployer READMEs, and 34 in archived per-version engineering notes. Archived links whose target was never migrated now keep the prose without a dead link, rather than pointing at a file that does not exist. - * Also corrected the "Return to Main" link in the Azure CLI and Terraform deployer READMEs, which pointed one directory too shallow. - * (Ref: `README.md`, `deployers/*/README.md`, `docs/explanation/features/`, `docs/explanation/fixes/`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) - -* **Recovered 27 Release Note Sections Missing From The Source File** - * `docs/explanation/release_notes.md` had been truncated from 46 version sections to 19, dropping every v0.260 entry along with v0.250.229 through v0.250.231. The published site still showed them, because the pages that render release notes are generated from this file and had not been rebuilt since the truncation. - * That left the repository one routine `build_release_notes_pages.py` run away from erasing roughly 2,400 lines of release history from the site with no obvious cause. The sections have been restored from history and the pages regenerated, so the source and the site agree again. - * (Ref: `docs/explanation/release_notes.md`, `scripts/build_release_notes_pages.py`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) - -#### Documentation - -* **Web Search Documentation Now Describes What Actually Happens** - * The web search guide still described the Bing Web Search API integration that was removed back in v0.229.001. Web search has since run through an Azure AI Foundry agent using the Grounding with Bing Search tool, which is why an admin has to configure a Foundry project and agent ID before the **Web** control appears. - * Added a dedicated **What leaves SimpleChat** section stating the egress boundary plainly: only the message the user just typed is sent to the external search service. Conversation history, workspace documents, attached file contents, system prompts, agent instructions, and workspace or document names are never included. This behavior was hardened in v0.241.022 but was previously mentioned only in passing. - * Documented the Deep Research nuance: it runs several planned queries instead of one, but every query is still derived from the current message alone, so no conversation history is introduced. - * Added the Grounding with Bing Search compliance-boundary notice to the user-facing guide, replaced the placeholder text in the admin Web Search settings table with real descriptions, and reused the existing web search flow diagram instead of leaving a "recording planned" video card. - * (Ref: `docs/guides/use-web-search.md`, `docs/admin/knowledge.md`, `docs/reference/chat-controls.md`, `build_web_search_query_text`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) - -* **Documentation Link Rot Now Fails A Test** - * Added `functional_tests/test_docs_link_integrity.py`, which fails when any relative markdown link in the README, `docs/`, or `deployers/` points at a missing file, when a Jekyll `relative_url` page link does not resolve, or when a media include names an unregistered slot. Outstanding screenshots are reported but never fail the run. - * Added `functional_tests/test_docs_web_search_accuracy.py`, which ties the published privacy claim to the implementation. If the web search query builder ever starts folding conversation history back into the outbound query, the test fails and forces the documentation to be corrected with it. - * (Ref: `test_docs_link_integrity.py`, `test_docs_web_search_accuracy.py`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) - -* **First Batch Of Documentation Screenshots** - * Filled 54 empty screenshot slots, taking documentation screenshot coverage from 18 of 122 to 72 of 122. The Administration group is now fully illustrated. - * Added the four admin settings overviews (Backup & Recovery, Data Lifecycle, Governance, Workflow), six chat control references (conversation list, conversation header, composer, selectors, grounded search, and advanced conversation search), thirty-three task guide steps, and configuration panes for eleven action types (Azure Maps, Blob Storage, Chart, Cosmos Query, Databricks, Document Search, Log Analytics, MCP, Microsoft Graph, OpenAPI, RocksDB, and SimpleChat). - * The web search screenshot captures the live data notice, so the guide's claim that only the current message is sent is now visible rather than only asserted. - * Action configuration panes were captured without saving any action, so every credential field shows only its placeholder text and no tenant values were recorded. Where an admin settings pane already held real values, those fields were replaced with example values before capture and the page was reloaded without saving. - * Replaced the generated placeholder alt text on every filled slot with a description of what the reader actually learns from the image. - * (Ref: `docs/images/admin/`, `docs/images/reference/`, `docs/images/guides/`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) diff --git a/docs/explanation/release-notes/v0.261.md b/docs/explanation/release-notes/v0.261.md index addf486c1..5775a786b 100644 --- a/docs/explanation/release-notes/v0.261.md +++ b/docs/explanation/release-notes/v0.261.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.261 series" -description: "SimpleChat release notes for 0.261.002 – 0.261.001." +description: "SimpleChat release notes for 0.261.003 – 0.261.001." section: "Reference" layout: page --- @@ -11,6 +11,43 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.261.003)** + +#### Bug Fixes + +* **Broken Documentation Links Repaired** + * Clicking the upgrade guide, Docker customization, or enterprise networking links from the repository README or the deployer READMEs led to a "page not found". Those pages were reorganized from `docs/how-to/.md` to `docs/guides/.md`, and the site kept redirects, but redirects do not apply when browsing files on GitHub. No documentation was ever lost, only mislinked. + * Repaired 46 broken relative links in total: 12 in the README and deployer READMEs, and 34 in archived per-version engineering notes. Archived links whose target was never migrated now keep the prose without a dead link, rather than pointing at a file that does not exist. + * Also corrected the "Return to Main" link in the Azure CLI and Terraform deployer READMEs, which pointed one directory too shallow. + * (Ref: `README.md`, `deployers/*/README.md`, `docs/explanation/features/`, `docs/explanation/fixes/`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) + +* **Recovered 27 Release Note Sections Missing From The Source File** + * `docs/explanation/release_notes.md` had been truncated from 46 version sections to 19, dropping every v0.260 entry along with v0.250.229 through v0.250.231. The published site still showed them, because the pages that render release notes are generated from this file and had not been rebuilt since the truncation. + * That left the repository one routine `build_release_notes_pages.py` run away from erasing roughly 2,400 lines of release history from the site with no obvious cause. The sections have been restored from history and the pages regenerated, so the source and the site agree again. + * (Ref: `docs/explanation/release_notes.md`, `scripts/build_release_notes_pages.py`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) + +#### Documentation + +* **Web Search Documentation Now Describes What Actually Happens** + * The web search guide still described the Bing Web Search API integration that was removed back in v0.229.001. Web search has since run through an Azure AI Foundry agent using the Grounding with Bing Search tool, which is why an admin has to configure a Foundry project and agent ID before the **Web** control appears. + * Added a dedicated **What leaves SimpleChat** section stating the egress boundary plainly: only the message the user just typed is sent to the external search service. Conversation history, workspace documents, attached file contents, system prompts, agent instructions, and workspace or document names are never included. This behavior was hardened in v0.241.022 but was previously mentioned only in passing. + * Documented the Deep Research nuance: it runs several planned queries instead of one, but every query is still derived from the current message alone, so no conversation history is introduced. + * Added the Grounding with Bing Search compliance-boundary notice to the user-facing guide, replaced the placeholder text in the admin Web Search settings table with real descriptions, and reused the existing web search flow diagram instead of leaving a "recording planned" video card. + * (Ref: `docs/guides/use-web-search.md`, `docs/admin/knowledge.md`, `docs/reference/chat-controls.md`, `build_web_search_query_text`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) + +* **Documentation Link Rot Now Fails A Test** + * Added `functional_tests/test_docs_link_integrity.py`, which fails when any relative markdown link in the README, `docs/`, or `deployers/` points at a missing file, when a Jekyll `relative_url` page link does not resolve, or when a media include names an unregistered slot. Outstanding screenshots are reported but never fail the run. + * Added `functional_tests/test_docs_web_search_accuracy.py`, which ties the published privacy claim to the implementation. If the web search query builder ever starts folding conversation history back into the outbound query, the test fails and forces the documentation to be corrected with it. + * (Ref: `test_docs_link_integrity.py`, `test_docs_web_search_accuracy.py`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) + +* **First Batch Of Documentation Screenshots** + * Filled 54 empty screenshot slots, taking documentation screenshot coverage from 18 of 122 to 72 of 122. The Administration group is now fully illustrated. + * Added the four admin settings overviews (Backup & Recovery, Data Lifecycle, Governance, Workflow), six chat control references (conversation list, conversation header, composer, selectors, grounded search, and advanced conversation search), thirty-three task guide steps, and configuration panes for eleven action types (Azure Maps, Blob Storage, Chart, Cosmos Query, Databricks, Document Search, Log Analytics, MCP, Microsoft Graph, OpenAPI, RocksDB, and SimpleChat). + * The web search screenshot captures the live data notice, so the guide's claim that only the current message is sent is now visible rather than only asserted. + * Action configuration panes were captured without saving any action, so every credential field shows only its placeholder text and no tenant values were recorded. Where an admin settings pane already held real values, those fields were replaced with example values before capture and the page was reloaded without saving. + * Replaced the generated placeholder alt text on every filled slot with a description of what the reader actually learns from the image. + * (Ref: `docs/images/admin/`, `docs/images/reference/`, `docs/images/guides/`, [#1371](https://github.com/microsoft/simplechat/issues/1371)) + ### **(v0.261.002)** #### User Interface Enhancements diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index b2206b5b7..15b702385 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,18 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.016)** + +#### New Features + +* **On-Premises Custom Model Endpoints Now Work** + * The administrator gate named "allow private Custom endpoint hosts" did not actually permit the two most common on-premises address forms. An IP address such as `https://10.20.30.40/v1` and a short host name such as `https://llm-gateway/v1` were both rejected even with the gate enabled, and both were refused with a message claiming the URL was an IP address, which was wrong for the short host name. + * With the gate enabled, IP addresses, short host names, and hosts resolving to private ranges are now accepted. Loopback, link-local, and cloud metadata addresses remain rejected regardless of any setting, and every address is still revalidated at connection time. + * **Added a CA bundle setting.** Custom endpoints trust only public certificate authorities and deliberately ignore ambient environment variables, so an on-premises gateway using an internally issued certificate previously could not be trusted at all. An administrator can now name a PEM bundle. A bundle that cannot be loaded fails loudly rather than silently falling back to weaker trust. + * **Added a separate plaintext HTTP gate** for isolated networks where TLS cannot be terminated. It requires the private-hosts gate as well, and is labelled with its consequence: prompts and API keys travel unencrypted. + * Saving an endpoint no longer requires the host name to resolve from the application tier, so configuration can be seeded or restored from backup ahead of connectivity. Policy violations are still refused at save time, and the connection-time check is unchanged. + * (Ref: `functions_model_endpoint_validation.py`, `model_endpoint_clients.py`, `allow_insecure_custom_model_endpoints`, `custom_model_endpoint_ca_bundle_path`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.015)** #### Bug Fixes diff --git a/functional_tests/test_custom_model_endpoint_on_prem.py b/functional_tests/test_custom_model_endpoint_on_prem.py new file mode 100644 index 000000000..1d5261bbd --- /dev/null +++ b/functional_tests/test_custom_model_endpoint_on_prem.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +""" +Functional test for on-premises Custom model endpoint support. +Version: 0.261.016 +Implemented in: 0.261.016 + +On-premises inference was effectively unreachable. The administrator gate named +"allow private Custom endpoint hosts" did not permit the two most common +on-premises address forms -- an IP literal and a short single-label host name -- +and rejected both with a message claiming the URL was an IP address even when it +was not. Plaintext HTTP had no gate at all. + +Separately, the outbound transport pinned trust to certifi's public roots and +deliberately ignored ambient environment variables, so an on-premises gateway +with an internally issued certificate could never be trusted. + +These tests ensure that: + * with the gate off nothing changes, so the secure default is preserved, + * with the gate on, IP literals, short host names, private ranges, and -- with + a second explicit gate -- plaintext HTTP are accepted, + * loopback and cloud metadata addresses stay rejected even with the gate on, + * an administrator can name a CA bundle, a missing bundle fails loudly rather + than silently weakening trust, and ambient environment variables still cannot + widen what is trusted, + * saving configuration tolerates a name that does not resolve yet, while a + policy violation is never tolerated. +""" + +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + ) +) + +from test_support.versioning import assert_app_version_at_least + +from functions_model_endpoint_validation import ( + ModelEndpointUnresolvableError, + ModelEndpointValidationError, + validate_custom_model_endpoint_url, +) +from model_endpoint_clients import build_custom_endpoint_ssl_context + + +ON_PREM_URLS = [ + "https://10.20.30.40/v1", + "https://10.20.30.40:8443/v1", + "https://llm-gateway/v1", + "https://llm.corp.internal/v1", +] + +ALWAYS_BLOCKED_URLS = [ + "https://127.0.0.1/v1", + "https://169.254.169.254/v1", + "https://metadata.google.internal/v1", + "https://localhost/v1", +] + + +def _validate(url, *, private=False, insecure=False): + return validate_custom_model_endpoint_url( + url, + allow_private=private, + allow_insecure=insecure, + require_resolvable=False, + ) + + +def test_gate_off_preserves_the_secure_default(): + """With the gate off, every on-premises address form stays rejected.""" + print("Testing default-deny behaviour...") + try: + for url in ON_PREM_URLS + ["http://llm.corp.example.com/v1"]: + try: + _validate(url) + except ModelEndpointValidationError: + continue + raise AssertionError(f"{url} must be rejected when the gate is off.") + + # A public HTTPS endpoint is unaffected. + assert _validate("https://api.openai.com/v1") == "https://api.openai.com/v1" + + print(f"Default-deny held for {len(ON_PREM_URLS) + 1} address forms") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_gate_on_enables_on_prem_addresses(): + """With the gate on, on-premises address forms are accepted.""" + print("Testing on-premises address acceptance...") + try: + for url in ON_PREM_URLS: + resolved = _validate(url, private=True) + assert resolved, f"{url} must be accepted when the gate is on." + + # Plaintext HTTP needs its own second gate, not just the private gate. + plaintext = "http://llm.corp.example.com/v1" + try: + _validate(plaintext, private=True) + except ModelEndpointValidationError: + pass + else: + raise AssertionError("Plaintext HTTP must require its own explicit gate.") + + assert _validate(plaintext, private=True, insecure=True) == plaintext + + print(f"On-premises addresses accepted for {len(ON_PREM_URLS)} forms") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_dangerous_targets_stay_blocked_with_the_gate_on(): + """Loopback and cloud metadata must be rejected even with the gate on.""" + print("Testing that dangerous targets stay blocked...") + try: + for url in ALWAYS_BLOCKED_URLS: + try: + _validate(url, private=True, insecure=True) + except ModelEndpointValidationError: + continue + raise AssertionError( + f"{url} must stay blocked even with every gate enabled." + ) + + print(f"All {len(ALWAYS_BLOCKED_URLS)} dangerous targets stay blocked") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_rejection_messages_are_accurate(): + """A short host name must not be described as an IP address.""" + print("Testing rejection message accuracy...") + try: + try: + _validate("https://llm-gateway/v1") + except ModelEndpointValidationError as exc: + message = str(exc) + assert "not an IP address" not in message, ( + f"A short host name must not be called an IP address: {message!r}" + ) + assert "short host name" in message, ( + f"The message should say what to enable: {message!r}" + ) + else: + raise AssertionError("Expected a rejection for a short host name.") + + print("Rejection messages are accurate") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_custom_ca_bundle_is_honoured_and_fails_loudly(): + """An internal CA can be trusted explicitly, but never silently.""" + print("Testing CA bundle handling...") + try: + import certifi + + # The default context ignores ambient environment variables, so nothing + # outside SimpleChat's configuration can widen what is trusted. + original_cert_file = os.environ.get("SSL_CERT_FILE") + os.environ["SSL_CERT_FILE"] = os.path.join("C:" + os.sep, "nonexistent", "evil.pem") + try: + default_context = build_custom_endpoint_ssl_context("") + assert default_context.get_ca_certs(), "The default context must trust public roots." + finally: + if original_cert_file is None: + os.environ.pop("SSL_CERT_FILE", None) + else: + os.environ["SSL_CERT_FILE"] = original_cert_file + + # An explicitly named bundle is loaded. + explicit_context = build_custom_endpoint_ssl_context(certifi.where()) + assert explicit_context.get_ca_certs() + + # A missing bundle must not silently fall back to a weaker context. + try: + build_custom_endpoint_ssl_context( + os.path.join("C:" + os.sep, "nonexistent", "missing-ca.pem") + ) + except ModelEndpointValidationError: + pass + else: + raise AssertionError( + "A missing CA bundle must fail rather than silently fall back." + ) + + print("CA bundle honoured explicitly and fails loudly when missing") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_saving_tolerates_unresolvable_names_but_not_policy_violations(): + """Configuration may be saved before the host resolves; policy still applies.""" + print("Testing save-time resolution behaviour...") + try: + unresolvable = "https://not-a-real-host.invalid/v1" + + # Saving tolerates a name that does not resolve yet. + assert _validate(unresolvable, private=True) == unresolvable + + # Requiring resolution surfaces the failure with a distinct exception. + try: + validate_custom_model_endpoint_url( + unresolvable, + allow_private=True, + require_resolvable=True, + ) + except ModelEndpointUnresolvableError: + pass + else: + raise AssertionError("Expected an unresolvable-hostname error.") + + # A policy violation is never tolerated, regardless of resolvability. + try: + validate_custom_model_endpoint_url( + "https://127.0.0.1/v1", + allow_private=True, + require_resolvable=False, + ) + except ModelEndpointValidationError as exc: + assert not isinstance(exc, ModelEndpointUnresolvableError), ( + "A blocked address must be a policy violation, not a resolution failure." + ) + else: + raise AssertionError("A loopback address must always be rejected.") + + print("Save tolerates unresolvable names while policy still applies") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_version_bumped(): + """On-premises support ships at or after its implementation version.""" + print("Testing config version...") + try: + assert_app_version_at_least("0.261.016") + print("Config version check passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +if __name__ == "__main__": + tests = [ + test_gate_off_preserves_the_secure_default, + test_gate_on_enables_on_prem_addresses, + test_dangerous_targets_stay_blocked_with_the_gate_on, + test_rejection_messages_are_accurate, + test_custom_ca_bundle_is_honoured_and_fails_loudly, + test_saving_tolerates_unresolvable_names_but_not_policy_violations, + test_version_bumped, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + results.append(test()) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_custom_model_endpoint_provider.py b/functional_tests/test_custom_model_endpoint_provider.py index 57610a296..2d1f0cf64 100644 --- a/functional_tests/test_custom_model_endpoint_provider.py +++ b/functional_tests/test_custom_model_endpoint_provider.py @@ -192,7 +192,7 @@ def test_custom_endpoint_url_policy(): ("http://models.example.com", "must use HTTPS"), ("https://user:password@models.example.com", "embedded credentials"), ("https://models.example.com?key=value", "query string or fragment"), - ("https://127.0.0.1", "not an IP address"), + ("https://127.0.0.1", "fully qualified domain name"), ("https://single-label", "fully qualified domain name"), ("https://localhost", "hostname is blocked"), ): @@ -585,6 +585,7 @@ def test_custom_runtime_client_construction(): validate_url.assert_called_with( "https://models.example.com", allow_private=True, + allow_insecure=False, ) openai_client._client.close() From 51178c36cafe7756c7739bc84ddbe44e81615146 Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Fri, 4 Sep 2026 08:42:23 -0400 Subject: [PATCH 8/9] Add bearer, OAuth2, and mTLS authentication to Custom endpoints Custom endpoints accepted one scheme: an API key sent in whichever header the built-in providers happened to use. That covers OpenAI and Anthropic and nothing else, so a gateway reading the key from a different header, a corporate gateway issuing short-lived tokens, and an appliance requiring a client certificate were all unreachable. The API key header name and value prefix are now configurable, which is what lets one scheme cover Authorization: Bearer, Anthropic's x-api-key, Google's x-goog-api-key, and bespoke gateway headers. Each provider supplies its own default so the common case needs no configuration. Static bearer tokens are supported, as is the OAuth2 client credentials grant. Tokens are cached per token URL, client, and scope, and refreshed ahead of expiry so one cannot lapse mid-request. Two things about the token endpoint matter. It is a different host from the inference endpoint, so it is validated against the same outbound policy and cannot become an unchecked request target: a token URL pointing at cloud metadata is refused exactly as an inference endpoint would be. And its error body routinely echoes the client id or secret, so failures are sanitized to the browser and recorded server-side with credentials redacted. mTLS is modelled as a transport concern rather than an auth type, because a client certificate combines with any scheme. Certificates are referenced by file path, never by value, so a private key is mounted into the deployment instead of being written to the configuration database and replicated with it. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_model_endpoint_auth.py | 262 +++++++++++ .../functions_model_endpoint_providers.py | 21 +- .../functions_model_endpoint_runtime.py | 56 ++- .../functions_model_endpoint_validation.py | 43 +- .../single_app/model_endpoint_clients.py | 37 +- .../features/CUSTOM_MODEL_ENDPOINT_AUTH.md | 134 ++++++ docs/explanation/release-notes/index.md | 26 +- docs/explanation/release-notes/v0.261.md | 13 +- docs/explanation/release_notes.md | 12 + .../test_custom_model_endpoint_auth.py | 422 ++++++++++++++++++ 11 files changed, 994 insertions(+), 34 deletions(-) create mode 100644 application/single_app/functions_model_endpoint_auth.py create mode 100644 docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md create mode 100644 functional_tests/test_custom_model_endpoint_auth.py diff --git a/application/single_app/config.py b/application/single_app/config.py index dd9553058..9f7c1460d 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.016" +VERSION = "0.261.017" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_endpoint_auth.py b/application/single_app/functions_model_endpoint_auth.py new file mode 100644 index 000000000..2c228731d --- /dev/null +++ b/application/single_app/functions_model_endpoint_auth.py @@ -0,0 +1,262 @@ +# functions_model_endpoint_auth.py +"""Authentication schemes for Custom model endpoints. + +Custom endpoints originally supported one scheme: an API key sent in whichever +header the built-in providers happened to use. That covers OpenAI and Anthropic +and nothing else. A gateway that expects "x-goog-api-key", a corporate gateway +that issues short-lived OAuth2 tokens, and an on-premises appliance that requires +a client certificate were all unreachable. + +This module adds those schemes without widening what the browser can see: every +secret stays server-side, and OAuth2 token responses are never surfaced to a +caller beyond the token itself. + +mTLS is deliberately modelled as a transport concern rather than an auth "type", +because a client certificate combines with any of the schemes below. Certificates +are referenced by file path rather than stored in settings, so a private key is +mounted into the deployment and never written to the configuration database. +""" + +import threading +import time +from typing import Any, Dict, Tuple + +import httpx + +from functions_model_endpoint_diagnostics import build_sanitized_model_endpoint_error + + +AUTH_TYPE_API_KEY = "api_key" +AUTH_TYPE_KEY = "key" +AUTH_TYPE_BEARER = "bearer" +AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" + +CUSTOM_ENDPOINT_AUTH_TYPES = ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, +) + +# Refresh slightly before expiry so a token cannot lapse mid-request. +OAUTH2_EXPIRY_SKEW_SECONDS = 60 +OAUTH2_DEFAULT_EXPIRY_SECONDS = 3600 +OAUTH2_REQUEST_TIMEOUT_SECONDS = 30 + +_TOKEN_CACHE: Dict[Tuple[str, str, str], Tuple[str, float]] = {} +_TOKEN_CACHE_LOCK = threading.Lock() + + +def normalize_custom_endpoint_auth_type(auth_type: Any) -> str: + """Return a supported Custom endpoint auth type, defaulting to API key.""" + normalized = str(auth_type or "").strip().lower() + if normalized == AUTH_TYPE_KEY: + return AUTH_TYPE_API_KEY + return normalized if normalized in CUSTOM_ENDPOINT_AUTH_TYPES else "" + + +def resolve_api_key_header( + auth: Dict[str, Any], + default_header: str = "", + default_prefix: str = "", +) -> Tuple[str, str]: + """Return the header name and value prefix used to send an API key. + + Providers disagree about this. OpenAI uses "Authorization: Bearer", Anthropic + uses "x-api-key" with no prefix, Google uses "x-goog-api-key", and gateways + invent their own. Making both configurable is what lets one auth type cover + all of them. + + An administrator override wins over the provider default. An override that + names a header but no prefix means exactly that, so the provider's prefix is + not reapplied. + """ + auth = auth or {} + override_header = str(auth.get("api_key_header") or "").strip() + if override_header: + return override_header, str(auth.get("api_key_prefix") or "").strip() + + header_name = str(default_header or "").strip() + prefix = str(auth.get("api_key_prefix") or default_prefix or "").strip() + return header_name, prefix + + +def build_api_key_headers( + auth: Dict[str, Any], + default_header: str = "", + default_prefix: str = "", +) -> Dict[str, str]: + """Build the request headers that carry a configured API key.""" + api_key = str((auth or {}).get("api_key") or "").strip() + if not api_key: + raise ValueError("Selected model endpoint is missing an API key.") + + header_name, prefix = resolve_api_key_header(auth, default_header, default_prefix) + if not header_name: + return {} + header_value = f"{prefix} {api_key}".strip() if prefix else api_key + return {header_name: header_value} + + +def build_bearer_headers(auth: Dict[str, Any]) -> Dict[str, str]: + """Build the request headers for a static bearer token.""" + token = str((auth or {}).get("bearer_token") or "").strip() + if not token: + raise ValueError("Selected model endpoint is missing a bearer token.") + return {"Authorization": f"Bearer {token}"} + + +def _token_cache_key(auth: Dict[str, Any]) -> Tuple[str, str, str]: + return ( + str(auth.get("token_url") or "").strip(), + str(auth.get("client_id") or "").strip(), + str(auth.get("scope") or "").strip(), + ) + + +def clear_oauth2_token_cache() -> None: + """Drop every cached OAuth2 token.""" + with _TOKEN_CACHE_LOCK: + _TOKEN_CACHE.clear() + + +def fetch_oauth2_client_credentials_token( + auth: Dict[str, Any], + *, + verify: Any = True, + http_client_factory=None, +) -> str: + """Return an OAuth2 client-credentials access token, using the cache when valid. + + The token endpoint is normally a different host from the inference endpoint, + and commonly redirects, so it is fetched with an ordinary client rather than + the no-redirect pinned transport used for inference. + """ + token_url = str(auth.get("token_url") or "").strip() + client_id = str(auth.get("client_id") or "").strip() + client_secret = str(auth.get("client_secret") or "").strip() + if not token_url or not client_id or not client_secret: + raise ValueError( + "OAuth2 model endpoints require a token URL, client ID, and client secret." + ) + + cache_key = _token_cache_key(auth) + now = time.monotonic() + with _TOKEN_CACHE_LOCK: + cached = _TOKEN_CACHE.get(cache_key) + if cached and cached[1] > now: + return cached[0] + + payload = { + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + } + scope = str(auth.get("scope") or "").strip() + if scope: + payload["scope"] = scope + + client = (http_client_factory or httpx.Client)( + timeout=OAUTH2_REQUEST_TIMEOUT_SECONDS, + verify=verify, + trust_env=False, + ) + try: + response = client.post(token_url, data=payload) + status_code = response.status_code + if status_code >= 400: + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token request failed.", + request_url=token_url, + status_code=status_code, + detail=response.text, + ) + token_payload = response.json() + except Exception as exc: + if isinstance(exc, RuntimeError): + raise + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token request failed.", + exc, + request_url=token_url, + ) from None + finally: + client.close() + + access_token = str(token_payload.get("access_token") or "").strip() + if not access_token: + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token response did not contain an access token.", + request_url=token_url, + ) + + try: + expires_in = int(token_payload.get("expires_in") or OAUTH2_DEFAULT_EXPIRY_SECONDS) + except (TypeError, ValueError): + expires_in = OAUTH2_DEFAULT_EXPIRY_SECONDS + expires_at = time.monotonic() + max(1, expires_in - OAUTH2_EXPIRY_SKEW_SECONDS) + + with _TOKEN_CACHE_LOCK: + _TOKEN_CACHE[cache_key] = (access_token, expires_at) + return access_token + + +def resolve_custom_endpoint_credentials( + auth: Dict[str, Any], + *, + default_api_key_header: str = "", + default_api_key_prefix: str = "", + verify: Any = True, +) -> Tuple[str, Dict[str, str]]: + """Resolve one Custom endpoint's credentials. + + Returns the value to hand to an SDK that takes an api_key argument, plus any + additional headers the scheme requires. An SDK that sends "Authorization: + Bearer" natively needs only the first; a scheme using a different header name + supplies the second and passes a placeholder for the first. + """ + auth = auth or {} + auth_type = normalize_custom_endpoint_auth_type(auth.get("type")) + + if auth_type == AUTH_TYPE_BEARER: + token = str(auth.get("bearer_token") or "").strip() + if not token: + raise ValueError("Selected model endpoint is missing a bearer token.") + return token, {} + + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: + return fetch_oauth2_client_credentials_token(auth, verify=verify), {} + + if auth_type == AUTH_TYPE_API_KEY: + api_key = str(auth.get("api_key") or "").strip() + if not api_key: + raise ValueError("Selected model endpoint is missing an API key.") + header_name, _ = resolve_api_key_header( + auth, + default_api_key_header, + default_api_key_prefix, + ) + # An explicit non-Authorization header is sent alongside the SDK's own + # credential argument, because the SDK cannot express it. + if header_name and header_name.lower() != "authorization": + return api_key, build_api_key_headers( + auth, + default_api_key_header, + default_api_key_prefix, + ) + return api_key, {} + + raise ValueError("Custom model endpoints do not support the selected authentication type.") + + +def resolve_client_certificate(connection: Dict[str, Any]): + """Return the mTLS client certificate for httpx, or None when not configured. + + Certificates are referenced by path so that a private key is mounted into the + deployment rather than stored in the configuration database. + """ + connection = connection or {} + cert_path = str(connection.get("client_cert_path") or "").strip() + if not cert_path: + return None + key_path = str(connection.get("client_key_path") or "").strip() + return (cert_path, key_path) if key_path else cert_path diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py index e7a987fff..a855b2ea9 100644 --- a/application/single_app/functions_model_endpoint_providers.py +++ b/application/single_app/functions_model_endpoint_providers.py @@ -69,6 +69,16 @@ def normalize_custom_endpoint_url_mode(url_mode: Any) -> str: DEFAULT_ANTHROPIC_VERSION = "2023-06-01" AUTH_TYPE_API_KEY = "api_key" +AUTH_TYPE_BEARER = "bearer" +AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" + +# Every registered provider accepts these. API key remains the default and the +# only one required; the others are opt-in for gateways that need them. +DEFAULT_CUSTOM_AUTH_TYPES = ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, +) class ModelEndpointProvider: @@ -82,7 +92,9 @@ def __init__( model_identifier: str, url_policy: str, *, - auth_types: Tuple[str, ...] = (AUTH_TYPE_API_KEY,), + auth_types: Tuple[str, ...] = DEFAULT_CUSTOM_AUTH_TYPES, + default_api_key_header: str = "Authorization", + default_api_key_prefix: str = "Bearer", requires_api_version: bool = False, version_field: str = "", default_version: str = "", @@ -97,6 +109,8 @@ def __init__( self.model_identifier = model_identifier self.url_policy = url_policy self.auth_types = auth_types + self.default_api_key_header = default_api_key_header + self.default_api_key_prefix = default_api_key_prefix self.requires_api_version = requires_api_version self.version_field = version_field self.default_version = default_version @@ -120,6 +134,8 @@ def to_ui_option(self) -> Dict[str, Any]: "versionField": self.version_field, "defaultVersion": self.default_version, "authTypes": list(self.auth_types), + "defaultApiKeyHeader": self.default_api_key_header, + "defaultApiKeyPrefix": self.default_api_key_prefix, "description": self.description, } @@ -157,6 +173,9 @@ def to_ui_option(self) -> Dict[str, Any]: url_policy=URL_POLICY_ANTHROPIC_MESSAGES, version_field="anthropic_version", default_version=DEFAULT_ANTHROPIC_VERSION, + # Anthropic reads the key from x-api-key with no value prefix. + default_api_key_header="x-api-key", + default_api_key_prefix="", description="Anthropic's messages API, direct or through a gateway.", ), ModelEndpointProvider( diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 17dec8fb4..8d1592cd0 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -8,7 +8,14 @@ from config import cognitive_services_scope from foundry_agent_runtime import resolve_authority from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers -from functions_model_endpoint_providers import normalize_custom_endpoint_url_mode +from functions_model_endpoint_auth import ( + normalize_custom_endpoint_auth_type, + resolve_custom_endpoint_credentials, +) +from functions_model_endpoint_providers import ( + get_model_endpoint_provider, + normalize_custom_endpoint_url_mode, +) from functions_model_endpoint_types import ( DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_PROVIDER_CUSTOM, @@ -184,8 +191,27 @@ def build_model_endpoint_sync_chat_client( api_type, ) auth_type = str(auth_settings.get('type') or 'managed_identity').strip().lower() - if direct_custom and auth_type not in ('api_key', 'key'): - raise ValueError('Custom model endpoints support API key authentication only.') + if direct_custom: + normalized_custom_auth = normalize_custom_endpoint_auth_type(auth_type) + if not normalized_custom_auth: + raise ValueError( + 'Custom model endpoints support API key, bearer token, or OAuth2 ' + 'client credentials authentication.' + ) + registered_provider = get_model_endpoint_provider(api_type) + credential, credential_headers = resolve_custom_endpoint_credentials( + auth_settings, + default_api_key_header=( + registered_provider.default_api_key_header if registered_provider else '' + ), + default_api_key_prefix=( + registered_provider.default_api_key_prefix if registered_provider else '' + ), + ) + if credential_headers: + extra_headers = {**(extra_headers or {}), **credential_headers} + auth_type = 'api_key' + auth_settings = {**auth_settings, 'type': 'api_key', 'api_key': credential} if auth_type in ('api_key', 'key'): api_key = auth_settings.get('api_key') @@ -452,13 +478,33 @@ def build_semantic_kernel_chat_service_for_model( api_type, ) auth_type = str(auth_settings.get('type') or 'managed_identity').lower() - if direct_custom and auth_type not in ('api_key', 'key'): - raise ValueError('Custom model endpoints support API key authentication only.') extra_headers = build_model_endpoint_identity_headers( settings, endpoint_config=resolved_model_endpoint, identity_context=model_context, ) + if direct_custom: + normalized_custom_auth = normalize_custom_endpoint_auth_type(auth_type) + if not normalized_custom_auth: + raise ValueError( + 'Custom model endpoints support API key, bearer token, or OAuth2 ' + 'client credentials authentication.' + ) + registered_provider = get_model_endpoint_provider(api_type) + credential, credential_headers = resolve_custom_endpoint_credentials( + auth_settings, + default_api_key_header=( + registered_provider.default_api_key_header if registered_provider else '' + ), + default_api_key_prefix=( + registered_provider.default_api_key_prefix if registered_provider else '' + ), + ) + if credential_headers: + extra_headers = {**(extra_headers or {}), **credential_headers} + auth_type = 'api_key' + auth_settings = {**auth_settings, 'type': 'api_key', 'api_key': credential} + if auth_type in ('api_key', 'key'): api_key = auth_settings.get('api_key') if not api_key: diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py index 886d1c63d..6dd11152a 100644 --- a/application/single_app/functions_model_endpoint_validation.py +++ b/application/single_app/functions_model_endpoint_validation.py @@ -7,6 +7,12 @@ from typing import Any, Dict, Iterable from urllib.parse import urlparse, urlunparse +from functions_model_endpoint_auth import ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + normalize_custom_endpoint_auth_type, +) from functions_model_endpoint_providers import get_model_endpoint_provider from functions_model_endpoint_types import ( DEFAULT_ANTHROPIC_VERSION, @@ -297,13 +303,40 @@ def validate_custom_model_endpoint( registered_provider = get_model_endpoint_provider(api_type) auth = endpoint.get("auth") if isinstance(endpoint.get("auth"), dict) else {} - auth_type = str(auth.get("type") or "").strip().lower() - if auth_type not in {"api_key", "key"}: + auth_type = normalize_custom_endpoint_auth_type(auth.get("type")) + if not auth_type: + raise ModelEndpointValidationError( + "Custom endpoints support API key, bearer token, or OAuth2 " + "client credentials authentication." + ) + if registered_provider is not None and auth_type not in registered_provider.auth_types: raise ModelEndpointValidationError( - "Custom endpoints support API key authentication only." + f"{registered_provider.display_name} does not support the selected " + "authentication type." + ) + if require_api_key: + if auth_type == AUTH_TYPE_API_KEY and not auth.get("api_key"): + raise ModelEndpointValidationError("Custom endpoint API key is required.") + if auth_type == AUTH_TYPE_BEARER and not auth.get("bearer_token"): + raise ModelEndpointValidationError("Custom endpoint bearer token is required.") + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS and not all( + str(auth.get(field) or "").strip() + for field in ("token_url", "client_id", "client_secret") + ): + raise ModelEndpointValidationError( + "Custom endpoint OAuth2 authentication requires a token URL, " + "client ID, and client secret." + ) + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS and auth.get("token_url"): + # The token endpoint is a separate host and must satisfy the same policy + # as the inference endpoint, otherwise it becomes an unchecked outbound + # request target. + validate_custom_model_endpoint_url( + auth.get("token_url"), + allow_private=bool((settings or {}).get("allow_private_custom_model_endpoints", False)), + allow_insecure=bool((settings or {}).get("allow_insecure_custom_model_endpoints", False)), + require_resolvable=False, ) - if require_api_key and not auth.get("api_key"): - raise ModelEndpointValidationError("Custom endpoint API key is required.") connection = ( endpoint.get("connection") diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index ad478c699..dc765cffd 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -488,7 +488,7 @@ async def sleep(self, seconds): await self._backend.sleep(seconds) -def build_custom_endpoint_ssl_context(ca_bundle_path: Any = ""): +def build_custom_endpoint_ssl_context(ca_bundle_path: Any = "", client_cert: Any = None): """Return the TLS context for Custom endpoint requests. The default context trusts only certifi's public roots, and deliberately does @@ -496,26 +496,43 @@ def build_custom_endpoint_ssl_context(ca_bundle_path: Any = ""): what SimpleChat trusts. That leaves an on-premises gateway with an enterprise-issued certificate untrustable, so an administrator may name a CA bundle explicitly. Naming a bundle is an explicit decision, not an ambient one. + + ``client_cert`` supplies an mTLS client certificate, as either a combined PEM + path or a (certificate, key) pair of paths. """ bundle_path = str(ca_bundle_path or "").strip() if bundle_path: try: - return ssl.create_default_context(cafile=bundle_path) + context = ssl.create_default_context(cafile=bundle_path) except (OSError, ssl.SSLError): # A missing or unreadable bundle must not silently fall back to a # weaker context, so the failure is surfaced to the caller. raise ModelEndpointValidationError( "The configured Custom endpoint CA bundle could not be loaded." ) from None - return httpx.create_ssl_context(verify=True, trust_env=False) + else: + context = httpx.create_ssl_context(verify=True, trust_env=False) + + if client_cert: + try: + if isinstance(client_cert, (tuple, list)): + context.load_cert_chain(*client_cert) + else: + context.load_cert_chain(client_cert) + except (OSError, ssl.SSLError): + raise ModelEndpointValidationError( + "The configured Custom endpoint client certificate could not be loaded." + ) from None + + return context class _PinnedCustomEndpointHTTPTransport(httpx.HTTPTransport): """HTTPX transport whose TCP connection uses the validated DNS results.""" - def __init__(self, *, allow_private=False, ca_bundle_path=""): + def __init__(self, *, allow_private=False, ca_bundle_path="", client_cert=None): self._pool = httpcore.ConnectionPool( - ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path), + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, @@ -528,9 +545,9 @@ def __init__(self, *, allow_private=False, ca_bundle_path=""): class _PinnedCustomEndpointAsyncHTTPTransport(httpx.AsyncHTTPTransport): """Async HTTPX transport whose TCP connection uses validated DNS results.""" - def __init__(self, *, allow_private=False, ca_bundle_path=""): + def __init__(self, *, allow_private=False, ca_bundle_path="", client_cert=None): self._pool = httpcore.AsyncConnectionPool( - ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path), + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, @@ -540,24 +557,26 @@ def __init__(self, *, allow_private=False, ca_bundle_path=""): ) -def build_custom_openai_sync_http_client(*, allow_private=False, ca_bundle_path=""): +def build_custom_openai_sync_http_client(*, allow_private=False, ca_bundle_path="", client_cert=None): """Return a no-redirect SDK transport pinned to validated DNS addresses.""" return DefaultHttpxClient( transport=_PinnedCustomEndpointHTTPTransport( allow_private=allow_private, ca_bundle_path=ca_bundle_path, + client_cert=client_cert, ), follow_redirects=False, trust_env=False, ) -def build_custom_openai_async_http_client(*, allow_private=False, ca_bundle_path=""): +def build_custom_openai_async_http_client(*, allow_private=False, ca_bundle_path="", client_cert=None): """Return an async no-redirect transport pinned to validated DNS addresses.""" return DefaultAsyncHttpxClient( transport=_PinnedCustomEndpointAsyncHTTPTransport( allow_private=allow_private, ca_bundle_path=ca_bundle_path, + client_cert=client_cert, ), follow_redirects=False, trust_env=False, diff --git a/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md new file mode 100644 index 000000000..e01a08abc --- /dev/null +++ b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md @@ -0,0 +1,134 @@ +# Custom Model Endpoint Authentication + +## Overview + +A Custom model endpoint has to authenticate to whatever it is pointed at. The +first implementation supported exactly one scheme — an API key, sent in whichever +header the built-in providers happened to use — which covers OpenAI and Anthropic +and nothing else. + +That left three common cases unreachable: a gateway that reads the key from a +different header, a corporate gateway that issues short-lived OAuth2 tokens, and +an appliance that requires a client certificate. + +**Implemented in version: 0.261.017** + +## Schemes + +| Scheme | `auth.type` | Use it for | +|---|---|---| +| API key | `api_key` | Any provider or gateway that reads a static key from a header | +| Bearer token | `bearer` | A long-lived token issued out of band | +| OAuth2 client credentials | `oauth2_client_credentials` | A gateway that issues short-lived tokens | + +mTLS is deliberately **not** a scheme. A client certificate combines with any of +the above, so it is configured on the connection rather than the auth block. + +## API key + +The header name and value prefix are both configurable, which is what lets one +scheme cover every convention: + +| Provider | Header | Prefix | Result | +|---|---|---|---| +| OpenAI | `Authorization` | `Bearer` | `Authorization: Bearer sk-...` | +| Anthropic | `x-api-key` | none | `x-api-key: sk-ant-...` | +| Google | `x-goog-api-key` | none | `x-goog-api-key: AIza...` | +| Custom gateway | anything | anything | `X-Corp-Key: Token abc123` | + +Each registered provider supplies its own default, so nothing needs configuring +for the common case. Override `auth.api_key_header` and `auth.api_key_prefix` +only when a gateway differs. + +An override that names a header but no prefix means exactly that — the provider's +default prefix is not silently reapplied. + +## Bearer token + +```json +{ "type": "bearer", "bearer_token": "..." } +``` + +Sent as `Authorization: Bearer `. + +## OAuth2 client credentials + +```json +{ + "type": "oauth2_client_credentials", + "token_url": "https://auth.example.com/oauth2/token", + "client_id": "...", + "client_secret": "...", + "scope": "inference.read" +} +``` + +Behaviour worth knowing: + +- **Tokens are cached** per token URL, client ID, and scope, and refreshed 60 + seconds before expiry so a token cannot lapse part-way through a request. A + response without `expires_in` is treated as one hour. +- **The token endpoint is policy checked.** It is a different host from the + inference endpoint, so it is validated against the same outbound rules. A token + URL pointing at a cloud metadata address is refused, exactly as an inference + endpoint would be. Without this, the token URL would be an unchecked outbound + request target. +- **The token endpoint is fetched with an ordinary client**, not the no-redirect + pinned transport used for inference, because token endpoints commonly redirect. +- **Failures are sanitized.** A token endpoint's error body frequently echoes the + client ID or secret, so the browser sees a generic message with a correlation + id while the real response is recorded server-side with credentials redacted. + +## mTLS client certificates + +Set these on the endpoint's `connection`: + +```json +{ + "client_cert_path": "/etc/ssl/certs/client.pem", + "client_key_path": "/etc/ssl/private/client.key" +} +``` + +A single combined PEM may be supplied through `client_cert_path` alone. + +**Certificates are referenced by path, never by value.** A private key pasted into +a settings field would be written to the configuration database and replicated +wherever that database goes. Mounting the key into the deployment and naming its +path keeps the key material out of application storage entirely. + +A certificate that cannot be loaded fails the request rather than silently +continuing without one. + +## What has not changed + +- Custom endpoints still refuse managed identity and service principal + authentication; those are for Azure-hosted providers. +- The outbound protections are unchanged. Every scheme runs over the same + validated-DNS, no-redirect transport described in the on-premises documentation. + +## Testing and validation + +`functional_tests/test_custom_model_endpoint_auth.py` covers: + +- API key header customization across four conventions, plus the per-provider + defaults declared by the registry; +- bearer and API key credential resolution, including which schemes need an + explicit header and which ride on the SDK's own credential argument; +- OAuth2 tokens being fetched once, served from cache on the second call, and + refetched after the cache is cleared, with the request payload asserted; +- a failing token endpoint leaking neither its error body nor the client details, + while still offering a correlation id; +- a token endpoint pointing at a cloud metadata address being refused; +- unsupported and incomplete auth configurations being rejected; +- mTLS certificates resolving by path only, with the module asserted to offer no + way to supply key material inline. + +## Known limitations + +- OAuth2 supports the client credentials grant only. Authorization code and + on-behalf-of flows are not implemented. +- The token cache is per process. A multi-worker deployment fetches one token per + worker, which is correct but not maximally efficient. +- Client certificate paths are not yet editable in the endpoint editor UI; they + are set on the endpoint's connection record. diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index e5c87f954..f460d309c 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.017 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.016 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.015 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -31,7 +32,7 @@ This page includes the latest release notes inline. Older release sections are s | v0.261.007 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.006 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.005 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.261.004 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.004 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.003 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.002 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.001 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | @@ -83,6 +84,18 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.017)** + +#### New Features + +* **Custom Model Endpoints Support Bearer Tokens, OAuth2, And Client Certificates** + * Custom endpoints accepted one authentication scheme: an API key sent in whichever header the built-in providers happened to use. That covers OpenAI and Anthropic and nothing else, so a gateway expecting `x-goog-api-key`, a corporate gateway issuing short-lived tokens, and an appliance requiring a client certificate were all unreachable. + * **The API key header name and value prefix are now configurable**, so a single scheme covers `Authorization: Bearer`, Anthropic's `x-api-key`, Google's `x-goog-api-key`, and any bespoke gateway header. + * **Added static bearer token authentication.** + * **Added OAuth2 client credentials**, with token caching and refresh ahead of expiry so a token cannot lapse mid-request. The token endpoint is validated against the same outbound policy as the inference endpoint, so it cannot become an unchecked request target, and a failing token response is sanitized before it reaches the browser. + * **Added mTLS client certificates.** Certificates are referenced by file path so a private key is mounted into the deployment and never written to the configuration database. + * (Ref: `functions_model_endpoint_auth.py`, `functions_model_endpoint_providers.py`, `functions_model_endpoint_validation.py`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.016)** #### New Features @@ -204,14 +217,3 @@ This page includes the latest release notes inline. Older release sections are s * The temporary upload summary no longer labels unconfirmed browser upload requests as final document failures. This avoids misleading summaries such as `Uploaded 77/204, Failed: 127` when the document list later shows that most documents were queued and processed successfully. * Personal, group, and public workspace uploads now use `Queued` for confirmed upload requests and direct users to the refreshed document list for final processing status. * (Ref: workspace upload progress summary, `workspace-documents.js`, `public_workspace.js`, `group_workspaces.html`, [Workspace Upload Status Counter Fix](fixes/WORKSPACE_UPLOAD_STATUS_COUNTER_FIX.md)) - -### **(v0.261.004)** - -#### Bug Fixes - -* **Large Workspace Uploads No Longer Fail On Search Write Gate Contention** - * Fixed partial failures when uploading many small Markdown, JSON, or YAML files to personal, group, or public workspaces at once. Document processing could fail with a message that the Data Management Search write gate changed too often to reserve a write slot. - * The shared write gate now waits within the existing request timeout budget, briefly backs off after transient Cosmos ETag conflicts, and serializes Search writes inside each worker process. This prevents local upload threads from stampeding the same gate document while preserving the migration freeze protection for Azure AI Search writes. - * Markdown processing now batches its chunk embeddings and Search upload instead of reserving the gate once per chunk, which reduces contention and avoids the intermittent `OrderedDict mutated during iteration` failures seen during concurrent Markdown ingestion. - * Added regression coverage for repeated transient gate conflicts, local worker serialization, and Markdown use of the batch chunk writer. - * (Ref: `functions_data_management_search_write_fence.py`, `functions_documents.py`, `test_data_management_search_write_fence.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) diff --git a/docs/explanation/release-notes/v0.261.md b/docs/explanation/release-notes/v0.261.md index 5775a786b..686df5a8c 100644 --- a/docs/explanation/release-notes/v0.261.md +++ b/docs/explanation/release-notes/v0.261.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.261 series" -description: "SimpleChat release notes for 0.261.003 – 0.261.001." +description: "SimpleChat release notes for 0.261.004 – 0.261.001." section: "Reference" layout: page --- @@ -11,6 +11,17 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.261.004)** + +#### Bug Fixes + +* **Large Workspace Uploads No Longer Fail On Search Write Gate Contention** + * Fixed partial failures when uploading many small Markdown, JSON, or YAML files to personal, group, or public workspaces at once. Document processing could fail with a message that the Data Management Search write gate changed too often to reserve a write slot. + * The shared write gate now waits within the existing request timeout budget, briefly backs off after transient Cosmos ETag conflicts, and serializes Search writes inside each worker process. This prevents local upload threads from stampeding the same gate document while preserving the migration freeze protection for Azure AI Search writes. + * Markdown processing now batches its chunk embeddings and Search upload instead of reserving the gate once per chunk, which reduces contention and avoids the intermittent `OrderedDict mutated during iteration` failures seen during concurrent Markdown ingestion. + * Added regression coverage for repeated transient gate conflicts, local worker serialization, and Markdown use of the batch chunk writer. + * (Ref: `functions_data_management_search_write_fence.py`, `functions_documents.py`, `test_data_management_search_write_fence.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) + ### **(v0.261.003)** #### Bug Fixes diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 15b702385..db4164481 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,18 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.017)** + +#### New Features + +* **Custom Model Endpoints Support Bearer Tokens, OAuth2, And Client Certificates** + * Custom endpoints accepted one authentication scheme: an API key sent in whichever header the built-in providers happened to use. That covers OpenAI and Anthropic and nothing else, so a gateway expecting `x-goog-api-key`, a corporate gateway issuing short-lived tokens, and an appliance requiring a client certificate were all unreachable. + * **The API key header name and value prefix are now configurable**, so a single scheme covers `Authorization: Bearer`, Anthropic's `x-api-key`, Google's `x-goog-api-key`, and any bespoke gateway header. + * **Added static bearer token authentication.** + * **Added OAuth2 client credentials**, with token caching and refresh ahead of expiry so a token cannot lapse mid-request. The token endpoint is validated against the same outbound policy as the inference endpoint, so it cannot become an unchecked request target, and a failing token response is sanitized before it reaches the browser. + * **Added mTLS client certificates.** Certificates are referenced by file path so a private key is mounted into the deployment and never written to the configuration database. + * (Ref: `functions_model_endpoint_auth.py`, `functions_model_endpoint_providers.py`, `functions_model_endpoint_validation.py`, [#1228](https://github.com/microsoft/simplechat/pull/1228)) + ### **(v0.261.016)** #### New Features diff --git a/functional_tests/test_custom_model_endpoint_auth.py b/functional_tests/test_custom_model_endpoint_auth.py new file mode 100644 index 000000000..c32b6a781 --- /dev/null +++ b/functional_tests/test_custom_model_endpoint_auth.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +""" +Functional test for Custom model endpoint authentication schemes. +Version: 0.261.017 +Implemented in: 0.261.017 + +Custom endpoints supported one authentication scheme: an API key sent in whichever +header the built-in providers happened to use. That covers OpenAI and Anthropic +and nothing else. A gateway expecting "x-goog-api-key", a corporate gateway +issuing short-lived OAuth2 tokens, and an appliance requiring a client +certificate were all unreachable. + +These tests ensure that: + * the API key header name and value prefix are configurable, so one scheme + covers OpenAI, Anthropic, Google, and bespoke gateway headers, + * static bearer tokens work, + * OAuth2 client credentials are fetched, cached, and refreshed before expiry, + * an OAuth2 token endpoint is validated against the same outbound policy as the + inference endpoint, so it cannot become an unchecked request target, + * mTLS client certificates are referenced by path, never stored in settings, + * unsupported and incomplete auth configurations are still rejected. +""" + +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) +sys.path.append( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + ) +) + +from test_support.versioning import assert_app_version_at_least + +import functions_model_endpoint_auth as endpoint_auth +from functions_model_endpoint_auth import ( + build_api_key_headers, + clear_oauth2_token_cache, + fetch_oauth2_client_credentials_token, + normalize_custom_endpoint_auth_type, + resolve_client_certificate, + resolve_custom_endpoint_credentials, +) +from functions_model_endpoint_providers import ( + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_OPENAI, + get_model_endpoint_provider, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_model_endpoint, +) + + +class _FakeResponse: + def __init__(self, payload, status_code=200, text=""): + self._payload = payload + self.status_code = status_code + self.text = text + + def json(self): + return self._payload + + +class _FakeTokenClient: + """Stand-in for httpx.Client that records token requests.""" + + instances = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.requests = [] + self.response = _FakeResponse({"access_token": "issued-token", "expires_in": 3600}) + _FakeTokenClient.instances.append(self) + + def post(self, url, data=None): + self.requests.append((url, data)) + return self.response + + def close(self): + pass + + +def test_api_key_header_is_configurable(): + """One API key scheme must cover every provider's header convention.""" + print("Testing API key header customization...") + try: + cases = [ + ("Anthropic default", {"api_key": "sk-ant"}, "x-api-key", "", {"x-api-key": "sk-ant"}), + ( + "OpenAI default", + {"api_key": "sk-abc"}, + "Authorization", + "Bearer", + {"Authorization": "Bearer sk-abc"}, + ), + ( + "Google override", + {"api_key": "AIza", "api_key_header": "x-goog-api-key"}, + "Authorization", + "Bearer", + {"x-goog-api-key": "AIza"}, + ), + ( + "Gateway with prefix", + {"api_key": "k", "api_key_header": "X-Corp-Key", "api_key_prefix": "Token"}, + "Authorization", + "Bearer", + {"X-Corp-Key": "Token k"}, + ), + ] + for label, auth, default_header, default_prefix, expected in cases: + headers = build_api_key_headers(auth, default_header, default_prefix) + assert headers == expected, f"{label}: got {headers}, expected {expected}" + + # The registry supplies the per-provider defaults. + anthropic = get_model_endpoint_provider(MODEL_ENDPOINT_API_TYPE_ANTHROPIC) + assert anthropic.default_api_key_header == "x-api-key" + assert anthropic.default_api_key_prefix == "" + openai = get_model_endpoint_provider(MODEL_ENDPOINT_API_TYPE_OPENAI) + assert openai.default_api_key_header == "Authorization" + assert openai.default_api_key_prefix == "Bearer" + + print(f"API key header customization correct for {len(cases)} conventions") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_bearer_and_api_key_credentials_resolve(): + """Bearer tokens and API keys must resolve to the right SDK inputs.""" + print("Testing credential resolution...") + try: + credential, headers = resolve_custom_endpoint_credentials( + {"type": "bearer", "bearer_token": "tok-123"} + ) + assert credential == "tok-123" and headers == {} + + # An Authorization-header key rides on the SDK's own credential argument. + credential, headers = resolve_custom_endpoint_credentials( + {"type": "api_key", "api_key": "sk-abc"}, + default_api_key_header="Authorization", + default_api_key_prefix="Bearer", + ) + assert credential == "sk-abc" and headers == {} + + # A non-Authorization header must be sent explicitly. + credential, headers = resolve_custom_endpoint_credentials( + {"type": "api_key", "api_key": "sk-ant"}, + default_api_key_header="x-api-key", + ) + assert credential == "sk-ant" + assert headers == {"x-api-key": "sk-ant"} + + assert normalize_custom_endpoint_auth_type("key") == "api_key" + assert normalize_custom_endpoint_auth_type("managed_identity") == "" + + print("Credential resolution passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_oauth2_tokens_are_fetched_and_cached(): + """OAuth2 tokens must be fetched once and reused until near expiry.""" + print("Testing OAuth2 client credentials...") + try: + clear_oauth2_token_cache() + _FakeTokenClient.instances = [] + + auth = { + "type": "oauth2_client_credentials", + "token_url": "https://auth.example.com/oauth2/token", + "client_id": "client-1", + "client_secret": "secret-1", + "scope": "inference.read", + } + + token = fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + assert token == "issued-token" + assert len(_FakeTokenClient.instances) == 1 + + request_url, payload = _FakeTokenClient.instances[0].requests[0] + assert request_url == auth["token_url"] + assert payload["grant_type"] == "client_credentials" + assert payload["scope"] == "inference.read" + + # A second call is served from the cache, so no new request is made. + cached_token = fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + assert cached_token == "issued-token" + assert len(_FakeTokenClient.instances) == 1, "Token must be cached." + + # Clearing the cache forces a new request. + clear_oauth2_token_cache() + fetch_oauth2_client_credentials_token(auth, http_client_factory=_FakeTokenClient) + assert len(_FakeTokenClient.instances) == 2 + + clear_oauth2_token_cache() + print("OAuth2 fetch, cache, and refresh passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_oauth2_failures_are_sanitized(): + """A failing token endpoint must not leak its response to the caller.""" + print("Testing OAuth2 failure sanitization...") + try: + clear_oauth2_token_cache() + + class FailingTokenClient(_FakeTokenClient): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.response = _FakeResponse( + {}, status_code=401, text='{"error":"invalid_client","secret":"leak-me"}' + ) + + auth = { + "type": "oauth2_client_credentials", + "token_url": "https://auth.example.com/oauth2/token", + "client_id": "client-1", + "client_secret": "secret-1", + } + try: + fetch_oauth2_client_credentials_token( + auth, http_client_factory=FailingTokenClient + ) + except RuntimeError as exc: + assert "leak-me" not in str(exc) + assert "invalid_client" not in str(exc) + assert "reference" in str(exc), "A correlation id should be offered." + else: + raise AssertionError("A failing token endpoint must raise.") + + clear_oauth2_token_cache() + print("OAuth2 failures sanitized") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_oauth2_token_endpoint_is_policy_checked(): + """The token endpoint is a separate host and must satisfy the same policy.""" + print("Testing OAuth2 token endpoint policy...") + try: + endpoint = { + "name": "Gateway", + "provider": "custom", + "api_type": "openai", + "connection": {"endpoint": "https://api.example.com/v1"}, + "auth": { + "type": "oauth2_client_credentials", + # A token endpoint pointing at cloud metadata must be refused. + "token_url": "https://169.254.169.254/token", + "client_id": "c", + "client_secret": "s", + }, + "models": [{"modelName": "gpt-4o"}], + } + try: + validate_custom_model_endpoint(endpoint, {}) + except ModelEndpointValidationError: + pass + else: + raise AssertionError("A metadata-address token endpoint must be refused.") + + print("Token endpoint is policy checked") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_incomplete_and_unsupported_auth_is_rejected(): + """Missing credentials and unsupported schemes must still be refused.""" + print("Testing auth rejection...") + try: + for auth in ( + {"type": "managed_identity"}, + {"type": "api_key"}, + {"type": "bearer"}, + ): + try: + resolve_custom_endpoint_credentials(auth) + except ValueError: + continue + raise AssertionError(f"{auth} must be rejected.") + + base_endpoint = { + "name": "Gateway", + "provider": "custom", + "api_type": "openai", + "connection": {"endpoint": "https://api.example.com/v1"}, + "models": [{"modelName": "gpt-4o"}], + } + + # Managed identity is not a Custom endpoint scheme. + try: + validate_custom_model_endpoint( + {**base_endpoint, "auth": {"type": "managed_identity"}}, {} + ) + except ModelEndpointValidationError: + pass + else: + raise AssertionError("Managed identity must be refused for Custom endpoints.") + + # OAuth2 without its required fields is incomplete. + try: + validate_custom_model_endpoint( + {**base_endpoint, "auth": {"type": "oauth2_client_credentials"}}, {} + ) + except ModelEndpointValidationError: + pass + else: + raise AssertionError("Incomplete OAuth2 configuration must be refused.") + + print("Unsupported and incomplete auth rejected") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_mtls_certificates_are_referenced_by_path(): + """A client private key must never be stored in settings.""" + print("Testing mTLS certificate handling...") + try: + assert resolve_client_certificate({}) is None + assert resolve_client_certificate({"client_cert_path": "/etc/ssl/client.pem"}) == ( + "/etc/ssl/client.pem" + ) + assert resolve_client_certificate( + {"client_cert_path": "/c.pem", "client_key_path": "/k.pem"} + ) == ("/c.pem", "/k.pem") + + # The module must not offer any way to supply key material inline, so a + # private key cannot end up written to the configuration database. + source = open( + os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + "functions_model_endpoint_auth.py", + ), + encoding="utf-8", + ).read() + assert "client_key_pem" not in source + assert "private_key" not in source + + print("mTLS certificates referenced by path only") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +def test_version_bumped(): + """Authentication schemes ship at or after their implementation version.""" + print("Testing config version...") + try: + assert_app_version_at_least("0.261.017") + print("Config version check passed") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + +if __name__ == "__main__": + tests = [ + test_api_key_header_is_configurable, + test_bearer_and_api_key_credentials_resolve, + test_oauth2_tokens_are_fetched_and_cached, + test_oauth2_failures_are_sanitized, + test_oauth2_token_endpoint_is_policy_checked, + test_incomplete_and_unsupported_auth_is_rejected, + test_mtls_certificates_are_referenced_by_path, + test_version_bumped, + ] + + results = [] + for test in tests: + print(f"\nRunning {test.__name__}...") + results.append(test()) + + print(f"\nResults: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) From a20d08b525618f32c3ac7b7b474c0f34cbd1191e Mon Sep 17 00:00:00 2001 From: Paul Lizer Date: Sat, 5 Sep 2026 11:02:31 -0400 Subject: [PATCH 9/9] Address code scanning findings on the Custom endpoint work Validate the OAuth2 token endpoint when the token is fetched, not only when the endpoint is saved. Checking at save time leaves the request itself unguarded, because settings can be written by another path, restored from backup, or changed after validation, so code scanning was right to call this a server-side request forgery. The token URL is now revalidated against the same outbound policy as the inference endpoint and the request runs on the same pinned transport, so its addresses are validated at connection time. That transport refuses redirects, which the previous code avoided on the stated grounds that token endpoints commonly redirect. That reasoning was wrong: redirects belong to the browser-based authorization-code flow, while a client-credentials token endpoint answers a server-to-server POST with a JSON body. A regression test now asserts a blocked token URL is refused before any HTTP client is constructed. Remove backtracking from the version-segment pattern. Its optional suffix could begin with a digit, making it ambiguous with the preceding digits and quadratic on a long run of them: 8,000 characters took about 0.19s to reject and now takes 0.0003s. Requiring the suffix to begin with a letter matches exactly the same segments. Move the auth-type constants and their normalizer to the provider registry so validation no longer imports the authentication module. That removes the real coupling behind several cyclic-import findings and lets the token fetch import validation normally rather than through a deferred import whose comment would otherwise have become untrue. Drop genuinely dead imports, and declare the deliberate re-exports in functions_model_endpoint_types with __all__, since functions_settings and the provider tests import those constants through it. Close file handles in tests and stop importing the same module both ways. The remaining cyclic-import findings are pre-existing: the same single nine-module cycle is present on Development, and none of the modules in this change participate in an import-time cycle. Refs #1228 Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- application/single_app/config.py | 2 +- .../functions_model_endpoint_auth.py | 75 ++++++---- .../functions_model_endpoint_providers.py | 14 ++ .../functions_model_endpoint_runtime.py | 6 + .../functions_model_endpoint_types.py | 16 ++- .../functions_model_endpoint_validation.py | 7 +- .../single_app/model_endpoint_clients.py | 10 +- .../features/CUSTOM_MODEL_ENDPOINT_AUTH.md | 20 ++- docs/explanation/release-notes/index.md | 27 ++-- docs/explanation/release-notes/v0.261.md | 11 +- docs/explanation/release_notes.md | 15 ++ .../test_custom_model_endpoint_auth.py | 134 +++++++++++++----- .../test_custom_model_endpoint_diagnostics.py | 14 +- ...stom_model_endpoint_synthetic_streaming.py | 17 ++- 14 files changed, 261 insertions(+), 107 deletions(-) diff --git a/application/single_app/config.py b/application/single_app/config.py index 9f7c1460d..6a28de8fe 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.017" +VERSION = "0.261.018" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_model_endpoint_auth.py b/application/single_app/functions_model_endpoint_auth.py index 2c228731d..5facf031c 100644 --- a/application/single_app/functions_model_endpoint_auth.py +++ b/application/single_app/functions_model_endpoint_auth.py @@ -21,21 +21,18 @@ import time from typing import Any, Dict, Tuple -import httpx - from functions_model_endpoint_diagnostics import build_sanitized_model_endpoint_error - - -AUTH_TYPE_API_KEY = "api_key" -AUTH_TYPE_KEY = "key" -AUTH_TYPE_BEARER = "bearer" -AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" - -CUSTOM_ENDPOINT_AUTH_TYPES = ( +from functions_model_endpoint_providers import ( AUTH_TYPE_API_KEY, AUTH_TYPE_BEARER, AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + DEFAULT_CUSTOM_AUTH_TYPES, + normalize_custom_endpoint_auth_type, ) +from functions_model_endpoint_validation import validate_custom_model_endpoint_url + + +CUSTOM_ENDPOINT_AUTH_TYPES = DEFAULT_CUSTOM_AUTH_TYPES # Refresh slightly before expiry so a token cannot lapse mid-request. OAUTH2_EXPIRY_SKEW_SECONDS = 60 @@ -46,14 +43,6 @@ _TOKEN_CACHE_LOCK = threading.Lock() -def normalize_custom_endpoint_auth_type(auth_type: Any) -> str: - """Return a supported Custom endpoint auth type, defaulting to API key.""" - normalized = str(auth_type or "").strip().lower() - if normalized == AUTH_TYPE_KEY: - return AUTH_TYPE_API_KEY - return normalized if normalized in CUSTOM_ENDPOINT_AUTH_TYPES else "" - - def resolve_api_key_header( auth: Dict[str, Any], default_header: str = "", @@ -122,14 +111,22 @@ def clear_oauth2_token_cache() -> None: def fetch_oauth2_client_credentials_token( auth: Dict[str, Any], *, - verify: Any = True, + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", http_client_factory=None, ) -> str: """Return an OAuth2 client-credentials access token, using the cache when valid. - The token endpoint is normally a different host from the inference endpoint, - and commonly redirects, so it is fetched with an ordinary client rather than - the no-redirect pinned transport used for inference. + The token endpoint is a separate, administrator-supplied host, so it is an + outbound request target in its own right and is held to the same policy as the + inference endpoint: the URL is revalidated here rather than trusted from + configuration time, the connection is pinned to the validated addresses, and + redirects are refused. + + Refusing redirects is safe for this grant. Redirects belong to the browser-based + authorization-code flow; a client-credentials token endpoint answers a + server-to-server POST with a JSON body. """ token_url = str(auth.get("token_url") or "").strip() client_id = str(auth.get("client_id") or "").strip() @@ -139,6 +136,17 @@ def fetch_oauth2_client_credentials_token( "OAuth2 model endpoints require a token URL, client ID, and client secret." ) + # Imported here rather than at module scope: the transport lives with the + # model endpoint clients, which pull in the OpenAI and Semantic Kernel SDKs. + # Deferring keeps this module importable without that cost. + from model_endpoint_clients import build_custom_openai_sync_http_client + + token_url = validate_custom_model_endpoint_url( + token_url, + allow_private=allow_private, + allow_insecure=allow_insecure, + ) + cache_key = _token_cache_key(auth) now = time.monotonic() with _TOKEN_CACHE_LOCK: @@ -155,11 +163,13 @@ def fetch_oauth2_client_credentials_token( if scope: payload["scope"] = scope - client = (http_client_factory or httpx.Client)( - timeout=OAUTH2_REQUEST_TIMEOUT_SECONDS, - verify=verify, - trust_env=False, - ) + if http_client_factory is not None: + client = http_client_factory(timeout=OAUTH2_REQUEST_TIMEOUT_SECONDS) + else: + client = build_custom_openai_sync_http_client( + allow_private=allow_private, + ca_bundle_path=ca_bundle_path, + ) try: response = client.post(token_url, data=payload) status_code = response.status_code @@ -205,7 +215,9 @@ def resolve_custom_endpoint_credentials( *, default_api_key_header: str = "", default_api_key_prefix: str = "", - verify: Any = True, + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", ) -> Tuple[str, Dict[str, str]]: """Resolve one Custom endpoint's credentials. @@ -224,7 +236,12 @@ def resolve_custom_endpoint_credentials( return token, {} if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: - return fetch_oauth2_client_credentials_token(auth, verify=verify), {} + return fetch_oauth2_client_credentials_token( + auth, + allow_private=allow_private, + allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, + ), {} if auth_type == AUTH_TYPE_API_KEY: api_key = str(auth.get("api_key") or "").strip() diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py index a855b2ea9..103310c30 100644 --- a/application/single_app/functions_model_endpoint_providers.py +++ b/application/single_app/functions_model_endpoint_providers.py @@ -69,6 +69,7 @@ def normalize_custom_endpoint_url_mode(url_mode: Any) -> str: DEFAULT_ANTHROPIC_VERSION = "2023-06-01" AUTH_TYPE_API_KEY = "api_key" +AUTH_TYPE_KEY = "key" AUTH_TYPE_BEARER = "bearer" AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" @@ -81,6 +82,19 @@ def normalize_custom_endpoint_url_mode(url_mode: Any) -> str: ) +def normalize_custom_endpoint_auth_type(auth_type: Any) -> str: + """Return a supported Custom endpoint auth type, or "" when unsupported. + + This lives with the registry rather than with the authentication code so that + validation can classify an auth type without importing the module that + performs authentication. + """ + normalized = str(auth_type or "").strip().lower() + if normalized == AUTH_TYPE_KEY: + return AUTH_TYPE_API_KEY + return normalized if normalized in DEFAULT_CUSTOM_AUTH_TYPES else "" + + class ModelEndpointProvider: """One Custom endpoint API type and everything the app needs to know about it.""" diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 8d1592cd0..25633f43e 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -207,6 +207,9 @@ def build_model_endpoint_sync_chat_client( default_api_key_prefix=( registered_provider.default_api_key_prefix if registered_provider else '' ), + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, ) if credential_headers: extra_headers = {**(extra_headers or {}), **credential_headers} @@ -499,6 +502,9 @@ def build_semantic_kernel_chat_service_for_model( default_api_key_prefix=( registered_provider.default_api_key_prefix if registered_provider else '' ), + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, ) if credential_headers: extra_headers = {**(extra_headers or {}), **credential_headers} diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py index 6a310cb0d..61b3c0646 100644 --- a/application/single_app/functions_model_endpoint_types.py +++ b/application/single_app/functions_model_endpoint_types.py @@ -13,7 +13,6 @@ DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_API_TYPE_ANTHROPIC, MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, - MODEL_ENDPOINT_API_TYPE_GEMINI, MODEL_ENDPOINT_API_TYPE_OPENAI, MODEL_ENDPOINT_CUSTOM_API_TYPES, MODEL_ENDPOINT_PROVIDER_CUSTOM, @@ -22,6 +21,21 @@ ) +# Callers have long imported these constants from this module rather than from the +# registry that now owns them, so they are re-exported deliberately. +__all__ = [ + "DEFAULT_ANTHROPIC_VERSION", + "MODEL_ENDPOINT_API_TYPE_ANTHROPIC", + "MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI", + "MODEL_ENDPOINT_API_TYPE_OPENAI", + "MODEL_ENDPOINT_CUSTOM_API_TYPES", + "MODEL_ENDPOINT_PROVIDER_CUSTOM", + "get_model_endpoint_api_type", + "normalize_model_endpoint_api_type", + "resolve_model_endpoint_request_model", +] + + def normalize_model_endpoint_api_type(provider: Any, api_type: Any) -> str: """Return a supported explicit API type for Custom endpoints.""" normalized_provider = str(provider or "").strip().lower() diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py index 6dd11152a..2b5dafabb 100644 --- a/application/single_app/functions_model_endpoint_validation.py +++ b/application/single_app/functions_model_endpoint_validation.py @@ -7,17 +7,14 @@ from typing import Any, Dict, Iterable from urllib.parse import urlparse, urlunparse -from functions_model_endpoint_auth import ( +from functions_model_endpoint_providers import ( AUTH_TYPE_API_KEY, AUTH_TYPE_BEARER, AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + get_model_endpoint_provider, normalize_custom_endpoint_auth_type, ) -from functions_model_endpoint_providers import get_model_endpoint_provider from functions_model_endpoint_types import ( - DEFAULT_ANTHROPIC_VERSION, - MODEL_ENDPOINT_API_TYPE_ANTHROPIC, - MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, MODEL_ENDPOINT_PROVIDER_CUSTOM, get_model_endpoint_api_type, resolve_model_endpoint_request_model, diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index dc765cffd..36cb9717a 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -44,8 +44,6 @@ from functions_model_endpoint_types import ( DEFAULT_ANTHROPIC_VERSION, MODEL_ENDPOINT_API_TYPE_ANTHROPIC, - MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, - MODEL_ENDPOINT_API_TYPE_OPENAI, MODEL_ENDPOINT_PROVIDER_CUSTOM, normalize_model_endpoint_api_type, ) @@ -216,7 +214,13 @@ def normalize_openai_style_base_url(raw_endpoint: Any) -> str: CUSTOM_OPENAI_OPERATION_SUFFIXES = ("/chat/completions", "/responses", "/models") -CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN = re.compile(r"^v\d+[a-z0-9]*$", re.IGNORECASE) +# The optional suffix must start with a letter. Allowing it to start with a digit +# would make it ambiguous with the preceding \d+, which backtracks quadratically +# on a long run of digits. +CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN = re.compile( + r"^v\d+(?:[a-z][a-z0-9]*)?$", + re.IGNORECASE, +) def _endpoint_path_names_a_version(endpoint: str) -> bool: diff --git a/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md index e01a08abc..1804dfb5c 100644 --- a/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md +++ b/docs/explanation/features/CUSTOM_MODEL_ENDPOINT_AUTH.md @@ -68,13 +68,17 @@ Behaviour worth knowing: - **Tokens are cached** per token URL, client ID, and scope, and refreshed 60 seconds before expiry so a token cannot lapse part-way through a request. A response without `expires_in` is treated as one hour. -- **The token endpoint is policy checked.** It is a different host from the - inference endpoint, so it is validated against the same outbound rules. A token - URL pointing at a cloud metadata address is refused, exactly as an inference - endpoint would be. Without this, the token URL would be an unchecked outbound - request target. -- **The token endpoint is fetched with an ordinary client**, not the no-redirect - pinned transport used for inference, because token endpoints commonly redirect. +- **The token endpoint is policy checked, at request time as well as at save + time.** It is a different host from the inference endpoint, so it is validated + against the same outbound rules, and revalidated when the token is actually + fetched rather than trusted from configuration. Settings can be written by + another path, restored from backup, or changed after validation, so a + save-time-only check would leave the request unguarded. +- **The request runs on the same pinned transport as inference**, so its + addresses are validated at connection time and redirects are refused. That is + safe for this grant: redirects belong to the browser-based authorization-code + flow, whereas a client-credentials token endpoint answers a server-to-server + POST with a JSON body. - **Failures are sanitized.** A token endpoint's error body frequently echoes the client ID or secret, so the browser sees a generic message with a correlation id while the real response is recorded server-side with credentials redacted. @@ -117,6 +121,8 @@ continuing without one. explicit header and which ride on the SDK's own credential argument; - OAuth2 tokens being fetched once, served from cache on the second call, and refetched after the cache is cleared, with the request payload asserted; +- a blocked token URL being refused when the token is fetched, not only when the + endpoint is saved, and refused before any HTTP client is constructed; - a failing token endpoint leaking neither its error body nor the client details, while still offering a correlation id; - a token endpoint pointing at a cloud metadata address being refused; diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index f460d309c..bb0677f66 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.261.018 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.017 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.016 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.015 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -31,7 +32,7 @@ This page includes the latest release notes inline. Older release sections are s | v0.261.009 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.007 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.261.006 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.261.005 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.261.005 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.004 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.003 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | | v0.261.002 | [Release notes 0.261 series]({{ '/explanation/release-notes/v0.261/' | relative_url }}) | @@ -84,6 +85,21 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.261.018)** + +#### Bug Fixes + +* **OAuth2 Token Endpoints Are Now Validated When The Token Is Fetched** + * A Custom endpoint's OAuth2 token URL was checked when the endpoint was saved, but not when the token was actually requested. Validating only at save time leaves the request itself unguarded, since settings can be written by another path, restored from backup, or changed after validation. Code scanning correctly identified this as a server-side request forgery. + * The token URL is now revalidated at request time against the same outbound policy as the inference endpoint, and the request runs on the same pinned transport, so its addresses are validated at connection time and redirects are refused. + * Refusing redirects is safe for this grant: redirects belong to the browser-based authorization-code flow, whereas a client-credentials token endpoint answers a server-to-server POST with a JSON body. The previous code allowed them based on an incorrect assumption. + * (Ref: `functions_model_endpoint_auth.py`, [#1437](https://github.com/microsoft/simplechat/pull/1437)) + +* **Endpoint URL Version Matching No Longer Backtracks** + * The pattern recognising a version path segment allowed its optional suffix to begin with a digit, making it ambiguous with the preceding digits and quadratic on a long run of them. A 8,000-character segment took roughly 0.19 seconds to reject; it now takes 0.0003 seconds. + * The suffix must now begin with a letter, which removes the ambiguity while matching exactly the same version segments. + * (Ref: `model_endpoint_clients.py`, [#1437](https://github.com/microsoft/simplechat/pull/1437)) + ### **(v0.261.017)** #### New Features @@ -208,12 +224,3 @@ This page includes the latest release notes inline. Older release sections are s * Markdown document processing now retries the known transient `OrderedDict mutated during iteration` parser failure before marking a document failed. * The retry is limited to this specific Markdown failure signature, so unrelated parsing, validation, or service errors still fail normally with their original error. * (Ref: Markdown upload processing, `functions_documents.py`, `test_markdown_processing_batches_search_writes.py`, [Search Write Gate Upload Contention Fix](fixes/SEARCH_WRITE_GATE_UPLOAD_CONTENTION_FIX.md)) - -### **(v0.261.005)** - -#### User Interface Enhancements - -* **Workspace Upload Progress Now Separates Request Status From Document Processing Status** - * The temporary upload summary no longer labels unconfirmed browser upload requests as final document failures. This avoids misleading summaries such as `Uploaded 77/204, Failed: 127` when the document list later shows that most documents were queued and processed successfully. - * Personal, group, and public workspace uploads now use `Queued` for confirmed upload requests and direct users to the refreshed document list for final processing status. - * (Ref: workspace upload progress summary, `workspace-documents.js`, `public_workspace.js`, `group_workspaces.html`, [Workspace Upload Status Counter Fix](fixes/WORKSPACE_UPLOAD_STATUS_COUNTER_FIX.md)) diff --git a/docs/explanation/release-notes/v0.261.md b/docs/explanation/release-notes/v0.261.md index 686df5a8c..66f57d904 100644 --- a/docs/explanation/release-notes/v0.261.md +++ b/docs/explanation/release-notes/v0.261.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.261 series" -description: "SimpleChat release notes for 0.261.004 – 0.261.001." +description: "SimpleChat release notes for 0.261.005 – 0.261.001." section: "Reference" layout: page --- @@ -11,6 +11,15 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.261.005)** + +#### User Interface Enhancements + +* **Workspace Upload Progress Now Separates Request Status From Document Processing Status** + * The temporary upload summary no longer labels unconfirmed browser upload requests as final document failures. This avoids misleading summaries such as `Uploaded 77/204, Failed: 127` when the document list later shows that most documents were queued and processed successfully. + * Personal, group, and public workspace uploads now use `Queued` for confirmed upload requests and direct users to the refreshed document list for final processing status. + * (Ref: workspace upload progress summary, `workspace-documents.js`, `public_workspace.js`, `group_workspaces.html`, [Workspace Upload Status Counter Fix](fixes/WORKSPACE_UPLOAD_STATUS_COUNTER_FIX.md)) + ### **(v0.261.004)** #### Bug Fixes diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index db4164481..66a35e152 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,21 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.261.018)** + +#### Bug Fixes + +* **OAuth2 Token Endpoints Are Now Validated When The Token Is Fetched** + * A Custom endpoint's OAuth2 token URL was checked when the endpoint was saved, but not when the token was actually requested. Validating only at save time leaves the request itself unguarded, since settings can be written by another path, restored from backup, or changed after validation. Code scanning correctly identified this as a server-side request forgery. + * The token URL is now revalidated at request time against the same outbound policy as the inference endpoint, and the request runs on the same pinned transport, so its addresses are validated at connection time and redirects are refused. + * Refusing redirects is safe for this grant: redirects belong to the browser-based authorization-code flow, whereas a client-credentials token endpoint answers a server-to-server POST with a JSON body. The previous code allowed them based on an incorrect assumption. + * (Ref: `functions_model_endpoint_auth.py`, [#1437](https://github.com/microsoft/simplechat/pull/1437)) + +* **Endpoint URL Version Matching No Longer Backtracks** + * The pattern recognising a version path segment allowed its optional suffix to begin with a digit, making it ambiguous with the preceding digits and quadratic on a long run of them. A 8,000-character segment took roughly 0.19 seconds to reject; it now takes 0.0003 seconds. + * The suffix must now begin with a letter, which removes the ambiguity while matching exactly the same version segments. + * (Ref: `model_endpoint_clients.py`, [#1437](https://github.com/microsoft/simplechat/pull/1437)) + ### **(v0.261.017)** #### New Features diff --git a/functional_tests/test_custom_model_endpoint_auth.py b/functional_tests/test_custom_model_endpoint_auth.py index c32b6a781..0e6b8d728 100644 --- a/functional_tests/test_custom_model_endpoint_auth.py +++ b/functional_tests/test_custom_model_endpoint_auth.py @@ -22,7 +22,9 @@ """ import os +import socket import sys +from unittest.mock import patch sys.path.append(os.path.dirname(os.path.abspath(__file__))) sys.path.append( @@ -35,7 +37,6 @@ from test_support.versioning import assert_app_version_at_least -import functions_model_endpoint_auth as endpoint_auth from functions_model_endpoint_auth import ( build_api_key_headers, clear_oauth2_token_cache, @@ -66,7 +67,7 @@ def json(self): class _FakeTokenClient: - """Stand-in for httpx.Client that records token requests.""" + """Stand-in for the pinned HTTP client that records token requests.""" instances = [] @@ -84,6 +85,21 @@ def close(self): pass +# The token endpoint is now revalidated at request time, which resolves the +# hostname. These tests use documentation hostnames that do not resolve, so DNS is +# stubbed to a public address; the policy checks themselves still run for real. +PUBLIC_ADDRESS_INFO = [ + (socket.AF_INET, socket.SOCK_STREAM, socket.IPPROTO_TCP, "", ("93.184.216.34", 443)) +] + + +def _patch_public_dns(): + return patch( + "functions_model_endpoint_validation.socket.getaddrinfo", + return_value=PUBLIC_ADDRESS_INFO, + ) + + def test_api_key_header_is_configurable(): """One API key scheme must cover every provider's header convention.""" print("Testing API key header customization...") @@ -187,29 +203,31 @@ def test_oauth2_tokens_are_fetched_and_cached(): "scope": "inference.read", } - token = fetch_oauth2_client_credentials_token( - auth, http_client_factory=_FakeTokenClient - ) - assert token == "issued-token" - assert len(_FakeTokenClient.instances) == 1 + with _patch_public_dns(): + token = fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + assert token == "issued-token" + assert len(_FakeTokenClient.instances) == 1 - request_url, payload = _FakeTokenClient.instances[0].requests[0] - assert request_url == auth["token_url"] - assert payload["grant_type"] == "client_credentials" - assert payload["scope"] == "inference.read" + request_url, payload = _FakeTokenClient.instances[0].requests[0] + assert request_url == auth["token_url"] + assert payload["grant_type"] == "client_credentials" + assert payload["scope"] == "inference.read" - # A second call is served from the cache, so no new request is made. - cached_token = fetch_oauth2_client_credentials_token( - auth, http_client_factory=_FakeTokenClient - ) - assert cached_token == "issued-token" - assert len(_FakeTokenClient.instances) == 1, "Token must be cached." - - # Clearing the cache forces a new request. - clear_oauth2_token_cache() - fetch_oauth2_client_credentials_token(auth, http_client_factory=_FakeTokenClient) - assert len(_FakeTokenClient.instances) == 2 + # A second call is served from the cache, so no new request is made. + cached_token = fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + assert cached_token == "issued-token" + assert len(_FakeTokenClient.instances) == 1, "Token must be cached." + # Clearing the cache forces a new request. + clear_oauth2_token_cache() + fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + assert len(_FakeTokenClient.instances) == 2 clear_oauth2_token_cache() print("OAuth2 fetch, cache, and refresh passed") return True @@ -241,9 +259,10 @@ def __init__(self, **kwargs): "client_secret": "secret-1", } try: - fetch_oauth2_client_credentials_token( - auth, http_client_factory=FailingTokenClient - ) + with _patch_public_dns(): + fetch_oauth2_client_credentials_token( + auth, http_client_factory=FailingTokenClient + ) except RuntimeError as exc: assert "leak-me" not in str(exc) assert "invalid_client" not in str(exc) @@ -262,6 +281,53 @@ def __init__(self, **kwargs): return False +def test_token_endpoint_is_revalidated_at_request_time(): + """A blocked token URL must be refused when the token is fetched, not only when saved. + + Validating only at save time leaves the request itself unguarded: settings can + be written by another path, restored from backup, or changed after validation. + CodeQL flagged this as a server-side request forgery, correctly. + """ + print("Testing request-time token endpoint validation...") + try: + clear_oauth2_token_cache() + + for blocked_url in ( + "https://169.254.169.254/token", + "https://127.0.0.1/token", + "https://metadata.google.internal/token", + "http://auth.example.com/token", + ): + auth = { + "type": "oauth2_client_credentials", + "token_url": blocked_url, + "client_id": "client-1", + "client_secret": "secret-1", + } + _FakeTokenClient.instances = [] + try: + fetch_oauth2_client_credentials_token( + auth, http_client_factory=_FakeTokenClient + ) + except Exception: + # The request must be refused before any client is constructed. + assert not _FakeTokenClient.instances, ( + f"{blocked_url} reached the network before being refused." + ) + continue + raise AssertionError(f"{blocked_url} must be refused at request time.") + + clear_oauth2_token_cache() + print("Token endpoint revalidated at request time") + return True + except Exception as e: + print(f"Test failed: {e}") + import traceback + + traceback.print_exc() + return False + + def test_oauth2_token_endpoint_is_policy_checked(): """The token endpoint is a separate host and must satisfy the same policy.""" print("Testing OAuth2 token endpoint policy...") @@ -364,15 +430,14 @@ def test_mtls_certificates_are_referenced_by_path(): # The module must not offer any way to supply key material inline, so a # private key cannot end up written to the configuration database. - source = open( - os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "application", - "single_app", - "functions_model_endpoint_auth.py", - ), - encoding="utf-8", - ).read() + auth_module_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + "functions_model_endpoint_auth.py", + ) + with open(auth_module_path, encoding="utf-8") as source_file: + source = source_file.read() assert "client_key_pem" not in source assert "private_key" not in source @@ -407,6 +472,7 @@ def test_version_bumped(): test_bearer_and_api_key_credentials_resolve, test_oauth2_tokens_are_fetched_and_cached, test_oauth2_failures_are_sanitized, + test_token_endpoint_is_revalidated_at_request_time, test_oauth2_token_endpoint_is_policy_checked, test_incomplete_and_unsupported_auth_is_rejected, test_mtls_certificates_are_referenced_by_path, diff --git a/functional_tests/test_custom_model_endpoint_diagnostics.py b/functional_tests/test_custom_model_endpoint_diagnostics.py index 5fde63691..349aa71a6 100644 --- a/functional_tests/test_custom_model_endpoint_diagnostics.py +++ b/functional_tests/test_custom_model_endpoint_diagnostics.py @@ -37,11 +37,10 @@ from test_support.versioning import assert_app_version_at_least import functions_model_endpoint_diagnostics as diagnostics -from functions_model_endpoint_diagnostics import ( - build_sanitized_model_endpoint_error, - log_custom_model_endpoint_failure, - redact_model_endpoint_secrets, -) + +build_sanitized_model_endpoint_error = diagnostics.build_sanitized_model_endpoint_error +log_custom_model_endpoint_failure = diagnostics.log_custom_model_endpoint_failure +redact_model_endpoint_secrets = diagnostics.redact_model_endpoint_secrets SECRET_SAMPLES = [ @@ -192,9 +191,10 @@ def test_no_failure_path_discards_its_cause(): """No Custom endpoint failure path may raise a bare sanitized error any more.""" print("Testing that no failure path discards its cause...") try: - source = open( + with open( os.path.join(APP_DIR, "model_endpoint_clients.py"), encoding="utf-8" - ).read() + ) as source_file: + source = source_file.read() discarded = re.findall( r'raise RuntimeError\(\s*\n?\s*"Custom[^"]*"\s*\n?\s*\) from None', diff --git a/functional_tests/test_custom_model_endpoint_synthetic_streaming.py b/functional_tests/test_custom_model_endpoint_synthetic_streaming.py index 3238c0191..cd38bcfb1 100644 --- a/functional_tests/test_custom_model_endpoint_synthetic_streaming.py +++ b/functional_tests/test_custom_model_endpoint_synthetic_streaming.py @@ -218,15 +218,14 @@ def test_stream_options_are_kept_where_supported(): "OpenAI accepts stream_options.include_usage, which reports token usage." ) - source = open( - os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "application", - "single_app", - "model_endpoint_clients.py", - ), - encoding="utf-8", - ).read() + source_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "application", + "single_app", + "model_endpoint_clients.py", + ) + with open(source_path, encoding="utf-8") as source_file: + source = source_file.read() assert ( 'request_kwargs = dict(kwargs)\n request_kwargs.pop("stream_options", None)' not in source