diff --git a/application/single_app/app.py b/application/single_app/app.py index 36afa5685..8865a9382 100644 --- a/application/single_app/app.py +++ b/application/single_app/app.py @@ -105,6 +105,7 @@ from route_plugin_logging import bpl as plugin_logging_bp from functions_custom_pages import get_custom_pages_nav from functions_debug import debug_print +from functions_model_endpoint_providers import get_model_endpoint_provider_ui_options from functions_terms_of_use import has_terms_of_use_acceptance from functions_mcp_server_auth import inbound_mcp_required_blueprint @@ -589,6 +590,7 @@ def inject_settings(): idle_timeout_enabled=idle_timeout_enabled, idle_timeout_minutes=idle_timeout_minutes, idle_warning_minutes=idle_warning_minutes, + model_endpoint_api_types=get_model_endpoint_provider_ui_options(), mcp_ui_enabled=is_mcp_ui_enabled() ) diff --git a/application/single_app/config.py b/application/single_app/config.py index 42be1c9c5..9c3f1527b 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -97,7 +97,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.261.012" +VERSION = "0.261.021" IS_DEVELOPMENT = is_development_env_enabled() # Opt-out for deployments where App Service Easy Auth is active but the platform diff --git a/application/single_app/functions_documents.py b/application/single_app/functions_documents.py index a055c13b4..99f767106 100644 --- a/application/single_app/functions_documents.py +++ b/application/single_app/functions_documents.py @@ -37,6 +37,11 @@ from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers from functions_model_endpoint_runtime import MODEL_ENDPOINT_PROVIDER_ALLOWLIST, build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from model_endpoint_clients import MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, infer_model_endpoint_protocol import azure.cognitiveservices.speech as speechsdk _AUDIO_RUNTIME_CAPABILITIES_CACHE = None @@ -142,6 +147,9 @@ def _build_model_endpoint_client( api_version, deployment_name, *, + api_type='', + anthropic_version='', + allow_private_custom_endpoints=False, settings=None, endpoint_config=None, identity_context=None, @@ -152,6 +160,9 @@ def _build_model_endpoint_client( endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, settings=settings, endpoint_config=endpoint_config, identity_context=identity_context, @@ -191,14 +202,24 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): provider = str(endpoint_cfg.get("provider") or selection["provider"] or "aoai").lower() connection = endpoint_cfg.get("connection", {}) or {} auth_settings = endpoint_cfg.get("auth", {}) or {} - deployment = str(model_cfg.get("deploymentName") or model_cfg.get("deployment") or "").strip() + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) endpoint = str(connection.get("endpoint") or "").strip() api_version = str(connection.get("openai_api_version") or connection.get("api_version") or "").strip() + api_type = get_model_endpoint_api_type(endpoint_cfg) + anthropic_version = str(connection.get("anthropic_version") or "").strip() + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) if provider not in MODEL_ENDPOINT_PROVIDER_ALLOWLIST: raise ValueError(f"Selected metadata extraction provider '{provider}' is not supported.") - if not endpoint or not api_version or not deployment: - raise ValueError("Selected metadata extraction endpoint is missing endpoint, API version, or deployment configuration.") + if not endpoint or not deployment or ( + runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version + ): + raise ValueError("Selected metadata extraction endpoint is incomplete.") return _build_model_endpoint_client( auth_settings, @@ -206,6 +227,11 @@ def _resolve_metadata_extraction_client(settings, identity_context=None): endpoint, api_version, deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get("allow_private_custom_model_endpoints", False) + ), settings=settings, endpoint_config=endpoint_cfg, identity_context=identity_context, diff --git a/application/single_app/functions_model_capabilities.py b/application/single_app/functions_model_capabilities.py index aecb13dab..9520bba65 100644 --- a/application/single_app/functions_model_capabilities.py +++ b/application/single_app/functions_model_capabilities.py @@ -1,12 +1,27 @@ # functions_model_capabilities.py +"""Model capability resolution backed by the SimpleChat model capability catalog. +Capability answers resolve through a precedence chain so that a model which is not +present in the shipped catalog -- a customer's on-premises or bespoke model -- can +still be described accurately instead of being guessed at from its name: + + per-model override -> endpoint override -> catalog entry -> name heuristic + +Only stdlib imports are used here on purpose. This module sits below the settings, +logging, and route layers, so pulling those in would risk import cycles. +""" + +import json +import os import re +import threading from collections.abc import Mapping MODEL_IDENTIFIER_SEPARATOR_PATTERN = re.compile(r"[\s_.]+") GPT_VISION_MODEL_PATTERN = re.compile(r"(?:^|-)gpt-(?:[5-9]|\d{2,})(?:-|$)") O_SERIES_MODEL_PATTERN = re.compile(r"(?:^|-)o\d+(?:-|$)") +REASONING_MODEL_PATTERN = re.compile(r"(?:^|-)(?:o\d+|gpt-(?:[5-9]|\d{2,}))(?:-|$)") MODEL_IDENTIFIER_FIELDS = ( "modelName", "displayName", @@ -15,6 +30,37 @@ "name", ) +CATALOG_RELATIVE_PATH = ("static", "json", "model_capabilities.json") + +CAPABILITY_PROCESSES_IMAGES = "processesImages" +CAPABILITY_TOOL_CALLING = "toolCalling" +CAPABILITY_STRUCTURED_OUTPUT = "structuredOutput" +CAPABILITY_SUPPORTS_STREAMING = "supportsStreaming" +CAPABILITY_REASONING = "reasoning" + +CAPABILITY_FIELD_NAMES = ( + "processesText", + "generatesText", + CAPABILITY_PROCESSES_IMAGES, + "generatesImages", + "processesAudio", + "generatesAudio", + "processesVideo", + "generatesVideo", + "processesBinaryFiles", + "optimizedForCoding", + CAPABILITY_TOOL_CALLING, + CAPABILITY_STRUCTURED_OUTPUT, + CAPABILITY_SUPPORTS_STREAMING, + CAPABILITY_REASONING, +) + +CATALOG_CONTEXT_LIMIT_FIELDS = ("inputTokenLimit", "contextWindow", "maxInputTokens") +CATALOG_OUTPUT_LIMIT_FIELDS = ("outputTokenLimit", "maxOutputTokens", "maxCompletionTokens") + +_CATALOG_LOCK = threading.Lock() +_CATALOG_CACHE = None + def _normalize_model_identifier(value): return MODEL_IDENTIFIER_SEPARATOR_PATTERN.sub( @@ -23,10 +69,154 @@ def _normalize_model_identifier(value): ) -def is_vision_capable_model_name(*model_names): - """Return whether any supplied identifier names a supported vision model.""" - for model_name in model_names: - normalized_name = _normalize_model_identifier(model_name) +def get_model_capability_catalog_path(): + """Return the absolute path of the shipped model capability catalog.""" + return os.path.join(os.path.dirname(__file__), *CATALOG_RELATIVE_PATH) + + +def reset_model_capability_catalog_cache(): + """Clear the cached catalog so a later read picks the file up again.""" + global _CATALOG_CACHE + with _CATALOG_LOCK: + _CATALOG_CACHE = None + + +def load_model_capability_catalog(): + """Return the parsed catalog, caching it after the first successful read.""" + global _CATALOG_CACHE + with _CATALOG_LOCK: + if _CATALOG_CACHE is not None: + return _CATALOG_CACHE + try: + with open(get_model_capability_catalog_path(), "r", encoding="utf-8") as catalog_file: + catalog = json.load(catalog_file) + except (OSError, json.JSONDecodeError): + catalog = {} + if not isinstance(catalog, dict): + catalog = {} + _CATALOG_CACHE = catalog + return _CATALOG_CACHE + + +def get_model_capability_catalog_records(): + """Return every model record defined by the catalog.""" + catalog = load_model_capability_catalog() + return [record for record in catalog.get("models") or [] if isinstance(record, dict)] + + +def _get_record_field(record, field_name): + if isinstance(record, Mapping): + return record.get(field_name) + return getattr(record, field_name, None) + + +def _iter_model_identifiers(model): + """Yield every normalized identifier that could name the supplied model.""" + if model is None: + return + if isinstance(model, str): + normalized = _normalize_model_identifier(model) + if normalized: + yield normalized + return + for field_name in MODEL_IDENTIFIER_FIELDS: + normalized = _normalize_model_identifier(_get_record_field(model, field_name)) + if normalized: + yield normalized + + +def _iter_catalog_record_identifiers(record): + """Yield every normalized identifier a catalog record answers to. + + "family" is deliberately excluded. It is a grouping attribute rather than an + identifier, and members of one family disagree on capabilities -- "phi-4" + covers both the multimodal and the text-only Phi models, and the "gpt-5.x" + families each contain a non-vision "-chat" member. Matching on it would let a + model inherit a sibling's capabilities. + """ + for field_name in ("id", "displayName"): + normalized = _normalize_model_identifier(record.get(field_name)) + if normalized: + yield normalized + aliases = record.get("aliases") + if isinstance(aliases, (list, tuple)): + for alias in aliases: + normalized = _normalize_model_identifier(alias) + if normalized: + yield normalized + + +def _is_variant_suffix_match(requested_identifier, record_identifier): + """Return whether requested is a variant of record rather than a later version. + + Identifier normalization collapses "." and "-" to the same separator, so + "gpt-5.3" becomes "gpt-5-3" and would otherwise look like a suffixed variant of + "gpt-5". A remainder that starts with a digit is a version continuation, not a + variant, so it is rejected. A remainder starting with a letter -- the "eastus" + in "gpt-5.6-sol-eastus", or the "mini" in "gpt-4o-mini" -- is a real variant. + """ + prefix = f"{record_identifier}-" + if not requested_identifier.startswith(prefix): + return False + remainder = requested_identifier[len(prefix):] + return bool(remainder) and not remainder[0].isdigit() + + +def find_model_catalog_record(model): + """Return the catalog record naming this model, or None when it is unknown. + + An exact identifier match always wins. Otherwise the longest matching + identifier prefix wins, so a deployment named "gpt-5.6-sol-eastus" resolves to + "gpt-5.6-sol", and "gpt-5.1-chat-v2" resolves to "gpt-5.1-chat" rather than to + the shorter, and differently capable, "gpt-5.1". + """ + requested_identifiers = list(_iter_model_identifiers(model)) + if not requested_identifiers: + return None + + requested_identifier_set = set(requested_identifiers) + best_prefix_match = None + best_prefix_length = 0 + for record in get_model_capability_catalog_records(): + record_identifiers = list(_iter_catalog_record_identifiers(record)) + if requested_identifier_set.intersection(record_identifiers): + return record + for record_identifier in record_identifiers: + if len(record_identifier) <= best_prefix_length: + continue + for requested_identifier in requested_identifiers: + if _is_variant_suffix_match(requested_identifier, record_identifier): + best_prefix_match = record + best_prefix_length = len(record_identifier) + break + return best_prefix_match + + +def _read_declared_capabilities(source): + """Return the explicit capability map declared on a model or endpoint record.""" + if source is None: + return {} + capabilities = _get_record_field(source, "capabilities") + if not isinstance(capabilities, Mapping): + return {} + declared = {} + for capability_name, capability_value in capabilities.items(): + if isinstance(capability_value, bool): + declared[str(capability_name)] = capability_value + return declared + + +def _heuristic_capability(capability_name, model): + """Return the legacy name-based answer for the capabilities that have one.""" + if capability_name == CAPABILITY_PROCESSES_IMAGES: + return _heuristic_is_vision_capable(model) + if capability_name == CAPABILITY_REASONING: + return _heuristic_is_reasoning_model(model) + return None + + +def _heuristic_is_vision_capable(model): + for normalized_name in _iter_model_identifiers(model): if ( "vision" in normalized_name or "gpt-4o" in normalized_name @@ -36,18 +226,139 @@ def is_vision_capable_model_name(*model_names): or O_SERIES_MODEL_PATTERN.search(normalized_name) ): return True + return False + +def _heuristic_is_reasoning_model(model): + for normalized_name in _iter_model_identifiers(model): + if REASONING_MODEL_PATTERN.search(normalized_name) or "gpt-5" in normalized_name: + return True return False -def is_vision_capable_model(model): +def resolve_model_capability(capability_name, model=None, endpoint=None, default=None): + """Resolve one capability through the override, catalog, then heuristic chain.""" + declared_model_capabilities = _read_declared_capabilities(model) + if capability_name in declared_model_capabilities: + return declared_model_capabilities[capability_name] + + declared_endpoint_capabilities = _read_declared_capabilities(endpoint) + if capability_name in declared_endpoint_capabilities: + return declared_endpoint_capabilities[capability_name] + + catalog_record = find_model_catalog_record(model) + if catalog_record is not None: + catalog_capabilities = catalog_record.get("capabilities") + if isinstance(catalog_capabilities, Mapping): + catalog_value = catalog_capabilities.get(capability_name) + if isinstance(catalog_value, bool): + return catalog_value + + heuristic_value = _heuristic_capability(capability_name, model) + if heuristic_value is not None: + return heuristic_value + return default + + +def resolve_model_capabilities(model=None, endpoint=None): + """Return every known capability for a model as a name to boolean-or-None map.""" + return { + capability_name: resolve_model_capability(capability_name, model, endpoint) + for capability_name in CAPABILITY_FIELD_NAMES + } + + +def _read_token_limit(record, field_names): + for field_name in field_names: + value = _get_record_field(record, field_name) + try: + normalized_value = int(value) + except (TypeError, ValueError): + continue + if normalized_value > 0: + return normalized_value + return None + + +def resolve_model_token_limits(model=None, endpoint=None): + """Return the (context, output) token limits for a model, or None when unknown.""" + for source in (model, endpoint): + if source is None or isinstance(source, str): + continue + context_limit = _read_token_limit(source, CATALOG_CONTEXT_LIMIT_FIELDS) + output_limit = _read_token_limit(source, CATALOG_OUTPUT_LIMIT_FIELDS) + if context_limit or output_limit: + return context_limit, output_limit + + catalog_record = find_model_catalog_record(model) + if catalog_record is None: + return None, None + return ( + _read_token_limit(catalog_record, CATALOG_CONTEXT_LIMIT_FIELDS), + _read_token_limit(catalog_record, CATALOG_OUTPUT_LIMIT_FIELDS), + ) + + +def resolve_model_output_token_limit(model=None, endpoint=None, default=None): + """Return the output token limit for a model, falling back to the supplied default.""" + _, output_limit = resolve_model_token_limits(model, endpoint) + return output_limit or default + + +def is_vision_capable_model_name(*model_names): + """Return whether any supplied identifier names a supported vision model.""" + for model_name in model_names: + if model_name in (None, ""): + continue + if resolve_model_capability(CAPABILITY_PROCESSES_IMAGES, model_name, default=False): + return True + + return False + + +def is_vision_capable_model(model, endpoint=None): """Return whether a model record or identifier names a supported vision model.""" - if isinstance(model, str): - return is_vision_capable_model_name(model) + return bool( + resolve_model_capability( + CAPABILITY_PROCESSES_IMAGES, + model, + endpoint, + default=False, + ) + ) + + +def is_reasoning_model(model, endpoint=None): + """Return whether a model uses reasoning-style response length parameters.""" + return bool( + resolve_model_capability( + CAPABILITY_REASONING, + model, + endpoint, + default=False, + ) + ) - if isinstance(model, Mapping): - model_names = [model.get(field_name) for field_name in MODEL_IDENTIFIER_FIELDS] - else: - model_names = [getattr(model, field_name, None) for field_name in MODEL_IDENTIFIER_FIELDS] - return is_vision_capable_model_name(*model_names) \ No newline at end of file +def supports_streaming(model=None, endpoint=None): + """Return whether a model can stream. Unknown models are assumed to stream.""" + return bool( + resolve_model_capability( + CAPABILITY_SUPPORTS_STREAMING, + model, + endpoint, + default=True, + ) + ) + + +def supports_tool_calling(model=None, endpoint=None, default=True): + """Return whether a model supports tool or function calling.""" + return bool( + resolve_model_capability( + CAPABILITY_TOOL_CALLING, + model, + endpoint, + default=default, + ) + ) diff --git a/application/single_app/functions_model_endpoint_auth.py b/application/single_app/functions_model_endpoint_auth.py new file mode 100644 index 000000000..5facf031c --- /dev/null +++ b/application/single_app/functions_model_endpoint_auth.py @@ -0,0 +1,279 @@ +# functions_model_endpoint_auth.py +"""Authentication schemes for Custom model endpoints. + +Custom endpoints originally supported one scheme: an API key sent in whichever +header the built-in providers happened to use. That covers OpenAI and Anthropic +and nothing else. A gateway that expects "x-goog-api-key", a corporate gateway +that issues short-lived OAuth2 tokens, and an on-premises appliance that requires +a client certificate were all unreachable. + +This module adds those schemes without widening what the browser can see: every +secret stays server-side, and OAuth2 token responses are never surfaced to a +caller beyond the token itself. + +mTLS is deliberately modelled as a transport concern rather than an auth "type", +because a client certificate combines with any of the schemes below. Certificates +are referenced by file path rather than stored in settings, so a private key is +mounted into the deployment and never written to the configuration database. +""" + +import threading +import time +from typing import Any, Dict, Tuple + +from functions_model_endpoint_diagnostics import build_sanitized_model_endpoint_error +from functions_model_endpoint_providers import ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + DEFAULT_CUSTOM_AUTH_TYPES, + normalize_custom_endpoint_auth_type, +) +from functions_model_endpoint_validation import validate_custom_model_endpoint_url + + +CUSTOM_ENDPOINT_AUTH_TYPES = DEFAULT_CUSTOM_AUTH_TYPES + +# Refresh slightly before expiry so a token cannot lapse mid-request. +OAUTH2_EXPIRY_SKEW_SECONDS = 60 +OAUTH2_DEFAULT_EXPIRY_SECONDS = 3600 +OAUTH2_REQUEST_TIMEOUT_SECONDS = 30 + +_TOKEN_CACHE: Dict[Tuple[str, str, str], Tuple[str, float]] = {} +_TOKEN_CACHE_LOCK = threading.Lock() + + +def resolve_api_key_header( + auth: Dict[str, Any], + default_header: str = "", + default_prefix: str = "", +) -> Tuple[str, str]: + """Return the header name and value prefix used to send an API key. + + Providers disagree about this. OpenAI uses "Authorization: Bearer", Anthropic + uses "x-api-key" with no prefix, Google uses "x-goog-api-key", and gateways + invent their own. Making both configurable is what lets one auth type cover + all of them. + + An administrator override wins over the provider default. An override that + names a header but no prefix means exactly that, so the provider's prefix is + not reapplied. + """ + auth = auth or {} + override_header = str(auth.get("api_key_header") or "").strip() + if override_header: + return override_header, str(auth.get("api_key_prefix") or "").strip() + + header_name = str(default_header or "").strip() + prefix = str(auth.get("api_key_prefix") or default_prefix or "").strip() + return header_name, prefix + + +def build_api_key_headers( + auth: Dict[str, Any], + default_header: str = "", + default_prefix: str = "", +) -> Dict[str, str]: + """Build the request headers that carry a configured API key.""" + api_key = str((auth or {}).get("api_key") or "").strip() + if not api_key: + raise ValueError("Selected model endpoint is missing an API key.") + + header_name, prefix = resolve_api_key_header(auth, default_header, default_prefix) + if not header_name: + return {} + header_value = f"{prefix} {api_key}".strip() if prefix else api_key + return {header_name: header_value} + + +def build_bearer_headers(auth: Dict[str, Any]) -> Dict[str, str]: + """Build the request headers for a static bearer token.""" + token = str((auth or {}).get("bearer_token") or "").strip() + if not token: + raise ValueError("Selected model endpoint is missing a bearer token.") + return {"Authorization": f"Bearer {token}"} + + +def _token_cache_key(auth: Dict[str, Any]) -> Tuple[str, str, str]: + return ( + str(auth.get("token_url") or "").strip(), + str(auth.get("client_id") or "").strip(), + str(auth.get("scope") or "").strip(), + ) + + +def clear_oauth2_token_cache() -> None: + """Drop every cached OAuth2 token.""" + with _TOKEN_CACHE_LOCK: + _TOKEN_CACHE.clear() + + +def fetch_oauth2_client_credentials_token( + auth: Dict[str, Any], + *, + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", + http_client_factory=None, +) -> str: + """Return an OAuth2 client-credentials access token, using the cache when valid. + + The token endpoint is a separate, administrator-supplied host, so it is an + outbound request target in its own right and is held to the same policy as the + inference endpoint: the URL is revalidated here rather than trusted from + configuration time, the connection is pinned to the validated addresses, and + redirects are refused. + + Refusing redirects is safe for this grant. Redirects belong to the browser-based + authorization-code flow; a client-credentials token endpoint answers a + server-to-server POST with a JSON body. + """ + token_url = str(auth.get("token_url") or "").strip() + client_id = str(auth.get("client_id") or "").strip() + client_secret = str(auth.get("client_secret") or "").strip() + if not token_url or not client_id or not client_secret: + raise ValueError( + "OAuth2 model endpoints require a token URL, client ID, and client secret." + ) + + # Imported here rather than at module scope: the transport lives with the + # model endpoint clients, which pull in the OpenAI and Semantic Kernel SDKs. + # Deferring keeps this module importable without that cost. + from model_endpoint_clients import build_custom_openai_sync_http_client + + token_url = validate_custom_model_endpoint_url( + token_url, + allow_private=allow_private, + allow_insecure=allow_insecure, + ) + + cache_key = _token_cache_key(auth) + now = time.monotonic() + with _TOKEN_CACHE_LOCK: + cached = _TOKEN_CACHE.get(cache_key) + if cached and cached[1] > now: + return cached[0] + + payload = { + "grant_type": "client_credentials", + "client_id": client_id, + "client_secret": client_secret, + } + scope = str(auth.get("scope") or "").strip() + if scope: + payload["scope"] = scope + + if http_client_factory is not None: + client = http_client_factory(timeout=OAUTH2_REQUEST_TIMEOUT_SECONDS) + else: + client = build_custom_openai_sync_http_client( + allow_private=allow_private, + ca_bundle_path=ca_bundle_path, + ) + try: + response = client.post(token_url, data=payload) + status_code = response.status_code + if status_code >= 400: + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token request failed.", + request_url=token_url, + status_code=status_code, + detail=response.text, + ) + token_payload = response.json() + except Exception as exc: + if isinstance(exc, RuntimeError): + raise + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token request failed.", + exc, + request_url=token_url, + ) from None + finally: + client.close() + + access_token = str(token_payload.get("access_token") or "").strip() + if not access_token: + raise build_sanitized_model_endpoint_error( + "Custom model endpoint token response did not contain an access token.", + request_url=token_url, + ) + + try: + expires_in = int(token_payload.get("expires_in") or OAUTH2_DEFAULT_EXPIRY_SECONDS) + except (TypeError, ValueError): + expires_in = OAUTH2_DEFAULT_EXPIRY_SECONDS + expires_at = time.monotonic() + max(1, expires_in - OAUTH2_EXPIRY_SKEW_SECONDS) + + with _TOKEN_CACHE_LOCK: + _TOKEN_CACHE[cache_key] = (access_token, expires_at) + return access_token + + +def resolve_custom_endpoint_credentials( + auth: Dict[str, Any], + *, + default_api_key_header: str = "", + default_api_key_prefix: str = "", + allow_private: bool = False, + allow_insecure: bool = False, + ca_bundle_path: str = "", +) -> Tuple[str, Dict[str, str]]: + """Resolve one Custom endpoint's credentials. + + Returns the value to hand to an SDK that takes an api_key argument, plus any + additional headers the scheme requires. An SDK that sends "Authorization: + Bearer" natively needs only the first; a scheme using a different header name + supplies the second and passes a placeholder for the first. + """ + auth = auth or {} + auth_type = normalize_custom_endpoint_auth_type(auth.get("type")) + + if auth_type == AUTH_TYPE_BEARER: + token = str(auth.get("bearer_token") or "").strip() + if not token: + raise ValueError("Selected model endpoint is missing a bearer token.") + return token, {} + + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS: + return fetch_oauth2_client_credentials_token( + auth, + allow_private=allow_private, + allow_insecure=allow_insecure, + ca_bundle_path=ca_bundle_path, + ), {} + + if auth_type == AUTH_TYPE_API_KEY: + api_key = str(auth.get("api_key") or "").strip() + if not api_key: + raise ValueError("Selected model endpoint is missing an API key.") + header_name, _ = resolve_api_key_header( + auth, + default_api_key_header, + default_api_key_prefix, + ) + # An explicit non-Authorization header is sent alongside the SDK's own + # credential argument, because the SDK cannot express it. + if header_name and header_name.lower() != "authorization": + return api_key, build_api_key_headers( + auth, + default_api_key_header, + default_api_key_prefix, + ) + return api_key, {} + + raise ValueError("Custom model endpoints do not support the selected authentication type.") + + +def resolve_client_certificate(connection: Dict[str, Any]): + """Return the mTLS client certificate for httpx, or None when not configured. + + Certificates are referenced by path so that a private key is mounted into the + deployment rather than stored in the configuration database. + """ + connection = connection or {} + cert_path = str(connection.get("client_cert_path") or "").strip() + if not cert_path: + return None + key_path = str(connection.get("client_key_path") or "").strip() + return (cert_path, key_path) if key_path else cert_path diff --git a/application/single_app/functions_model_endpoint_diagnostics.py b/application/single_app/functions_model_endpoint_diagnostics.py new file mode 100644 index 000000000..9733c54fd --- /dev/null +++ b/application/single_app/functions_model_endpoint_diagnostics.py @@ -0,0 +1,146 @@ +# functions_model_endpoint_diagnostics.py +"""Server-side diagnostics for Custom model endpoint failures. + +Custom endpoint errors are sanitized before they reach the browser, because an +upstream error body can echo back a URL, a header, or an API key. The first +implementation achieved that by discarding the cause entirely: + + raise RuntimeError("Custom model request failed.") from None + +That is safe and undebuggable. An administrator saw the same sentence for a +wrong path, a wrong key, a wrong model name, a TLS failure, and a blocked +address, with nothing in the log to tell them apart. + +This module keeps the browser message generic while recording the real cause +server-side, and stamps both with a short correlation id so an administrator can +join the message they were shown to the log entry that explains it. +""" + +import logging +import re +import uuid +from typing import Any, Dict + +from functions_appinsights import log_event + + +CORRELATION_ID_LENGTH = 8 + +# Credentials can appear in an upstream error body, in a repeated request URL, or +# in a header dump. Redact them before anything is written to the log. +_REDACTION_PATTERNS = ( + re.compile(r"(?i)(api[-_]?key\"?\s*[:=]\s*\"?)([^\"\s,&]+)"), + re.compile(r"(?i)(authorization\"?\s*[:=]\s*\"?)([^\"\s,&]+)"), + re.compile(r"(?i)(bearer\s+)([A-Za-z0-9\-._~+/]+=*)"), + re.compile(r"(?i)([?&](?:key|api[-_]?key|access[-_]?token)=)([^&\s\"]+)"), + re.compile(r"(?i)(x-api-key\"?\s*[:=]\s*\"?)([^\"\s,&]+)"), + re.compile(r"(?i)(x-goog-api-key\"?\s*[:=]\s*\"?)([^\"\s,&]+)"), + re.compile(r"(sk-[A-Za-z0-9\-_]{8,})"), +) + +MAX_LOGGED_DETAIL_LENGTH = 2000 + + +def redact_model_endpoint_secrets(value: Any) -> str: + """Return text with credential-looking values replaced by a redaction marker.""" + text = str(value or "") + if not text: + return "" + for pattern in _REDACTION_PATTERNS: + if pattern.groups >= 2: + text = pattern.sub(lambda match: f"{match.group(1)}[REDACTED]", text) + else: + text = pattern.sub("[REDACTED]", text) + if len(text) > MAX_LOGGED_DETAIL_LENGTH: + text = f"{text[:MAX_LOGGED_DETAIL_LENGTH]}...[truncated]" + return text + + +def new_model_endpoint_correlation_id() -> str: + """Return a short id that links a sanitized message to its log entry.""" + return uuid.uuid4().hex[:CORRELATION_ID_LENGTH] + + +def _build_log_context( + correlation_id: str, + *, + api_type: Any = "", + protocol: Any = "", + request_url: Any = "", + status_code: Any = None, + detail: Any = "", +) -> Dict[str, Any]: + context: Dict[str, Any] = {"correlation_id": correlation_id} + if api_type: + context["api_type"] = str(api_type) + if protocol: + context["protocol"] = str(protocol) + if request_url: + # The resolved URL is the single most useful diagnostic, because URL + # normalization can rewrite what the administrator typed. + context["request_url"] = redact_model_endpoint_secrets(request_url) + if status_code is not None: + context["status_code"] = status_code + if detail: + context["detail"] = redact_model_endpoint_secrets(detail) + return context + + +def log_custom_model_endpoint_failure( + summary: str, + exception: BaseException | None = None, + *, + api_type: Any = "", + protocol: Any = "", + request_url: Any = "", + status_code: Any = None, + detail: Any = "", +) -> str: + """Record a Custom endpoint failure server-side and return its correlation id.""" + correlation_id = new_model_endpoint_correlation_id() + context = _build_log_context( + correlation_id, + api_type=api_type, + protocol=protocol, + request_url=request_url, + status_code=status_code, + detail=detail, + ) + if exception is not None: + context["error_type"] = type(exception).__name__ + context["error"] = redact_model_endpoint_secrets(exception) + + try: + log_event( + f"[CUSTOM_MODEL_ENDPOINT] {summary} (correlation_id={correlation_id})", + extra=context, + level=logging.ERROR, + exceptionTraceback=exception is not None, + ) + except Exception: + # Diagnostics must never replace the original failure with a logging error. + pass + return correlation_id + + +def build_sanitized_model_endpoint_error( + message: str, + exception: BaseException | None = None, + *, + api_type: Any = "", + protocol: Any = "", + request_url: Any = "", + status_code: Any = None, + detail: Any = "", +) -> RuntimeError: + """Log the real cause and return the sanitized error to raise in its place.""" + correlation_id = log_custom_model_endpoint_failure( + message, + exception, + api_type=api_type, + protocol=protocol, + request_url=request_url, + status_code=status_code, + detail=detail, + ) + return RuntimeError(f"{message} (reference {correlation_id})") diff --git a/application/single_app/functions_model_endpoint_providers.py b/application/single_app/functions_model_endpoint_providers.py new file mode 100644 index 000000000..103310c30 --- /dev/null +++ b/application/single_app/functions_model_endpoint_providers.py @@ -0,0 +1,234 @@ +# functions_model_endpoint_providers.py +"""Registry of Custom model endpoint API types. + +Custom endpoints used to support exactly three API types, hard-coded in five +places: the api-type allowlist, the request-model resolver, the protocol +inference if-chain, the admin template's option list, and the admin JavaScript. +Adding a provider meant editing all five and hoping none were missed. + +This module makes an API type a single declarative record. A provider entry +carries everything the rest of the application needs to know: which wire protocol +to speak, which field names the model identifier, how to turn the configured URL +into a request URL, which auth types are accepted, and which optional version +field applies. + +Transport tiers +--------------- +Providers are tiered by whether SimpleChat can control the outbound connection: + + Tier A reached through an OpenAI-compatible or Anthropic HTTP surface, so the + request goes through the validated-DNS pinned transport. + Tier B would require a vendor SDK with its own transport (gRPC or botocore), + which the pinned transport cannot wrap. + +Only Tier A providers are registered here. Google Gemini is reachable at Tier A +through its OpenAI-compatible surface, so it does not need a Tier B entry. + +Only stdlib imports are used so this module can sit below the client, runtime, +validation, and route layers without creating import cycles. +""" + +from typing import Any, Dict, Tuple + +MODEL_ENDPOINT_PROVIDER_CUSTOM = "custom" + +MODEL_ENDPOINT_API_TYPE_OPENAI = "openai" +MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_API_TYPE_ANTHROPIC = "anthropic" +MODEL_ENDPOINT_API_TYPE_GEMINI = "gemini" + +# Wire protocols. These mirror the MODEL_ENDPOINT_PROTOCOL_* values in +# model_endpoint_clients, which imports them from here to keep one definition. +MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" +MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE = "openai_style" +MODEL_ENDPOINT_PROTOCOL_ANTHROPIC = "anthropic" + +# How the configured endpoint URL becomes a request URL. +URL_POLICY_APPEND_V1_IF_MISSING = "append_v1_if_missing" +URL_POLICY_AS_GIVEN = "as_given" +URL_POLICY_AZURE_DEPLOYMENT = "azure_deployment" +URL_POLICY_ANTHROPIC_MESSAGES = "anthropic_messages" + +# An administrator can override the provider's URL policy per endpoint. "auto" +# uses the provider policy; "exact" forces the URL to be used exactly as entered, +# which covers gateways that mount the API at a path SimpleChat cannot infer. +CUSTOM_ENDPOINT_URL_MODE_AUTO = "auto" +CUSTOM_ENDPOINT_URL_MODE_EXACT = "exact" +CUSTOM_ENDPOINT_URL_MODES = (CUSTOM_ENDPOINT_URL_MODE_AUTO, CUSTOM_ENDPOINT_URL_MODE_EXACT) + + +def normalize_custom_endpoint_url_mode(url_mode: Any) -> str: + """Return a supported URL mode, defaulting to the provider's own policy.""" + normalized = str(url_mode or "").strip().lower() + return normalized if normalized in CUSTOM_ENDPOINT_URL_MODES else CUSTOM_ENDPOINT_URL_MODE_AUTO + +# Which model record field carries the identifier sent on the wire. +MODEL_IDENTIFIER_MODEL_NAME = "model_name" +MODEL_IDENTIFIER_DEPLOYMENT_NAME = "deployment_name" + +DEFAULT_ANTHROPIC_VERSION = "2023-06-01" + +AUTH_TYPE_API_KEY = "api_key" +AUTH_TYPE_KEY = "key" +AUTH_TYPE_BEARER = "bearer" +AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS = "oauth2_client_credentials" + +# Every registered provider accepts these. API key remains the default and the +# only one required; the others are opt-in for gateways that need them. +DEFAULT_CUSTOM_AUTH_TYPES = ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, +) + + +def normalize_custom_endpoint_auth_type(auth_type: Any) -> str: + """Return a supported Custom endpoint auth type, or "" when unsupported. + + This lives with the registry rather than with the authentication code so that + validation can classify an auth type without importing the module that + performs authentication. + """ + normalized = str(auth_type or "").strip().lower() + if normalized == AUTH_TYPE_KEY: + return AUTH_TYPE_API_KEY + return normalized if normalized in DEFAULT_CUSTOM_AUTH_TYPES else "" + + +class ModelEndpointProvider: + """One Custom endpoint API type and everything the app needs to know about it.""" + + def __init__( + self, + api_type: str, + display_name: str, + protocol: str, + model_identifier: str, + url_policy: str, + *, + auth_types: Tuple[str, ...] = DEFAULT_CUSTOM_AUTH_TYPES, + default_api_key_header: str = "Authorization", + default_api_key_prefix: str = "Bearer", + requires_api_version: bool = False, + version_field: str = "", + default_version: str = "", + supports_streaming: bool = True, + supports_tools: bool = True, + supports_stream_options: bool = False, + description: str = "", + ): + self.api_type = api_type + self.display_name = display_name + self.protocol = protocol + self.model_identifier = model_identifier + self.url_policy = url_policy + self.auth_types = auth_types + self.default_api_key_header = default_api_key_header + self.default_api_key_prefix = default_api_key_prefix + self.requires_api_version = requires_api_version + self.version_field = version_field + self.default_version = default_version + self.supports_streaming = supports_streaming + self.supports_tools = supports_tools + self.supports_stream_options = supports_stream_options + self.description = description + + @property + def uses_model_name(self) -> bool: + """Return whether this API type names models rather than deployments.""" + return self.model_identifier == MODEL_IDENTIFIER_MODEL_NAME + + def to_ui_option(self) -> Dict[str, Any]: + """Return the descriptor the admin UI needs to render and drive this type.""" + return { + "value": self.api_type, + "label": self.display_name, + "usesModelName": self.uses_model_name, + "requiresApiVersion": self.requires_api_version, + "versionField": self.version_field, + "defaultVersion": self.default_version, + "authTypes": list(self.auth_types), + "defaultApiKeyHeader": self.default_api_key_header, + "defaultApiKeyPrefix": self.default_api_key_prefix, + "description": self.description, + } + + +MODEL_ENDPOINT_PROVIDERS: Tuple[ModelEndpointProvider, ...] = ( + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_OPENAI, + display_name="OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_APPEND_V1_IF_MISSING, + # OpenAI accepts stream_options.include_usage, which is how a streaming + # response reports token usage. Providers that reject it keep the default. + supports_stream_options=True, + description=( + "OpenAI and any OpenAI-compatible surface, including gateways, " + "vLLM, and LiteLLM." + ), + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + display_name="Azure OpenAI API", + protocol=MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + model_identifier=MODEL_IDENTIFIER_DEPLOYMENT_NAME, + url_policy=URL_POLICY_AZURE_DEPLOYMENT, + requires_api_version=True, + version_field="api_version", + description="An Azure OpenAI resource addressed by deployment name.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + display_name="Anthropic", + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + url_policy=URL_POLICY_ANTHROPIC_MESSAGES, + version_field="anthropic_version", + default_version=DEFAULT_ANTHROPIC_VERSION, + # Anthropic reads the key from x-api-key with no value prefix. + default_api_key_header="x-api-key", + default_api_key_prefix="", + description="Anthropic's messages API, direct or through a gateway.", + ), + ModelEndpointProvider( + api_type=MODEL_ENDPOINT_API_TYPE_GEMINI, + display_name="Google Gemini (OpenAI-compatible)", + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + model_identifier=MODEL_IDENTIFIER_MODEL_NAME, + # Gemini's compatible surface already ends in /v1beta/openai/, so appending + # /v1 would produce a 404. The URL is used exactly as configured. + url_policy=URL_POLICY_AS_GIVEN, + description=( + "Google Gemini through its OpenAI-compatible surface, normally " + "https://generativelanguage.googleapis.com/v1beta/openai/." + ), + ), +) + +MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE: Dict[str, ModelEndpointProvider] = { + provider.api_type: provider for provider in MODEL_ENDPOINT_PROVIDERS +} + +MODEL_ENDPOINT_CUSTOM_API_TYPES = frozenset(MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE) + + +def normalize_api_type_value(api_type: Any) -> str: + """Return an api_type string in canonical form.""" + return str(api_type or "").strip().lower().replace("-", "_") + + +def get_model_endpoint_provider(api_type: Any) -> ModelEndpointProvider | None: + """Return the registered provider for an api_type, or None when unsupported.""" + return MODEL_ENDPOINT_PROVIDERS_BY_API_TYPE.get(normalize_api_type_value(api_type)) + + +def get_model_endpoint_provider_ui_options() -> list: + """Return every registered API type as an admin UI descriptor.""" + return [provider.to_ui_option() for provider in MODEL_ENDPOINT_PROVIDERS] + + +def is_supported_custom_api_type(api_type: Any) -> bool: + """Return whether an api_type names a registered provider.""" + return normalize_api_type_value(api_type) in MODEL_ENDPOINT_CUSTOM_API_TYPES diff --git a/application/single_app/functions_model_endpoint_runtime.py b/application/single_app/functions_model_endpoint_runtime.py index 4deb4fa7c..25633f43e 100644 --- a/application/single_app/functions_model_endpoint_runtime.py +++ b/application/single_app/functions_model_endpoint_runtime.py @@ -1,13 +1,28 @@ # functions_model_endpoint_runtime.py """Runtime helpers for configured model endpoint clients and Semantic Kernel services.""" -from openai import AsyncOpenAI, AzureOpenAI +from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI from azure.identity import ClientSecretCredential, DefaultAzureCredential, get_bearer_token_provider from semantic_kernel.connectors.ai.open_ai import AzureChatCompletion, OpenAIChatCompletion from config import cognitive_services_scope from foundry_agent_runtime import resolve_authority from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_auth import ( + normalize_custom_endpoint_auth_type, + resolve_custom_endpoint_credentials, +) +from functions_model_endpoint_providers import ( + get_model_endpoint_provider, + normalize_custom_endpoint_url_mode, +) +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import validate_custom_model_endpoint_url from functions_settings import resolve_model_endpoint_foundry_scope from model_endpoint_clients import ( MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, @@ -15,14 +30,27 @@ MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, AnthropicSemanticKernelChatCompletion, build_anthropic_chat_client, + build_custom_openai_async_http_client, + build_custom_openai_sync_http_client, build_openai_style_chat_client, infer_model_endpoint_protocol, + normalize_custom_openai_base_url, + resolve_custom_openai_base_url, normalize_openai_style_base_url, resolve_openai_style_request_api_version, + SanitizedCustomChatCompletionClient, + sanitize_custom_async_openai_client, ) -MODEL_ENDPOINT_PROVIDER_ALLOWLIST = {'aoai', 'aifoundry', 'new_foundry', 'anthropic', 'claude'} +MODEL_ENDPOINT_PROVIDER_ALLOWLIST = { + 'aoai', + 'aifoundry', + 'new_foundry', + 'anthropic', + 'claude', + MODEL_ENDPOINT_PROVIDER_CUSTOM, +} MODEL_CONTEXT_AUTH_FIELDS = ( 'type', 'tenant_id', @@ -62,9 +90,12 @@ def build_model_endpoint_context( endpoint=None, auth=None, api_version=None, + api_type=None, + anthropic_version=None, endpoint_id=None, model_id=None, model_deployment=None, + request_model=None, user_id=None, active_group_ids=None, ): @@ -73,9 +104,12 @@ def build_model_endpoint_context( 'provider': str(provider or '').strip().lower(), 'endpoint': str(endpoint or '').strip(), 'api_version': str(api_version or '').strip(), + 'api_type': str(api_type or '').strip().lower(), + 'anthropic_version': str(anthropic_version or '').strip(), 'endpoint_id': str(endpoint_id or '').strip(), 'model_id': str(model_id or '').strip(), 'model_deployment': str(model_deployment or '').strip(), + 'request_model': str(request_model or model_deployment or '').strip(), } normalized_user_id = str(user_id or '').strip() @@ -125,6 +159,12 @@ def build_model_endpoint_sync_chat_client( api_version, deployment_name='', *, + api_type='', + url_mode='', + anthropic_version=DEFAULT_ANTHROPIC_VERSION, + allow_private_custom_endpoints=False, + allow_insecure_custom_endpoints=False, + custom_endpoint_ca_bundle_path='', settings=None, endpoint_config=None, identity_context=None, @@ -137,8 +177,44 @@ def build_model_endpoint_sync_chat_client( identity_context=identity_context, ) normalized_provider = str(provider or 'aoai').strip().lower() - runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) + direct_custom = normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM + if direct_custom: + endpoint = validate_custom_model_endpoint_url( + endpoint, + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ) + runtime_protocol = infer_model_endpoint_protocol( + normalized_provider, + endpoint, + deployment_name, + api_type, + ) auth_type = str(auth_settings.get('type') or 'managed_identity').strip().lower() + if direct_custom: + normalized_custom_auth = normalize_custom_endpoint_auth_type(auth_type) + if not normalized_custom_auth: + raise ValueError( + 'Custom model endpoints support API key, bearer token, or OAuth2 ' + 'client credentials authentication.' + ) + registered_provider = get_model_endpoint_provider(api_type) + credential, credential_headers = resolve_custom_endpoint_credentials( + auth_settings, + default_api_key_header=( + registered_provider.default_api_key_header if registered_provider else '' + ), + default_api_key_prefix=( + registered_provider.default_api_key_prefix if registered_provider else '' + ), + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, + ) + if credential_headers: + extra_headers = {**(extra_headers or {}), **credential_headers} + auth_type = 'api_key' + auth_settings = {**auth_settings, 'type': 'api_key', 'api_key': credential} if auth_type in ('api_key', 'key'): api_key = auth_settings.get('api_key') @@ -148,6 +224,10 @@ def build_model_endpoint_sync_chat_client( return build_anthropic_chat_client( endpoint=endpoint, api_key=api_key, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: @@ -155,14 +235,33 @@ def build_model_endpoint_sync_chat_client( api_key, endpoint, api_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, default_headers=extra_headers, + api_type=api_type, + url_mode=url_mode, + ca_bundle_path=custom_endpoint_ca_bundle_path, ), runtime_protocol - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=api_key, - default_headers=extra_headers or None, - ), runtime_protocol + client_kwargs = { + 'api_version': api_version, + 'azure_endpoint': endpoint, + 'api_key': api_key, + } + if extra_headers: + client_kwargs['default_headers'] = extra_headers + if direct_custom: + client_kwargs['http_client'] = build_custom_openai_sync_http_client( + allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, + ) + client = AzureOpenAI(**client_kwargs) + if direct_custom: + client = SanitizedCustomChatCompletionClient( + client, + api_type=api_type, + request_url=endpoint, + ) + return client, runtime_protocol credential = resolve_credential_for_model_endpoint_auth(auth_settings) scope = cognitive_services_scope @@ -210,11 +309,15 @@ def resolve_model_endpoint_from_context(settings, model_context): model_context = model_context if isinstance(model_context, dict) else {} requested_endpoint_id = str(model_context.get('endpoint_id') or '').strip() requested_model_id = str(model_context.get('model_id') or '').strip() - requested_deployment = str(model_context.get('model_deployment') or '').strip() + requested_model_name = str( + model_context.get('request_model') + or model_context.get('model_deployment') + or '' + ).strip() requested_provider = str(model_context.get('provider') or '').strip().lower() if not settings.get('enable_multi_model_endpoints', False): return None - if not (requested_endpoint_id or requested_model_id or requested_deployment): + if not (requested_endpoint_id or requested_model_id or requested_model_name): return None endpoints = [] @@ -252,11 +355,11 @@ def resolve_model_endpoint_from_context(settings, model_context): models = endpoint_cfg.get('models', []) or [] matched_model = None for model_cfg in models: - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() if requested_model_id and str(model_cfg.get('id') or '').strip() == requested_model_id: matched_model = model_cfg break - if requested_deployment and deployment == requested_deployment: + request_model = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + if requested_model_name and request_model == requested_model_name: matched_model = model_cfg break if not matched_model or not matched_model.get('enabled', True): @@ -296,18 +399,38 @@ def build_semantic_kernel_chat_service_for_model( provider = str(model_context.get('provider') or '').strip().lower() endpoint = str(model_context.get('endpoint') or '').strip() api_version = str(model_context.get('api_version') or '').strip() + api_type = str(model_context.get('api_type') or '').strip().lower() + url_mode = normalize_custom_endpoint_url_mode(model_context.get('url_mode')) + anthropic_version = str( + model_context.get('anthropic_version') + or DEFAULT_ANTHROPIC_VERSION + ).strip() auth_settings = model_context.get('auth') if isinstance(model_context.get('auth'), dict) else {} - deployment_name = str(model_context.get('model_deployment') or gpt_model or '').strip() + request_model = str( + model_context.get('request_model') + or model_context.get('model_deployment') + or gpt_model + or '' + ).strip() if resolved_model_endpoint: provider = str(resolved_model_endpoint.get('provider') or provider or 'aoai').strip().lower() connection = resolved_model_endpoint.get('connection', {}) or {} endpoint = str(connection.get('endpoint') or endpoint).strip() + api_type = get_model_endpoint_api_type(resolved_model_endpoint) or api_type + url_mode = normalize_custom_endpoint_url_mode( + connection.get('url_mode') or url_mode + ) api_version = str( connection.get('openai_api_version') or connection.get('api_version') or api_version ).strip() + anthropic_version = str( + connection.get('anthropic_version') + or anthropic_version + or DEFAULT_ANTHROPIC_VERSION + ).strip() auth_settings = resolved_model_endpoint.get('auth', {}) or auth_settings resolved_models = resolved_model_endpoint.get('models', []) or [] requested_model_id = str(model_context.get('model_id') or '').strip() @@ -317,27 +440,77 @@ def build_semantic_kernel_chat_service_for_model( (model for model in resolved_models if str(model.get('id') or '').strip() == requested_model_id), None, ) - if matched_model is None and deployment_name: + if matched_model is None and request_model: matched_model = next( ( model for model in resolved_models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == deployment_name + if resolve_model_endpoint_request_model( + resolved_model_endpoint, + model, + ) == request_model ), None, ) if matched_model: - deployment_name = str( - matched_model.get('deploymentName') or matched_model.get('deployment') or deployment_name - ).strip() + request_model = resolve_model_endpoint_request_model( + resolved_model_endpoint, + matched_model, + ) - if provider and endpoint and deployment_name: - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) + if provider and endpoint and request_model: + direct_custom = provider == MODEL_ENDPOINT_PROVIDER_CUSTOM + allow_private_custom_endpoints = bool( + settings.get('allow_private_custom_model_endpoints', False) + ) + allow_insecure_custom_endpoints = bool( + settings.get('allow_insecure_custom_model_endpoints', False) + ) + custom_endpoint_ca_bundle_path = str( + settings.get('custom_model_endpoint_ca_bundle_path') or '' + ).strip() + if direct_custom: + endpoint = validate_custom_model_endpoint_url( + endpoint, + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + request_model, + api_type, + ) auth_type = str(auth_settings.get('type') or 'managed_identity').lower() extra_headers = build_model_endpoint_identity_headers( settings, endpoint_config=resolved_model_endpoint, identity_context=model_context, ) + if direct_custom: + normalized_custom_auth = normalize_custom_endpoint_auth_type(auth_type) + if not normalized_custom_auth: + raise ValueError( + 'Custom model endpoints support API key, bearer token, or OAuth2 ' + 'client credentials authentication.' + ) + registered_provider = get_model_endpoint_provider(api_type) + credential, credential_headers = resolve_custom_endpoint_credentials( + auth_settings, + default_api_key_header=( + registered_provider.default_api_key_header if registered_provider else '' + ), + default_api_key_prefix=( + registered_provider.default_api_key_prefix if registered_provider else '' + ), + allow_private=allow_private_custom_endpoints, + allow_insecure=allow_insecure_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, + ) + if credential_headers: + extra_headers = {**(extra_headers or {}), **credential_headers} + auth_type = 'api_key' + auth_settings = {**auth_settings, 'type': 'api_key', 'api_key': credential} + if auth_type in ('api_key', 'key'): api_key = auth_settings.get('api_key') if not api_key: @@ -345,29 +518,70 @@ def build_semantic_kernel_chat_service_for_model( if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: return AnthropicSemanticKernelChatCompletion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_key=api_key, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, extra_headers=extra_headers, ), runtime_protocol if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs = { 'api_key': api_key, - 'base_url': normalize_openai_style_base_url(endpoint), + 'base_url': ( + resolve_custom_openai_base_url(endpoint, api_type, url_mode) + if direct_custom + else normalize_openai_style_base_url(endpoint) + ), } + if direct_custom: + client_kwargs['http_client'] = build_custom_openai_async_http_client( + allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, + ) if extra_headers: client_kwargs['default_headers'] = extra_headers if request_api_version: client_kwargs['default_query'] = {'api-version': request_api_version} + async_client = AsyncOpenAI(**client_kwargs) + if direct_custom: + async_client = sanitize_custom_async_openai_client( + async_client, + api_type=api_type, + request_url=client_kwargs['base_url'], + ) return OpenAIChatCompletion( service_id=service_id, - ai_model_id=deployment_name, - async_client=AsyncOpenAI(**client_kwargs), + ai_model_id=request_model, + async_client=async_client, + ), runtime_protocol + if direct_custom: + async_client = AsyncAzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=api_key, + default_headers=extra_headers or None, + http_client=build_custom_openai_async_http_client( + allow_private=allow_private_custom_endpoints, + ca_bundle_path=custom_endpoint_ca_bundle_path, + ), + ) + async_client = sanitize_custom_async_openai_client( + async_client, + api_type=api_type, + request_url=endpoint, + ) + return AzureChatCompletion( + service_id=service_id, + deployment_name=request_model, + async_client=async_client, ), runtime_protocol return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_key=api_key, api_version=api_version, @@ -383,7 +597,7 @@ def build_semantic_kernel_chat_service_for_model( token = credential.get_token(scope).token return AnthropicSemanticKernelChatCompletion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, bearer_token=token, extra_headers=extra_headers, @@ -402,7 +616,7 @@ def build_semantic_kernel_chat_service_for_model( client_kwargs['default_query'] = {'api-version': request_api_version} return OpenAIChatCompletion( service_id=service_id, - ai_model_id=deployment_name, + ai_model_id=request_model, async_client=AsyncOpenAI(**client_kwargs), ), runtime_protocol @@ -410,7 +624,7 @@ def build_semantic_kernel_chat_service_for_model( try: return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_version=api_version, azure_ad_token_provider=token_provider, @@ -419,7 +633,7 @@ def build_semantic_kernel_chat_service_for_model( except TypeError: return _build_azure_chat_completion( service_id=service_id, - deployment_name=deployment_name, + deployment_name=request_model, endpoint=endpoint, api_version=api_version, ad_token_provider=token_provider, diff --git a/application/single_app/functions_model_endpoint_types.py b/application/single_app/functions_model_endpoint_types.py new file mode 100644 index 000000000..61b3c0646 --- /dev/null +++ b/application/single_app/functions_model_endpoint_types.py @@ -0,0 +1,79 @@ +# functions_model_endpoint_types.py +"""Canonical provider, API type, and model identifier helpers. + +The supported API types and their per-type behaviour live in +functions_model_endpoint_providers. This module keeps the long-standing helper +names that the rest of the application imports, and delegates the decisions to +the registry so an API type is declared in exactly one place. +""" + +from typing import Any, Dict + +from functions_model_endpoint_providers import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_API_TYPE_OPENAI, + MODEL_ENDPOINT_CUSTOM_API_TYPES, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_provider, + normalize_api_type_value, +) + + +# Callers have long imported these constants from this module rather than from the +# registry that now owns them, so they are re-exported deliberately. +__all__ = [ + "DEFAULT_ANTHROPIC_VERSION", + "MODEL_ENDPOINT_API_TYPE_ANTHROPIC", + "MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI", + "MODEL_ENDPOINT_API_TYPE_OPENAI", + "MODEL_ENDPOINT_CUSTOM_API_TYPES", + "MODEL_ENDPOINT_PROVIDER_CUSTOM", + "get_model_endpoint_api_type", + "normalize_model_endpoint_api_type", + "resolve_model_endpoint_request_model", +] + + +def normalize_model_endpoint_api_type(provider: Any, api_type: Any) -> str: + """Return a supported explicit API type for Custom endpoints.""" + normalized_provider = str(provider or "").strip().lower() + if normalized_provider != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return "" + normalized_api_type = normalize_api_type_value(api_type) + return normalized_api_type if normalized_api_type in MODEL_ENDPOINT_CUSTOM_API_TYPES else "" + + +def get_model_endpoint_api_type(endpoint: Any) -> str: + """Return the canonical explicit API type from an endpoint record.""" + if not isinstance(endpoint, dict): + return "" + return normalize_model_endpoint_api_type(endpoint.get("provider"), endpoint.get("api_type")) + + +def resolve_model_endpoint_request_model(endpoint: Any, model: Any) -> str: + """Resolve the model identifier that must be sent to the configured API.""" + endpoint_data: Dict[str, Any] = endpoint if isinstance(endpoint, dict) else {} + model_data: Dict[str, Any] = model if isinstance(model, dict) else {} + provider = str(endpoint_data.get("provider") or "aoai").strip().lower() + + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + registered_provider = get_model_endpoint_provider(get_model_endpoint_api_type(endpoint_data)) + if registered_provider is None: + return "" + if registered_provider.uses_model_name: + return str(model_data.get("modelName") or model_data.get("name") or "").strip() + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or "" + ).strip() + + return str( + model_data.get("deploymentName") + or model_data.get("deployment") + or model_data.get("modelName") + or model_data.get("name") + or "" + ).strip() diff --git a/application/single_app/functions_model_endpoint_validation.py b/application/single_app/functions_model_endpoint_validation.py new file mode 100644 index 000000000..2b5dafabb --- /dev/null +++ b/application/single_app/functions_model_endpoint_validation.py @@ -0,0 +1,405 @@ +# functions_model_endpoint_validation.py +"""Validation and outbound-network safety for Custom model endpoints.""" + +import ipaddress +import re +import socket +from typing import Any, Dict, Iterable +from urllib.parse import urlparse, urlunparse + +from functions_model_endpoint_providers import ( + AUTH_TYPE_API_KEY, + AUTH_TYPE_BEARER, + AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS, + get_model_endpoint_provider, + normalize_custom_endpoint_auth_type, +) +from functions_model_endpoint_types import ( + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) + + +CUSTOM_ENDPOINT_MAX_URL_LENGTH = 2048 +CUSTOM_ENDPOINT_VERSION_PATTERN = re.compile(r"^[A-Za-z0-9._-]{1,64}$") +CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES = { + "instance-data.ec2.internal", + "localhost", + "localhost.localdomain", + "metadata.azure.com", + "metadata.google.internal", +} +CUSTOM_ENDPOINT_BLOCKED_IPS = { + ipaddress.ip_address("168.63.129.16"), + ipaddress.ip_address("169.254.169.254"), + ipaddress.ip_address("169.254.169.250"), + ipaddress.ip_address("169.254.169.251"), +} +CUSTOM_ENDPOINT_PRIVATE_NETWORKS = ( + ipaddress.ip_network("10.0.0.0/8"), + ipaddress.ip_network("172.16.0.0/12"), + ipaddress.ip_network("192.168.0.0/16"), + ipaddress.ip_network("fc00::/7"), +) + + +class ModelEndpointValidationError(ValueError): + """Raised when a model endpoint configuration violates the saved policy.""" + + +class ModelEndpointUnresolvableError(ModelEndpointValidationError): + """Raised when a Custom endpoint hostname cannot be resolved right now. + + This is distinct from a policy violation. A name that does not resolve yet is + tolerable when saving configuration -- the environment may not be reachable + from the application tier at configuration time -- but a policy violation + never is. + """ + + +def _is_ip_literal(hostname: str) -> bool: + try: + ipaddress.ip_address(hostname) + return True + except ValueError: + return False + + +def validate_custom_model_endpoint_address( + address: str, + *, + allow_private: bool = False, +) -> None: + """Validate one resolved Custom endpoint address against the outbound policy.""" + try: + ip_address = ipaddress.ip_address(address) + except ValueError as exc: + raise ModelEndpointValidationError( + "Custom endpoint hostname resolved to an invalid address." + ) from exc + + if ip_address in CUSTOM_ENDPOINT_BLOCKED_IPS: + raise ModelEndpointValidationError( + "Custom endpoint hostname resolves to a blocked platform address." + ) + if ip_address.is_loopback: + raise ModelEndpointValidationError( + "Custom endpoint hostname must not resolve to a loopback address." + ) + if ip_address.is_link_local: + raise ModelEndpointValidationError( + "Custom endpoint hostname must not resolve to a link-local address." + ) + if ip_address.is_multicast or ip_address.is_reserved or ip_address.is_unspecified: + raise ModelEndpointValidationError( + "Custom endpoint hostname must resolve to a usable network address." + ) + is_allowed_private_address = any( + ip_address in private_network + for private_network in CUSTOM_ENDPOINT_PRIVATE_NETWORKS + if ip_address.version == private_network.version + ) + if is_allowed_private_address: + if not allow_private: + raise ModelEndpointValidationError( + "Private Custom endpoint hosts are not enabled by the administrator." + ) + return + if not ip_address.is_global: + raise ModelEndpointValidationError( + "Custom endpoint hostname must resolve to a globally routable address." + ) + + +def resolve_custom_model_endpoint_addresses( + hostname: str, + port: int = 443, + *, + allow_private: bool = False, +) -> tuple[str, ...]: + """Resolve and validate every address before a Custom endpoint connection.""" + try: + resolved_addresses = socket.getaddrinfo( + hostname, + port, + type=socket.SOCK_STREAM, + ) + except socket.gaierror as exc: + raise ModelEndpointUnresolvableError( + "Custom endpoint hostname could not be resolved." + ) from exc + + if not resolved_addresses: + raise ModelEndpointUnresolvableError( + "Custom endpoint hostname did not resolve to an address." + ) + + validated_addresses = [] + seen_addresses = set() + for address_info in resolved_addresses: + address = address_info[4][0] + validate_custom_model_endpoint_address( + address, + allow_private=allow_private, + ) + if address not in seen_addresses: + seen_addresses.add(address) + validated_addresses.append(address) + return tuple(validated_addresses) + + +def validate_custom_model_endpoint_url( + endpoint: Any, + *, + allow_private: bool = False, + allow_insecure: bool = False, + require_resolvable: bool = True, +) -> str: + """Validate and normalize a Custom endpoint URL before an outbound request. + + ``allow_private`` is the administrator's on-premises gate. With it enabled, an + endpoint may be an IP literal, a single-label host, or a private-range + address, because those are how on-premises inference is normally addressed. + Connect-time address validation still applies on every request. + + ``require_resolvable`` is set to False on the configuration save path so that + an endpoint can be configured, seeded, or restored from backup before the + application tier can resolve it. The connect-time check is what actually + protects the request, and it always runs. + """ + endpoint_text = str(endpoint or "").strip() + if not endpoint_text: + raise ModelEndpointValidationError("Custom endpoint URL is required.") + if len(endpoint_text) > CUSTOM_ENDPOINT_MAX_URL_LENGTH: + raise ModelEndpointValidationError("Custom endpoint URL is too long.") + + try: + parsed_endpoint = urlparse(endpoint_text) + port = parsed_endpoint.port + except ValueError as exc: + raise ModelEndpointValidationError("Custom endpoint URL is invalid.") from exc + + scheme = parsed_endpoint.scheme.lower() + if scheme == "http": + if not (allow_private and allow_insecure): + raise ModelEndpointValidationError( + "Custom endpoint URL must use HTTPS. Plaintext HTTP requires the " + "administrator to enable both private hosts and insecure endpoints." + ) + elif scheme != "https": + raise ModelEndpointValidationError("Custom endpoint URL must use HTTPS.") + + if not parsed_endpoint.netloc or not parsed_endpoint.hostname: + raise ModelEndpointValidationError( + "Custom endpoint URL must include a host name." + ) + if parsed_endpoint.username or parsed_endpoint.password: + raise ModelEndpointValidationError( + "Custom endpoint URL must not include embedded credentials." + ) + if parsed_endpoint.query or parsed_endpoint.fragment: + raise ModelEndpointValidationError( + "Custom endpoint URL must not include a query string or fragment." + ) + + hostname = parsed_endpoint.hostname.strip().lower().rstrip(".") + try: + hostname = hostname.encode("idna").decode("ascii") + except UnicodeError as exc: + raise ModelEndpointValidationError( + "Custom endpoint hostname is invalid." + ) from exc + + if ( + hostname in CUSTOM_ENDPOINT_BLOCKED_HOSTNAMES + or hostname.endswith(".localhost") + ): + raise ModelEndpointValidationError("Custom endpoint hostname is blocked.") + + is_ip_literal = _is_ip_literal(hostname) + is_single_label = not is_ip_literal and "." not in hostname + + if is_ip_literal: + if not allow_private: + raise ModelEndpointValidationError( + "Custom endpoint URL must use a fully qualified domain name. " + "Enable private Custom endpoint hosts to use an IP address." + ) + # An IP literal skips DNS entirely, so validate the address directly. + validate_custom_model_endpoint_address(hostname, allow_private=True) + elif is_single_label: + if not allow_private: + raise ModelEndpointValidationError( + "Custom endpoint URL must use a fully qualified domain name. " + "Enable private Custom endpoint hosts to use a short host name." + ) + elif not allow_private and hostname.endswith((".internal", ".local")): + raise ModelEndpointValidationError( + "Private Custom endpoint hosts are not enabled by the administrator." + ) + + if not is_ip_literal: + default_port = 80 if scheme == "http" else 443 + try: + resolve_custom_model_endpoint_addresses( + hostname, + port or default_port, + allow_private=allow_private, + ) + except ModelEndpointUnresolvableError: + # A name that does not resolve yet is only fatal when the caller needs + # it resolvable now. Policy violations are a different exception and + # always propagate. The connect-time check re-resolves on every + # request, so nothing is skipped by tolerating this here. + if require_resolvable: + raise + + default_port = 80 if scheme == "http" else 443 + normalized_netloc = hostname + if port and port != default_port: + normalized_netloc = f"{hostname}:{port}" + return urlunparse(( + scheme, + normalized_netloc, + parsed_endpoint.path or "", + "", + "", + "", + )).rstrip("/") + + +def _validate_version(value: Any, field_label: str) -> str: + normalized_value = str(value or "").strip() + if not CUSTOM_ENDPOINT_VERSION_PATTERN.fullmatch(normalized_value): + raise ModelEndpointValidationError( + f"{field_label} must contain only letters, numbers, dots, underscores, or hyphens." + ) + return normalized_value + + +def validate_custom_model_endpoint( + endpoint: Any, + settings: Dict[str, Any] | None = None, + *, + require_api_key: bool = True, +) -> None: + """Validate a normalized Custom endpoint record.""" + if not isinstance(endpoint, dict): + raise ModelEndpointValidationError("Custom endpoint configuration is invalid.") + if str(endpoint.get("provider") or "").strip().lower() != MODEL_ENDPOINT_PROVIDER_CUSTOM: + return + + endpoint_name = str(endpoint.get("name") or "").strip() + if not endpoint_name: + raise ModelEndpointValidationError("Custom endpoint name is required.") + + api_type = get_model_endpoint_api_type(endpoint) + if not api_type: + raise ModelEndpointValidationError("Custom endpoint API type is not supported.") + registered_provider = get_model_endpoint_provider(api_type) + + auth = endpoint.get("auth") if isinstance(endpoint.get("auth"), dict) else {} + auth_type = normalize_custom_endpoint_auth_type(auth.get("type")) + if not auth_type: + raise ModelEndpointValidationError( + "Custom endpoints support API key, bearer token, or OAuth2 " + "client credentials authentication." + ) + if registered_provider is not None and auth_type not in registered_provider.auth_types: + raise ModelEndpointValidationError( + f"{registered_provider.display_name} does not support the selected " + "authentication type." + ) + if require_api_key: + if auth_type == AUTH_TYPE_API_KEY and not auth.get("api_key"): + raise ModelEndpointValidationError("Custom endpoint API key is required.") + if auth_type == AUTH_TYPE_BEARER and not auth.get("bearer_token"): + raise ModelEndpointValidationError("Custom endpoint bearer token is required.") + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS and not all( + str(auth.get(field) or "").strip() + for field in ("token_url", "client_id", "client_secret") + ): + raise ModelEndpointValidationError( + "Custom endpoint OAuth2 authentication requires a token URL, " + "client ID, and client secret." + ) + if auth_type == AUTH_TYPE_OAUTH2_CLIENT_CREDENTIALS and auth.get("token_url"): + # The token endpoint is a separate host and must satisfy the same policy + # as the inference endpoint, otherwise it becomes an unchecked outbound + # request target. + validate_custom_model_endpoint_url( + auth.get("token_url"), + allow_private=bool((settings or {}).get("allow_private_custom_model_endpoints", False)), + allow_insecure=bool((settings or {}).get("allow_insecure_custom_model_endpoints", False)), + require_resolvable=False, + ) + + connection = ( + endpoint.get("connection") + if isinstance(endpoint.get("connection"), dict) + else {} + ) + endpoint_settings = settings or {} + allow_private = bool(endpoint_settings.get("allow_private_custom_model_endpoints", False)) + allow_insecure = bool(endpoint_settings.get("allow_insecure_custom_model_endpoints", False)) + connection["endpoint"] = validate_custom_model_endpoint_url( + connection.get("endpoint"), + allow_private=allow_private, + allow_insecure=allow_insecure, + # Configuration may be saved before the application tier can resolve the + # host, so saving does not require the name to resolve right now. + require_resolvable=False, + ) + + if registered_provider is not None and registered_provider.version_field: + version_value = connection.get(registered_provider.version_field) + if registered_provider.default_version: + version_value = version_value or registered_provider.default_version + if registered_provider.requires_api_version or version_value: + _validate_version(version_value, f"{registered_provider.display_name} version") + + seen_model_names = set() + models: Iterable[Any] = endpoint.get("models") or [] + if not isinstance(models, list): + raise ModelEndpointValidationError("Custom endpoint models must be a list.") + if not models: + raise ModelEndpointValidationError( + "Custom endpoints require at least one manually configured model." + ) + for model in models: + if not isinstance(model, dict): + raise ModelEndpointValidationError("Custom endpoint model configuration is invalid.") + request_model = resolve_model_endpoint_request_model(endpoint, model) + if not request_model: + model_field = ( + "Model Name" + if registered_provider is not None and registered_provider.uses_model_name + else "Deployment Name" + ) + raise ModelEndpointValidationError( + f"Custom endpoint models require {model_field}." + ) + normalized_model_name = request_model.casefold() + if normalized_model_name in seen_model_names: + raise ModelEndpointValidationError( + "Custom endpoint model names must be unique." + ) + seen_model_names.add(normalized_model_name) + + +def validate_custom_model_endpoints( + endpoints: Any, + settings: Dict[str, Any] | None = None, + *, + require_api_key: bool = True, +) -> None: + """Validate every Custom endpoint in an endpoint list.""" + if not isinstance(endpoints, list): + raise ModelEndpointValidationError("Model endpoints must be a list.") + for endpoint in endpoints: + validate_custom_model_endpoint( + endpoint, + settings, + require_api_key=require_api_key, + ) diff --git a/application/single_app/functions_settings.py b/application/single_app/functions_settings.py index 431e22a01..daf2356f1 100644 --- a/application/single_app/functions_settings.py +++ b/application/single_app/functions_settings.py @@ -1,6 +1,7 @@ # functions_settings.py from functools import wraps +import logging from flask import g, has_request_context, jsonify, request, session @@ -21,6 +22,14 @@ normalize_model_endpoint_identity_header_value_type, ) from functions_mcp_server_config import INBOUND_MCP_SETTINGS_DEFAULTS, normalize_inbound_mcp_settings +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + normalize_model_endpoint_api_type, +) from functions_rate_limit import ( RATE_LIMIT_MESSAGE_DEFAULT, build_rate_limit_error_payload, @@ -1373,6 +1382,9 @@ def get_settings(use_cosmos=False, include_source=False): }, 'allow_user_agents': False, 'allow_user_custom_endpoints': False, + 'allow_private_custom_model_endpoints': False, + 'allow_insecure_custom_model_endpoints': False, + 'custom_model_endpoint_ca_bundle_path': '', 'allow_user_custom_agent_endpoints': False, 'allow_user_plugins': False, 'allow_user_workflows': False, @@ -2594,6 +2606,24 @@ def normalize_model_endpoint_auth_for_environment(endpoint_copy): provider = str(endpoint_copy.get("provider") or "").strip().lower() auth_type = str(auth.get("type") or "").strip().lower() + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + if auth.get("type") != "api_key": + auth["type"] = "api_key" + changed = True + for field_name in ( + "management_cloud", + "custom_authority", + "foundry_scope", + "tenant_id", + "client_id", + "client_secret", + "managed_identity_client_id", + ): + if field_name in auth: + auth.pop(field_name, None) + changed = True + return changed + current_cloud = normalize_model_endpoint_management_cloud(auth.get("management_cloud")) default_cloud = get_model_endpoint_management_cloud_for_environment() cloud_user_editable = is_model_endpoint_management_cloud_user_editable(provider, auth_type) @@ -2673,6 +2703,48 @@ def normalize_model_endpoints(endpoints): endpoint_copy.pop("has_api_key", None) endpoint_copy.pop("has_client_secret", None) connection = endpoint_copy.get("connection") or {} + provider = str(endpoint_copy.get("provider") or "aoai").strip().lower() + if endpoint_copy.get("provider") != provider: + endpoint_copy["provider"] = provider + changed = True + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + api_type = normalize_model_endpoint_api_type( + provider, + endpoint_copy.get("api_type"), + ) + if endpoint_copy.get("api_type") != api_type: + endpoint_copy["api_type"] = api_type + changed = True + if not isinstance(connection, dict): + connection = {} + changed = True + connection = json.loads(json.dumps(connection)) + if api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + if "anthropic_version" in connection: + connection.pop("anthropic_version", None) + changed = True + elif api_type == MODEL_ENDPOINT_API_TYPE_ANTHROPIC: + anthropic_version = str( + connection.get("anthropic_version") + or DEFAULT_ANTHROPIC_VERSION + ).strip() + if connection.get("anthropic_version") != anthropic_version: + connection["anthropic_version"] = anthropic_version + changed = True + for field_name in ("api_version", "openai_api_version"): + if field_name in connection: + connection.pop(field_name, None) + changed = True + else: + for field_name in ( + "api_version", + "openai_api_version", + "anthropic_version", + ): + if field_name in connection: + connection.pop(field_name, None) + changed = True + endpoint_copy["connection"] = connection identity_header = normalize_model_endpoint_identity_header_override(endpoint_copy.get("identity_header")) if endpoint_copy.get("identity_header") != identity_header: endpoint_copy["identity_header"] = identity_header @@ -2693,10 +2765,38 @@ def normalize_model_endpoints(endpoints): models = endpoint_copy.get("models") or [] normalized_models = [] + custom_api_type = get_model_endpoint_api_type(endpoint_copy) for model in models: if not isinstance(model, dict): continue model_copy = json.loads(json.dumps(model)) + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + if custom_api_type == MODEL_ENDPOINT_API_TYPE_AZURE_OPENAI: + deployment_name = str( + model_copy.get("deploymentName") + or model_copy.get("deployment") + or "" + ).strip() + if deployment_name and model_copy.get("deploymentName") != deployment_name: + model_copy["deploymentName"] = deployment_name + changed = True + for field_name in ("deployment", "modelName", "name"): + if field_name in model_copy: + model_copy.pop(field_name, None) + changed = True + else: + model_name = str( + model_copy.get("modelName") + or model_copy.get("name") + or "" + ).strip() + if model_name and model_copy.get("modelName") != model_name: + model_copy["modelName"] = model_name + changed = True + for field_name in ("deploymentName", "deployment", "name"): + if field_name in model_copy: + model_copy.pop(field_name, None) + changed = True if not model_copy.get("id"): model_id = ( model_copy.get("deploymentName") @@ -2742,7 +2842,12 @@ def normalize_model_endpoints(endpoints): def is_frontend_visible_model_endpoint_provider(provider): """Return whether the provider should be exposed in user-facing endpoint UIs.""" normalized_provider = (provider or "aoai").lower() - return normalized_provider in {"aoai", "aifoundry", "new_foundry"} + return normalized_provider in { + "aoai", + "aifoundry", + "new_foundry", + MODEL_ENDPOINT_PROVIDER_CUSTOM, + } def merge_model_endpoint_auth(existing_auth, incoming_auth): diff --git a/application/single_app/functions_workflow_runner.py b/application/single_app/functions_workflow_runner.py index 4c3d09afc..363c28f85 100644 --- a/application/single_app/functions_workflow_runner.py +++ b/application/single_app/functions_workflow_runner.py @@ -163,6 +163,10 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_notifications import create_workflow_priority_notification from functions_workflow_alerts import ( build_workflow_alert_facts, @@ -6019,14 +6023,11 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group connection = resolved_endpoint.get('connection', {}) if isinstance(resolved_endpoint, dict) else {} auth = resolved_endpoint.get('auth', {}) if isinstance(resolved_endpoint, dict) else {} provider = str(resolved_endpoint.get('provider') or endpoint_cfg.get('provider') or 'aoai').strip().lower() - deployment_name = ( - model_cfg.get('deploymentName') - or model_cfg.get('deployment') - or model_cfg.get('displayName') - or model_id - ) - api_version = connection.get('api_version') or connection.get('openai_api_version') or settings.get('azure_openai_gpt_api_version') + deployment_name = resolve_model_endpoint_request_model(resolved_endpoint, model_cfg) + api_version = connection.get('api_version') or connection.get('openai_api_version') or '' endpoint = connection.get('endpoint') + api_type = get_model_endpoint_api_type(resolved_endpoint) + anthropic_version = connection.get('anthropic_version') or '' auth_type = str(auth.get('type') or 'api_key').strip().lower() auth_settings = { **auth, @@ -6041,6 +6042,11 @@ def _build_multi_endpoint_client(user_id, endpoint_id, model_id, settings, group endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint, identity_context={'user_id': user_id}, diff --git a/application/single_app/model_endpoint_clients.py b/application/single_app/model_endpoint_clients.py index c40f8b9ee..36cb9717a 100644 --- a/application/single_app/model_endpoint_clients.py +++ b/application/single_app/model_endpoint_clients.py @@ -3,12 +3,22 @@ import json import asyncio +import re +import ssl from types import SimpleNamespace from typing import Any, Dict, Iterable, Iterator, List from urllib.parse import urlparse +import anyio +import httpcore +import httpx import requests -from openai import OpenAI +from openai import ( + DEFAULT_CONNECTION_LIMITS, + DefaultAsyncHttpxClient, + DefaultHttpxClient, + OpenAI, +) from pydantic import Field from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase from semantic_kernel.connectors.ai.function_calling_utils import update_settings_from_function_call_configuration @@ -24,6 +34,23 @@ from semantic_kernel.exceptions.service_exceptions import ServiceInvalidExecutionSettingsError from functions_debug import debug_print +from functions_model_endpoint_diagnostics import build_sanitized_model_endpoint_error +from functions_model_endpoint_providers import ( + CUSTOM_ENDPOINT_URL_MODE_EXACT, + URL_POLICY_APPEND_V1_IF_MISSING, + URL_POLICY_AS_GIVEN, + get_model_endpoint_provider, +) +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + normalize_model_endpoint_api_type, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + resolve_custom_model_endpoint_addresses, +) MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI = "azure_openai" @@ -131,9 +158,22 @@ def endpoint_uses_openai_style_protocol(endpoint: Any) -> bool: ) -def infer_model_endpoint_protocol(provider: Any, endpoint: Any, deployment_name: Any = "") -> str: +def infer_model_endpoint_protocol( + provider: Any, + endpoint: Any, + deployment_name: Any = "", + api_type: Any = "", +) -> str: """Infer the runtime protocol from provider, endpoint path, and deployment name.""" normalized_provider = str(provider or "aoai").strip().lower() + if normalized_provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + registered_provider = get_model_endpoint_provider( + normalize_model_endpoint_api_type(normalized_provider, api_type) + ) + if registered_provider is None: + raise ValueError("Custom model endpoints require a supported API type.") + return registered_provider.protocol + endpoint_path = get_endpoint_path(endpoint) if normalized_provider in ("anthropic", "claude"): @@ -173,13 +213,101 @@ def normalize_openai_style_base_url(raw_endpoint: Any) -> str: return endpoint.rstrip("/") + "/openai/v1/" -def normalize_anthropic_messages_url(raw_endpoint: Any) -> str: +CUSTOM_OPENAI_OPERATION_SUFFIXES = ("/chat/completions", "/responses", "/models") +# The optional suffix must start with a letter. Allowing it to start with a digit +# would make it ambiguous with the preceding \d+, which backtracks quadratically +# on a long run of digits. +CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN = re.compile( + r"^v\d+(?:[a-z][a-z0-9]*)?$", + re.IGNORECASE, +) + + +def _endpoint_path_names_a_version(endpoint: str) -> bool: + """Return whether the endpoint's last path segment is already a version.""" + try: + path = urlparse(endpoint).path + except ValueError: + return False + segments = [segment for segment in path.split("/") if segment] + if not segments: + return False + return bool(CUSTOM_OPENAI_VERSION_SEGMENT_PATTERN.fullmatch(segments[-1])) + + +def normalize_custom_openai_base_url(raw_endpoint: Any) -> str: + """Normalize a Custom OpenAI-compatible endpoint to its base URL. + + "/v1" is appended only when the configured URL does not already say where the + API lives. It is not appended when the last path segment is already a version + such as "v1", "v2", or "v1beta", and it is not appended when the administrator + pasted a full operation URL, because that URL states the base exactly. + """ + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ValueError("A Custom endpoint is required for OpenAI-compatible inference.") + + lowered_endpoint = endpoint.lower() + for suffix in CUSTOM_OPENAI_OPERATION_SUFFIXES: + if lowered_endpoint.endswith(suffix): + # A full operation URL states the base exactly, so trust it as given. + return endpoint[: -len(suffix)].rstrip("/") + "/" + + if _endpoint_path_names_a_version(endpoint): + return endpoint.rstrip("/") + "/" + return endpoint.rstrip("/") + "/v1/" + + +def resolve_custom_openai_base_url( + raw_endpoint: Any, + api_type: Any = "", + url_mode: Any = "", +) -> str: + """Resolve a Custom endpoint base URL using the provider's URL policy. + + Appending "/v1" is correct for OpenAI and OpenAI-compatible gateways, but wrong + for surfaces that already carry their own version segment. Google Gemini's + compatible base ends in "/v1beta/openai/", and appending "/v1" to it produces a + 404, so that provider declares the as-given policy instead. + + An administrator can also force the as-given policy for any API type by setting + the endpoint's url_mode to "exact", which covers gateways that mount the + OpenAI surface at a path SimpleChat cannot infer. + """ + provider = get_model_endpoint_provider(api_type) + url_policy = provider.url_policy if provider else URL_POLICY_APPEND_V1_IF_MISSING + if str(url_mode or "").strip().lower() == CUSTOM_ENDPOINT_URL_MODE_EXACT: + url_policy = URL_POLICY_AS_GIVEN + + if url_policy == URL_POLICY_AS_GIVEN: + endpoint = normalize_endpoint_text(raw_endpoint) + if not endpoint: + raise ValueError("A Custom endpoint is required for OpenAI-compatible inference.") + return endpoint.rstrip("/") + "/" + + return normalize_custom_openai_base_url(raw_endpoint) + + +def normalize_anthropic_messages_url( + raw_endpoint: Any, + *, + direct_custom: bool = False, +) -> str: """Normalize a Foundry endpoint to the Anthropic messages URL.""" endpoint = normalize_endpoint_text(raw_endpoint) if not endpoint: - raise ValueError("A Foundry endpoint is required for Anthropic inference.") + raise ValueError("An endpoint is required for Anthropic inference.") lowered_endpoint = endpoint.lower() + if direct_custom: + if lowered_endpoint.endswith("/v1/messages"): + return endpoint + if lowered_endpoint.endswith("/v1"): + return endpoint.rstrip("/") + "/messages" + if lowered_endpoint.endswith("/messages"): + return endpoint + return endpoint.rstrip("/") + "/v1/messages" + messages_index = lowered_endpoint.find("/anthropic/v1/messages") if messages_index >= 0: return endpoint[: messages_index + len("/anthropic/v1/messages")] @@ -196,10 +324,44 @@ def resolve_openai_style_request_api_version(raw_api_version: Any) -> str: return "" +SYNTHETIC_STREAM_CHUNK_CHARACTERS = 24 +_SYNTHETIC_STREAM_TOKEN_PATTERN = re.compile(r"\S+\s*|\s+") + + +def iter_synthetic_stream_text_chunks( + text: Any, + chunk_characters: int = SYNTHETIC_STREAM_CHUNK_CHARACTERS, +) -> Iterator[str]: + """Split a completed response into stream-sized chunks at word boundaries. + + SimpleChat only supports streaming responses, so a provider or code path that + can only return a completed answer still has to deliver it through the stream. + Emitting the whole answer as one chunk technically satisfies that, but the user + sees nothing and then everything at once, which reads as a hang. + + Chunking is lossless: concatenating every chunk reproduces the original text + exactly, including its whitespace, because the frontend accumulates chunks. + """ + normalized_text = str(text or "") + if not normalized_text: + return + if chunk_characters < 1: + yield normalized_text + return + + buffer = "" + for token in _SYNTHETIC_STREAM_TOKEN_PATTERN.findall(normalized_text): + buffer += token + if len(buffer) >= chunk_characters: + yield buffer + buffer = "" + if buffer: + yield buffer + + def normalize_chat_completion_text(content: Any) -> str: """Normalize text content returned by OpenAI-compatible chat responses.""" - if content is None: - return "" + if content is None: return "" if isinstance(content, str): return content if isinstance(content, (list, tuple)): @@ -231,36 +393,449 @@ def extract_chat_completion_response_text(response: Any) -> str: return normalize_chat_completion_text(getattr(message, "content", None)) +def _resolve_custom_connection_addresses(host, port, allow_private): + hostname = host.decode("ascii") if isinstance(host, bytes) else str(host) + try: + return resolve_custom_model_endpoint_addresses( + hostname, + port, + allow_private=allow_private, + ) + except ModelEndpointValidationError: + raise httpcore.ConnectError("Custom endpoint connection blocked.") from None + + +class _PinnedCustomEndpointSyncBackend(httpcore.NetworkBackend): + """Connect only to addresses returned by the validated DNS lookup.""" + + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.SyncBackend() + + def connect_tcp( + self, + host, + port, + timeout=None, + local_address=None, + socket_options=None, + ): + addresses = _resolve_custom_connection_addresses( + host, + port, + self._allow_private, + ) + last_error = None + for address in addresses: + try: + return self._backend.connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + if last_error: + raise last_error + raise httpcore.ConnectError("Custom endpoint connection failed.") + + def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + def sleep(self, seconds): + self._backend.sleep(seconds) + + +class _PinnedCustomEndpointAsyncBackend(httpcore.AsyncNetworkBackend): + """Async counterpart to the validated synchronous DNS backend.""" + + def __init__(self, *, allow_private=False): + self._allow_private = allow_private + self._backend = httpcore.AnyIOBackend() + + async def connect_tcp( + self, + host, + port, + timeout=None, + local_address=None, + socket_options=None, + ): + addresses = await anyio.to_thread.run_sync( + _resolve_custom_connection_addresses, + host, + port, + self._allow_private, + ) + last_error = None + for address in addresses: + try: + return await self._backend.connect_tcp( + address, + port, + timeout=timeout, + local_address=local_address, + socket_options=socket_options, + ) + except (httpcore.ConnectError, httpcore.ConnectTimeout) as exc: + last_error = exc + if last_error: + raise last_error + raise httpcore.ConnectError("Custom endpoint connection failed.") + + async def connect_unix_socket(self, path, timeout=None, socket_options=None): + raise httpcore.ConnectError("Custom endpoint UNIX sockets are not supported.") + + async def sleep(self, seconds): + await self._backend.sleep(seconds) + + +def build_custom_endpoint_ssl_context(ca_bundle_path: Any = "", client_cert: Any = None): + """Return the TLS context for Custom endpoint requests. + + The default context trusts only certifi's public roots, and deliberately does + not read SSL_CERT_FILE, so ambient environment variables cannot silently widen + what SimpleChat trusts. That leaves an on-premises gateway with an + enterprise-issued certificate untrustable, so an administrator may name a CA + bundle explicitly. Naming a bundle is an explicit decision, not an ambient one. + + ``client_cert`` supplies an mTLS client certificate, as either a combined PEM + path or a (certificate, key) pair of paths. + """ + bundle_path = str(ca_bundle_path or "").strip() + if bundle_path: + try: + context = ssl.create_default_context(cafile=bundle_path) + except (OSError, ssl.SSLError): + # A missing or unreadable bundle must not silently fall back to a + # weaker context, so the failure is surfaced to the caller. + raise ModelEndpointValidationError( + "The configured Custom endpoint CA bundle could not be loaded." + ) from None + else: + context = httpx.create_ssl_context(verify=True, trust_env=False) + + if client_cert: + try: + if isinstance(client_cert, (tuple, list)): + context.load_cert_chain(*client_cert) + else: + context.load_cert_chain(client_cert) + except (OSError, ssl.SSLError): + raise ModelEndpointValidationError( + "The configured Custom endpoint client certificate could not be loaded." + ) from None + + return context + + +class _PinnedCustomEndpointHTTPTransport(httpx.HTTPTransport): + """HTTPX transport whose TCP connection uses the validated DNS results.""" + + def __init__(self, *, allow_private=False, ca_bundle_path="", client_cert=None): + self._pool = httpcore.ConnectionPool( + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointSyncBackend( + allow_private=allow_private, + ), + ) + + +class _PinnedCustomEndpointAsyncHTTPTransport(httpx.AsyncHTTPTransport): + """Async HTTPX transport whose TCP connection uses validated DNS results.""" + + def __init__(self, *, allow_private=False, ca_bundle_path="", client_cert=None): + self._pool = httpcore.AsyncConnectionPool( + ssl_context=build_custom_endpoint_ssl_context(ca_bundle_path, client_cert), + max_connections=DEFAULT_CONNECTION_LIMITS.max_connections, + max_keepalive_connections=DEFAULT_CONNECTION_LIMITS.max_keepalive_connections, + keepalive_expiry=DEFAULT_CONNECTION_LIMITS.keepalive_expiry, + network_backend=_PinnedCustomEndpointAsyncBackend( + allow_private=allow_private, + ), + ) + + +def build_custom_openai_sync_http_client(*, allow_private=False, ca_bundle_path="", client_cert=None): + """Return a no-redirect SDK transport pinned to validated DNS addresses.""" + return DefaultHttpxClient( + transport=_PinnedCustomEndpointHTTPTransport( + allow_private=allow_private, + ca_bundle_path=ca_bundle_path, + client_cert=client_cert, + ), + follow_redirects=False, + trust_env=False, + ) + + +def build_custom_openai_async_http_client(*, allow_private=False, ca_bundle_path="", client_cert=None): + """Return an async no-redirect transport pinned to validated DNS addresses.""" + return DefaultAsyncHttpxClient( + transport=_PinnedCustomEndpointAsyncHTTPTransport( + allow_private=allow_private, + ca_bundle_path=ca_bundle_path, + client_cert=client_cert, + ), + follow_redirects=False, + trust_env=False, + ) + + def build_openai_style_chat_client( token_or_key: str, base_url: str, api_version: Any = "", default_headers: Dict[str, str] | None = None, + *, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, + api_type: Any = "", + url_mode: Any = "", + ca_bundle_path: Any = "", ): """Build an OpenAI-compatible chat client for Foundry data-plane endpoints.""" request_api_version = resolve_openai_style_request_api_version(api_version) client_kwargs: Dict[str, Any] = { "api_key": token_or_key, - "base_url": normalize_openai_style_base_url(base_url), + "base_url": ( + resolve_custom_openai_base_url(base_url, api_type, url_mode) + if direct_custom + else normalize_openai_style_base_url(base_url) + ), } + if direct_custom: + client_kwargs["http_client"] = build_custom_openai_sync_http_client( + allow_private=allow_private_custom_endpoints, + ca_bundle_path=ca_bundle_path, + ) if default_headers: client_kwargs["default_headers"] = default_headers if request_api_version: client_kwargs["default_query"] = {"api-version": request_api_version} - return OpenAIStyleChatCompletionClient(OpenAI(**client_kwargs)) + return OpenAIStyleChatCompletionClient( + OpenAI(**client_kwargs), + sanitize_errors=direct_custom, + api_type=api_type, + request_url=client_kwargs["base_url"], + ) class OpenAIStyleChatCompletionClient: """Small wrapper that makes OpenAI-compatible Foundry calls tolerant of Azure-only options.""" - def __init__(self, client: OpenAI): + def __init__( + self, + client: OpenAI, + *, + sanitize_errors: bool = False, + api_type: Any = "", + request_url: Any = "", + ): self._client = client + self._sanitize_errors = sanitize_errors + self._api_type = api_type + self._request_url = request_url + provider = get_model_endpoint_provider(api_type) if sanitize_errors else None + self._supports_stream_options = bool(provider and provider.supports_stream_options) self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): request_kwargs = dict(kwargs) - request_kwargs.pop("stream_options", None) - return self._client.chat.completions.create(**request_kwargs) + # stream_options is how a streaming response reports token usage. It is + # dropped only for surfaces that reject it, rather than for everyone. + if not self._supports_stream_options: + request_kwargs.pop("stream_options", None) + try: + response = self._client.chat.completions.create(**request_kwargs) + except Exception as exc: + if self._sanitize_errors: + raise build_sanitized_model_endpoint_error( + "Custom model request failed.", + exc, + api_type=self._api_type, + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + request_url=self._request_url, + status_code=getattr(exc, "status_code", None), + detail=getattr(exc, "message", "") or getattr(exc, "body", ""), + ) from None + raise + if self._sanitize_errors and request_kwargs.get("stream"): + return _SanitizedSyncIterator( + response, + api_type=self._api_type, + request_url=self._request_url, + ) + return response + + +class _SanitizedSyncIterator: + """Proxy a streaming response without exposing provider exception details.""" + + def __init__(self, iterator: Any, *, api_type: Any = "", request_url: Any = ""): + self._iterator = iterator + self._items = iter(iterator) + self._api_type = api_type + self._request_url = request_url + + def __iter__(self): + return self + + def __next__(self): + try: + return next(self._items) + except StopIteration: + raise + except Exception as exc: + raise build_sanitized_model_endpoint_error( + "Custom model stream failed.", + exc, + api_type=self._api_type, + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + request_url=self._request_url, + ) from None + + def __enter__(self): + enter = getattr(self._iterator, "__enter__", None) + if callable(enter): + enter() + return self + + def __exit__(self, exc_type, exc_value, traceback): + exit_method = getattr(self._iterator, "__exit__", None) + if callable(exit_method): + return exit_method(exc_type, exc_value, traceback) + return False + + def close(self): + close_method = getattr(self._iterator, "close", None) + if callable(close_method): + return close_method() + return None + + def __getattr__(self, name: str): + return getattr(self._iterator, name) + + +class _SanitizedAsyncIterator: + """Proxy an async streaming response without exposing provider exception details.""" + + def __init__(self, iterator: Any, *, api_type: Any = "", request_url: Any = ""): + self._iterator = iterator + self._items = iterator.__aiter__() + self._api_type = api_type + self._request_url = request_url + + def __aiter__(self): + return self + + async def __anext__(self): + try: + return await self._items.__anext__() + except StopAsyncIteration: + raise + except Exception as exc: + raise build_sanitized_model_endpoint_error( + "Custom model stream failed.", + exc, + api_type=self._api_type, + protocol=MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE, + request_url=self._request_url, + ) from None + + async def __aenter__(self): + enter = getattr(self._iterator, "__aenter__", None) + if callable(enter): + await enter() + return self + + async def __aexit__(self, exc_type, exc_value, traceback): + exit_method = getattr(self._iterator, "__aexit__", None) + if callable(exit_method): + return await exit_method(exc_type, exc_value, traceback) + return False + + async def close(self): + close_method = getattr(self._iterator, "close", None) + if callable(close_method): + result = close_method() + if asyncio.iscoroutine(result): + return await result + return None + + def __getattr__(self, name: str): + return getattr(self._iterator, name) + + +class SanitizedCustomChatCompletionClient: + """Expose an SDK chat client while replacing direct Custom provider errors.""" + + def __init__(self, client: Any, *, api_type: Any = "", request_url: Any = ""): + self._client = client + self._api_type = api_type + self._request_url = request_url + self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) + + def create(self, **kwargs: Any): + try: + response = self._client.chat.completions.create(**kwargs) + except Exception as exc: + raise build_sanitized_model_endpoint_error( + "Custom model request failed.", + exc, + api_type=self._api_type, + protocol=MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI, + request_url=self._request_url, + status_code=getattr(exc, "status_code", None), + detail=getattr(exc, "message", "") or getattr(exc, "body", ""), + ) from None + if kwargs.get("stream"): + return _SanitizedSyncIterator( + response, + api_type=self._api_type, + request_url=self._request_url, + ) + return response + + def __getattr__(self, name: str): + return getattr(self._client, name) + + +def sanitize_custom_async_openai_client(client: Any, *, api_type: Any = "", request_url: Any = ""): + """Replace async SDK chat errors with safe direct-Custom messages.""" + if getattr(client, "_simplechat_custom_errors_sanitized", False): + return client + + original_create = client.chat.completions.create + + async def sanitized_create(*args, **kwargs): + try: + response = await original_create(*args, **kwargs) + except Exception as exc: + raise build_sanitized_model_endpoint_error( + "Custom model request failed.", + exc, + api_type=api_type, + request_url=request_url, + status_code=getattr(exc, "status_code", None), + detail=getattr(exc, "message", "") or getattr(exc, "body", ""), + ) from None + if kwargs.get("stream"): + return _SanitizedAsyncIterator( + response, + api_type=api_type, + request_url=request_url, + ) + return response + + client.chat.completions.create = sanitized_create + client._simplechat_custom_errors_sanitized = True + return client def build_anthropic_chat_client( @@ -270,6 +845,10 @@ def build_anthropic_chat_client( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): """Build a chat-completions-shaped adapter over the Anthropic messages protocol.""" return AnthropicChatCompletionClient( @@ -278,6 +857,10 @@ def build_anthropic_chat_client( bearer_token=bearer_token, extra_headers=extra_headers, timeout=timeout, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, ) @@ -292,23 +875,40 @@ def __init__( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): - self.endpoint = normalize_anthropic_messages_url(endpoint) + self.endpoint = normalize_anthropic_messages_url( + endpoint, + direct_custom=direct_custom, + ) self.api_key = api_key self.bearer_token = bearer_token self.extra_headers = extra_headers or {} self.timeout = timeout + self.anthropic_version = str( + anthropic_version or DEFAULT_ANTHROPIC_VERSION + ).strip() + self.direct_custom = direct_custom + self.allow_private_custom_endpoints = allow_private_custom_endpoints + self.custom_endpoint_ca_bundle_path = custom_endpoint_ca_bundle_path self.chat = SimpleNamespace(completions=SimpleNamespace(create=self.create)) def create(self, **kwargs: Any): payload = self._build_payload(kwargs) stream = bool(kwargs.get("stream")) + if self.direct_custom: + return self._create_direct_custom(payload, stream=stream) + response = requests.post( self.endpoint, headers=self._build_headers(stream=stream), json=payload, timeout=(30, self.timeout), stream=stream, + allow_redirects=not self.direct_custom, ) if response.status_code >= 400: self._raise_response_error(response) @@ -318,16 +918,86 @@ def create(self, **kwargs: Any): return self._build_completion_response(response.json()) + def _create_direct_custom(self, payload, *, stream): + http_client = build_custom_openai_sync_http_client( + allow_private=self.allow_private_custom_endpoints, + ca_bundle_path=self.custom_endpoint_ca_bundle_path, + ) + request = http_client.build_request( + "POST", + self.endpoint, + headers=self._build_headers(stream=stream), + json=payload, + timeout=httpx.Timeout(self.timeout, connect=30), + ) + try: + response = http_client.send( + request, + stream=stream, + follow_redirects=False, + ) + except Exception as exc: + http_client.close() + raise build_sanitized_model_endpoint_error( + "Custom Anthropic model request failed.", + exc, + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + request_url=self.endpoint, + ) from None + + if response.status_code >= 400: + status_code = response.status_code + # Read the upstream body before closing so the log can explain the + # failure, even though the browser only ever sees the status code. + error_detail = "" + try: + if not stream: + error_detail = response.text + except Exception: + error_detail = "" + response.close() + http_client.close() + raise build_sanitized_model_endpoint_error( + f"Custom Anthropic model request failed with status {status_code}.", + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + request_url=self.endpoint, + status_code=status_code, + detail=error_detail, + ) + + if stream: + return self._iter_stream_chunks( + response, + http_client=http_client, + ) + + try: + return self._build_completion_response(response.json()) + except Exception as exc: + raise build_sanitized_model_endpoint_error( + "Custom Anthropic model returned an invalid response.", + exc, + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + request_url=self.endpoint, + ) from None + finally: + response.close() + http_client.close() + def _build_headers(self, *, stream: bool = False) -> Dict[str, str]: headers = { "Content-Type": "application/json", "Accept": "text/event-stream" if stream else "application/json", - "anthropic-version": "2023-06-01", + "anthropic-version": self.anthropic_version, } if self.bearer_token: headers["Authorization"] = f"Bearer {self.bearer_token}" elif self.api_key: - headers["api-key"] = self.api_key + if not self.direct_custom: + headers["api-key"] = self.api_key headers["x-api-key"] = self.api_key else: raise ValueError("Anthropic model endpoints require an API key or bearer token.") @@ -340,7 +1010,7 @@ def _build_headers(self, *, stream: bool = False) -> Dict[str, str]: def _build_payload(self, kwargs: Dict[str, Any]) -> Dict[str, Any]: model = str(kwargs.get("model") or "").strip() if not model: - raise ValueError("Anthropic model requests require a deployment name.") + raise ValueError("Anthropic model requests require a model name.") messages, system_prompt = self._convert_messages(kwargs.get("messages") or []) payload: Dict[str, Any] = { @@ -447,9 +1117,14 @@ def _normalize_content(self, content: Any) -> str | List[Dict[str, Any]]: text_parts.append(item) elif isinstance(item, dict): item_type = item.get("type") - if item_type in ("text", "tool_use", "tool_result"): + if item_type in ("text", "image", "tool_use", "tool_result"): normalized_blocks.append(item) continue + if item_type == "image_url": + normalized_blocks.append( + self._convert_openai_image_block(item) + ) + continue text_value = item.get("text") if isinstance(text_value, str): text_parts.append(text_value) @@ -462,6 +1137,33 @@ def _normalize_content(self, content: Any) -> str | List[Dict[str, Any]]: return "" return str(content) + def _convert_openai_image_block(self, image_block: Dict[str, Any]) -> Dict[str, Any]: + """Convert an OpenAI data-URL image block to Anthropic base64 content.""" + image_value = image_block.get("image_url") + image_url = ( + image_value.get("url") + if isinstance(image_value, dict) + else image_value + ) + image_url = str(image_url or "").strip() + if not image_url.startswith("data:") or ";base64," not in image_url: + raise ValueError( + "Anthropic image content requires a base64 data URL." + ) + + metadata, image_data = image_url.split(",", 1) + media_type = metadata[5:].split(";", 1)[0].strip().lower() + if not media_type.startswith("image/") or not image_data: + raise ValueError("Anthropic image content is invalid.") + return { + "type": "image", + "source": { + "type": "base64", + "media_type": media_type, + "data": image_data, + }, + } + def _content_to_text(self, content: Any) -> str: if isinstance(content, str): return content @@ -521,11 +1223,20 @@ def _extract_response_parts(self, response_payload: Dict[str, Any]) -> tuple[str )) return "".join(text_parts), tool_calls - def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: + def _iter_stream_chunks( + self, + response, + *, + http_client=None, + ) -> Iterator[Any]: prompt_tokens = 0 completion_tokens = 0 try: - for raw_line in response.iter_lines(decode_unicode=True): + try: + response_lines = response.iter_lines(decode_unicode=True) + except TypeError: + response_lines = response.iter_lines() + for raw_line in response_lines: if not raw_line: continue if isinstance(raw_line, bytes): @@ -540,7 +1251,10 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: try: event_payload = json.loads(event_data) except json.JSONDecodeError: - debug_print(f"[MODEL_ENDPOINT] Ignoring invalid Anthropic stream payload: {event_data[:200]}") + if self.direct_custom: + debug_print("[MODEL_ENDPOINT] Ignoring invalid Custom Anthropic stream payload.") + else: + debug_print(f"[MODEL_ENDPOINT] Ignoring invalid Anthropic stream payload: {event_data[:200]}") continue event_type = event_payload.get("type") @@ -550,6 +1264,8 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: error_message = error_payload.get("message") or error_payload.get("type") or str(error_payload) else: error_message = str(error_payload or event_payload) + if self.direct_custom: + raise RuntimeError("Custom Anthropic model stream failed.") raise RuntimeError(f"Anthropic model stream failed: {error_message}") if event_type == "message_start": usage = event_payload.get("message", {}).get("usage", {}) @@ -569,8 +1285,20 @@ def _iter_stream_chunks(self, response: requests.Response) -> Iterator[Any]: prompt_tokens = int(usage.get("input_tokens") or prompt_tokens or 0) completion_tokens = int(usage.get("output_tokens") or completion_tokens or 0) continue + except Exception as exc: + if self.direct_custom: + raise build_sanitized_model_endpoint_error( + "Custom Anthropic model stream failed.", + exc, + api_type=MODEL_ENDPOINT_API_TYPE_ANTHROPIC, + protocol=MODEL_ENDPOINT_PROTOCOL_ANTHROPIC, + request_url=self.endpoint, + ) from None + raise finally: response.close() + if http_client is not None: + http_client.close() if prompt_tokens or completion_tokens: yield SimpleNamespace( @@ -594,6 +1322,10 @@ def _raise_response_error(self, response: requests.Response) -> None: else: error_message = str(error_payload or payload) + if self.direct_custom: + raise RuntimeError( + f"Custom Anthropic model request failed with status {response.status_code}." + ) raise RuntimeError( f"Anthropic model request failed with status {response.status_code}: {error_message}" ) @@ -609,6 +1341,10 @@ class AnthropicSemanticKernelChatCompletion(ChatCompletionClientBase): bearer_token: str = "" extra_headers: Dict[str, str] = Field(default_factory=dict) timeout: int = 90 + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION + direct_custom: bool = False + allow_private_custom_endpoints: bool = False + custom_endpoint_ca_bundle_path: str = "" prompt_execution_settings: OpenAIChatPromptExecutionSettings | None = Field(default=None) def __init__( @@ -621,6 +1357,10 @@ def __init__( bearer_token: str = "", extra_headers: Dict[str, str] | None = None, timeout: int = 90, + anthropic_version: str = DEFAULT_ANTHROPIC_VERSION, + direct_custom: bool = False, + allow_private_custom_endpoints: bool = False, + custom_endpoint_ca_bundle_path: str = "", ): super().__init__( ai_model_id=deployment_name, @@ -630,6 +1370,10 @@ def __init__( bearer_token=bearer_token, extra_headers=extra_headers or {}, timeout=timeout, + anthropic_version=anthropic_version, + direct_custom=direct_custom, + allow_private_custom_endpoints=allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=custom_endpoint_ca_bundle_path, ) def get_prompt_execution_settings_class(self): @@ -714,11 +1458,18 @@ async def _inner_get_streaming_chat_message_contents( function_invoke_attempt: int = 0, ): if getattr(settings, "tools", None): + # Tool calling is answered without streaming, because a tool call has + # to arrive complete. The completed answer is still delivered through + # the stream, chunked so it reads like one. request_kwargs = self._build_request_kwargs(chat_history, settings, stream=False) client = self._build_client() response = await asyncio.to_thread(client.chat.completions.create, **request_kwargs) for message in self._create_chat_message_contents_from_response(response): - yield [self._to_streaming_chat_message_content(message, function_invoke_attempt)] + for streaming_message in self._iter_synthetic_stream_messages( + message, + function_invoke_attempt, + ): + yield [streaming_message] return request_kwargs = self._build_request_kwargs(chat_history, settings, stream=True) @@ -763,6 +1514,61 @@ async def _inner_get_streaming_chat_message_contents( ) ] + def _iter_synthetic_stream_messages( + self, + message: ChatMessageContent, + function_invoke_attempt: int, + ) -> Iterator[StreamingChatMessageContent]: + """Deliver a completed message through the streaming interface, in chunks. + + Text is split so the response arrives progressively. Non-text items, such + as function calls, must arrive whole, so they ride on the final message + alongside the finish reason and usage metadata. Emitting metadata only + once keeps token usage from being counted per chunk. + """ + text_items = [item for item in message.items or [] if isinstance(item, TextContent)] + other_items = [item for item in message.items or [] if not isinstance(item, TextContent)] + + combined_text = "".join(item.text or "" for item in text_items) + text_chunks = list(iter_synthetic_stream_text_chunks(combined_text)) + + # Every chunk before the last carries text only. + for chunk_text in text_chunks[:-1]: + yield StreamingChatMessageContent( + role=message.role, + items=[StreamingTextContent( + choice_index=0, + text=chunk_text, + ai_model_id=self.ai_model_id, + )], + choice_index=0, + ai_model_id=self.ai_model_id, + function_invoke_attempt=function_invoke_attempt, + ) + + final_items: List[Any] = [] + if text_chunks: + final_items.append(StreamingTextContent( + choice_index=0, + text=text_chunks[-1], + ai_model_id=self.ai_model_id, + inner_content=text_items[-1].inner_content if text_items else None, + metadata=text_items[-1].metadata if text_items else {}, + encoding=text_items[-1].encoding if text_items else None, + )) + final_items.extend(other_items) + + yield StreamingChatMessageContent( + role=message.role, + items=final_items, + choice_index=0, + ai_model_id=self.ai_model_id, + inner_content=message.inner_content, + metadata=message.metadata, + finish_reason=message.finish_reason, + function_invoke_attempt=function_invoke_attempt, + ) + def _to_streaming_chat_message_content( self, message: ChatMessageContent, @@ -799,6 +1605,10 @@ def _build_client(self): bearer_token=self.bearer_token, extra_headers=self.extra_headers, timeout=self.timeout, + anthropic_version=self.anthropic_version, + direct_custom=self.direct_custom, + allow_private_custom_endpoints=self.allow_private_custom_endpoints, + custom_endpoint_ca_bundle_path=self.custom_endpoint_ca_bundle_path, ) def _build_request_kwargs(self, chat_history, settings, *, stream: bool) -> Dict[str, Any]: diff --git a/application/single_app/route_backend_agents.py b/application/single_app/route_backend_agents.py index 30a967da0..9b03a9ff2 100644 --- a/application/single_app/route_backend_agents.py +++ b/application/single_app/route_backend_agents.py @@ -476,6 +476,8 @@ def _format_model_provider_label(provider): return 'Foundry (classic)' if normalized_provider == 'new_foundry': return 'New Foundry' + if normalized_provider == 'custom': + return 'Custom' return 'Azure OpenAI' diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index f403c2642..0bf133311 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -43,6 +43,10 @@ build_model_endpoint_sync_chat_client, build_semantic_kernel_chat_service_for_model, ) +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_mixed_source_orchestration import ( MixedSourceCancellationError, MixedSourceFinalizationError, @@ -13928,6 +13932,9 @@ def build_streaming_multi_endpoint_client( api_version, deployment_name='', *, + api_type='', + anthropic_version='', + allow_private_custom_endpoints=False, settings=None, endpoint_config=None, identity_context=None, @@ -13939,6 +13946,9 @@ def build_streaming_multi_endpoint_client( endpoint, api_version, deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, settings=settings, endpoint_config=endpoint_config, identity_context=identity_context, @@ -14090,7 +14100,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ model_cfg = next( ( model for model in models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == requested_deployment + if resolve_model_endpoint_request_model(resolved_endpoint_cfg, model) == requested_deployment ), None, ) @@ -14122,10 +14132,12 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = str(model_cfg.get('deploymentName') or model_cfg.get('deployment') or '').strip() + deployment = resolve_model_endpoint_request_model(resolved_endpoint_cfg, model_cfg) endpoint = str(connection.get('endpoint') or '').strip() api_version = str(connection.get('openai_api_version') or connection.get('api_version') or '').strip() - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + api_type = get_model_endpoint_api_type(resolved_endpoint_cfg) + anthropic_version = str(connection.get('anthropic_version') or '').strip() + runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment, api_type) model_icon = _normalize_model_icon_payload(model_cfg.get('icon')) model_response_length = normalize_model_response_length_from_model(model_cfg) model_behavior_name = _build_model_endpoint_behavior_name(model_cfg, deployment) @@ -14159,6 +14171,11 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ endpoint, api_version, deployment_name=deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint_cfg, identity_context={'user_id': user_id}, @@ -14166,7 +14183,7 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ debug_print( f"[STREAMING][Model Resolution] Resolved {selection_source} multi-endpoint model | " f"provider={provider} | endpoint_id={requested_endpoint_id} | model_id={model_cfg.get('id')} | " - f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol} | " + f"request_model={deployment} | api_version={api_version} | api_type={api_type} | protocol={runtime_protocol} | " f"response_length={model_response_length or ''} | " f"response_length_parameter={model_response_length_parameter or ''}" ) @@ -14177,6 +14194,8 @@ def resolve_streaming_multi_endpoint_gpt_config(settings, data, user_id, active_ endpoint, auth_settings, api_version, + api_type, + anthropic_version, requested_endpoint_id, str(model_cfg.get('id') or '').strip(), model_icon, @@ -16552,6 +16571,8 @@ def result_requires_message_reload(result: Any) -> bool: gpt_endpoint = None gpt_auth = None gpt_api_version = None + gpt_api_type = None + gpt_anthropic_version = None gpt_endpoint_id = None gpt_model_id = None gpt_model_icon = None @@ -16590,6 +16611,8 @@ def result_requires_message_reload(result: Any) -> bool: gpt_endpoint, gpt_auth, gpt_api_version, + gpt_api_type, + gpt_anthropic_version, gpt_endpoint_id, gpt_model_id, gpt_model_icon, @@ -16686,9 +16709,12 @@ def result_requires_message_reload(result: Any) -> bool: endpoint=gpt_endpoint, auth=gpt_auth, api_version=gpt_api_version, + api_type=gpt_api_type, + anthropic_version=gpt_anthropic_version, endpoint_id=gpt_endpoint_id or data.get('model_endpoint_id'), model_id=gpt_model_id or data.get('model_id'), model_deployment=gpt_model, + request_model=gpt_model, user_id=user_id, active_group_ids=active_group_ids, ) @@ -21006,6 +21032,8 @@ def collect_stream_response_conversation_metadata(): gpt_endpoint = None gpt_auth = None gpt_api_version = None + gpt_api_type = None + gpt_anthropic_version = None gpt_endpoint_id = None gpt_model_id = None gpt_model_icon = None @@ -21045,6 +21073,8 @@ def collect_stream_response_conversation_metadata(): gpt_endpoint, gpt_auth, gpt_api_version, + gpt_api_type, + gpt_anthropic_version, gpt_endpoint_id, gpt_model_id, gpt_model_icon, @@ -21124,9 +21154,12 @@ def collect_stream_response_conversation_metadata(): endpoint=gpt_endpoint, auth=gpt_auth, api_version=gpt_api_version, + api_type=gpt_api_type, + anthropic_version=gpt_anthropic_version, endpoint_id=gpt_endpoint_id or frontend_model_endpoint_id, model_id=gpt_model_id or frontend_model_id, model_deployment=gpt_model, + request_model=gpt_model, user_id=user_id, active_group_ids=active_group_ids, ) diff --git a/application/single_app/route_backend_conversation_export.py b/application/single_app/route_backend_conversation_export.py index 56af5c671..e175c1007 100644 --- a/application/single_app/route_backend_conversation_export.py +++ b/application/single_app/route_backend_conversation_export.py @@ -52,6 +52,11 @@ ) from functions_settings import * from functions_keyvault import SecretReturnType, keyvault_model_endpoint_get_helper +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_simplechat_operations import download_blob_content from functions_thoughts import get_thoughts_for_conversation from foundry_agent_runtime import resolve_authority @@ -1153,13 +1158,24 @@ def _get_summary_model_endpoint_candidates(settings: Dict[str, Any], user_id: st return candidates -def _summary_model_matches(model_cfg: Dict[str, Any], requested_model: str, requested_model_id: str) -> bool: +def _summary_model_matches( + endpoint_cfg: Dict[str, Any], + model_cfg: Dict[str, Any], + requested_model: str, + requested_model_id: str, +) -> bool: + request_model = '' + if _normalize_summary_model_value(endpoint_cfg.get('provider')).lower() == 'custom': + request_model = _normalize_summary_model_value( + resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) + ) model_values = { _normalize_summary_model_value(model_cfg.get('id')), _normalize_summary_model_value(model_cfg.get('deploymentName')), _normalize_summary_model_value(model_cfg.get('deployment')), _normalize_summary_model_value(model_cfg.get('modelName')), _normalize_summary_model_value(model_cfg.get('name')), + request_model, } model_values.discard('') @@ -1177,7 +1193,12 @@ def _find_summary_endpoint_model( for model_cfg in models: if not isinstance(model_cfg, dict) or not model_cfg.get('enabled', True): continue - if _summary_model_matches(model_cfg, requested_model, requested_model_id): + if _summary_model_matches( + endpoint_cfg, + model_cfg, + requested_model, + requested_model_id, + ): return model_cfg return None @@ -1214,6 +1235,9 @@ def _build_summary_model_endpoint_client( api_version: str, deployment_name: str, *, + api_type: str = '', + anthropic_version: str = '', + allow_private_custom_endpoints: bool = False, settings: Dict[str, Any] = None, endpoint_config: Dict[str, Any] = None, identity_context: Dict[str, Any] = None, @@ -1224,55 +1248,105 @@ def _build_summary_model_endpoint_client( endpoint_config=endpoint_config, identity_context=identity_context, ) - auth_type = _normalize_summary_model_value(auth_settings.get('type') or 'managed_identity').lower() normalized_provider = _normalize_summary_model_value(provider or 'aoai').lower() - runtime_protocol = infer_model_endpoint_protocol(normalized_provider, endpoint, deployment_name) + if normalized_provider != 'custom': + auth_type = _normalize_summary_model_value( + auth_settings.get('type') or 'managed_identity' + ).lower() + runtime_protocol = infer_model_endpoint_protocol( + normalized_provider, + endpoint, + deployment_name, + ) + + if auth_type in ('api_key', 'key'): + api_key = auth_settings.get('api_key') + if not api_key: + raise ValueError('Selected summary model endpoint is missing an API key.') + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: + return build_anthropic_chat_client( + endpoint=endpoint, + api_key=api_key, + extra_headers=extra_headers, + ) + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: + return build_openai_style_chat_client( + api_key, + endpoint, + api_version, + default_headers=extra_headers, + ) + return AzureOpenAI( + api_version=api_version, + azure_endpoint=endpoint, + api_key=api_key, + default_headers=extra_headers or None, + ) + + if auth_type == 'service_principal': + credential = ClientSecretCredential( + tenant_id=auth_settings.get('tenant_id'), + client_id=auth_settings.get('client_id'), + client_secret=auth_settings.get('client_secret'), + authority=resolve_authority(auth_settings), + ) + else: + managed_identity_client_id = auth_settings.get( + 'managed_identity_client_id' + ) or None + credential = DefaultAzureCredential( + managed_identity_client_id=managed_identity_client_id + ) + + scope = cognitive_services_scope + if ( + normalized_provider in ('aifoundry', 'new_foundry') + or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI + ): + scope = _resolve_summary_foundry_scope_for_auth( + auth_settings, + endpoint=endpoint, + ) - if auth_type in ('api_key', 'key'): - api_key = auth_settings.get('api_key') - if not api_key: - raise ValueError('Selected summary model endpoint is missing an API key.') if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - return build_anthropic_chat_client(endpoint=endpoint, api_key=api_key, extra_headers=extra_headers) + token = credential.get_token(scope).token + return build_anthropic_chat_client( + endpoint=endpoint, + bearer_token=token, + extra_headers=extra_headers, + ) + if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - return build_openai_style_chat_client(api_key, endpoint, api_version, default_headers=extra_headers) + token = credential.get_token(scope).token + return build_openai_style_chat_client( + token, + endpoint, + api_version, + default_headers=extra_headers, + ) + + token_provider = get_bearer_token_provider(credential, scope) return AzureOpenAI( api_version=api_version, azure_endpoint=endpoint, - api_key=api_key, + azure_ad_token_provider=token_provider, default_headers=extra_headers or None, ) - if auth_type == 'service_principal': - credential = ClientSecretCredential( - tenant_id=auth_settings.get('tenant_id'), - client_id=auth_settings.get('client_id'), - client_secret=auth_settings.get('client_secret'), - authority=resolve_authority(auth_settings), - ) - else: - managed_identity_client_id = auth_settings.get('managed_identity_client_id') or None - credential = DefaultAzureCredential(managed_identity_client_id=managed_identity_client_id) - - scope = cognitive_services_scope - if normalized_provider in ('aifoundry', 'new_foundry') or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: - scope = _resolve_summary_foundry_scope_for_auth(auth_settings, endpoint=endpoint) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - token = credential.get_token(scope).token - return build_anthropic_chat_client(endpoint=endpoint, bearer_token=token, extra_headers=extra_headers) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - token = credential.get_token(scope).token - return build_openai_style_chat_client(token, endpoint, api_version, default_headers=extra_headers) - - token_provider = get_bearer_token_provider(credential, scope) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - azure_ad_token_provider=token_provider, - default_headers=extra_headers or None, + client, _ = build_model_endpoint_sync_chat_client( + auth_settings, + provider, + endpoint, + api_version, + deployment_name=deployment_name, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=allow_private_custom_endpoints, + settings=settings, + endpoint_config=endpoint_config, + identity_context=identity_context, ) + return client def _resolve_summary_multi_endpoint_client( @@ -1336,12 +1410,34 @@ def _resolve_summary_multi_endpoint_client( provider = _normalize_summary_model_value(resolved_endpoint_cfg.get('provider') or requested_provider or 'aoai').lower() connection = resolved_endpoint_cfg.get('connection', {}) or {} auth_settings = resolved_endpoint_cfg.get('auth', {}) or {} - deployment = _normalize_summary_model_value( - model_cfg.get('deploymentName') or model_cfg.get('deployment') or model_cfg.get('id') - ) + if provider == 'custom': + deployment = resolve_model_endpoint_request_model( + resolved_endpoint_cfg, + model_cfg, + ) + else: + deployment = _normalize_summary_model_value( + model_cfg.get('deploymentName') + or model_cfg.get('deployment') + or model_cfg.get('modelName') + or model_cfg.get('name') + ) endpoint = _normalize_summary_model_value(connection.get('endpoint')) api_version = _normalize_summary_model_value(connection.get('openai_api_version') or connection.get('api_version')) - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment) + api_type = ( + get_model_endpoint_api_type(resolved_endpoint_cfg) + if provider == 'custom' + else '' + ) + anthropic_version = _normalize_summary_model_value( + connection.get('anthropic_version') + ) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + deployment, + api_type, + ) missing_required_config = not endpoint or not deployment or ( runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version @@ -1357,6 +1453,11 @@ def _resolve_summary_multi_endpoint_client( endpoint, api_version, deployment, + api_type=api_type, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint_cfg, identity_context={'user_id': user_id}, @@ -1364,7 +1465,7 @@ def _resolve_summary_multi_endpoint_client( debug_print( f"[SUMMARY][Model Resolution] Resolved {selection_source} multi-endpoint model | " f"provider={provider} | endpoint_id={endpoint_id} | model_id={model_cfg.get('id')} | " - f"deployment={deployment} | api_version={api_version} | protocol={runtime_protocol}" + f"request_model={deployment} | api_version={api_version} | api_type={api_type} | protocol={runtime_protocol}" ) return gpt_client, deployment diff --git a/application/single_app/route_backend_models.py b/application/single_app/route_backend_models.py index 652b1558f..2eae9c727 100644 --- a/application/single_app/route_backend_models.py +++ b/application/single_app/route_backend_models.py @@ -7,6 +7,18 @@ from functions_governance import ensure_governance_access from functions_group import assert_group_role, get_group_model_endpoints, require_active_group, update_group_model_endpoints from functions_keyvault import SecretReturnType, keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_get_helper, keyvault_model_endpoint_save_helper +from functions_model_endpoint_runtime import build_model_endpoint_sync_chat_client +from functions_model_endpoint_types import ( + DEFAULT_ANTHROPIC_VERSION, + MODEL_ENDPOINT_PROVIDER_CUSTOM, + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_model_endpoint, + validate_custom_model_endpoints, +) from functions_settings import * from foundry_agent_runtime import FoundryAgentUserAuthenticationRequired, list_foundry_agents_from_endpoint, list_foundry_workflows_from_endpoint, list_new_foundry_agents_from_endpoint, resolve_foundry_project_base, resolve_foundry_project_api_version, build_project_credential, resolve_authority from functions_appinsights import log_event @@ -17,6 +29,9 @@ build_anthropic_chat_client, build_openai_style_chat_client, infer_model_endpoint_protocol, + normalize_anthropic_messages_url, + normalize_openai_style_base_url, + resolve_custom_openai_base_url, ) from swagger_wrapper import swagger_route, get_auth_security from azure.identity import DefaultAzureCredential, ClientSecretCredential, get_bearer_token_provider @@ -169,7 +184,39 @@ def resolve_request_endpoint_payload(payload, scope="global"): # Persisted non-admin endpoints must resolve from stored configuration only. merged_payload = merge_model_endpoint_payload(persisted_endpoint, {}) if "model" in payload: - merged_payload["model"] = payload.get("model") + requested_model = payload.get("model") + if not isinstance(requested_model, dict): + raise LookupError("Model endpoint model not found.") + requested_model_id = str(requested_model.get("id") or "").strip() + requested_model_name = resolve_model_endpoint_request_model( + persisted_endpoint, + requested_model, + ) + persisted_model = next( + ( + model + for model in (persisted_endpoint.get("models") or []) + if isinstance(model, dict) + and model.get("enabled", True) + and ( + ( + requested_model_id + and str(model.get("id") or "").strip() == requested_model_id + ) + or ( + requested_model_name + and resolve_model_endpoint_request_model( + persisted_endpoint, + model, + ) == requested_model_name + ) + ) + ), + None, + ) + if not persisted_model: + raise LookupError("Model endpoint model not found.") + merged_payload["model"] = persisted_model else: merged_payload = merge_model_endpoint_payload(persisted_endpoint or {}, payload) @@ -268,54 +315,49 @@ def build_legacy_aoai_discovery_auth_settings(): "client_secret": MICROSOFT_PROVIDER_AUTHENTICATION_SECRET, } - def build_inference_client(endpoint, api_version, auth_settings, provider="aoai", deployment_name=""): - auth_type = (auth_settings.get("type") or "managed_identity").lower() - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) - if auth_type == "api_key": - api_key = auth_settings.get("api_key") - if not api_key: - raise ValueError("API key is required for API key authentication.") + def build_inference_client( + endpoint, + api_version, + auth_settings, + provider="aoai", + deployment_name="", + api_type="", + anthropic_version=DEFAULT_ANTHROPIC_VERSION, + url_mode="", + ): + client, runtime_protocol = build_model_endpoint_sync_chat_client( + auth_settings, + provider, + endpoint, + api_version, + deployment_name=deployment_name, + api_type=api_type, + url_mode=url_mode, + anthropic_version=anthropic_version, + allow_private_custom_endpoints=bool( + get_settings().get("allow_private_custom_model_endpoints", False) + ), + ) + log_models_debug( + f"Inference client provider={provider} protocol={runtime_protocol}" + ) + return client + + def describe_resolved_request_url(provider, endpoint, api_type, url_mode, runtime_protocol): + """Return the URL SimpleChat actually calls, for display after a test.""" + try: if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - return build_anthropic_chat_client(endpoint=endpoint, api_key=api_key) + return normalize_anthropic_messages_url( + endpoint, + direct_custom=provider == MODEL_ENDPOINT_PROVIDER_CUSTOM, + ) if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - return build_openai_style_chat_client(api_key, endpoint, api_version) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - api_key=api_key - ) - - if auth_type == "service_principal": - authority_override = resolve_authority(auth_settings) - credential = ClientSecretCredential( - tenant_id=auth_settings.get("tenant_id"), - client_id=auth_settings.get("client_id"), - client_secret=auth_settings.get("client_secret"), - authority=authority_override - ) - else: - managed_identity_client_id = auth_settings.get("managed_identity_client_id") or None - credential = DefaultAzureCredential(managed_identity_client_id=managed_identity_client_id) - - scope = cognitive_services_scope - if provider in ("aifoundry", "new_foundry") or runtime_protocol != MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI: - scope = resolve_foundry_scope(auth_settings) - log_models_debug(f"Inference token scope={scope} provider={provider} protocol={runtime_protocol}") - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_ANTHROPIC: - token = credential.get_token(scope).token - return build_anthropic_chat_client(endpoint=endpoint, bearer_token=token) - - if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_OPENAI_STYLE: - token = credential.get_token(scope).token - return build_openai_style_chat_client(token, endpoint, api_version) - - token_provider = get_bearer_token_provider(credential, scope) - return AzureOpenAI( - api_version=api_version, - azure_endpoint=endpoint, - azure_ad_token_provider=token_provider - ) + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + return resolve_custom_openai_base_url(endpoint, api_type, url_mode) + return normalize_openai_style_base_url(endpoint) + return str(endpoint or "") + except Exception: + return str(endpoint or "") def fetch_foundry_project_deployments(endpoint, api_version, auth_settings, project_name=None): if not endpoint: @@ -373,6 +415,12 @@ def handle_fetch_model_list(scope="global"): f" resource_group_present={bool(management.get('resource_group'))}" ) + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + return build_safe_error_response( + "Model discovery is not available for Custom endpoints. Add models manually.", + 400, + ) + if provider in ("aifoundry", "new_foundry"): endpoint = connection.get("endpoint") api_version = connection.get("project_api_version") or connection.get("api_version") or "v1" @@ -464,23 +512,48 @@ def handle_test_model_connection(scope="global"): endpoint = connection.get("endpoint") or "" api_version = connection.get("openai_api_version") or connection.get("api_version") or "" - deployment_name = model.get("deploymentName") or "" - runtime_protocol = infer_model_endpoint_protocol(provider, endpoint, deployment_name) + api_type = get_model_endpoint_api_type(data) + anthropic_version = ( + connection.get("anthropic_version") + or DEFAULT_ANTHROPIC_VERSION + ) + request_model = resolve_model_endpoint_request_model(data, model) + runtime_protocol = infer_model_endpoint_protocol( + provider, + endpoint, + request_model, + api_type, + ) auth_type = (auth_settings.get("type") or "managed_identity").lower() log_models_debug( "Test model request" f" provider={provider} auth_type={auth_type}" - f" endpoint={endpoint} deployment={deployment_name}" + f" endpoint={endpoint} model={request_model}" ) - if not endpoint or not deployment_name: - return jsonify({"error": "Endpoint and deployment name are required."}), 400 + if provider == MODEL_ENDPOINT_PROVIDER_CUSTOM: + validation_endpoint = dict(data) + validation_endpoint["models"] = [model] + validate_custom_model_endpoint( + validation_endpoint, + get_settings(), + ) + + if not endpoint or not request_model: + return jsonify({"error": "Endpoint and model identifier are required."}), 400 if runtime_protocol == MODEL_ENDPOINT_PROTOCOL_AZURE_OPENAI and not api_version: - return jsonify({"error": "Endpoint, API version, and deployment name are required."}), 400 - - if provider not in ("aoai", "aifoundry", "new_foundry", "anthropic", "claude"): + return jsonify({"error": "Endpoint, API version, and model identifier are required."}), 400 + + if provider not in ( + "aoai", + "aifoundry", + "new_foundry", + "anthropic", + "claude", + MODEL_ENDPOINT_PROVIDER_CUSTOM, + ): return jsonify({"error": "Model provider not found."}), 400 gpt_client = build_inference_client( @@ -488,15 +561,35 @@ def handle_test_model_connection(scope="global"): api_version, auth_settings, provider=provider, - deployment_name=deployment_name, + deployment_name=request_model, + api_type=api_type, + anthropic_version=anthropic_version, + url_mode=connection.get("url_mode") or "", ) response = gpt_client.chat.completions.create( - model=deployment_name, + model=request_model, messages=[{"role": "user", "content": "Testing access."}] ) if response: - return jsonify({"success": True}), 200 + # Report what was actually called. URL normalization can rewrite + # the configured endpoint, and that rewrite was previously + # invisible, so a working test could still hide a surprise. + return jsonify({ + "success": True, + "resolved": { + "request_url": describe_resolved_request_url( + provider, + endpoint, + api_type, + connection.get("url_mode") or "", + runtime_protocol, + ), + "protocol": runtime_protocol, + "api_type": api_type, + "request_model": request_model, + }, + }), 200 return jsonify({"error": "No response returned from model."}), 400 @@ -820,6 +913,16 @@ def save_user_model_endpoints(): merged = merge_model_endpoints_with_existing(incoming, existing) normalized, _ = normalize_model_endpoints(merged) + try: + validate_custom_model_endpoints(normalized, get_settings()) + except ModelEndpointValidationError as exc: + log_models_exception( + "Personal model endpoint validation failed", + exc, + extra={"scope": "user"}, + level=logging.WARNING, + ) + return build_safe_error_response(str(exc), 400) existing_by_id = { endpoint.get("id"): endpoint for endpoint in existing @@ -861,7 +964,10 @@ def save_user_model_endpoints(): keyvault_model_endpoint_delete_helper(endpoint, endpoint_id, scope="user") update_user_settings(user_id, {"personal_model_endpoints": saved_endpoints}) - return jsonify({"success": True}) + return jsonify({ + "success": True, + "endpoints": sanitize_model_endpoints_for_frontend(saved_endpoints), + }) @bp.route('/api/group/model-endpoints', methods=['GET']) @@ -924,6 +1030,16 @@ def save_group_model_endpoints(): merged = merge_model_endpoints_with_existing(incoming, existing) normalized, _ = normalize_model_endpoints(merged) + try: + validate_custom_model_endpoints(normalized, get_settings()) + except ModelEndpointValidationError as exc: + log_models_exception( + "Group model endpoint validation failed", + exc, + extra={"scope": "group"}, + level=logging.WARNING, + ) + return build_safe_error_response(str(exc), 400) existing_by_id = { endpoint.get("id"): endpoint for endpoint in existing @@ -965,7 +1081,10 @@ def save_group_model_endpoints(): keyvault_model_endpoint_delete_helper(endpoint, endpoint_id, scope="group") update_group_model_endpoints(group_id, saved_endpoints) - return jsonify({"success": True}) + return jsonify({ + "success": True, + "endpoints": sanitize_model_endpoints_for_frontend(saved_endpoints), + }) @bp.route('/api/models/foundry/agents', methods=['POST']) diff --git a/application/single_app/route_backend_settings.py b/application/single_app/route_backend_settings.py index 1e6e4aa23..ebf577c63 100644 --- a/application/single_app/route_backend_settings.py +++ b/application/single_app/route_backend_settings.py @@ -11,6 +11,10 @@ resolve_model_endpoint_from_context, ) from functions_model_endpoint_identity_header import build_model_endpoint_identity_headers +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from functions_activity_logging import ( log_admin_feedback_email_submission, log_general_admin_action, @@ -1394,6 +1398,7 @@ def _test_multimodal_vision_connection(payload): # Create a simple test image (1x1 red pixel PNG) test_image_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" + is_custom_model_endpoint = False try: multi_endpoint_selection = payload.get('multi_endpoint') if isinstance(payload.get('multi_endpoint'), dict) else None @@ -1413,6 +1418,9 @@ def _test_multimodal_vision_connection(payload): resolved_endpoint = resolve_model_endpoint_from_context(settings, model_context) if not resolved_endpoint: return jsonify({'error': 'Selected vision model endpoint could not be resolved from saved settings'}), 400 + is_custom_model_endpoint = ( + str(resolved_endpoint.get('provider') or '').strip().lower() == 'custom' + ) resolved_models = resolved_endpoint.get('models', []) or [] matched_model = next( @@ -1426,18 +1434,14 @@ def _test_multimodal_vision_connection(payload): matched_model = next( ( model for model in resolved_models - if str(model.get('deploymentName') or model.get('deployment') or '').strip() == model_context['model_deployment'] + if resolve_model_endpoint_request_model(resolved_endpoint, model) == model_context['model_deployment'] ), None, ) if not matched_model: return jsonify({'error': 'Selected vision model could not be resolved from saved settings'}), 400 - vision_model = str( - matched_model.get('deploymentName') - or matched_model.get('deployment') - or model_context['model_deployment'] - ).strip() + vision_model = resolve_model_endpoint_request_model(resolved_endpoint, matched_model) vision_model_name = str(matched_model.get('modelName') or vision_model).strip() connection = resolved_endpoint.get('connection', {}) or {} gpt_client, _ = build_model_endpoint_sync_chat_client( @@ -1446,6 +1450,11 @@ def _test_multimodal_vision_connection(payload): connection.get('endpoint'), connection.get('openai_api_version') or connection.get('api_version'), deployment_name=vision_model, + api_type=get_model_endpoint_api_type(resolved_endpoint), + anthropic_version=connection.get('anthropic_version') or '', + allow_private_custom_endpoints=bool( + settings.get('allow_private_custom_model_endpoints', False) + ), settings=settings, endpoint_config=resolved_endpoint, identity_context=identity_context, @@ -1547,6 +1556,15 @@ def _test_multimodal_vision_connection(payload): }), 200 except Exception as e: + if is_custom_model_endpoint: + log_event( + "[MODEL_ENDPOINT] Custom vision model test failed", + extra={"exception_type": type(e).__name__}, + level=logging.WARNING, + ) + return jsonify({ + 'error': 'The Custom vision model test failed. Review the endpoint and model configuration.' + }), 500 return jsonify({'error': f'Vision test failed: {str(e)}'}), 500 def get_index_client() -> SearchIndexClient: diff --git a/application/single_app/route_frontend_admin_settings.py b/application/single_app/route_frontend_admin_settings.py index 2417dcab1..f7e137abe 100644 --- a/application/single_app/route_frontend_admin_settings.py +++ b/application/single_app/route_frontend_admin_settings.py @@ -9,6 +9,11 @@ from flask import current_app, jsonify, request from functions_keyvault import keyvault_model_endpoint_cleanup_helper, keyvault_model_endpoint_delete_helper, keyvault_model_endpoint_save_helper, redact_model_endpoint_secret_values +from functions_model_endpoint_types import resolve_model_endpoint_request_model +from functions_model_endpoint_validation import ( + ModelEndpointValidationError, + validate_custom_model_endpoints, +) from functions_settings import * from functions_content_safety import normalize_content_safety_violation_message from functions_rate_limit import normalize_rate_limit_message @@ -780,6 +785,8 @@ def admin_settings(): settings['allow_user_agents'] = False if 'allow_user_custom_endpoints' not in settings: settings['allow_user_custom_endpoints'] = settings.get('allow_user_custom_agent_endpoints', False) + if 'allow_private_custom_model_endpoints' not in settings: + settings['allow_private_custom_model_endpoints'] = False if 'allow_user_plugins' not in settings: settings['allow_user_plugins'] = False if 'allow_user_workflows' not in settings: @@ -1722,6 +1729,26 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul parsed_model_endpoints = merge_model_endpoints_with_existing(parsed_model_endpoints, existing_model_endpoints) parsed_model_endpoints, _ = normalize_model_endpoints(parsed_model_endpoints) + custom_endpoint_validation_settings = dict(settings) + custom_endpoint_validation_settings['allow_private_custom_model_endpoints'] = ( + form_data.get('allow_private_custom_model_endpoints') == 'on' + ) + custom_endpoint_validation_settings['allow_insecure_custom_model_endpoints'] = ( + form_data.get('allow_insecure_custom_model_endpoints') == 'on' + ) + try: + validate_custom_model_endpoints( + parsed_model_endpoints, + custom_endpoint_validation_settings, + ) + except ModelEndpointValidationError as exc: + log_event( + "[MODEL_ENDPOINT] Custom model endpoint validation failed", + extra={"exception_type": type(exc).__name__}, + level=logging.WARNING, + ) + flash(str(exc), 'danger') + return redirect(url_for('frontend_admin_settings.admin_settings')) existing_endpoints_by_id = { endpoint.get('id'): endpoint @@ -1884,9 +1911,10 @@ def parse_admin_int(raw_value, fallback_value, field_name="unknown", hard_defaul if endpoint_provider: normalized_metadata_model_selection['provider'] = endpoint_provider metadata_extraction_model_deployment = str( - model_cfg.get('deploymentName') - or model_cfg.get('deployment') - or '' + resolve_model_endpoint_request_model( + endpoint_cfg, + model_cfg, + ) ).strip() else: normalized_metadata_model_selection = { @@ -2455,6 +2483,15 @@ def is_valid_url(url): 'gpt_model': gpt_model_obj, 'enable_multi_model_endpoints': enable_multi_model_endpoints, 'model_endpoints': parsed_model_endpoints, + 'allow_private_custom_model_endpoints': ( + form_data.get('allow_private_custom_model_endpoints') == 'on' + ), + 'allow_insecure_custom_model_endpoints': ( + form_data.get('allow_insecure_custom_model_endpoints') == 'on' + ), + 'custom_model_endpoint_ca_bundle_path': ( + form_data.get('custom_model_endpoint_ca_bundle_path', '').strip() + ), 'model_endpoint_identity_header_enabled': model_endpoint_identity_header_enabled, 'model_endpoint_identity_header_name': model_endpoint_identity_header_name, 'model_endpoint_identity_header_value_type': model_endpoint_identity_header_value_type, diff --git a/application/single_app/route_frontend_chats.py b/application/single_app/route_frontend_chats.py index 4e700465b..b4a2caee6 100644 --- a/application/single_app/route_frontend_chats.py +++ b/application/single_app/route_frontend_chats.py @@ -5,6 +5,7 @@ from functions_authentication import * from functions_content import * from functions_settings import * +from functions_model_endpoint_types import resolve_model_endpoint_request_model from functions_agent_catalog import build_accessible_agent_catalog from functions_ai_notice import get_ai_notice_config, is_ai_notice_dismissed from functions_collaboration import ( @@ -465,15 +466,22 @@ def serialize_option(option): selection_key = _normalize_chat_model_value(option.get('selection_key')) model_id = _normalize_chat_model_value(option.get('model_id')) display_name = _normalize_chat_model_value( - option.get('display_name') or option.get('deployment_name') or option.get('model_id') + option.get('display_name') + or option.get('request_model') + or option.get('deployment_name') + or option.get('model_id') ) or 'Select a Model' deployment_name = _normalize_chat_model_value(option.get('deployment_name')) + request_model = _normalize_chat_model_value( + option.get('request_model') or deployment_name + ) scope_type = _normalize_chat_model_value(option.get('scope_type')) scope_name = _normalize_chat_model_value(option.get('scope_name')) search_parts = [ display_name, model_id, + request_model, deployment_name, scope_name or scope_type, ] @@ -481,6 +489,7 @@ def serialize_option(option): 'selection_key': selection_key, 'model_id': model_id, 'display_name': display_name, + 'request_model': request_model, 'deployment_name': deployment_name, 'endpoint_id': _normalize_chat_model_value(option.get('endpoint_id')), 'provider': _normalize_chat_model_value(option.get('provider')), @@ -488,23 +497,28 @@ def serialize_option(option): 'scope_id': _normalize_chat_model_value(option.get('scope_id')), 'scope_name': scope_name, 'icon': option.get('icon') if isinstance(option.get('icon'), dict) else {}, - 'option_value': deployment_name or model_id or selection_key, + 'option_value': request_model or deployment_name or model_id or selection_key, 'search_text': ' '.join(part for part in search_parts if part), } def sort_key(option): scope_type = _normalize_chat_model_value(option.get('scope_type')) display_name = _normalize_chat_model_value( - option.get('display_name') or option.get('deployment_name') or option.get('model_id') + option.get('display_name') + or option.get('request_model') + or option.get('deployment_name') + or option.get('model_id') ).lower() scope_name = _normalize_chat_model_value(option.get('scope_name')).lower() model_id = _normalize_chat_model_value(option.get('model_id')).lower() deployment_name = _normalize_chat_model_value(option.get('deployment_name')).lower() + request_model = _normalize_chat_model_value(option.get('request_model')).lower() return ( scope_order.get(scope_type, 99), scope_name, display_name, model_id, + request_model, deployment_name, ) @@ -526,7 +540,13 @@ def sort_key(option): if normalized_preferred_model_deployment: for option in sorted_options: deployment_name = _normalize_chat_model_value(option.get('deployment_name')) - if deployment_name == normalized_preferred_model_deployment: + request_model = _normalize_chat_model_value( + option.get('request_model') or deployment_name + ) + if ( + deployment_name == normalized_preferred_model_deployment + or request_model == normalized_preferred_model_deployment + ): return serialize_option(option) return serialize_option(sorted_options[0]) @@ -556,13 +576,15 @@ def append_models(endpoints, scope_type, scope_id=None, scope_name=None): model_id = model.get('id') or model.get('deploymentName') or model.get('deployment') or model.get('modelName') or model.get('name') or '' deployment_name = model.get('deploymentName') or model.get('deployment') or '' - display_name = model.get('displayName') or model.get('modelName') or deployment_name or model.get('name') or model_id - selection_key = f"{scope_type}:{scope_id or ''}:{endpoint_id}:{model_id or deployment_name}" + request_model = resolve_model_endpoint_request_model(endpoint, model) + display_name = model.get('displayName') or model.get('modelName') or request_model or deployment_name or model.get('name') or model_id + selection_key = f"{scope_type}:{scope_id or ''}:{endpoint_id}:{model_id or deployment_name or request_model}" catalog.append({ 'selection_key': selection_key, 'model_id': model_id, 'display_name': display_name, + 'request_model': request_model, 'deployment_name': deployment_name, 'endpoint_id': endpoint_id, 'provider': provider, @@ -780,6 +802,7 @@ def chats(): multi_endpoint_models.append({ "id": model.get("id"), "display_name": model.get("displayName") or model.get("deploymentName") or model.get("modelName") or "", + "request_model": resolve_model_endpoint_request_model(endpoint, model), "deployment_name": model.get("deploymentName") or "", "endpoint_id": endpoint.get("id"), "provider": endpoint.get("provider"), diff --git a/application/single_app/semantic_kernel_loader.py b/application/single_app/semantic_kernel_loader.py index ec7ba2b6a..2c38cad7c 100644 --- a/application/single_app/semantic_kernel_loader.py +++ b/application/single_app/semantic_kernel_loader.py @@ -27,6 +27,11 @@ from semantic_kernel_plugins.chart_plugin import ChartPlugin from semantic_kernel_plugins.tabular_processing_plugin import TabularProcessingPlugin from functions_settings import get_settings, get_user_settings, is_tabular_processing_enabled, resolve_model_endpoint_foundry_scope +from functions_model_endpoint_runtime import build_semantic_kernel_chat_service_for_model +from functions_model_endpoint_types import ( + get_model_endpoint_api_type, + resolve_model_endpoint_request_model, +) from foundry_agent_runtime import ( AzureAIFoundryChatCompletionAgent, AzureAIFoundryNewChatCompletionAgent, @@ -165,6 +170,7 @@ def resolve_agent_endpoint_protocol(agent_config): agent_config.get("model_provider") or agent_config.get("provider") or "aoai", agent_config.get("endpoint"), agent_config.get("deployment"), + agent_config.get("api_type"), ) @@ -179,11 +185,31 @@ def resolve_agent_endpoint_token(agent_config): return "" -def create_model_endpoint_chat_completion_service(agent_config, service_id): +def create_model_endpoint_chat_completion_service(agent_config, service_id, settings=None): """Create the correct Semantic Kernel chat service for an endpoint-bound agent.""" if not agent_config.get("endpoint") or not agent_config.get("deployment"): return None + provider = str( + agent_config.get("model_provider") or agent_config.get("provider") or "aoai" + ).strip().lower() + if provider == "custom": + chat_service, _ = build_semantic_kernel_chat_service_for_model( + agent_config["deployment"], + settings or {}, + service_id=service_id, + model_context={ + "provider": provider, + "endpoint": agent_config["endpoint"], + "api_version": agent_config.get("api_version") or "", + "api_type": agent_config.get("api_type") or "", + "anthropic_version": agent_config.get("anthropic_version") or "", + "auth": agent_config.get("auth") or {}, + "request_model": agent_config["deployment"], + }, + ) + return chat_service + runtime_protocol = resolve_agent_endpoint_protocol(agent_config) token_or_key = resolve_agent_endpoint_token(agent_config) if not token_or_key: @@ -576,13 +602,15 @@ def resolve_multi_endpoint_agent_binding(endpoint_candidates, endpoint_id, model provider = (endpoint_cfg.get("provider") or "aoai").lower() connection = endpoint_cfg.get("connection", {}) or {} auth = endpoint_cfg.get("auth", {}) or {} - deployment = model_cfg.get("deploymentName") or model_cfg.get("deployment") or "" + deployment = resolve_model_endpoint_request_model(endpoint_cfg, model_cfg) api_version = connection.get("openai_api_version") or connection.get("api_version") endpoint = connection.get("endpoint") return { "provider": provider, "endpoint": endpoint, "api_version": api_version, + "api_type": get_model_endpoint_api_type(endpoint_cfg), + "anthropic_version": connection.get("anthropic_version") or "", "deployment": deployment, "auth": auth, "model": model_cfg, @@ -808,7 +836,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): if not per_user_enabled: try: token_provider = None - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -816,13 +844,15 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): deployment = multi_endpoint_config.get("deployment") api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" - if auth_type != "api_key": + if auth_type not in ("api_key", "key"): token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) return { "endpoint": endpoint, "key": key, "deployment": deployment, "api_version": api_version, + "api_type": multi_endpoint_config.get("api_type") or "", + "anthropic_version": multi_endpoint_config.get("anthropic_version") or "", "instructions": agent.get("instructions", ""), "actions_to_load": agent.get("actions_to_load", []), "additional_settings": agent.get("additional_settings", {}), @@ -843,6 +873,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "auth": auth, } if global_apim_enabled: g_apim = get_global_apim() @@ -885,7 +916,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): can_use_agent_endpoints = allow_custom_agent_endpoints user_apim_allowed = user_apim_enabled and can_use_agent_endpoints - if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow"): + if multi_endpoint_config and multi_endpoint_config.get("provider") in ("aoai", "aifoundry", "new_foundry", "foundry_workflow", "custom"): auth = multi_endpoint_config.get("auth", {}) or {} auth_type = (auth.get("type") or "managed_identity").lower() provider = multi_endpoint_config.get("provider") @@ -894,13 +925,15 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): api_version = multi_endpoint_config.get("api_version") key = auth.get("api_key") or "" token_provider = None - if auth_type != "api_key": + if auth_type not in ("api_key", "key"): token_provider = build_token_provider(auth, provider=provider, endpoint=endpoint) result = { "endpoint": endpoint, "key": key, "deployment": deployment, "api_version": api_version, + "api_type": multi_endpoint_config.get("api_type") or "", + "anthropic_version": multi_endpoint_config.get("anthropic_version") or "", "instructions": agent.get("instructions", ""), "actions_to_load": agent.get("actions_to_load", []), "additional_settings": agent.get("additional_settings", {}), @@ -921,6 +954,7 @@ def enrich_foundry_settings(foundry_settings, endpoint_cfg): "model_endpoint_id": agent.get("model_endpoint_id", ""), "model_id": agent.get("model_id", ""), "model_provider": provider, + "auth": auth, } return result @@ -1797,7 +1831,7 @@ def load_single_agent_for_kernel(kernel, agent_cfg, settings, context_obj, redis apim_enabled = settings.get("enable_gpt_apim", False) def create_chat_completion_service(): - return create_model_endpoint_chat_completion_service(agent_config, service_id) + return create_model_endpoint_chat_completion_service(agent_config, service_id, settings) if agent_type in {"aifoundry", "new_foundry", "foundry_workflow"}: if agent_type == "foundry_workflow": @@ -2925,7 +2959,7 @@ def load_semantic_kernel(kernel: Kernel, settings): }, level=logging.INFO ) - chat_service = create_model_endpoint_chat_completion_service(agent_config, service_id) + chat_service = create_model_endpoint_chat_completion_service(agent_config, service_id, settings) if should_apply_prompt_settings(orchestrator_config, settings): if orchestrator_config.get('max_completion_tokens', -1) > 0: print(f"[SK_LOADER] Using {orchestrator_config['max_completion_tokens']} max_completion_tokens for {orchestrator_config['name']}") @@ -3023,7 +3057,7 @@ def load_semantic_kernel(kernel: Kernel, settings): }, level=logging.INFO ) - chat_service = create_model_endpoint_chat_completion_service(orchestrator_config, service_id) + chat_service = create_model_endpoint_chat_completion_service(orchestrator_config, service_id, settings) if should_apply_prompt_settings(agent_config, settings): if agent_config.get('max_completion_tokens', -1) > 0: print(f"[SK_LOADER] Using {agent_config['max_completion_tokens']} max_completion_tokens for {agent_config['name']}") diff --git a/application/single_app/static/js/admin/admin_model_endpoints.js b/application/single_app/static/js/admin/admin_model_endpoints.js index d62f67114..eb599e561 100644 --- a/application/single_app/static/js/admin/admin_model_endpoints.js +++ b/application/single_app/static/js/admin/admin_model_endpoints.js @@ -43,6 +43,10 @@ const endpointModal = endpointModalEl && window.bootstrap ? bootstrap.Modal.getO const endpointIdInput = document.getElementById("model-endpoint-id"); const endpointNameInput = document.getElementById("model-endpoint-name"); const endpointProviderSelect = document.getElementById("model-endpoint-provider"); +const endpointApiTypeGroup = document.getElementById("model-endpoint-api-type-group"); +const endpointUrlModeGroup = document.getElementById("model-endpoint-url-mode-group"); +const endpointUrlModeExactInput = document.getElementById("model-endpoint-url-mode-exact"); +const endpointApiTypeSelect = document.getElementById("model-endpoint-api-type"); const endpointUrlInput = document.getElementById("model-endpoint-endpoint"); const endpointUrlLabel = document.getElementById("model-endpoint-endpoint-label"); const endpointUrlHelp = document.getElementById("model-endpoint-endpoint-help"); @@ -54,6 +58,8 @@ const endpointProjectApiVersionCustomInput = document.getElementById("model-endp const endpointOpenAiApiVersionGroup = document.getElementById("model-endpoint-openai-api-version-group"); const endpointOpenAiApiVersionInput = document.getElementById("model-endpoint-openai-api-version"); const endpointOpenAiApiVersionCustomInput = document.getElementById("model-endpoint-openai-api-version-custom"); +const endpointAnthropicVersionGroup = document.getElementById("model-endpoint-anthropic-version-group"); +const endpointAnthropicVersionInput = document.getElementById("model-endpoint-anthropic-version"); const endpointSubscriptionGroup = document.getElementById("model-endpoint-subscription-group"); const endpointResourceGroup = document.getElementById("model-endpoint-resource-group-group"); const endpointSubscriptionInput = document.getElementById("model-endpoint-subscription-id"); @@ -66,6 +72,7 @@ const endpointCustomAuthorityInput = document.getElementById("model-endpoint-cus const endpointFoundryScopeGroup = document.getElementById("model-endpoint-foundry-scope-group"); const endpointFoundryScopeInput = document.getElementById("model-endpoint-foundry-scope"); const apiKeyNote = document.getElementById("model-endpoint-api-key-note"); +const apiKeyNoteText = document.getElementById("model-endpoint-api-key-note-text"); const miTypeGroup = document.getElementById("model-endpoint-mi-type-group"); const miClientGroup = document.getElementById("model-endpoint-mi-client-group"); @@ -108,6 +115,7 @@ let migrationSelectedKeys = new Set(); const DEFAULT_AOAI_OPENAI_API_VERSION = "2024-05-01-preview"; const DEFAULT_FOUNDRY_OPENAI_API_VERSION = "v1"; const DEFAULT_FOUNDRY_PROJECT_API_VERSION = "v1"; +const DEFAULT_ANTHROPIC_VERSION = "2023-06-01"; const CUSTOM_VERSION_VALUE = "custom"; const IDENTITY_HEADER_MODES = new Set(["inherit", "enabled", "disabled"]); const IDENTITY_HEADER_VALUE_TYPES = new Set(["", "user_oid_tenant_id", "user_oid", "user_upn_tenant_id", "user_upn"]); @@ -149,6 +157,77 @@ function isFoundryProvider(provider) { return provider === "aifoundry" || provider === "new_foundry"; } +function isCustomProvider(provider = endpointProviderSelect?.value) { + return provider === "custom"; +} + +// The API type registry is rendered server-side from +// functions_model_endpoint_providers so the option list, the model identifier +// field, and the version field are declared in exactly one place. +let customApiTypeRegistry = null; + +function getCustomApiTypeRegistry() { + if (customApiTypeRegistry) { + return customApiTypeRegistry; + } + customApiTypeRegistry = {}; + try { + const rawRegistry = endpointApiTypeSelect?.dataset?.apiTypes; + if (rawRegistry) { + JSON.parse(rawRegistry).forEach((apiType) => { + customApiTypeRegistry[apiType.value] = apiType; + }); + } + } catch (error) { + console.error("Unable to parse the model endpoint API type registry.", error); + } + return customApiTypeRegistry; +} + +function getCustomApiTypeDescriptor(apiType = getCustomApiType()) { + return getCustomApiTypeRegistry()[apiType] || null; +} + +function getCustomApiType() { + return endpointApiTypeSelect?.value || "openai"; +} + +function customApiTypeUsesModelName(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return descriptor ? Boolean(descriptor.usesModelName) : true; +} + +function customApiTypeRequiresApiVersion(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return Boolean(descriptor?.requiresApiVersion); +} + +function customApiTypeVersionField(apiType = getCustomApiType()) { + return getCustomApiTypeDescriptor(apiType)?.versionField || ""; +} + +function getModelRequestName(model) { + if (isCustomProvider() && customApiTypeUsesModelName()) { + return String(model?.modelName || "").trim(); + } + return String(model?.deploymentName || model?.deployment || "").trim(); +} + +function setModelRequestName(model, value) { + const requestName = String(value || "").trim(); + if (isCustomProvider() && customApiTypeUsesModelName()) { + model.modelName = requestName; + delete model.deploymentName; + delete model.deployment; + return; + } + model.deploymentName = requestName; + if (isCustomProvider()) { + delete model.modelName; + delete model.name; + } +} + function endpointIncludesProject(endpoint) { return String(endpoint || "").toLowerCase().includes("/api/projects/"); } @@ -198,9 +277,13 @@ function syncEndpointCopyForProvider() { : "Endpoint Fully Qualified Domain Name (FQDN)"; } if (endpointUrlHelp) { - endpointUrlHelp.textContent = isFoundryProvider(provider) - ? "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name." - : "For Azure OpenAI, paste the resource endpoint."; + if (isFoundryProvider(provider)) { + endpointUrlHelp.textContent = "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name."; + } else if (isCustomProvider(provider)) { + endpointUrlHelp.textContent = "Enter the HTTPS FQDN for the Custom endpoint."; + } else { + endpointUrlHelp.textContent = "For Azure OpenAI, paste the resource endpoint."; + } } } @@ -392,6 +475,9 @@ function formatProviderLabel(provider) { if (provider === "new_foundry") { return "New Foundry"; } + if (provider === "custom") { + return "Custom"; + } return "Azure OpenAI"; } @@ -417,7 +503,7 @@ function syncOpenAiApiVersionForProvider() { return; } - if (!currentValue) { + if (!currentValue || currentValue === DEFAULT_FOUNDRY_OPENAI_API_VERSION) { setSelectedVersionValue( endpointOpenAiApiVersionInput, endpointOpenAiApiVersionCustomInput, @@ -776,18 +862,38 @@ function handleMetadataExtractionModelChange() { } function updateAuthVisibility() { - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const modelsPlaceholder = document.getElementById("model-endpoint-models-placeholder"); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + if (customProvider && endpointAuthTypeSelect) { + endpointAuthTypeSelect.value = "api_key"; + } + if (endpointAuthTypeSelect) { + endpointAuthTypeSelect.disabled = customProvider; + } + setElementVisibility(endpointApiTypeGroup, customProvider); + // The exact-URL escape hatch only applies to URL-built protocols, not to the + // Azure resource endpoint, which the SDK consumes as given. + setElementVisibility( + endpointUrlModeGroup, + customProvider && customApiTypeVersionField(getCustomApiType()) !== "api_version" + ); + + const apiType = getCustomApiType(); + const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; - const isFoundry = isFoundryProvider(provider); + const isFoundry = !customProvider && isFoundryProvider(provider); + const showOpenAiVersion = !customProvider || customApiTypeRequiresApiVersion(apiType); + const showAnthropicVersion = customProvider && customApiTypeVersionField(apiType) === "anthropic_version"; const projectNameFromEndpoint = syncProjectNameFromEndpoint(); syncEndpointCopyForProvider(); syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, true); - setElementVisibility(endpointSubscriptionGroup, provider === "aoai" && !isApiKey); - setElementVisibility(endpointResourceGroup, provider === "aoai" && !isApiKey); + setElementVisibility(endpointOpenAiApiVersionGroup, showOpenAiVersion); + setElementVisibility(endpointAnthropicVersionGroup, showAnthropicVersion); + setElementVisibility(endpointSubscriptionGroup, !customProvider && provider === "aoai" && !isApiKey); + setElementVisibility(endpointResourceGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); setElementVisibility(miClientGroup, authType === "managed_identity" && (miTypeSelect?.value === "user_assigned")); setElementVisibility(tenantGroup, authType === "service_principal"); @@ -797,9 +903,27 @@ function updateAuthVisibility() { setElementVisibility(endpointManagementCloudGroup, authType === "service_principal" && isFoundry); setElementVisibility(endpointCustomAuthorityGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); setElementVisibility(endpointFoundryScopeGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); - setElementVisibility(apiKeyNote, authType === "api_key"); - setElementVisibility(addModelBtn, authType === "api_key"); - setElementVisibility(fetchBtn, authType !== "api_key"); + setElementVisibility(apiKeyNote, customProvider || authType === "api_key"); + setElementVisibility(addModelBtn, customProvider || authType === "api_key"); + setElementVisibility(fetchBtn, !customProvider && authType !== "api_key"); + + if (customProvider) { + if (apiKeyNoteText) { + apiKeyNoteText.textContent = "Custom endpoints use API key authentication and manual model entry. Model discovery is unavailable."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = "Add a model manually."; + } + } else { + if (apiKeyNoteText) { + apiKeyNoteText.textContent = "API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = authType === "api_key" + ? "Add a model manually, or switch authentication to discover deployments." + : "Fetch models or add a model manually."; + } + } } function resetModal() { @@ -809,6 +933,7 @@ function resetModal() { if (endpointIdInput) endpointIdInput.value = ""; if (endpointNameInput) endpointNameInput.value = ""; if (endpointProviderSelect) endpointProviderSelect.value = "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = "openai"; if (endpointUrlInput) endpointUrlInput.value = ""; if (endpointProjectInput) endpointProjectInput.value = ""; setSelectedVersionValue( @@ -821,6 +946,7 @@ function resetModal() { endpointOpenAiApiVersionCustomInput, getDefaultOpenAiApiVersion("aoai") ); + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = DEFAULT_ANTHROPIC_VERSION; if (endpointSubscriptionInput) endpointSubscriptionInput.value = ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = "managed_identity"; @@ -840,7 +966,7 @@ function resetModal() { if (endpointIdentityValueTypeSelect) endpointIdentityValueTypeSelect.value = ""; modalModels = []; - if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; + if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; updateAuthVisibility(); } @@ -856,6 +982,10 @@ function openModalForEndpoint(endpoint) { if (endpointIdInput) endpointIdInput.value = endpoint.id || ""; if (endpointNameInput) endpointNameInput.value = endpoint.name || ""; if (endpointProviderSelect) endpointProviderSelect.value = endpoint.provider || "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = endpoint.api_type || "openai"; + if (endpointUrlModeExactInput) { + endpointUrlModeExactInput.checked = (endpoint.connection?.url_mode || "") === "exact"; + } if (endpointUrlInput) endpointUrlInput.value = endpoint.connection?.endpoint || ""; if (endpointProjectInput) endpointProjectInput.value = endpoint.connection?.project_name || ""; setSelectedVersionValue( @@ -868,6 +998,9 @@ function openModalForEndpoint(endpoint) { endpointOpenAiApiVersionCustomInput, endpoint.connection?.openai_api_version || endpoint.connection?.api_version || getDefaultOpenAiApiVersion(endpoint.provider || "aoai") ); + if (endpointAnthropicVersionInput) { + endpointAnthropicVersionInput.value = endpoint.connection?.anthropic_version || DEFAULT_ANTHROPIC_VERSION; + } if (endpointSubscriptionInput) endpointSubscriptionInput.value = endpoint.management?.subscription_id || ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = endpoint.management?.resource_group || ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = endpoint.auth?.type || "managed_identity"; @@ -1202,7 +1335,7 @@ function renderModalModels(models) { } if (!models || !models.length) { - modelsListEl.innerHTML = "

No models loaded yet.

"; + modelsListEl.innerHTML = "

No models loaded yet.

"; return; } @@ -1210,12 +1343,15 @@ function renderModalModels(models) { models.forEach((model) => { const wrapper = document.createElement("div"); wrapper.className = "border rounded p-2 mb-2"; - const deploymentName = model.deploymentName || ""; + const requestName = getModelRequestName(model); const modelName = model.modelName || ""; - const displayName = model.displayName || deploymentName; + const displayName = model.displayName || requestName; const description = model.description || ""; const responseLength = getModelResponseLength(model); - const deploymentReadonly = model.isDiscovered ? "readonly" : ""; + const requestNameReadonly = model.isDiscovered && !isCustomProvider(); + const requestNameLabel = isCustomProvider() && customApiTypeUsesModelName() + ? "Model Name" + : "Deployment Name"; const modelId = model.id || generateId(); model.id = modelId; @@ -1226,8 +1362,8 @@ function renderModalModels(models) { checkbox.dataset.modelId = modelId; checkbox.checked = !!model.enabled; const checkboxLabel = createElement("label", "form-check-label"); - checkboxLabel.appendChild(document.createTextNode(deploymentName)); - if (modelName) { + checkboxLabel.appendChild(document.createTextNode(requestName)); + if (!isCustomProvider() && modelName) { checkboxLabel.appendChild(document.createTextNode(" ")); const modelNameLabel = createElement("span", "text-muted"); modelNameLabel.textContent = `(${modelName})`; @@ -1238,8 +1374,8 @@ function renderModalModels(models) { const fieldsRow = createElement("div", "row g-2"); const deploymentCol = createElement("div", "col-md-4"); - deploymentCol.appendChild(createSmallLabel("Deployment Name")); - deploymentCol.appendChild(createModelTextInput(modelId, "deploymentNameFor", deploymentName, Boolean(deploymentReadonly))); + deploymentCol.appendChild(createSmallLabel(requestNameLabel)); + deploymentCol.appendChild(createModelTextInput(modelId, "requestModelFor", requestName, requestNameReadonly)); const displayCol = createElement("div", "col-md-4"); displayCol.appendChild(createSmallLabel("Display Name")); displayCol.appendChild(createModelTextInput(modelId, "displayNameFor", displayName)); @@ -1298,7 +1434,7 @@ function collectModalModels() { const updated = modalModels.map((model) => ({ ...model })); updated.forEach((model) => { const checkbox = modelsListEl.querySelector(`input[data-model-id="${model.id}"]`); - const deploymentInput = modelsListEl.querySelector(`input[data-deployment-name-for="${model.id}"]`); + const requestModelInput = modelsListEl.querySelector(`input[data-request-model-for="${model.id}"]`); const displayInput = modelsListEl.querySelector(`input[data-display-name-for="${model.id}"]`); const descriptionInput = modelsListEl.querySelector(`input[data-description-for="${model.id}"]`); const responseLengthInput = modelsListEl.querySelector(`input[data-response-length-for="${model.id}"]`); @@ -1308,7 +1444,7 @@ function collectModalModels() { throw new Error("Response length must be a positive whole number."); } model.enabled = checkbox ? checkbox.checked : model.enabled; - model.deploymentName = deploymentInput ? deploymentInput.value.trim() : model.deploymentName; + setModelRequestName(model, requestModelInput ? requestModelInput.value : getModelRequestName(model)); model.displayName = displayInput ? displayInput.value.trim() : model.displayName; model.icon = iconEditor ? getIconPayload(iconEditor, MODEL_ICON_CONTROL_CONFIG) : model.icon || {}; model.description = descriptionInput ? descriptionInput.value.trim() : model.description; @@ -1323,16 +1459,17 @@ function collectModalModels() { async function testModelConnection(model) { const payload = buildEndpointPayload(); - if (!payload || !model?.deploymentName) { - showToast("Model deployment name is required for testing.", "warning"); + const requestModel = getModelRequestName(model); + if (!payload || !requestModel) { + showToast(`${isCustomProvider() && customApiTypeUsesModelName() ? "Model" : "Deployment"} name is required for testing.`, "warning"); return; } + const testModel = {}; + setModelRequestName(testModel, requestModel); const requestBody = { ...payload, - model: { - deploymentName: model.deploymentName - } + model: testModel }; try { @@ -1345,7 +1482,15 @@ async function testModelConnection(model) { if (!response.ok) { throw new Error(data.error || "Connection test failed."); } - showToast("Model connection successful.", "success"); + // Report the URL that was actually called. Normalization can rewrite the + // configured endpoint, and that rewrite was previously invisible. + const resolvedUrl = data.resolved?.request_url || ""; + showToast( + resolvedUrl + ? `Model connection successful. Called ${resolvedUrl}` + : "Model connection successful.", + "success" + ); } catch (error) { console.error("Model connection failed", error); showToast(error.message || "Model connection failed.", "danger"); @@ -1353,6 +1498,10 @@ async function testModelConnection(model) { } async function fetchModels() { + if (isCustomProvider()) { + showToast("Model discovery is unavailable for Custom endpoints. Add models manually.", "warning"); + return; + } const payload = buildEndpointPayload(); if (!payload) { return; @@ -1418,6 +1567,8 @@ function buildEndpointPayload() { const name = endpointNameInput.value.trim(); const endpoint = endpointUrlInput.value.trim(); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + const apiType = getCustomApiType(); const projectNameFromEndpoint = isFoundryProvider(provider) ? syncProjectNameFromEndpoint() : ""; const projectName = projectNameFromEndpoint || endpointProjectInput?.value.trim() || ""; const projectApiVersion = getSelectedVersionValue( @@ -1432,7 +1583,7 @@ function buildEndpointPayload() { ); const subscriptionId = endpointSubscriptionInput?.value.trim() || ""; const resourceGroup = endpointResourceGroupInput?.value.trim() || ""; - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const authType = customProvider ? "api_key" : (endpointAuthTypeSelect?.value || "managed_identity"); const existingEndpoint = modelEndpoints.find((savedEndpoint) => savedEndpoint.id === endpointId); const identityHeader = normalizeEndpointIdentityHeaderOverride({ mode: endpointIdentityModeSelect?.value || "inherit", @@ -1440,8 +1591,18 @@ function buildEndpointPayload() { value_type: endpointIdentityValueTypeSelect?.value || "" }); - if (!name || !endpoint || !openAiApiVersion) { - showToast("Endpoint name, URL, and OpenAI API version are required.", "warning"); + if (!name || !endpoint) { + showToast("Endpoint name and URL are required.", "warning"); + return null; + } + + if (customProvider && !/^https:\/\//i.test(endpoint)) { + showToast("Custom endpoint URLs must use HTTPS.", "warning"); + return null; + } + + if ((!customProvider || customApiTypeRequiresApiVersion(apiType)) && !openAiApiVersion) { + showToast("OpenAI API version is required.", "warning"); return null; } @@ -1460,7 +1621,7 @@ function buildEndpointPayload() { return null; } - const auth = { + let auth = { type: authType, managed_identity_type: miTypeSelect?.value || "system_assigned", managed_identity_client_id: miClientIdInput?.value.trim() || "", @@ -1472,6 +1633,12 @@ function buildEndpointPayload() { custom_authority: endpointCustomAuthorityInput?.value.trim() || "", foundry_scope: endpointFoundryScopeInput?.value.trim() || "" }; + if (customProvider) { + auth = { + type: "api_key", + api_key: apiKeyInput?.value.trim() || "" + }; + } const hasStoredApiKey = authType === "api_key" && Boolean(existingEndpoint?.has_api_key); const hasStoredClientSecret = authType === "service_principal" && Boolean(existingEndpoint?.has_client_secret); @@ -1497,17 +1664,25 @@ function buildEndpointPayload() { return null; } - const management = provider === "aoai" ? { + const management = !customProvider && provider === "aoai" ? { subscription_id: subscriptionId, resource_group: resourceGroup } : {}; - const connection = { - endpoint, - openai_api_version: openAiApiVersion - }; + const connection = { endpoint }; + const versionField = customProvider ? customApiTypeVersionField(apiType) : ""; + if (customProvider && endpointUrlModeExactInput?.checked) { + connection.url_mode = "exact"; + } + if (customProvider && versionField === "api_version") { + connection.api_version = openAiApiVersion; + } else if (customProvider && versionField === "anthropic_version") { + connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; + } else if (!customProvider) { + connection.openai_api_version = openAiApiVersion; + } - if (isFoundryProvider(provider)) { + if (!customProvider && isFoundryProvider(provider)) { connection.project_api_version = projectApiVersion; if (projectName) { connection.project_name = projectName; @@ -1517,6 +1692,7 @@ function buildEndpointPayload() { return { id: endpointId, provider, + ...(customProvider ? { api_type: apiType } : {}), name, connection, management, @@ -1543,6 +1719,7 @@ function saveEndpoint() { id: endpointId, name: payload.name, provider: payload.provider, + ...(payload.api_type ? { api_type: payload.api_type } : {}), enabled: endpointModalEl?.dataset.duplicateDisabledDefault === 'true' ? false : (existingEndpoint ? existingEndpoint.enabled !== false : true), @@ -1574,16 +1751,16 @@ function saveEndpoint() { } function addManualModel() { - modalModels.push({ + const model = { id: generateId(), - deploymentName: "", - modelName: "", displayName: "", icon: {}, description: "", enabled: true, isDiscovered: false - }); + }; + setModelRequestName(model, ""); + modalModels.push(model); renderModalModels(modalModels); } @@ -2072,8 +2249,28 @@ function init() { } if (endpointProviderSelect) { endpointProviderSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the endpoint provider.", "danger"); + return; + } + syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); updateAuthVisibility(); + }); + } + if (endpointApiTypeSelect) { + endpointApiTypeSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the API type.", "danger"); + return; + } syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); + updateAuthVisibility(); }); } if (endpointUrlInput) { diff --git a/application/single_app/static/js/agent_modal_stepper.js b/application/single_app/static/js/agent_modal_stepper.js index 782ad5f3c..83ada36b5 100644 --- a/application/single_app/static/js/agent_modal_stepper.js +++ b/application/single_app/static/js/agent_modal_stepper.js @@ -4777,7 +4777,9 @@ export class AgentModalStepper { // Using global model - need to set at least one deployment field // We'll use the selected model as the deployment name for now if (formData.model) { - const deploymentName = selectedModelOption?.dataset?.deploymentName || formData.model; + const deploymentName = selectedModelOption?.dataset?.requestModel + || selectedModelOption?.dataset?.deploymentName + || formData.model; formData.azure_openai_gpt_deployment = deploymentName; } } diff --git a/application/single_app/static/js/agents_common.js b/application/single_app/static/js/agents_common.js index 169e4ca7c..01b71de73 100644 --- a/application/single_app/static/js/agents_common.js +++ b/application/single_app/static/js/agents_common.js @@ -704,17 +704,22 @@ export function getAvailableModels({ apimEnabled, settings, agent }) { return; } const endpointId = endpoint.id || ''; + const apiType = (endpoint.api_type || '').toLowerCase(); const endpointModels = endpoint.models || []; endpointModels.forEach(model => { if (!model || model.enabled === false) return; const modelId = model.id || model.deploymentName || model.deployment || model.modelName || model.name || ''; const deploymentName = model.deploymentName || model.deployment || ''; const modelName = model.modelName || model.name || ''; - const displayName = model.displayName || deploymentName || modelName || modelId; + const requestModel = provider === 'custom' && window.simplechatCustomApiTypeUsesModelName?.(apiType) + ? modelName + : deploymentName || modelName; + const displayName = model.displayName || requestModel || modelId; if (!displayName) return; models.push({ id: modelId, - deployment: deploymentName, + deployment: requestModel, + request_model: requestModel, name: modelName, display_name: displayName, endpoint_id: endpointId, @@ -832,7 +837,10 @@ export function populateGlobalModelDropdown(selectEl, models, selectedModel) { if (model.deployment) { opt.dataset.deploymentName = model.deployment; } - if (selectedModel && (model.name === selectedModel || model.deployment === selectedModel || model.id === selectedModel)) { + if (model.request_model) { + opt.dataset.requestModel = model.request_model; + } + if (selectedModel && (model.name === selectedModel || model.request_model === selectedModel || model.deployment === selectedModel || model.id === selectedModel)) { opt.selected = true; } selectEl.appendChild(opt); diff --git a/application/single_app/static/js/chat/chat-messages.js b/application/single_app/static/js/chat/chat-messages.js index ffd0c8504..97b839e0b 100644 --- a/application/single_app/static/js/chat/chat-messages.js +++ b/application/single_app/static/js/chat/chat-messages.js @@ -6755,7 +6755,10 @@ function getCurrentModelSelection() { modelId = selectedOption?.dataset?.modelId || selectedOption?.value || null; modelEndpointId = selectedOption?.dataset?.endpointId || null; modelProvider = selectedOption?.dataset?.provider || null; - modelDeployment = selectedOption?.dataset?.deploymentName || null; + modelDeployment = selectedOption?.dataset?.requestModel + || selectedOption?.value + || selectedOption?.dataset?.deploymentName + || null; modelIcon = parseSafeJsonObject(selectedOption?.dataset?.modelIcon || ''); } @@ -6916,7 +6919,14 @@ function buildCollaborativeModelTarget(option = {}) { return null; } - const modelDeployment = String(dataset.deploymentName || option.deployment_name || option.value || '').trim() || null; + const modelDeployment = String( + dataset.requestModel + || option.request_model + || dataset.deploymentName + || option.deployment_name + || option.value + || '' + ).trim() || null; const modelId = String(dataset.modelId || option.model_id || option.value || '').trim() || null; const modelEndpointId = String(dataset.endpointId || option.endpoint_id || '').trim() || null; const modelProvider = String(dataset.provider || option.provider || '').trim() || null; diff --git a/application/single_app/static/js/chat/chat-model-selector.js b/application/single_app/static/js/chat/chat-model-selector.js index 5b96f2456..b339aa5a5 100644 --- a/application/single_app/static/js/chat/chat-model-selector.js +++ b/application/single_app/static/js/chat/chat-model-selector.js @@ -44,13 +44,14 @@ function getSortedGroups() { } function getModelDisplayName(option) { - return (option.display_name || option.model_id || option.deployment_name || 'Unnamed Model').trim() || 'Unnamed Model'; + return (option.display_name || option.request_model || option.model_id || option.deployment_name || 'Unnamed Model').trim() || 'Unnamed Model'; } function getModelSearchText(option, sectionLabel) { return [ getModelDisplayName(option), option.model_id || '', + option.request_model || '', option.deployment_name || '', sectionLabel, ].join(' ').trim(); @@ -71,7 +72,7 @@ function getModelOptionLabel(option, duplicateCounts) { return displayName; } - return `${displayName} (${option.deployment_name || option.model_id || 'model'})`; + return `${displayName} (${option.request_model || option.deployment_name || option.model_id || 'model'})`; } function getKnownGroupIds() { @@ -260,6 +261,7 @@ function getSelectionSnapshot() { value: null, selectionKey: null, modelId: null, + requestModel: null, deploymentName: null, }; } @@ -269,6 +271,7 @@ function getSelectionSnapshot() { value: modelSelect.value || null, selectionKey: selectedOption?.dataset?.selectionKey || null, modelId: selectedOption?.dataset?.modelId || null, + requestModel: selectedOption?.dataset?.requestModel || null, deploymentName: selectedOption?.dataset?.deploymentName || null, }; } @@ -316,14 +319,20 @@ function resolveSelectedSelectionKey(options, restoreOptions = {}) { } if (preferredModelDeployment) { - const deploymentOption = matchBy(option => option.deployment_name === preferredModelDeployment); + const deploymentOption = matchBy(option => ( + option.request_model === preferredModelDeployment + || option.deployment_name === preferredModelDeployment + )); if (deploymentOption) { return deploymentOption.selection_key; } } - if (preserveCurrentSelection && currentSelection?.deploymentName) { - const currentDeploymentOption = matchBy(option => option.deployment_name === currentSelection.deploymentName); + if (preserveCurrentSelection && (currentSelection?.requestModel || currentSelection?.deploymentName)) { + const currentDeploymentOption = matchBy(option => ( + option.request_model === currentSelection.requestModel + || option.deployment_name === currentSelection.deploymentName + )); if (currentDeploymentOption) { return currentDeploymentOption.selection_key; } @@ -376,11 +385,12 @@ function rebuildModelOptions(sections, restoreOptions = {}) { section.options.forEach(option => { const modelOption = document.createElement('option'); - modelOption.value = option.deployment_name || option.model_id || option.selection_key; + modelOption.value = option.request_model || option.deployment_name || option.model_id || option.selection_key; modelOption.textContent = option.optionLabel; modelOption.dataset.selectionKey = option.selection_key || ''; modelOption.dataset.modelId = option.model_id || ''; modelOption.dataset.displayName = option.display_name || ''; + modelOption.dataset.requestModel = option.request_model || ''; modelOption.dataset.deploymentName = option.deployment_name || ''; modelOption.dataset.endpointId = option.endpoint_id || ''; modelOption.dataset.provider = option.provider || ''; diff --git a/application/single_app/static/js/workspace/workspace_model_endpoints.js b/application/single_app/static/js/workspace/workspace_model_endpoints.js index 3658f5d94..09c23fc18 100644 --- a/application/single_app/static/js/workspace/workspace_model_endpoints.js +++ b/application/single_app/static/js/workspace/workspace_model_endpoints.js @@ -14,6 +14,10 @@ const endpointModal = endpointModalEl && window.bootstrap ? bootstrap.Modal.getO const endpointIdInput = document.getElementById("model-endpoint-id"); const endpointNameInput = document.getElementById("model-endpoint-name"); const endpointProviderSelect = document.getElementById("model-endpoint-provider"); +const endpointApiTypeGroup = document.getElementById("model-endpoint-api-type-group"); +const endpointUrlModeGroup = document.getElementById("model-endpoint-url-mode-group"); +const endpointUrlModeExactInput = document.getElementById("model-endpoint-url-mode-exact"); +const endpointApiTypeSelect = document.getElementById("model-endpoint-api-type"); const endpointUrlInput = document.getElementById("model-endpoint-endpoint"); const endpointUrlLabel = document.getElementById("model-endpoint-endpoint-label"); const endpointUrlHelp = document.getElementById("model-endpoint-endpoint-help"); @@ -25,6 +29,8 @@ const endpointProjectApiVersionCustomInput = document.getElementById("model-endp const endpointOpenAiApiVersionGroup = document.getElementById("model-endpoint-openai-api-version-group"); const endpointOpenAiApiVersionInput = document.getElementById("model-endpoint-openai-api-version"); const endpointOpenAiApiVersionCustomInput = document.getElementById("model-endpoint-openai-api-version-custom"); +const endpointAnthropicVersionGroup = document.getElementById("model-endpoint-anthropic-version-group"); +const endpointAnthropicVersionInput = document.getElementById("model-endpoint-anthropic-version"); const endpointSubscriptionGroup = document.getElementById("model-endpoint-subscription-group"); const endpointResourceGroup = document.getElementById("model-endpoint-resource-group-group"); const endpointSubscriptionInput = document.getElementById("model-endpoint-subscription-id"); @@ -37,6 +43,7 @@ const endpointCustomAuthorityInput = document.getElementById("model-endpoint-cus const endpointFoundryScopeGroup = document.getElementById("model-endpoint-foundry-scope-group"); const endpointFoundryScopeInput = document.getElementById("model-endpoint-foundry-scope"); const apiKeyNote = document.getElementById("model-endpoint-api-key-note"); +const apiKeyNoteText = document.getElementById("model-endpoint-api-key-note-text"); const miTypeGroup = document.getElementById("model-endpoint-mi-type-group"); const miClientGroup = document.getElementById("model-endpoint-mi-client-group"); @@ -70,6 +77,7 @@ let modalModels = []; const DEFAULT_AOAI_OPENAI_API_VERSION = "2024-05-01-preview"; const DEFAULT_FOUNDRY_OPENAI_API_VERSION = "v1"; const DEFAULT_FOUNDRY_PROJECT_API_VERSION = "v1"; +const DEFAULT_ANTHROPIC_VERSION = "2023-06-01"; const CUSTOM_VERSION_VALUE = "custom"; const MODEL_ICON_CLASS_PATTERN = /^bi-[a-z0-9][a-z0-9-]{0,80}$/; const MODEL_ICON_CONTROL_CONFIG = Object.freeze({ @@ -124,6 +132,77 @@ function isFoundryProvider(provider) { return provider === "aifoundry" || provider === "new_foundry"; } +function isCustomProvider(provider = endpointProviderSelect?.value) { + return provider === "custom"; +} + +// The API type registry is rendered server-side from +// functions_model_endpoint_providers so the option list, the model identifier +// field, and the version field are declared in exactly one place. +let customApiTypeRegistry = null; + +function getCustomApiTypeRegistry() { + if (customApiTypeRegistry) { + return customApiTypeRegistry; + } + customApiTypeRegistry = {}; + try { + const rawRegistry = endpointApiTypeSelect?.dataset?.apiTypes; + if (rawRegistry) { + JSON.parse(rawRegistry).forEach((apiType) => { + customApiTypeRegistry[apiType.value] = apiType; + }); + } + } catch (error) { + console.error("Unable to parse the model endpoint API type registry.", error); + } + return customApiTypeRegistry; +} + +function getCustomApiTypeDescriptor(apiType = getCustomApiType()) { + return getCustomApiTypeRegistry()[apiType] || null; +} + +function getCustomApiType() { + return endpointApiTypeSelect?.value || "openai"; +} + +function customApiTypeUsesModelName(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return descriptor ? Boolean(descriptor.usesModelName) : true; +} + +function customApiTypeRequiresApiVersion(apiType = getCustomApiType()) { + const descriptor = getCustomApiTypeDescriptor(apiType); + return Boolean(descriptor?.requiresApiVersion); +} + +function customApiTypeVersionField(apiType = getCustomApiType()) { + return getCustomApiTypeDescriptor(apiType)?.versionField || ""; +} + +function getModelRequestName(model) { + if (isCustomProvider() && customApiTypeUsesModelName()) { + return String(model?.modelName || "").trim(); + } + return String(model?.deploymentName || model?.deployment || "").trim(); +} + +function setModelRequestName(model, value) { + const requestName = String(value || "").trim(); + if (isCustomProvider() && customApiTypeUsesModelName()) { + model.modelName = requestName; + delete model.deploymentName; + delete model.deployment; + return; + } + model.deploymentName = requestName; + if (isCustomProvider()) { + delete model.modelName; + delete model.name; + } +} + function endpointIncludesProject(endpoint) { return String(endpoint || "").toLowerCase().includes("/api/projects/"); } @@ -173,9 +252,13 @@ function syncEndpointCopyForProvider() { : "Endpoint Fully Qualified Domain Name (FQDN)"; } if (endpointUrlHelp) { - endpointUrlHelp.textContent = isFoundryProvider(provider) - ? "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name." - : "For Azure OpenAI, paste the resource endpoint."; + if (isFoundryProvider(provider)) { + endpointUrlHelp.textContent = "Paste the Project endpoint from Azure AI Foundry. It can include /api/projects/; Claude deployments are detected from the model name."; + } else if (isCustomProvider(provider)) { + endpointUrlHelp.textContent = "Enter the HTTPS FQDN for the Custom endpoint."; + } else { + endpointUrlHelp.textContent = "For Azure OpenAI, paste the resource endpoint."; + } } } @@ -242,7 +325,7 @@ function syncOpenAiApiVersionForProvider() { return; } - if (!currentValue) { + if (!currentValue || currentValue === DEFAULT_FOUNDRY_OPENAI_API_VERSION) { setSelectedVersionValue( endpointOpenAiApiVersionInput, endpointOpenAiApiVersionCustomInput, @@ -258,6 +341,9 @@ function formatProviderLabel(provider) { if (provider === "new_foundry") { return "New Foundry"; } + if (provider === "custom") { + return "Custom"; + } return "Azure OpenAI"; } @@ -318,18 +404,36 @@ function renderEndpoints() { } function updateAuthVisibility() { - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const modelsPlaceholder = document.getElementById("model-endpoint-models-placeholder"); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + if (customProvider && endpointAuthTypeSelect) { + endpointAuthTypeSelect.value = "api_key"; + } + if (endpointAuthTypeSelect) { + endpointAuthTypeSelect.disabled = customProvider; + } + setElementVisibility(endpointApiTypeGroup, customProvider); + // The exact-URL escape hatch only applies to URL-built protocols, not to the + // Azure resource endpoint, which the SDK consumes as given. + setElementVisibility( + endpointUrlModeGroup, + customProvider && customApiTypeVersionField(getCustomApiType()) !== "api_version" + ); + + const apiType = getCustomApiType(); + const authType = endpointAuthTypeSelect?.value || "managed_identity"; const isApiKey = authType === "api_key"; - const isFoundry = isFoundryProvider(provider); + const isFoundry = !customProvider && isFoundryProvider(provider); const projectNameFromEndpoint = syncProjectNameFromEndpoint(); syncEndpointCopyForProvider(); syncVersionCustomVisibility(); setElementVisibility(endpointProjectGroup, isFoundry && !projectNameFromEndpoint); setElementVisibility(endpointProjectApiVersionGroup, isFoundry); - setElementVisibility(endpointOpenAiApiVersionGroup, true); - setElementVisibility(endpointSubscriptionGroup, provider === "aoai" && !isApiKey); - setElementVisibility(endpointResourceGroup, provider === "aoai" && !isApiKey); + setElementVisibility(endpointOpenAiApiVersionGroup, !customProvider || customApiTypeRequiresApiVersion(apiType)); + setElementVisibility(endpointAnthropicVersionGroup, customProvider && customApiTypeVersionField(apiType) === "anthropic_version"); + setElementVisibility(endpointSubscriptionGroup, !customProvider && provider === "aoai" && !isApiKey); + setElementVisibility(endpointResourceGroup, !customProvider && provider === "aoai" && !isApiKey); setElementVisibility(miTypeGroup, authType === "managed_identity"); setElementVisibility(miClientGroup, authType === "managed_identity" && (miTypeSelect?.value === "user_assigned")); setElementVisibility(tenantGroup, authType === "service_principal"); @@ -339,15 +443,28 @@ function updateAuthVisibility() { setElementVisibility(endpointManagementCloudGroup, authType === "service_principal" && isFoundry); setElementVisibility(endpointCustomAuthorityGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); setElementVisibility(endpointFoundryScopeGroup, authType === "service_principal" && isFoundry && endpointManagementCloudSelect?.value === "custom"); - setElementVisibility(apiKeyNote, authType === "api_key"); - setElementVisibility(addModelBtn, authType === "api_key"); - setElementVisibility(fetchBtn, authType !== "api_key"); + setElementVisibility(apiKeyNote, customProvider || authType === "api_key"); + setElementVisibility(addModelBtn, customProvider || authType === "api_key"); + setElementVisibility(fetchBtn, !customProvider && authType !== "api_key"); + if (apiKeyNoteText) { + apiKeyNoteText.textContent = customProvider + ? "Custom endpoints use API key authentication and manual model entry. Model discovery is unavailable." + : "API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry."; + } + if (modelsPlaceholder) { + modelsPlaceholder.textContent = customProvider + ? "Add a model manually." + : (authType === "api_key" + ? "Add a model manually, or switch authentication to discover deployments." + : "Fetch models or add a model manually."); + } } function resetModal() { if (endpointIdInput) endpointIdInput.value = ""; if (endpointNameInput) endpointNameInput.value = ""; if (endpointProviderSelect) endpointProviderSelect.value = "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = "openai"; if (endpointUrlInput) endpointUrlInput.value = ""; if (endpointProjectInput) endpointProjectInput.value = ""; setSelectedVersionValue( @@ -360,6 +477,7 @@ function resetModal() { endpointOpenAiApiVersionCustomInput, getDefaultOpenAiApiVersion("aoai") ); + if (endpointAnthropicVersionInput) endpointAnthropicVersionInput.value = DEFAULT_ANTHROPIC_VERSION; if (endpointSubscriptionInput) endpointSubscriptionInput.value = ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = "managed_identity"; @@ -376,7 +494,7 @@ function resetModal() { if (apiKeyInput) apiKeyInput.placeholder = ""; modalModels = []; - if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; + if (modelsListEl) modelsListEl.innerHTML = "

Fetch models to begin selection.

"; updateAuthVisibility(); } @@ -392,6 +510,10 @@ function openModalForEndpoint(endpoint) { if (endpointIdInput) endpointIdInput.value = endpoint.id || ""; if (endpointNameInput) endpointNameInput.value = endpoint.name || ""; if (endpointProviderSelect) endpointProviderSelect.value = endpoint.provider || "aoai"; + if (endpointApiTypeSelect) endpointApiTypeSelect.value = endpoint.api_type || "openai"; + if (endpointUrlModeExactInput) { + endpointUrlModeExactInput.checked = (endpoint.connection?.url_mode || "") === "exact"; + } if (endpointUrlInput) endpointUrlInput.value = endpoint.connection?.endpoint || ""; if (endpointProjectInput) endpointProjectInput.value = endpoint.connection?.project_name || ""; setSelectedVersionValue( @@ -404,6 +526,9 @@ function openModalForEndpoint(endpoint) { endpointOpenAiApiVersionCustomInput, endpoint.connection?.openai_api_version || endpoint.connection?.api_version || getDefaultOpenAiApiVersion(endpoint.provider || "aoai") ); + if (endpointAnthropicVersionInput) { + endpointAnthropicVersionInput.value = endpoint.connection?.anthropic_version || DEFAULT_ANTHROPIC_VERSION; + } if (endpointSubscriptionInput) endpointSubscriptionInput.value = endpoint.management?.subscription_id || ""; if (endpointResourceGroupInput) endpointResourceGroupInput.value = endpoint.management?.resource_group || ""; if (endpointAuthTypeSelect) endpointAuthTypeSelect.value = endpoint.auth?.type || "managed_identity"; @@ -458,6 +583,44 @@ function createModelTextInput(modelId, datasetKey, value, disabled = false) { return input; } +function normalizeModelResponseLength(value) { + const valueText = String(value ?? "").trim(); + if (!valueText) { + return ""; + } + if (!/^\d+$/.test(valueText)) { + return null; + } + + const parsedValue = Number.parseInt(valueText, 10); + return parsedValue > 0 ? parsedValue : null; +} + +function getModelResponseLength(model) { + return normalizeModelResponseLength( + model.responseLength + ?? model.response_length + ?? model.maxTokens + ?? model.max_tokens + ?? model.maxCompletionTokens + ?? model.max_completion_tokens + ); +} + +function createModelResponseLengthInput(modelId, value) { + const input = document.createElement("input"); + input.type = "number"; + input.className = "form-control form-control-sm"; + input.min = "1"; + input.step = "1"; + input.placeholder = "Optional"; + input.dataset.responseLengthFor = modelId; + input.id = getModelIconDomId(modelId, "response-length"); + input.value = value || ""; + input.setAttribute("aria-describedby", getModelIconDomId(modelId, "response-length-help")); + return input; +} + function getModelIconDomId(modelId, suffix) { const safeModelId = String(modelId || "model").replace(/[^A-Za-z0-9_-]/g, "-"); return `model-${safeModelId}-${suffix}`; @@ -596,7 +759,7 @@ function renderModalModels(models) { } if (!models || !models.length) { - modelsListEl.innerHTML = "

No models loaded yet.

"; + modelsListEl.innerHTML = "

No models loaded yet.

"; return; } @@ -604,10 +767,14 @@ function renderModalModels(models) { models.forEach((model) => { const wrapper = document.createElement("div"); wrapper.className = "border rounded p-2 mb-2"; - const deploymentName = model.deploymentName || ""; + const requestName = getModelRequestName(model); const modelName = model.modelName || ""; - const displayName = model.displayName || deploymentName; + const displayName = model.displayName || requestName; const description = model.description || ""; + const responseLength = getModelResponseLength(model); + const requestNameLabel = isCustomProvider() && customApiTypeUsesModelName() + ? "Model Name" + : "Deployment Name"; const modelId = model.id || generateId(); model.id = modelId; @@ -624,18 +791,29 @@ function renderModalModels(models) { const fieldsRow = createElement("div", "row g-2 mt-2"); const deploymentCol = createElement("div", "col-md-4"); - deploymentCol.appendChild(createSmallLabel("Deployment")); - deploymentCol.appendChild(createModelTextInput(modelId, "deploymentNameFor", deploymentName)); + deploymentCol.appendChild(createSmallLabel(requestNameLabel)); + deploymentCol.appendChild(createModelTextInput(modelId, "requestModelFor", requestName)); const displayCol = createElement("div", "col-md-4"); displayCol.appendChild(createSmallLabel("Display Name")); displayCol.appendChild(createModelTextInput(modelId, "displayNameFor", displayName)); - const modelNameCol = createElement("div", "col-md-4"); - modelNameCol.appendChild(createSmallLabel("Model Name")); - const modelNameInput = createModelTextInput(modelId, "modelNameFor", modelName, true); - modelNameCol.appendChild(modelNameInput); + const responseLengthCol = createElement("div", "col-md-4"); + const responseLengthLabel = createSmallLabel("Response Length"); + responseLengthLabel.htmlFor = getModelIconDomId(modelId, "response-length"); + responseLengthCol.appendChild(responseLengthLabel); + responseLengthCol.appendChild(createModelResponseLengthInput(modelId, responseLength)); + const responseLengthHelp = createElement("div", "form-text"); + responseLengthHelp.id = getModelIconDomId(modelId, "response-length-help"); + responseLengthHelp.textContent = "Optional output token ceiling for standard chat responses."; + responseLengthCol.appendChild(responseLengthHelp); fieldsRow.appendChild(deploymentCol); fieldsRow.appendChild(displayCol); - fieldsRow.appendChild(modelNameCol); + fieldsRow.appendChild(responseLengthCol); + if (!isCustomProvider()) { + const modelNameCol = createElement("div", "col-md-4"); + modelNameCol.appendChild(createSmallLabel("Model Name")); + modelNameCol.appendChild(createModelTextInput(modelId, "modelNameFor", modelName, true)); + fieldsRow.appendChild(modelNameCol); + } const descriptionWrapper = createElement("div", "mt-2"); descriptionWrapper.appendChild(createSmallLabel("Description")); @@ -650,10 +828,27 @@ function renderModalModels(models) { iconWrapper.appendChild(createSmallLabel("Icon")); iconWrapper.appendChild(createModelIconEditor(model, modelId)); + const actions = createElement("div", "d-flex gap-2 mt-2"); + const testButton = document.createElement("button"); + testButton.type = "button"; + testButton.className = "btn btn-sm btn-outline-secondary"; + testButton.dataset.action = "test-model"; + testButton.dataset.modelId = modelId; + testButton.textContent = "Test Connection"; + const removeButton = document.createElement("button"); + removeButton.type = "button"; + removeButton.className = "btn btn-sm btn-outline-danger"; + removeButton.dataset.action = "remove-model"; + removeButton.dataset.modelId = modelId; + removeButton.textContent = "Remove"; + actions.appendChild(testButton); + actions.appendChild(removeButton); + wrapper.appendChild(checkWrapper); wrapper.appendChild(fieldsRow); wrapper.appendChild(descriptionWrapper); wrapper.appendChild(iconWrapper); + wrapper.appendChild(actions); fragment.appendChild(wrapper); }); @@ -670,31 +865,42 @@ function collectModalModels() { const updated = modalModels.map((model) => ({ ...model })); updated.forEach((model) => { const checkbox = modelsListEl.querySelector(`input[data-model-id="${model.id}"]`); - const deploymentInput = modelsListEl.querySelector(`input[data-deployment-name-for="${model.id}"]`); + const requestModelInput = modelsListEl.querySelector(`input[data-request-model-for="${model.id}"]`); const displayInput = modelsListEl.querySelector(`input[data-display-name-for="${model.id}"]`); const descriptionInput = modelsListEl.querySelector(`textarea[data-description-for="${model.id}"]`); + const responseLengthInput = modelsListEl.querySelector(`input[data-response-length-for="${model.id}"]`); const iconEditor = findModelEditor(model.id); + const responseLength = responseLengthInput ? normalizeModelResponseLength(responseLengthInput.value) : ""; + if (responseLength === null) { + throw new Error("Response length must be a positive whole number."); + } model.enabled = checkbox ? checkbox.checked : model.enabled; - model.deploymentName = deploymentInput ? deploymentInput.value.trim() : model.deploymentName; + setModelRequestName(model, requestModelInput ? requestModelInput.value : getModelRequestName(model)); model.displayName = displayInput ? displayInput.value.trim() : model.displayName; model.icon = iconEditor ? getIconPayload(iconEditor, MODEL_ICON_CONTROL_CONFIG) : model.icon || {}; model.description = descriptionInput ? descriptionInput.value.trim() : model.description; + if (responseLength) { + model.responseLength = responseLength; + } else { + delete model.responseLength; + } }); return updated; } async function testModelConnection(model) { const payload = buildEndpointPayload(); - if (!payload || !model?.deploymentName) { - showToast("Model deployment name is required for testing.", "warning"); + const requestModel = getModelRequestName(model); + if (!payload || !requestModel) { + showToast(`${isCustomProvider() && customApiTypeUsesModelName() ? "Model" : "Deployment"} name is required for testing.`, "warning"); return; } + const testModel = {}; + setModelRequestName(testModel, requestModel); const requestBody = { ...payload, - model: { - deploymentName: model.deploymentName - } + model: testModel }; try { @@ -707,7 +913,15 @@ async function testModelConnection(model) { if (!response.ok) { throw new Error(data.error || "Connection test failed."); } - showToast("Model connection successful.", "success"); + // Report the URL that was actually called. Normalization can rewrite the + // configured endpoint, and that rewrite was previously invisible. + const resolvedUrl = data.resolved?.request_url || ""; + showToast( + resolvedUrl + ? `Model connection successful. Called ${resolvedUrl}` + : "Model connection successful.", + "success" + ); } catch (error) { console.error("Model connection failed", error); showToast(error.message || "Model connection failed.", "danger"); @@ -715,6 +929,10 @@ async function testModelConnection(model) { } async function fetchModels() { + if (isCustomProvider()) { + showToast("Model discovery is unavailable for Custom endpoints. Add models manually.", "warning"); + return; + } const payload = buildEndpointPayload(); if (!payload) { return; @@ -780,6 +998,8 @@ function buildEndpointPayload() { const name = endpointNameInput.value.trim(); const endpoint = endpointUrlInput.value.trim(); const provider = endpointProviderSelect?.value || "aoai"; + const customProvider = isCustomProvider(provider); + const apiType = getCustomApiType(); const projectNameFromEndpoint = isFoundryProvider(provider) ? syncProjectNameFromEndpoint() : ""; const projectName = projectNameFromEndpoint || endpointProjectInput?.value.trim() || ""; const projectApiVersion = getSelectedVersionValue( @@ -794,11 +1014,21 @@ function buildEndpointPayload() { ); const subscriptionId = endpointSubscriptionInput?.value.trim() || ""; const resourceGroup = endpointResourceGroupInput?.value.trim() || ""; - const authType = endpointAuthTypeSelect?.value || "managed_identity"; + const authType = customProvider ? "api_key" : (endpointAuthTypeSelect?.value || "managed_identity"); const existingEndpoint = workspaceEndpoints.find((savedEndpoint) => savedEndpoint.id === endpointId); - if (!name || !endpoint || !openAiApiVersion) { - showToast("Endpoint name, URL, and OpenAI API version are required.", "warning"); + if (!name || !endpoint) { + showToast("Endpoint name and URL are required.", "warning"); + return null; + } + + if (customProvider && !/^https:\/\//i.test(endpoint)) { + showToast("Custom endpoint URLs must use HTTPS.", "warning"); + return null; + } + + if ((!customProvider || customApiTypeRequiresApiVersion(apiType)) && !openAiApiVersion) { + showToast("OpenAI API version is required.", "warning"); return null; } @@ -817,7 +1047,7 @@ function buildEndpointPayload() { return null; } - const auth = { + let auth = { type: authType, managed_identity_type: miTypeSelect?.value || "system_assigned", managed_identity_client_id: miClientIdInput?.value.trim() || "", @@ -829,6 +1059,12 @@ function buildEndpointPayload() { custom_authority: endpointCustomAuthorityInput?.value.trim() || "", foundry_scope: endpointFoundryScopeInput?.value.trim() || "" }; + if (customProvider) { + auth = { + type: "api_key", + api_key: apiKeyInput?.value.trim() || "" + }; + } const hasStoredApiKey = authType === "api_key" && Boolean(existingEndpoint?.has_api_key); const hasStoredClientSecret = authType === "service_principal" && Boolean(existingEndpoint?.has_client_secret); @@ -854,17 +1090,25 @@ function buildEndpointPayload() { return null; } - const management = provider === "aoai" ? { + const management = !customProvider && provider === "aoai" ? { subscription_id: subscriptionId, resource_group: resourceGroup } : {}; - const connection = { - endpoint, - openai_api_version: openAiApiVersion - }; + const connection = { endpoint }; + const versionField = customProvider ? customApiTypeVersionField(apiType) : ""; + if (customProvider && endpointUrlModeExactInput?.checked) { + connection.url_mode = "exact"; + } + if (customProvider && versionField === "api_version") { + connection.api_version = openAiApiVersion; + } else if (customProvider && versionField === "anthropic_version") { + connection.anthropic_version = endpointAnthropicVersionInput?.value.trim() || DEFAULT_ANTHROPIC_VERSION; + } else if (!customProvider) { + connection.openai_api_version = openAiApiVersion; + } - if (isFoundryProvider(provider)) { + if (!customProvider && isFoundryProvider(provider)) { connection.project_api_version = projectApiVersion; if (projectName) { connection.project_name = projectName; @@ -874,6 +1118,7 @@ function buildEndpointPayload() { return { id: endpointId, provider, + ...(customProvider ? { api_type: apiType } : {}), name, connection, management, @@ -881,7 +1126,8 @@ function buildEndpointPayload() { }; } -function saveEndpoint() { +async function saveEndpoint() { + const previousEndpoints = [...workspaceEndpoints]; try { const payload = buildEndpointPayload(); if (!payload) { @@ -899,7 +1145,8 @@ function saveEndpoint() { id: endpointId, name: payload.name, provider: payload.provider, - enabled: true, + ...(payload.api_type ? { api_type: payload.api_type } : {}), + enabled: existingEndpoint ? existingEndpoint.enabled !== false : true, auth: payload.auth, connection: payload.connection, management: payload.management, @@ -915,41 +1162,60 @@ function saveEndpoint() { workspaceEndpoints.push(endpointData); } - persistEndpoints(); + await persistEndpoints(); renderEndpoints(); endpointModal.hide(); showToast("Endpoint saved successfully.", "success"); } catch (error) { + workspaceEndpoints = previousEndpoints; console.error("Error saving endpoint", error); showToast(error.message || "Failed to save endpoint.", "danger"); } } -function persistEndpoints() { - fetch(endpointsApi, { +async function persistEndpoints() { + const response = await fetch(endpointsApi, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ endpoints: workspaceEndpoints }) - }).catch((error) => { - console.error("Failed to save endpoints", error); - showToast("Failed to save endpoints.", "danger"); }); + const data = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(data.error || "Failed to save endpoints."); + } + if (Array.isArray(data.endpoints)) { + workspaceEndpoints = [...data.endpoints]; + } } -function toggleEndpoint(endpointId) { +async function toggleEndpoint(endpointId) { const endpoint = workspaceEndpoints.find((item) => item.id === endpointId); if (!endpoint) { return; } + const previousEnabled = endpoint.enabled; endpoint.enabled = !endpoint.enabled; - persistEndpoints(); - renderEndpoints(); + try { + await persistEndpoints(); + renderEndpoints(); + } catch (error) { + endpoint.enabled = previousEnabled; + console.error("Failed to update endpoint", error); + showToast(error.message || "Failed to update endpoint.", "danger"); + } } -function deleteEndpoint(endpointId) { +async function deleteEndpoint(endpointId) { + const previousEndpoints = workspaceEndpoints; workspaceEndpoints = workspaceEndpoints.filter((item) => item.id !== endpointId); - persistEndpoints(); - renderEndpoints(); + try { + await persistEndpoints(); + renderEndpoints(); + } catch (error) { + workspaceEndpoints = previousEndpoints; + console.error("Failed to delete endpoint", error); + showToast(error.message || "Failed to delete endpoint.", "danger"); + } } function handleTableClick(event) { @@ -979,18 +1245,48 @@ function handleTableClick(event) { function addManualModel() { modalModels = collectModalModels(); - modalModels.push({ + const model = { id: generateId(), - deploymentName: "", - modelName: "", displayName: "", icon: {}, description: "", enabled: true - }); + }; + setModelRequestName(model, ""); + modalModels.push(model); renderModalModels(modalModels); } +function handleModelListClick(event) { + const button = event.target.closest("button[data-action]"); + if (!button) { + return; + } + + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the model.", "danger"); + return; + } + + const modelId = button.dataset.modelId; + const model = modalModels.find((item) => item.id === modelId); + if (!model) { + return; + } + + if (button.dataset.action === "remove-model") { + modalModels = modalModels.filter((item) => item.id !== modelId); + renderModalModels(modalModels); + return; + } + + if (button.dataset.action === "test-model") { + testModelConnection(model); + } +} + function escapeHtml(value) { if (!value) return ""; return value.replace(/[&<>"']/g, (char) => ({ @@ -1054,8 +1350,29 @@ function initialize() { if (endpointProviderSelect) { endpointProviderSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the endpoint provider.", "danger"); + return; + } + syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); updateAuthVisibility(); + }); + } + + if (endpointApiTypeSelect) { + endpointApiTypeSelect.addEventListener("change", () => { + try { + modalModels = collectModalModels(); + } catch (error) { + showToast(error?.message || "Unable to update the API type.", "danger"); + return; + } syncOpenAiApiVersionForProvider(); + renderModalModels(modalModels); + updateAuthVisibility(); }); } @@ -1094,6 +1411,10 @@ function initialize() { if (addModelBtn) { addModelBtn.addEventListener("click", addManualModel); } + + if (modelsListEl) { + modelsListEl.addEventListener("click", handleModelListClick); + } } if (document.readyState === "loading") { diff --git a/application/single_app/static/json/model_capabilities.json b/application/single_app/static/json/model_capabilities.json index d44d19ad7..b2a016d59 100644 --- a/application/single_app/static/json/model_capabilities.json +++ b/application/single_app/static/json/model_capabilities.json @@ -1,6 +1,6 @@ { "$schema": "https://simplechat.local/schemas/model-capabilities.schema.json", - "schemaVersion": 1, + "schemaVersion": 2, "lastUpdated": "2026-08-04", "description": "SimpleChat model capability catalog. Capability flags remain data-only; optional model token-limit fields are consumed by durable tabular batch planning when present.", "capabilityFields": { @@ -15,14 +15,17 @@ "processesBinaryFiles": "Accepts uploaded files or binary/document payloads through the provider API.", "optimizedForCoding": "Documented or positioned for coding, agentic software tasks, code generation, or code understanding.", "toolCalling": "Supports function/tool calling or provider-equivalent tool use.", - "structuredOutput": "Supports structured outputs, JSON-schema outputs, or provider-equivalent constrained structured responses." + "structuredOutput": "Supports structured outputs, JSON-schema outputs, or provider-equivalent constrained structured responses.", + "supportsStreaming": "Supports incremental token streaming for chat responses. SimpleChat wraps models without streaming support so they still deliver through the stream.", + "reasoning": "Performs extended reasoning or thinking before responding." }, "coverageNotes": [ "OpenAI coverage starts at GPT-5.0 model families and includes Azure OpenAI GPT-5.x model IDs that SimpleChat commonly sees through Foundry.", "Claude coverage includes current, legacy, deprecated, and recently retired Claude models that fall within the requested two-year window.", "Meta coverage focuses on public Llama model families with clear model cards for text, vision, and coding support.", "xAI coverage includes Grok chat/coding models plus documented Imagine and Voice model SKUs.", - "Microsoft coverage includes public Phi and MAI model cards with clear capability statements." + "Microsoft coverage includes public Phi and MAI model cards with clear capability statements.", + "Google coverage includes the generally available Gemini chat model tiers that expose generateContent and streamGenerateContent." ], "sources": [ { @@ -167,7 +170,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Frontier GPT-5.6 tier; text and image input with text output."], "sourceIds": ["openai-gpt5-6", "azure-openai-gpt5"] @@ -192,7 +197,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["GPT-5.6 lower-latency tier; Azure catalog documents text and image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -217,7 +224,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["GPT-5.6 smallest tier; Azure catalog documents text and image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -242,7 +251,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents reasoning, Responses API, structured outputs, text and image processing, and tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -267,7 +278,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Azure catalog describes this preview model as Chat Completions/Responses with structured outputs and tools."], "sourceIds": ["azure-openai-gpt5"] @@ -292,7 +305,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus tools."], "sourceIds": ["azure-openai-gpt5"] @@ -317,7 +332,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API, text and image processing, and functions/tools."], "sourceIds": ["azure-openai-gpt5"] @@ -342,7 +359,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus parallel tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -367,7 +386,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus parallel tool calling."], "sourceIds": ["azure-openai-gpt5"] @@ -392,7 +413,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents optimization for Codex CLI and Codex VS Code extension."], "sourceIds": ["azure-openai-gpt5"] @@ -417,7 +440,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog does not list image processing for this entry."], "sourceIds": ["azure-openai-gpt5"] @@ -442,7 +467,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents optimization for Codex CLI and Codex VS Code extension."], "sourceIds": ["azure-openai-gpt5"] @@ -467,7 +494,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents text and image processing plus tools."], "sourceIds": ["azure-openai-gpt5"] @@ -492,7 +521,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog does not list image processing for this entry."], "sourceIds": ["azure-openai-gpt5"] @@ -517,7 +548,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5-1", "azure-openai-gpt5"] @@ -542,7 +575,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Preview chat model; Azure catalog documents tools and structured outputs but not image processing."], "sourceIds": ["azure-openai-gpt5"] @@ -567,7 +602,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -592,7 +629,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -617,7 +656,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only, Codex optimization, and xhigh reasoning effort."], "sourceIds": ["azure-openai-gpt5"] @@ -642,7 +683,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -667,7 +710,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API, text and image processing, and functions/tools."], "sourceIds": ["azure-openai-gpt5"] @@ -692,7 +737,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Azure catalog documents Responses API only and Codex optimization."], "sourceIds": ["azure-openai-gpt5"] @@ -717,7 +764,9 @@ "processesBinaryFiles": true, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI and Azure docs document text and image input with text output."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -742,7 +791,9 @@ "processesBinaryFiles": true, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["OpenAI positions this as fastest and cost-efficient for summarization and classification."], "sourceIds": ["openai-gpt5", "azure-openai-gpt5"] @@ -767,7 +818,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Azure catalog explicitly lists input as text/image and output as text only."], "sourceIds": ["azure-openai-gpt5"] @@ -792,7 +845,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic describes this as next-generation intelligence for long-running agents; current Claude models support text and image input with text output."], "sourceIds": ["anthropic-models"] @@ -817,7 +872,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Invitation-only Project Glasswing model sharing Fable 5 specs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -842,7 +899,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic positions Opus 5 for complex agentic coding and enterprise work."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -867,7 +926,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Anthropic positions Sonnet 5 as a speed/intelligence balance."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -892,7 +953,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -917,7 +980,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -942,7 +1007,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -967,7 +1034,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Opus 4.5 model listed in Anthropic lifecycle docs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -992,7 +1061,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Sonnet 4.x model with text and image input support."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1017,7 +1088,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Current Sonnet 4.5 model listed in Anthropic lifecycle docs."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1042,7 +1115,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Fastest current Claude model; current Claude models support vision."], "sourceIds": ["anthropic-models", "anthropic-deprecations"] @@ -1067,7 +1142,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Deprecated; scheduled retirement listed by Anthropic."], "sourceIds": ["anthropic-deprecations"] @@ -1092,7 +1169,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Recently retired; included for two-year Claude coverage."], "sourceIds": ["anthropic-deprecations"] @@ -1117,7 +1196,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Recently retired; included for two-year Claude coverage."], "sourceIds": ["anthropic-deprecations"] @@ -1142,7 +1223,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1167,7 +1250,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1192,7 +1277,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Retired; included because it falls within the requested two-year Claude window."], "sourceIds": ["anthropic-deprecations"] @@ -1217,7 +1304,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text and image input with multilingual text and code output."], "sourceIds": ["meta-llama4"] @@ -1242,7 +1331,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text and image input with multilingual text and code output."], "sourceIds": ["meta-llama4"] @@ -1267,7 +1358,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card lists multilingual text input and multilingual text/code output; Transformers examples document tool use."], "sourceIds": ["meta-llama33"] @@ -1292,7 +1385,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes text + image input with text output."], "sourceIds": ["meta-llama32-vision"] @@ -1317,7 +1412,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes the 11B and 90B Llama 3.2 Vision sizes as text + image input with text output."], "sourceIds": ["meta-llama32-vision"] @@ -1342,7 +1439,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes Code Llama as text-only input/output designed for code synthesis and understanding."], "sourceIds": ["meta-codellama"] @@ -1367,7 +1466,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["xAI describes Grok 4.5 as text,image -> text and recommends it for code and agentic software tasks."], "sourceIds": ["xai-grok45", "xai-models"] @@ -1392,7 +1493,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI text API pricing; xAI overview documents image input constraints for image-input models."], "sourceIds": ["xai-models"] @@ -1417,7 +1520,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Listed in xAI text API pricing as a reasoning model."], "sourceIds": ["xai-models"] @@ -1442,7 +1547,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI text API pricing as a non-reasoning model."], "sourceIds": ["xai-models"] @@ -1467,7 +1574,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI model pricing; Grok 4.5 page aliases grok-build-latest to Grok 4.5."], "sourceIds": ["xai-models", "xai-grok45"] @@ -1492,7 +1601,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": true, - "structuredOutput": true + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Listed in xAI model pricing as a multi-agent model."], "sourceIds": ["xai-models"] @@ -1517,7 +1628,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI documents modalities as text,image -> image."], "sourceIds": ["xai-imagine-image"] @@ -1542,7 +1655,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI overview lists this as an Imagine image model."], "sourceIds": ["xai-models", "xai-imagine-image"] @@ -1567,7 +1682,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI documents modalities as text,image -> video."], "sourceIds": ["xai-imagine-video"] @@ -1592,7 +1709,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": false, + "reasoning": false }, "notes": ["xAI overview lists this as an Imagine video model."], "sourceIds": ["xai-models", "xai-imagine-video"] @@ -1617,7 +1736,9 @@ "processesBinaryFiles": true, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["xAI Voice API documents speech-to-speech, speech-to-text, and text-to-speech powered by Grok."], "sourceIds": ["xai-voice", "xai-models"] @@ -1642,7 +1763,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card says it processes text, image, and audio inputs, generates text outputs, and supports multi-image or video clip summarization."], "sourceIds": ["microsoft-phi4-multimodal"] @@ -1667,7 +1790,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": true, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card documents instruction following and function calling for a text model."], "sourceIds": ["microsoft-phi4-mini"] @@ -1692,7 +1817,9 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Model card says Phi-4 reasoning is trained for math, science, and coding skills with text input and text output."], "sourceIds": ["microsoft-phi4-reasoning"] @@ -1717,7 +1844,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": true }, "notes": ["Model card positions this compact model for math reasoning, not general multimodal or coding use."], "sourceIds": ["microsoft-phi4-mini"] @@ -1742,7 +1871,9 @@ "processesBinaryFiles": false, "optimizedForCoding": false, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes visual and text input, multi-image comparison, and video clip summarization."], "sourceIds": ["microsoft-phi35-vision"] @@ -1767,10 +1898,282 @@ "processesBinaryFiles": false, "optimizedForCoding": true, "toolCalling": false, - "structuredOutput": false + "structuredOutput": false, + "supportsStreaming": true, + "reasoning": false }, "notes": ["Model card describes broad text generation, reasoning, problem solving, code generation, and code comprehension."], "sourceIds": ["microsoft-mai-ds-r1"] + }, + { + "id": "gemini-3.8-flash", + "provider": "google", + "displayName": "Gemini 3.8 Flash", + "aliases": [], + "family": "gemini-3.8", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Current Gemini 3.8 Flash tier; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.7-flash", + "provider": "google", + "displayName": "Gemini 3.7 Flash", + "aliases": [], + "family": "gemini-3.7", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.7 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.6-flash", + "provider": "google", + "displayName": "Gemini 3.6 Flash", + "aliases": [], + "family": "gemini-3.6", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.6 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.5-flash", + "provider": "google", + "displayName": "Gemini 3.5 Flash", + "aliases": [], + "family": "gemini-3.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.5 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.5-flash-lite", + "provider": "google", + "displayName": "Gemini 3.5 Flash-Lite", + "aliases": [], + "family": "gemini-3.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Cost-optimized Gemini 3.5 Flash-Lite tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-3.1-pro-preview", + "provider": "google", + "displayName": "Gemini 3.1 Pro Preview", + "aliases": ["gemini-3.1-pro"], + "family": "gemini-3.1", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 3.1 Pro preview tier; strongest Gemini 3.1 reasoning and coding tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-pro", + "provider": "google", + "displayName": "Gemini 2.5 Pro", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 2.5 Pro; multimodal input with text output and thinking support."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-flash", + "provider": "google", + "displayName": "Gemini 2.5 Flash", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": true, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Gemini 2.5 Flash; balanced multimodal tier with thinking support."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.5-flash-lite", + "provider": "google", + "displayName": "Gemini 2.5 Flash-Lite", + "aliases": [], + "family": "gemini-2.5", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": true + }, + "notes": ["Cost-optimized Gemini 2.5 Flash-Lite tier."], + "sourceIds": ["google-gemini-api"] + }, + { + "id": "gemini-2.0-flash", + "provider": "google", + "displayName": "Gemini 2.0 Flash", + "aliases": [], + "family": "gemini-2.0", + "lifecycle": "current", + "releaseDate": null, + "capabilities": { + "processesText": true, + "generatesText": true, + "processesImages": true, + "generatesImages": false, + "processesAudio": true, + "generatesAudio": false, + "processesVideo": true, + "generatesVideo": false, + "processesBinaryFiles": true, + "optimizedForCoding": false, + "toolCalling": true, + "structuredOutput": true, + "supportsStreaming": true, + "reasoning": false + }, + "notes": ["Gemini 2.0 Flash; multimodal input with text output."], + "sourceIds": ["google-gemini-api"] } ] } diff --git a/application/single_app/static/json/schemas/model_capabilities.schema.json b/application/single_app/static/json/schemas/model_capabilities.schema.json new file mode 100644 index 000000000..0043f3040 --- /dev/null +++ b/application/single_app/static/json/schemas/model_capabilities.schema.json @@ -0,0 +1,150 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://simplechat.local/schemas/model-capabilities.schema.json", + "title": "SimpleChat model capability catalog", + "description": "Schema for static/json/model_capabilities.json. The catalog is the source of truth for per-model capability answers, so that models are described by data rather than guessed at from their names.", + "type": "object", + "required": ["schemaVersion", "capabilityFields", "models"], + "properties": { + "$schema": { + "type": "string" + }, + "schemaVersion": { + "type": "integer", + "minimum": 2 + }, + "lastUpdated": { + "type": ["string", "null"] + }, + "description": { + "type": "string" + }, + "capabilityFields": { + "type": "object", + "description": "Human-readable description of every capability flag a model record may declare.", + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "coverageNotes": { + "type": "array", + "items": { + "type": "string" + } + }, + "sources": { + "type": "array", + "items": { + "type": "object" + } + }, + "models": { + "type": "array", + "minItems": 1, + "items": { + "$ref": "#/$defs/modelRecord" + } + } + }, + "$defs": { + "modelRecord": { + "type": "object", + "required": [ + "id", + "provider", + "displayName", + "aliases", + "family", + "lifecycle", + "capabilities" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "provider": { + "type": "string", + "minLength": 1 + }, + "displayName": { + "type": "string", + "minLength": 1 + }, + "aliases": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + } + }, + "family": { + "type": "string", + "minLength": 1 + }, + "lifecycle": { + "type": "string", + "enum": [ + "current", + "preview", + "limited-availability", + "legacy", + "deprecated", + "retired" + ] + }, + "releaseDate": { + "type": ["string", "null"] + }, + "capabilities": { + "$ref": "#/$defs/capabilityMap" + }, + "notes": { + "type": "array", + "items": { + "type": "string" + } + }, + "sourceIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "inputTokenLimit": { + "type": ["integer", "null"], + "minimum": 1 + }, + "outputTokenLimit": { + "type": ["integer", "null"], + "minimum": 1 + } + }, + "additionalProperties": false + }, + "capabilityMap": { + "type": "object", + "description": "Every flag is required so that a model is never silently missing a capability answer.", + "required": [ + "processesText", + "generatesText", + "processesImages", + "generatesImages", + "processesAudio", + "generatesAudio", + "processesVideo", + "generatesVideo", + "processesBinaryFiles", + "optimizedForCoding", + "toolCalling", + "structuredOutput", + "supportsStreaming", + "reasoning" + ], + "additionalProperties": { + "type": "boolean" + } + } + } +} diff --git a/application/single_app/templates/_multiendpoint_modal.html b/application/single_app/templates/_multiendpoint_modal.html index ed14026bc..b3277c48a 100644 --- a/application/single_app/templates/_multiendpoint_modal.html +++ b/application/single_app/templates/_multiendpoint_modal.html @@ -21,6 +21,7 @@
Identity se
  • Azure OpenAI: assign Reader plus Cognitive Services OpenAI User on the Azure OpenAI resource when using managed identity or service principal model discovery.
  • Foundry (classic): assign Foundry User, or Azure AI User where older role names still appear, on the target Foundry project or backing resource.
  • New Foundry: use the same Foundry project access as classic Foundry, then select the New Foundry provider and the project endpoint in this modal.
    • +
  • Custom: select an API type, enter an HTTPS endpoint and API key, then add models manually.
  • Provider setup: for Foundry project model endpoints, keep OpenAI API Version at endpoint default v1. Use separate endpoints when Grok, Meta/Llama, DeepSeek, OpenAI-compatible, or other model families need different project settings, auth, or manual deployment rows.
  • API Key: use for inference-only endpoints or APIM paths. Model and Foundry project discovery requires managed identity or service principal RBAC.
    • @@ -38,15 +39,37 @@
    Identity se +
    - For APIM, choose the matching provider with API key auth. If using classic Foundry, use Foundry (classic). If using the application-based runtime, use New Foundry. + Choose Custom for a manually configured API type and model list.
    +
    + + +
    The API type controls request paths, model identifiers, and headers for this Custom endpoint.
    +
    For Azure OpenAI, paste the resource endpoint.
    +
    + + +
    + SimpleChat normally appends /v1 when the URL does not already + say where the API lives. Select this when your gateway serves the API at a + path that cannot be inferred. Test Connection reports the URL actually called. +
    +
    @@ -73,6 +96,11 @@
    Identity se
    For Foundry project endpoints, Project API Version controls discovery and usually stays v1. OpenAI API Version controls the normalized /openai/v1 inference client and should stay Endpoint default (v1); the /v1 path does not allow an api-version query. Split model families into separate endpoints when they need different project settings or auth. Claude deployments are detected from the model name and use the Anthropic messages protocol.
    +
    + + +
    Sent as the anthropic-version request header.
    +
    @@ -144,7 +172,7 @@
    Identity se
    - API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry. + API key authentication is for inference only. Use a managed identity or service principal for model discovery, or use Add Model and enter the deployment name manually exactly as it appears in Foundry.
    @@ -241,6 +269,11 @@
    Provider selection
    Application-based Foundry runtime, New Foundry agents, and project model deployments. Use the New Foundry project endpoint. Set Project API Version for discovery, usually v1, and keep OpenAI API Version at endpoint default v1 for the normalized /openai/v1 inference path. + + Custom + Manually configured OpenAI API, Azure OpenAI API, or Anthropic models. + Use an HTTPS endpoint with API key authentication, select the API type, and add models manually. +
    diff --git a/application/single_app/templates/admin/_panes/extraction.html b/application/single_app/templates/admin/_panes/extraction.html index 0a12f26b3..890fa93e9 100644 --- a/application/single_app/templates/admin/_panes/extraction.html +++ b/application/single_app/templates/admin/_panes/extraction.html @@ -681,14 +681,14 @@
    {% for endpoint in settings.model_endpoints if endpoint.enabled %} {% for m in endpoint.models if m.enabled %} {% if is_vision_capable_model is defined and is_vision_capable_model(m) %} - {% set option_value = m.deploymentName %} + {% set option_value = m.modelName or m.deploymentName %} {% endif %} {% endfor %} diff --git a/application/single_app/templates/admin/_panes/model-endpoints.html b/application/single_app/templates/admin/_panes/model-endpoints.html index 5c9683ad4..ca2accd85 100644 --- a/application/single_app/templates/admin/_panes/model-endpoints.html +++ b/application/single_app/templates/admin/_panes/model-endpoints.html @@ -9,7 +9,7 @@
    Model Endpoints

    - Manage multiple AI model endpoints (Azure OpenAI and Azure AI Foundry). When enabled, model selection in chat is driven by these endpoints. + Manage Azure OpenAI, Foundry, New Foundry, and Custom model endpoints. When enabled, model selection in chat is driven by these endpoints.

    {% if not settings.enable_multi_model_endpoints %} @@ -19,6 +19,55 @@
    {% endif %} +
    + + +
    + Enable this for on-premises inference. It permits IP addresses, short host names, and hosts resolving to private ranges. Loopback, link-local, and cloud metadata addresses are always rejected, and every address is revalidated when the connection is made. +
    +
    + +
    + + +
    + Prompts and API keys travel unencrypted. Only for an isolated network where TLS cannot be terminated. Requires private Custom endpoint hosts to also be enabled. +
    +
    + +
    + + +
    + Custom endpoints trust only public certificate authorities, and deliberately ignore ambient environment variables so nothing can silently widen that trust. Name a PEM bundle here to trust an internally issued certificate, which an on-premises gateway normally uses. +
    +
    + {% if settings.enable_semantic_kernel %}
    diff --git a/application/single_app/templates/base.html b/application/single_app/templates/base.html index f5899222a..fc59212e7 100644 --- a/application/single_app/templates/base.html +++ b/application/single_app/templates/base.html @@ -691,6 +691,13 @@