Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,16 @@ KIMI_API_KEY=
KIMI_BASE_URL=https://api.moonshot.ai/v1
KIMI_K2_6_MODEL=kimi-k2.6
KIMI_K3_MODEL=kimi-k3
DEEPSEEK_API_KEY=
DEEPSEEK_BASE_URL=https://api.deepseek.com

# UI allowlist and backend task policies. A user's UI choice applies to ordinary chat.
# Claim review, MCP tool routing, and vision can be pinned independently by the backend.
CHAT_MODEL=qwen-plus
CHAT_MODELS=qwen-plus,qwen-max,qwen3-max,gpt-5-mini,claude-sonnet-4,kimi-k2.6,kimi-k3
CHAT_MODELS=qwen-plus,qwen-max,qwen3.8-max,deepseek-v4-pro,gpt-5-mini,claude-sonnet-4,kimi-k2.6,kimi-k3
CLAIM_REVIEW_MODEL=qwen-max
TOOL_MODEL=qwen-plus
VISION_MODEL=qwen-vl-plus
VISION_MODEL=qwen3-vl-plus
VISION_PROVIDER=qwen

# Storage provider:
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:

env:
CHAT_MODEL: qwen-plus
CHAT_MODELS: qwen-plus,qwen-max,qwen3-max
CHAT_MODELS: qwen-plus,qwen-max,qwen3.8-max,deepseek-v4-pro
EMBEDDING_API_KEY: ci-placeholder
EMBEDDING_BASE_URL: https://example.invalid/v1
DATABASE_URL: postgresql://supportagent:supportagent@localhost:5432/supportagent
Expand Down
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ language, p95 latency and cost per request from live token usage:

```bash
python -m supportagent.evaluation.online_runner \
--models qwen3-max,kimi-k2.6,claude-sonnet-4 \
--models qwen3.8-max,kimi-k2.6,claude-sonnet-4 \
--trials 3 --min-pass-rate 0.75 --confirm-live
```

Expand Down Expand Up @@ -470,7 +470,7 @@ Start with two cases and one trial per model:

```bash
python -m supportagent.evaluation.online_runner \
--models qwen3-max,kimi-k2.6,claude-sonnet-4 \
--models qwen3.8-max,kimi-k2.6,claude-sonnet-4 \
--cases household-standard-coverage,weather-out-of-domain-refusal \
--trials 1 \
--confirm-live
Expand All @@ -480,7 +480,7 @@ After the smoke run succeeds, run the interview benchmark with three trials:

```bash
python -m supportagent.evaluation.online_runner \
--models qwen3-max,kimi-k2.6,claude-sonnet-4 \
--models qwen3.8-max,kimi-k2.6,claude-sonnet-4 \
--trials 3 \
--min-pass-rate 0.75 \
--confirm-live
Expand Down Expand Up @@ -514,7 +514,7 @@ for the configured account, and run:

```bash
python -m supportagent.evaluation.online_runner \
--models qwen3-max,kimi-k2.6,claude-sonnet-4 \
--models qwen3.8-max,kimi-k2.6,claude-sonnet-4 \
--trials 3 \
--pricing evals/model_pricing.local.json \
--confirm-live
Expand Down
6 changes: 4 additions & 2 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,13 @@ services:
KIMI_BASE_URL: ${KIMI_BASE_URL:-https://api.moonshot.ai/v1}
KIMI_K2_6_MODEL: ${KIMI_K2_6_MODEL:-kimi-k2.6}
KIMI_K3_MODEL: ${KIMI_K3_MODEL:-kimi-k3}
DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-}
DEEPSEEK_BASE_URL: ${DEEPSEEK_BASE_URL:-https://api.deepseek.com}
CHAT_MODEL: ${CHAT_MODEL:-qwen-plus}
CHAT_MODELS: ${CHAT_MODELS:-qwen-plus,qwen-max,qwen3-max,gpt-5-mini,claude-sonnet-4,kimi-k2.6,kimi-k3}
CHAT_MODELS: ${CHAT_MODELS:-qwen-plus,qwen-max,qwen3.8-max,deepseek-v4-pro,gpt-5-mini,claude-sonnet-4,kimi-k2.6,kimi-k3}
CLAIM_REVIEW_MODEL: ${CLAIM_REVIEW_MODEL:-qwen-max}
TOOL_MODEL: ${TOOL_MODEL:-qwen-plus}
VISION_MODEL: ${VISION_MODEL:-qwen-vl-plus}
VISION_MODEL: ${VISION_MODEL:-qwen3-vl-plus}
VISION_PROVIDER: ${VISION_PROVIDER:-qwen}
FRONTEND_URL: ${FRONTEND_URL:?set FRONTEND_URL}
MICROSOFT_CLIENT_ID: ${MICROSOFT_CLIENT_ID:-}
Expand Down
22 changes: 18 additions & 4 deletions src/backend/supportagent/llm/registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,12 +39,12 @@ def _base_profiles() -> dict[str, ModelProfile]:
description="Higher-quality Qwen model for complex questions.",
),
ModelProfile(
id="qwen3-max",
label="Qwen3 Max",
id="qwen3.8-max",
label="Qwen3.8 Max",
provider="qwen",
provider_model="qwen3-max",
provider_model="qwen3.8-max",
capabilities=frozenset({"text", "tools"}),
description="Advanced Qwen model for demanding agent tasks.",
description="Latest Qwen model for demanding agent tasks.",
),
ModelProfile(
id="gpt-5-mini",
Expand Down Expand Up @@ -81,6 +81,14 @@ def _base_profiles() -> dict[str, ModelProfile]:
capabilities=frozenset({"text", "tools"}),
description="Latest Moonshot model for demanding agent tasks.",
),
ModelProfile(
id="deepseek-v4-pro",
label="DeepSeek V4 Pro",
provider="deepseek",
provider_model="deepseek-v4-pro",
capabilities=frozenset({"text", "tools"}),
description="DeepSeek model for reasoning and agentic tasks.",
),
]
vision_model = os.environ.get("VISION_MODEL", "").strip()
if vision_model and vision_model not in {profile.id for profile in profiles}:
Expand Down Expand Up @@ -127,6 +135,12 @@ def get_provider_settings(provider: ProviderName) -> ProviderSettings:
api_key=os.environ.get("ANTHROPIC_API_KEY", ""),
base_url=os.environ.get("ANTHROPIC_BASE_URL") or None,
)
if provider == "deepseek":
return ProviderSettings(
name=provider,
api_key=os.environ.get("DEEPSEEK_API_KEY", ""),
base_url=os.environ.get("DEEPSEEK_BASE_URL") or "https://api.deepseek.com",
)
return ProviderSettings(
name=provider,
api_key=os.environ.get("KIMI_API_KEY", ""),
Expand Down
2 changes: 1 addition & 1 deletion src/backend/supportagent/llm/schemas.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from dataclasses import dataclass, field
from typing import Any, Literal

ProviderName = Literal["qwen", "openai", "anthropic", "kimi"]
ProviderName = Literal["qwen", "openai", "anthropic", "kimi", "deepseek"]
LLMTask = Literal["chat", "claim_review", "tool", "vision"]
ModelCapability = Literal["text", "tools", "vision"]

Expand Down
51 changes: 51 additions & 0 deletions tests/unit/llm/test_llm_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from supportagent.llm.registry import (
ModelConfigurationError,
get_model_options,
get_provider_settings,
resolve_model,
)

Expand All @@ -15,6 +16,8 @@ def clear_model_environment(monkeypatch):
"CLAIM_REVIEW_MODEL",
"DASHSCOPE_API_KEY",
"DASHSCOPE_BASE_URL",
"DEEPSEEK_API_KEY",
"DEEPSEEK_BASE_URL",
"EMBEDDING_API_KEY",
"EMBEDDING_BASE_URL",
"KIMI_API_KEY",
Expand Down Expand Up @@ -130,3 +133,51 @@ def test_unknown_model_in_allowlist_is_rejected(monkeypatch):

with pytest.raises(ModelConfigurationError, match="Unknown model ids"):
get_model_options()


def test_deepseek_uses_its_own_api_key(monkeypatch):
clear_model_environment(monkeypatch)
monkeypatch.setenv("CHAT_MODELS", "qwen-plus,deepseek-v4-pro")
monkeypatch.setenv("DASHSCOPE_API_KEY", "qwen-key")
monkeypatch.setenv("DASHSCOPE_BASE_URL", "https://qwen.example/v1")
monkeypatch.setenv("DEEPSEEK_API_KEY", "deepseek-key")

selected = resolve_model("deepseek-v4-pro", task="chat")
settings = get_provider_settings("deepseek")

assert selected.provider == "deepseek"
assert selected.provider_model == "deepseek-v4-pro"
assert settings.api_key == "deepseek-key"
assert settings.base_url == "https://api.deepseek.com"


def test_deepseek_is_hidden_without_its_api_key(monkeypatch):
clear_model_environment(monkeypatch)
monkeypatch.setenv("CHAT_MODELS", "qwen-plus,deepseek-v4-pro")
monkeypatch.setenv("DASHSCOPE_API_KEY", "qwen-key")
monkeypatch.setenv("DASHSCOPE_BASE_URL", "https://qwen.example/v1")

options = get_model_options()

assert [option.id for option in options] == ["qwen-plus"]


def test_latest_qwen_max_is_available(monkeypatch):
clear_model_environment(monkeypatch)
monkeypatch.setenv("CHAT_MODELS", "qwen-plus,qwen3.8-max")
monkeypatch.setenv("DASHSCOPE_API_KEY", "qwen-key")
monkeypatch.setenv("DASHSCOPE_BASE_URL", "https://qwen.example/v1")

selected = resolve_model("qwen3.8-max", task="chat")

assert selected.provider_model == "qwen3.8-max"


def test_retired_qwen3_max_is_rejected(monkeypatch):
clear_model_environment(monkeypatch)
monkeypatch.setenv("CHAT_MODELS", "qwen-plus,qwen3-max")
monkeypatch.setenv("DASHSCOPE_API_KEY", "qwen-key")
monkeypatch.setenv("DASHSCOPE_BASE_URL", "https://qwen.example/v1")

with pytest.raises(ModelConfigurationError, match="qwen3-max"):
get_model_options()
Loading