Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 16 additions & 1 deletion codec_bridges.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,21 @@ def call_llm(channel, text, llm_cfg, conversation_history=None,
bridge degradation). `chat_template_kwargs` is filtered out of
`llm_cfg["kwargs"]` so codec_llm's enable_thinking=False is preserved."""
import codec_llm
if system_prompt_override:
# A persona model (Fréd / M Corpus / Mwen / the 8B reformat voice — config
# extra_models[].system_prompt*) owns the turn on every surface, so when it
# is the active model the bridge speaks as it, not as the channel persona,
# and applies its sampling. This is what makes "switch the model in CODEC,
# then use it on Telegram" behave the same as the dashboard chat.
_persona = {"system_prompt": None, "sampling": {}}
try:
import codec_models
_persona = codec_models.model_extras(llm_cfg.get("model"))
except Exception:
pass

if _persona.get("system_prompt"):
sys_prompt = _persona["system_prompt"] # persona wins over channel
elif system_prompt_override:
sys_prompt = system_prompt_override
else:
now_str = datetime.now().strftime("%A %B %d, %Y at %H:%M")
Expand All @@ -125,6 +139,7 @@ def call_llm(channel, text, llm_cfg, conversation_history=None,
messages.append({"role": "user", "content": text})

extra = {k: v for k, v in llm_cfg["kwargs"].items() if k != "chat_template_kwargs"}
extra.update(_persona.get("sampling") or {}) # per-model sampling
content = codec_llm.call(
messages, base_url=llm_cfg["base_url"], model=llm_cfg["model"],
api_key=llm_cfg["api_key"], max_tokens=1500, temperature=0.7,
Expand Down
6 changes: 5 additions & 1 deletion codec_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,7 +211,7 @@ def model_extras(model_id: Optional[str] = None,
"""
cfg = config if config is not None else _load_config()
mid = model_id or get_active(cfg)
out: Dict[str, Any] = {"system_prompt": None, "sampling": {}}
out: Dict[str, Any] = {"system_prompt": None, "sampling": {}, "no_think": False}
for em in (cfg.get("extra_models") or []):
if not isinstance(em, dict) or em.get("id") != mid:
continue
Expand All @@ -229,6 +229,10 @@ def model_extras(model_id: Optional[str] = None,
if isinstance(sampling, dict):
out["sampling"] = {k: v for k, v in sampling.items()
if isinstance(k, str) and isinstance(v, (int, float, str, bool))}
# A reformat/persona fine-tune that should answer directly (no <think>
# block) declares `"no_think": true`. Chat forces enable_thinking=False;
# the bridge path is already thinking-off via codec_llm.
out["no_think"] = bool(em.get("no_think", False))
break
return out

Expand Down
9 changes: 9 additions & 0 deletions routes/chat.py
Original file line number Diff line number Diff line change
Expand Up @@ -1147,6 +1147,15 @@ async def _skill_stream():
# Dashboard chat & Vibe benefit from thinking mode (deeper answers).
# Frontend can send thinking=false to override for speed.
thinking = body.get("thinking", True)
# A reformat/persona model declaring no_think answers directly — force
# thinking off regardless of the Think toggle (a <think> block would eat
# the budget and delay the rewrite).
try:
import codec_models as _cm_nt
if _cm_nt.model_extras(model).get("no_think"):
thinking = False
except Exception as _e:
log.debug("no_think lookup failed: %s", _e)
# Train-of-thought reveal: when the frontend's Thoughts toggle is ON,
# also stream the model's <think> reasoning as separate SSE `think`
# events. Off by default → no think frames → identical to before.
Expand Down
15 changes: 14 additions & 1 deletion tests/test_model_switching.py
Original file line number Diff line number Diff line change
Expand Up @@ -400,8 +400,21 @@ def test_model_extras_reads_persona_and_sampling(cfg, tmp_path):
x = codec_models.model_extras()
assert x["system_prompt"].startswith("You are Fred.")
assert x["sampling"] == {"repetition_penalty": 1.2, "repetition_context_size": 512}
assert x["no_think"] is False # not declared → defaults off
# a model with no entry gets no override
assert codec_models.model_extras("mlx-community/A") == {"system_prompt": None, "sampling": {}}
assert codec_models.model_extras("mlx-community/A") == {
"system_prompt": None, "sampling": {}, "no_think": False}


def test_model_extras_reads_no_think_flag(cfg, tmp_path):
"""A reformat model declares no_think:true so chat forces thinking off."""
import json as _json
data = _json.loads(cfg.read_text())
data["llm_model"] = "/abs/reformat-8b"
data["extra_models"] = [{"id": "/abs/reformat-8b", "label": "Reformat",
"system_prompt": "Rewrite it.", "no_think": True}]
cfg.write_text(_json.dumps(data))
assert codec_models.model_extras()["no_think"] is True


def test_model_extras_survives_missing_prompt_file(cfg):
Expand Down