From 290aea9562fb60fe16b9836690f06030eb16766a Mon Sep 17 00:00:00 2001 From: JustVugg Date: Wed, 4 Nov 2026 19:49:19 +0100 Subject: [PATCH] coli tune: the qwen36 family's prompts rendered by the gateway, each checkpoint with its own template Tune rendered every qwen36-family checkpoint's prompts with one template that ends in an open . The family holds checkpoints with different templates (Qwen3.6, Qwen3-Coder, Flint, Muse Spark, LFM3, the dense Qwen3), and Qwen3-Coder has no thinking mode: after that it ended its turn on the first token, the engine reported no decode speed and the tune failed at its baseline (#2590). Measured on Qwen3-Coder-30B-A3B int4 with CUDA (RTX PRO 4000), the same message as a raw completion: tune's form 0 tokens, finish stop; the gateway's rendering 16 tokens of an answer. openai_server.tune_chat_prompt renders one user message as the gateway does for a chat request that sets nothing: the checkpoint's template (detect_chat_flavor) and its default reasoning (default_reasoning_effort and template_thinks, factored out of the request handler, which uses them). The qwen36 family says tune_renders_chat and `coli tune` uses it there; the other families keep their template. On the RTX PRO 4000: `coli tune` on Qwen3-Coder-30B-A3B fails at its baseline on dev and runs to the end here (baseline 2 requests, omp-14, omp-7, baseline kept). On Qwen3.6-35B-A3B Q4_K_M both run to the end; dev times a request that thinks (114.2 tok/s), this branch the one coli serve is sent by default, without (121.8 tok/s). --- CHANGELOG.md | 8 ++++ c/coli | 6 ++- c/family_registry.py | 9 +++- c/openai_server.py | 46 +++++++++++++++++---- c/tests/test_family_registry.py | 8 ++-- c/tests/test_tune_chat_prompt.py | 71 ++++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 16 deletions(-) create mode 100644 c/tests/test_tune_chat_prompt.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f5f831a16..08903094e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -122,6 +122,14 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). ### Fixed +- **`coli tune` on Qwen3-Coder: "engine reported no decode speed"** (#2590). Tune put every + qwen36-family checkpoint's prompts in one form, ending in an open ``; Qwen3-Coder has + no thinking mode and ended its turn on the first token there (0 tokens, measured on + Qwen3-Coder-30B-A3B with CUDA), so the baseline had no speed and the tune stopped. The + family's prompts are now rendered by the gateway (`openai_server.tune_chat_prompt`), each + checkpoint with its own template and the default reasoning a chat request gets: the request + tune times is the one `coli serve` is sent. Qwen3-Coder tunes to the end; Qwen3.6 is timed + without thinking, as it is served unless asked. - **Qwen3.8 with a card on an ARM host (DGX Spark, GH200, Jetson): the build.** The CUDA chain groups a verify's pooled slots of one expert by `GQ_FUSED_ROWS`, which only the x86 AVX2 kernels defined, so `qwen38` with `CUDA=1` did not compile without them. Every build has it diff --git a/c/coli b/c/coli index a4641d561..5b246a9cb 100755 --- a/c/coli +++ b/c/coli @@ -1646,7 +1646,11 @@ def cmd_tune(a): raw_prompts=[a.prompt,*(a.rotate_prompt or [_TUNE_ROTATION_PROMPT])] if len(set(raw_prompts)) < 2: sys.exit(f"{C.yel}tuning rotation needs at least two distinct prompts{C.r}") - prompts=[tuning_replay_prompt(family,item) for item in raw_prompts] + if family.tune_renders_chat: # the checkpoint's own template, as coli serve renders it + from openai_server import tune_chat_prompt + prompts=[tune_chat_prompt(a.model,item) for item in raw_prompts] + else: + prompts=[tuning_replay_prompt(family,item) for item in raw_prompts] prompt=prompts[0] banner("tune ยท measured execution profile", model=a.model) if arch=="glm": diff --git a/c/family_registry.py b/c/family_registry.py index d907b53eb..03ed4db39 100644 --- a/c/family_registry.py +++ b/c/family_registry.py @@ -133,6 +133,9 @@ class FamilyDescriptor: has_gateway_adapter: bool = False has_cli_adapter: bool = False tune_prompt_template: str = "{prompt}" + # `coli tune` renders its prompts through the gateway (openai_server.tune_chat_prompt) + # instead of tune_prompt_template: the family's checkpoints ship different templates + tune_renders_chat: bool = False # The environment variable the engine reads for extra shard folders, a model # split across drives (st_init_multi: the shards of every listed folder, by # file name). The launcher looks there too before it calls a model incomplete @@ -1589,8 +1592,10 @@ def _qwen38_resident_inventory(name, size, _config, dtype=None): # the answer. False gives the user "use coli chat or coli serve", which # is true and actionable; chat/serve/web all work through the gateway. has_cli_adapter=False, - tune_prompt_template=( - "<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n\n"), + # Qwen3.6, Qwen3-Coder, Flint, Muse Spark, LFM3 and the dense Qwen3 each ship their + # own template (detect_chat_flavor): one fixed form put Qwen3-Coder, which has no + # thinking mode, after an open , and it answered nothing (#2590) + tune_renders_chat=True, ), FamilyDescriptor( id="qwen38", diff --git a/c/openai_server.py b/c/openai_server.py index 61bf4bd38..d3827e6d9 100644 --- a/c/openai_server.py +++ b/c/openai_server.py @@ -4768,6 +4768,42 @@ def resolve_generation_prompt(messages, body): return False +def default_reasoning_effort(): + """The reasoning_effort of a chat request that names neither reasoning_effort nor + enable_thinking: Qwen3.8's and Muse Spark's templates think at xhigh unless told, + MiMo-V2.6's and LFM3's think too; the other families keep the older opt-in default + (none), which COLI_THINK=1 turns on for all of them.""" + if chat_flavor() in ("qwen38", "muse"): + return "xhigh" + if ARCH == "mimo" or chat_flavor() == "lfm3" or os.environ.get("COLI_THINK", "0") == "1": + return "high" + return None + + +def template_thinks(): + """False for the templates with no thinking mode at all: Qwen3-Coder's (no + anywhere) and OLMoE's (enable_thinking accepted but unused).""" + return not (ARCH == "olmoe" or chat_flavor() == "qwen3_coder") + + +def tune_chat_prompt(model, text): + """TEXT as one user message, rendered as this gateway renders a chat request for MODEL + that sets nothing else: the checkpoint's own template (detect_chat_flavor) and its + default reasoning. `coli tune` measures with it, so the request it times is the one + `coli serve` is sent. A family-wide template opened for every qwen36 + checkpoint, and Qwen3-Coder, which has no thinking mode, ended its turn on the first + token there: no decode speed, and the tune failed (#2590). Qwen3.6 with thinking off + gets the template's pre-closed block, a state it was trained on (the bare + `assistant\n` is not, render_chat_qwen). Sets ARCH and CHAT_FLAVOR as a server does.""" + global ARCH, CHAT_FLAVOR + family = resolve_model(model).descriptor + ARCH = family.id + CHAT_FLAVOR = detect_chat_flavor(family.id, model) + effort = default_reasoning_effort() + thinking = effort not in (None, "none") and template_thinks() + return render_chat_for_arch([{"role": "user", "content": text}], thinking, effort) + + def render_chat_for_arch(messages, enable_thinking=False, reasoning_effort=None, tools=None, tool_choice=None, audio_out=None, add_generation_prompt=True, preserve_thinking=False): @@ -9675,17 +9711,11 @@ def chat_completion(self, body, request_id): # nor enable_thinking (a global switch, like the old server's --think). An explicit # client value always wins. Default off => exact OpenAI-standard behavior. if reasoning_effort is None and "enable_thinking" not in body: - # Qwen3.8's official template defaults to enabled xhigh thinking; - # preserve the older opt-in default for the other families. - if chat_flavor() in ("qwen38", "muse"): # Muse Spark's template: xhigh unless told - reasoning_effort = "xhigh" - elif ARCH == "mimo" or chat_flavor() == "lfm3" or os.environ.get("COLI_THINK", "0") == "1": - # MiMo-V2.6's and LFM3's templates think unless told not to (enable_thinking false) - reasoning_effort = "high" + reasoning_effort = default_reasoning_effort() enable_thinking = body.get("enable_thinking", reasoning_effort not in (None, "none")) if not isinstance(enable_thinking, bool): raise APIError(400, "`enable_thinking` must be a boolean.", "enable_thinking") - if (ARCH == "olmoe" or chat_flavor() == "qwen3_coder") and enable_thinking: + if not template_thinks() and enable_thinking: # Qwen3-Coder's template has no thinking mode either (no anywhere). # OLMoE's template has no thinking mode (render_chat_olmoe: "accepted # but unused"), so the engine never emits /. Left on, diff --git a/c/tests/test_family_registry.py b/c/tests/test_family_registry.py index 8ba456690..b40773197 100644 --- a/c/tests/test_family_registry.py +++ b/c/tests/test_family_registry.py @@ -1145,11 +1145,9 @@ def test_tuning_replay_prompts_are_registry_owned(self): "inkling": "<|user|>hello {world}<|assistant|>", "kimi": "K3CHAT1\nM user 13\nhello {world}G 0\n\n", "olmoe": "<|user|>\nhello {world}\n<|assistant|>\n", - # Qwen3.6's generation prompt MUST open : the model was - # never trained on the bare "assistant\\n" state and greedy argmax - # there lands on an EOS special (measured gen=0). - "qwen36": "<|im_start|>user\nhello {world}<|im_end|>\n" - "<|im_start|>assistant\n\n", + # qwen36's checkpoints ship different templates: `coli tune` renders + # through the gateway (tune_renders_chat, tests/test_tune_chat_prompt.py) + "qwen36": "hello {world}", "qwen38": "<|im_start|>system\nReasoning effort is set to xhigh. Please think carefully " "through the task, validate key assumptions, consider plausible alternatives, " "and prioritize correctness, consistency, and clarity in the final answer." diff --git a/c/tests/test_tune_chat_prompt.py b/c/tests/test_tune_chat_prompt.py new file mode 100644 index 000000000..27115d26c --- /dev/null +++ b/c/tests/test_tune_chat_prompt.py @@ -0,0 +1,71 @@ +"""`coli tune` times the request `coli serve` is sent (#2590). + +The qwen36 family's checkpoints ship different chat templates (Qwen3.6, Qwen3-Coder, Flint, +Muse Spark, LFM3, the dense Qwen3), and tune used one fixed form for all of them, with an +open . Qwen3-Coder has no thinking mode: after that it ended its turn on the +first token (measured on Qwen3-Coder-30B-A3B int4, CUDA: 0 tokens, where the gateway's own +rendering of the same message gets an answer), the engine reported no decode speed and the +tune failed at its baseline. tune_chat_prompt renders through the gateway instead, so each +checkpoint gets its own template and the default reasoning a chat request gets. +""" +import json +import os +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +C_DIR = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(C_DIR)) +import openai_server # noqa: E402 +from family_registry import FAMILIES # noqa: E402 + +CODER_TEMPLATE = C_DIR / "tests" / "fixtures" / "qwen3_coder_chat_template.jinja" + + +class TuneChatPrompt(unittest.TestCase): + def setUp(self): + self.saved = (openai_server.ARCH, openai_server.CHAT_FLAVOR) + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def tearDown(self): + openai_server.ARCH, openai_server.CHAT_FLAVOR = self.saved + + def checkpoint(self, model_type, template=None): + d = Path(self.tmp.name) / model_type + d.mkdir() + (d / "config.json").write_text(json.dumps({"model_type": model_type}), encoding="utf-8") + if template is not None: + (d / "chat_template.jinja").write_bytes(template.read_bytes()) + return str(d) + + def test_qwen3_coder_gets_its_template_without_think(self): + model = self.checkpoint("qwen3_moe", CODER_TEMPLATE) + prompt = openai_server.tune_chat_prompt(model, "Write a palindrome check.") + self.assertNotIn("", prompt) + self.assertTrue(prompt.endswith("Write a palindrome check.<|im_end|>\n<|im_start|>assistant\n"), prompt) + openai_server.ARCH, openai_server.CHAT_FLAVOR = "qwen36", "qwen3_coder" + self.assertEqual(prompt, openai_server.render_chat_qwen3_coder( + [{"role": "user", "content": "Write a palindrome check."}])) + + def test_qwen36_gets_the_closed_block_a_chat_request_gets(self): + model = self.checkpoint("qwen3_5_moe") + with mock.patch.dict(os.environ, {"COLI_THINK": "0"}): + prompt = openai_server.tune_chat_prompt(model, "hello") + self.assertEqual(prompt, "<|im_start|>user\nhello<|im_end|>\n<|im_start|>assistant\n\n\n\n\n") + + def test_coli_think_opens_the_block_for_qwen36_and_not_for_the_coder(self): + with mock.patch.dict(os.environ, {"COLI_THINK": "1"}): + self.assertTrue(openai_server.tune_chat_prompt(self.checkpoint("qwen3_5_moe"), "hello") + .endswith("<|im_start|>assistant\n\n")) + self.assertNotIn("", openai_server.tune_chat_prompt( + self.checkpoint("qwen3_moe", CODER_TEMPLATE), "hello")) + + def test_only_the_qwen36_family_renders_through_the_gateway(self): + self.assertEqual([f.id for f in FAMILIES if f.tune_renders_chat], ["qwen36"]) + + +if __name__ == "__main__": + unittest.main()